filebeat安装、配置及测试

一、filebeat安装、配置及测试

1、安装filebeat

# yum install filebeat-6.6.1-x86_64.rpm

2、配置filebeat收集系统日志输出到文件中（/etc/filebeat/filebeat.yml）

filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/*.log
    - /var/log/messages
  exclude_lines: ["^DBG","^$"]
  document_type: system-log-5612
output.file:
  path: "/tmp"
  filename: "filebeat.txt"

3、启动filebeat服务

systemctl start filebeat

4、向系统（/var/log/messages）日志插入数据，然后通过查看filebeat.txt文件是是否收集到了数据。

5、配置filebeat收集系统日志输出到redis中（/etc/filebeat/filebeat.yml）

# grep -v "#" /etc/filebeat/filebeat.yml | grep -v "^$"
filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/*.log
    - /var/log/messages
  exclude_lines: ["^DBG","^$"]
  document_type: system-log-5612
output.redis:
  hosts: "192.168.56.12"
  db: "3"
  port: "6379"
  password: "123456"
  key: "system-log-5612"
  
# systemctl restart filebeat
# 向/var/log/messages中插入数据

# redis中验证数据是否存在

6、将redis中存放的系统日志输出到elasticsearch中

# cat redis-elasticsearch.conf
input {
    redis {
        data_type => "list"
        host => "192.168.56.12"
        db => "3"
        port => "6379"
        password => "123456"
        key => "system-log-5612"    
    }
}

output {
    elasticsearch {
        hosts => ["192.168.56.11:9200"]
        index => "system-log-5612-%{+YYYY.MM.dd}"
    }
}

# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-elasticsearch.conf -t
# systemctl restart logstash

7、测试

# echo "aaaaaaaaaaaa" >> /var/log/messages              
# echo "bbbbbbbbbbbb" >> /var/log/messages

二、filebeat实验配置信息

环境信息：

服务器描述	IP地址	应用
web服务器	192.168.56.100	nginx、filebeat
redis服务器	192.168.56.12	redis
logstash服务器端	192.168.56.11	logstash
elasticsearch服务集群	192.168.56.15/16	java、elasticsearch
kibana服务器	192.168.56.12	kibana、nginx反向代理认证

 

1、filebeat配置文件，filebeat收集nginx日志并输出到redis数据库服务器

# grep -v "#" /etc/filebeat/filebeat.yml |grep -v "^$"

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
output.redis:
    hosts: ["192.168.56.12"]
    port: 6379
    key: "nginx-log"

2、logstash server端配置文件，从redis中读取数据输出到elasticsearch服务中

# cat /etc/logstash/conf.d/redis-es-logstash-nginx.conf

input {
    redis {
        data_type => "list"
        host => "192.168.56.12"
        db => "0"
        port => "6379"
        key => "nginx-log"
    }
}

output {
    elasticsearch {
        hosts => ["192.168.56.15:9200"]
        index => "nginx-log-%{+YYYY.MM.dd}"
    }
}

3、kibana配置文件

# grep -Evi "^#|^$" /etc/kibana/kibana.yml
server.port: 5601
server.host: "192.168.56.12"
elasticsearch.hosts: ["http://192.168.56.15:9200","http://192.168.56.16:9200"]

4、nginx 反向代理kibana配置文件

# cat /etc/nginx/nginx.conf

worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    log_format access_log_json '{"user_ip":"$http_x_forwarded_for","lan_ip":"$remote_addr","log_time":"$time_iso8601","user_rqp":"$request","http_code":"$status","body_bytes_sent":"$body_bytes_sent","req_time":"$request_time","user_ua":"$http_user_agent"}';
    sendfile        on;
    keepalive_timeout  65;
    include conf.d/*.conf;
}


# cat /etc/nginx/conf.d/http-www.conf
server {
    listen       81;
    server_name  localhost;
    auth_basic "User Authentication";
    auth_basic_user_file /etc/nginx/conf.d/kibana.passwd;
    access_log  /var/log/nginx/http-access.log  access_log_json;
    location / {
       proxy_set_header Host $host;
       proxy_set_header x-for $remote_addr;
       proxy_set_header x-server $host;
       proxy_set_header x-agent $http_user_agent;
       proxy_pass http://kibana;
    }
}

# cat /etc/nginx/conf.d/upstream.conf
upstream kibana {
    server 192.168.56.12:5601;
}

# cat /etc/nginx/conf.d/kibana.passwd
admin:$apr1$21NJ.Fx/$gmT0bwS4GoW1gmsHDRq911　　

三、filebeat 收集多日志文件

# 1、filebeat收集nginx访问日志、系统日志，输出到redis服务器中。
# grep -v "#" /etc/filebeat/filebeat.yml |grep -v "^$"

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  tags: ["nginx-log-56-100"]

- type: log
  enabled: true
  paths:
    - /var/log/messages
  tags: ["system-messages-log-56-100"]
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
output.redis:
    hosts: ["192.168.56.12"]
    port: 6379
    timeout: 5
    key: "default_list"

# 2、logstash服务端从redis数据库中读取数据并输出到Elasticsearch服务器中。
# cat redis-es-logstash-nginx.conf
input {
    redis {
        data_type => "list"
        host => "192.168.56.12"
        db => "0"
        port => "6379"
        key => "default_list"
    }
}

output {
    if "nginx-log-56-100" in [tags] {
    elasticsearch {
        hosts => ["192.168.56.15:9200"]
        index => "nginx-log-56100-%{+YYYY.MM.dd}"
        }
    }

    if "system-messages-log-56-100" in [tags] {
    elasticsearch {
        hosts => ["192.168.56.15:9200"]
        index => "system-messages-log-56100-%{+YYYY.MM.dd}"
        }
    }
}

四、filebeat 收集多日志文件（syslog、nginx、java 多行合并）

# grep -v "#" /etc/filebeat/filebeat.yml |grep -v "^$"

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  tags: ["nginx-log-56-100"]

- type: log
  enabled: true
  paths:
    - /var/log/messages
  tags: ["system-messages-log-56-100"]
- type: log
  enabled: true
  paths:
    - /data/tomcat/logs/catalina.out
  tags: ["tomcat-catalina-log-56-100"]
  multiline:
    pattern: '^\['
    negate: true
    match: after
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
output.redis:
    hosts: ["192.168.56.12"]
    port: 6379
    timeout: 5
    key: "default_list"


# cat redis-es-logstash-nginx-system-tomcat.conf
input {
    redis {
        data_type => "list"
        host => "192.168.56.12"
        db => "0"
        port => "6379"
        key => "default_list"
    }
}

output {
    if "nginx-log-56-100" in [tags] {
    elasticsearch {
        hosts => ["192.168.56.15:9200"]
        index => "nginx-log-56100-%{+YYYY.MM.dd}"
        }
    }

    if "system-messages-log-56-100" in [tags] {
    elasticsearch {
        hosts => ["192.168.56.15:9200"]
        index => "system-messages-log-56100-%{+YYYY.MM.dd}"
        }
    }
    if "tomcat-catalina-log-56-100" in [tags] {
    elasticsearch {
        hosts => ["192.168.56.15:9200"]
        index => "tomcat-catalina-log-56100-%{+YYYY.MM.dd}"
        }
    }
}