一 背景說明
我現在安裝了logstash,只配置了一個文件，想同時收集nginx和java的日志，nginx要顯示ip,國家城市，狀態碼，等，java要顯示日志的具體內容。
二 nginx filebeat設置
filebeat.inputs:
# 收集 nginx 日志
- type: log
enabled: true
paths:
- /var/log/nginx/*.log
tags: ["nginx_logs"]
# 日志是json開啟這個
json.keys_under_root: true
json.overwrite_keys: true
json.add_error_key: true
# 如果值為ture，那么fields存儲在輸出文檔的頂級位置
fields_under_root: true
fields:
app: easydong
logtype: nginx_logs
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.logstash:
hosts: ["172.17.199.231:5044"]
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
三 java filebeat設置
filebeat.inputs:
- type: log
enabled: true
paths:
- /xs/logs/app-front1/easydong-app/app.log
encoding: utf-8
tail_files: true
fields:
app: easydong
type: applog-V1-appfront1
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
enabled: true
paths:
- /xs/logs/app-front1/easydong-admin/admin.log
encoding: utf-8
tail_files: true
fields:
app: easydong
type: adminlog-V1-appfront1
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
enabled: true
paths:
- /xs/logs/app-front2/easydong-admin/admin.log
encoding: utf-8
tail_files: true
fields:
app: easydong
type: adminlog-V1-appfront2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
enabled: true
paths:
- /xs/logs/app-front2/easydong-app/app.log
encoding: utf-8
tail_files: true
fields:
app: easydong
type: applog-V1-appfront2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.logstash:
hosts: ["39.96.179.187:5044"]
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
四 logstash設置
根據 if [logtype] == "nginx_logs 去判斷是nginx日志還是java日志，
input {
beats {
port => 5044
}
}
filter{
if [type] == "adminlog-V1-appfront1"{
grok {
match => ["message", "%{SYSLOGBASE} %{GREEDYDATA:message}"]
overwrite => ["message"]
}
}
if [logtype] == "nginx_logs"{
mutate {
convert => [ "status","integer" ]
convert => [ "size","integer" ]
convert => [ "upstreatime","float" ]
convert => ["[geoip][coordinates]", "float"]
remove_field => "message"
}
date {
match => [ "timestamp" ,"dd/MMM/YYYY:HH:mm:ss Z" ]
}
geoip {
source => "client" ##日志格式里的ip來源，這里是client這個字段（client":"$remote_addr"）
target => "geoip"
database =>"/usr/share/logstash/GeoLite2-City.mmdb" ##### 下載GeoIP庫
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
# mutate {
# remove_field => "timestamp"
# }
}
}
output {
if [app] == "easydong" {
if [type] == "applog-V1-appfront1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
password => "111111"
user => "elastic"
index => "applog-v1-appfront1-%{+YYYY.MM.dd}"
}
}
else if [type] == "adminlog-V1-appfront1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
password => "111111"
user => "elastic"
index => "adminlog-v1-appfront1-%{+YYYY.MM.dd}"
}
}
else if [type] == "applog-V1-appfront2" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "applog-v1-appfront2-%{+YYYY.MM.dd}"
}
}
else if [type] == "adminlog-V1-appfront2" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "adminlog-v1-appfront2-%{+YYYY.MM.dd}"
}
}
else if [type] == "applog-V2-appfront1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "applog-v2-appfront1-%{+YYYY.MM.dd}"
}
}
else if [type] == "adminlog-V2-appfront1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "adminlog-v2-appfront1-%{+YYYY.MM.dd}"
}
}
else if [type] == "applog-V2-appfront2" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "applog-v2-appfront2-%{+YYYY.MM.dd}"
}
}
else if [type] == "adminlog-V2-appfront2" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "adminlog-v2-appfront2-%{+YYYY.MM.dd}"
}
}
else if [type] == "indonesia-adminlog-1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "indonesia-adminlog-1-%{+YYYY.MM.dd}"
}
}
if [app] == "easydong" {
if [logtype] == "nginx_logs"{
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
index => "logstash-sanwenqian-nginx-%{+YYYY-MM}"
}
}
}
}
stdout { codec=> rubydebug }
最后重啟logstash看能否正常啟動，可以去/var/log/message查看是否有新的日志產生。