Logstash配置與使用

host	ip
node1	192.168.79.103
node2	192.168.79.101

1、在命令行執行logstash

• 標准輸入和標准輸出，輸入什么顯示什么

[root@node1 ~]# cd /opt/logstash/bin
[root@node1 bin]# ./logstash -e 'input { stdin{} } output { stdout{} }'
Settings: Default pipeline workers: 2
Pipeline main started
hello world
2018-08-23T03:27:42.798Z node1 hello world

• 以rubydebug格式輸出

[root@node1 bin]# ./logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug }}'
Settings: Default pipeline workers: 2
Pipeline main started
hello world
{
       "message" => "hello world",
      "@version" => "1",
    "@timestamp" => "2018-08-23T03:31:04.556Z",
          "host" => "node1"
}

• 從stdin輸入，輸出值elasticsearch

[root@node1 bin]# ./logstash -e 'input { stdin{} } output { elasticsearch { hosts => ["192.168.79.103"] index => "logstash-%{+YYYY.MM.dd}" } }'

• 即輸出至elasticsearch也輸出至stdout

[root@node1 bin]# ./logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug }elasticsearch { hosts => ["192.168.79.103"] index => "logstash-%{+YYYY.MM.dd}" } }'
Settings: Default pipeline workers: 2
Pipeline main started
hello world
{
       "message" => "hello world",
      "@version" => "1",
    "@timestamp" => "2018-08-23T03:46:28.131Z",
          "host" => "node1"
}

2、通過配置文件輸入、輸出

logstash配置文件目錄/etc/logstash/conf.d

[root@node1 etc]# cat demo.conf
input{
	stdin{}
}

filter{

}

output{
	elasticsearch {
		hosts => ["192.168.79.103:9200"]
		index => "logstash-%{+YYYY.MM.dd}"
	}
	stdout {
		codec => rubydebug
	}
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/demo.conf

• 收集系統日志

[root@node1 conf.d]# cat file.conf
input{
    file{
	path => ["/var/log/messages","/var/log/secure"]
	type => "system-log"
	start_position => "beginning"
    }
}

filter{
}

output{
    	elasticsearch {
		hosts => ["192.168.79.103:9200"]
		index => "system-log-%{+YYYY.MM}"
	}
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/file.conf

• 收集Java日志

[root@node1 conf.d]# cat file.conf
input{
    file{
	path => ["/var/log/messages","/var/log/secure"]
	type => "system-log"
	start_position => "beginning"
    }
    file{
	path => "/var/log/elasticsearch/myes.log"
	type => "es-log"
	start_position => "beginning"
    }
}

filter{
}

output{
	if [type] == "system-log" {
    	    elasticsearch {
		hosts => ["192.168.79.103:9200"]
		index => "system-log-%{+YYYY.MM}"
	    }
	}
	if [type] == "es-log" {
    	    elasticsearch {
		hosts => ["192.168.79.103:9200"]
		index => "es-log-%{+YYYY.MM}"
	    }
	}
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/file.conf

上述配置文件瀏覽器訪問，發現java日志按行分隔，不好閱讀，所以加入多行codec

[root@node1 conf.d]# cat file.conf
input{
    file{
	path => ["/var/log/messages","/var/log/secure"]
	type => "system-log"
	start_position => "beginning"
    }
    file{
	path => "/var/log/elasticsearch/myes.log"
	type => "es-log"
	start_position => "beginning"
	codec => multiline {
            pattern => "^\["
            negate => true
            what => "previous"
	}
    }
}

filter{
}

output{
	if [type] == "system-log" {
    	    elasticsearch {
		hosts => ["192.168.79.103:9200"]
		index => "system-log-%{+YYYY.MM}"
	    }
	}
	if [type] == "es-log" {
    	    elasticsearch {
		hosts => ["192.168.79.103:9200"]
		index => "es-log-%{+YYYY.MM}"
	    }
	}
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f file.conf

• 收集nginx Jason格式日志

  • 方法1：nginx日志改成json輸出
  • 方法2：文件直接收取redis，python腳本讀取redis，寫成json，寫入es

方法1：

# nginx日志配置
log_format  access_log_json  '{"user_ip":"$http_x_real_ip","lan_ip":"$remote_addr","log_time":"$time_iso8601","user_req":"$request","http_code":"$status","body_bytes_sent":"$body_bytes_sent","req_time":"$request_time","user_ua":"$http_user_agent"}';

# 收集日志輸出至es
[root@node1 conf.d]# cat nginx.conf
input{
	file{
		path => "/var/log/nginx/access_json.log"
		codec => "json"
	}
}

filter{
}

output{
	elasticsearch {
		hosts => ["192.168.79.103:9200"]
		index => "nginx-access-log-%{+YYYY.MM.dd}"
	}
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f nginx.conf

方法2：

• input插件rsyslog

輸出至屏幕

[root@node1 conf.d]# cat syslog.conf
input{
    syslog{
    	type => "system-syslog"
	port => 514
    }
}

filter{

}

output{
    stdout{
	codec => rubydebug
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f syslog.conf

修改rsyslog配置文件

[root@node1 ~]# vim /etc/rsyslog.conf
*.* @@192.168.79.103:514
[root@node1 ~]# systemctl restart rsyslog

輸出至es

[root@node1 conf.d]# cat syslog.conf
input{
    syslog{
    	type => "system-syslog"
	port => 514
    }
}

filter{

}

output{
    elasticsearch{
	hosts => ["192.168.79.103:9200"]
	index => "system-syslog-%{+YYYY.MM}"
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f syslog.conf