一、logstash結合filebeat收集日志寫入到數據庫中
框架圖
原理：由於部分重要日志需要按照開發的要求將日志存儲到數據庫中，也是個比較重點以及難點的問題，此實驗是在ELK之十的基礎上進行搭建的，需要將ELK之十搭建完成才能搭建此環境：ELK之十鏈接：https://www.cnblogs.com/struggle-1216/p/12502928.html
1、安裝mariadb-server數據庫
1、安裝數據庫並啟動
# yum install mariadb-server -y # systemctl start mariadb
2、創建一個數據庫並授權
[root@filebeat-1 ~]# mysql Welcome to the MariaDB monitor. Commands end with ; or \g. Your MariaDB connection id is 3 Server version: 5.5.64-MariaDB MariaDB Server Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. MariaDB [(none)]> create database elk character set utf8 collate utf8_bin; Query OK, 1 row affected (0.00 sec) MariaDB [(none)]> grant all privileges on elk.* to elk@"192.168.7.%" identified by '123456'; Query OK, 0 rows affected (0.00 sec) MariaDB [(none)]>
3、在另一台mysql客戶端主機進行測試連接
[root@logstash-1 conf.d]# mysql -uelk -p123456 -h192.168.7.105 Welcome to the MariaDB monitor. Commands end with ; or \g. Your MariaDB connection id is 4 Server version: 5.5.64-MariaDB MariaDB Server Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. MariaDB [(none)]>
2、在logstash主機上安裝jdbc插件
1、安裝logstash-output-jdbc插件
[root@logstash-1 conf.d]# logstash-plugin install logstash-output-jdbc
2、在官網下載jar包
MySQL Connector/J是MySQL官方JDBC驅動程序JDBC（Java Data Base Connectivity,java數據庫連接）是一種用於執行SQL語句的Java API，可以為多種關系數據庫提供統一訪問，它由一組用Java語言編寫的類和接口組成。
官方下載地址：https://dev.mysql.com/downloads/
下載對應系統版本的包：mysql-connector-java-8.0.19-1.el7.noarch
選擇對應版本的包進行下載
3、創建一個存放jar包的目錄，將下載的包進行安裝，並將生成的jar復制到指定創建的目錄下。
[root@logstash-1 jdbc]# yum install mysql-connector-java-8.0.19-1.el7.noarch.rpm -y [root@logstash-1 ~]# mkdir /usr/share/logstash/vendor/jar/jdbc -p [root@logstash-1 ~]# cd /usr/share/logstash/vendor/jar/jdbc [root@logstash-1 jdbc]# ll /usr/share/java/mysql-connector-java.jar -rw-r--r-- 1 root root 2348827 Dec 4 20:06 /usr/share/java/mysql-connector-java.jar # 將此jar包存在上面創建的目錄下 [root@logstash-1 jdbc]# cp /usr/share/java/mysql-connector-java.jar /usr/share/logstash/vendor/jar/jdbc/
4、重啟logstash服務，要保證此時的logstash服務要能夠啟動
[root@logstash-1 jdbc]# systemctl restart logstash
[root@logstash-1 jdbc]# tail -f /var/log/logstash/logstash-plain.log
[2020-03-16T14:48:43,692][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600} # 此時的logstash已經啟動
[root@logstash-1 jdbc]# logstash-plugin list | grep logstash-output-jdbc # 查看此時的輸出插件是否存在
logstash-output-jdbc
3、在mariadb數據庫創建收集日志表
1、在數據庫中創建一個收集日志的elklog表
[root@filebeat-1 ~]# mysql Welcome to the MariaDB monitor. Commands end with ; or \g. Your MariaDB connection id is 13 Server version: 5.5.64-MariaDB MariaDB Server Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. MariaDB [(none)]> use elk; Reading table information for completion of table and column names You can turn off this feature to get a quicker startup with -A Database changed MariaDB [elk]> create table elklog (clientip varchar(64),responsetime varchar(16),http_host varchar(64),http_user_agent varchar(256),url varchar(128),time timestamp(0)); # 創建收集日志的elklog表，需要定義字段名稱
4、在logstash服務器上將nginx日志傳遞到數據庫中
1、在logstash服務器的/etc/logstash/conf.d目錄下創建一個收集nginx日志的配置文件
[root@logstash-1 conf.d]# cat filebeat-to-logstash.conf
input {
beats {
host => "192.168.7.101"
port => 5044
codec => "json"
}
}
output {
if [fields][app] == "syslog" {
redis {
host => "192.168.7.104"
port => "6379"
db => "0"
data_type => "list"
password => "123456"
key => "filebeat-syslog-7-103"
codec => "json"
}}
if [fields][app] == "nginx" {
redis {
host => "192.168.7.104"
port => "6379"
db => "0"
data_type => "list"
password => "123456"
key => "filebeat-nginx-log-7-103"
codec => "json"
}
jdbc { # 使用mysql模塊將日志傳遞到數據庫中
connection_string => "jdbc:mysql://192.168.7.105/elk?user=elk&password=123456&useUnicode=true&characterEncoding=UTF8" # IP地址為數據庫地址
statement => ["INSERT INTO elklog(clientip,responsetime,http_host,http_user_agent,url,status) VALUES(?,?,?,?,?,?)", "clientip","responsetime","http_host","http_user_agent","url","status"] # 定義字段時，與上面創建elklog對應，？為有幾個字段就寫幾個
}}
}
2、重啟logstash服務
# systemctl restart logstash
5、在數據庫中查看收集到的nginx日志
MariaDB [elk]> select * from elklog\G;
*************************** 1. row ***************************
clientip: 192.168.7.1
responsetime: 0.0
http_host: 192.168.7.103
http_user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
url: /index.html
status: 304
time: 2020-03-16 16:34:05
*************************** 2. row ***************************
clientip: 192.168.7.1
responsetime: 0.0
http_host: 192.168.7.103
http_user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
url: /index.html
status: 304
time: 2020-03-16 16:34:05
*************************** 3. row ***************************
clientip: 192.168.7.1
responsetime: 0.0
http_host: 192.168.7.103
http_user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
url: /index.html
status: 304
time: 2020-03-16 16:34:05
*************************** 4. row ***************************
clientip: 192.168.7.1
responsetime: 0.0
http_host: 192.168.7.103
http_user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
url: /index.html
status: 304
time: 2020-03-16 16:34:05
*************************** 5. row ***************************
clientip: 192.168.7.1
responsetime: 0.0
http_host: 192.168.7.103
http_user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
url: /index.html
status: 304
time: 2020-03-16 16:34:05
*************************** 6. row ***************************
clientip: 192.168.7.1
responsetime: 0.0
http_host: 192.168.7.103
http_user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
url: /index.html
status: 304
time: 2020-03-16 16:34:05
*************************** 7. row ***************************
clientip: 192.168.7.1
responsetime: 0.0
http_host: 192.168.7.103
http_user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
url: /index.html
status: 304
time: 2020-03-16 16:34:05
*************************** 8. row ***************************
clientip: 192.168.7.1
responsetime: 0.0
http_host: 192.168.7.103
http_user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
url: /index.html
status: 304
time: 2020-03-16 16:34:05
二、通過地圖統計客戶IP地址所在城市
1、下載官網地圖
地圖官方下載地址：https://dev.maxmind.com/geoip/geoip2/geolite2/
1、下載官網的地圖軟件，並將軟件存放到/etc/logstash目錄下，對壓縮包文件進行解壓
[root@logstash logstash]# pwd /etc/logstash [root@logstash logstash]# tar xvf GeoLite2-City_20190723.tar.gz [root@logstash logstash]# ll GeoLite2-City_20190723/GeoLite2-City.mmdb -rw-r--r-- 1 root root 61762964 Jul 23 2019 GeoLite2-City_20190723/GeoLite2-City.mmdb # 此配置文件是我們需要引用
2、配置logstash主機配置文件，將日志傳遞到elasticsearch主機
1、修改Logstash的/etc/logstash/conf.d目錄下的配置文件，加入一個加載地址地圖的模塊
[root@logstash conf.d]# cat logstash-to-es.conf
input {
redis {
host => "192.168.7.104"
port => "6379"
db => "0"
password => "123456"
data_type => "list"
key => "filebeat-syslog-7-103"
codec => "json"
}
redis {
host => "192.168.7.104"
port => "6379"
db => "0"
password => "123456"
data_type => "list"
key => "filebeat-nginx-log-7-103"
codec => "json"
}
}
filter {
if [fields][app] == "nginx" { # 使用nginx服務的類型進行判斷
geoip {
source => "clientip"
target => "geoip"
database => "/etc/logstash/GeoLite2-City_20190723/GeoLite2-City.mmdb" # 修改指定的地址位置
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
}
output {
if [fields][app] == "syslog" {
elasticsearch {
hosts => ["192.168.7.100:9200"]
index => "logstash-syslog-7-103-%{+YYYY.MM.dd}"
}}
if [fields][app] == "nginx" {
elasticsearch {
hosts => ["192.168.7.100:9200"]
index => "logstash-nginx-accesslog-7-103-%{+YYYY.MM.dd}"
}}
}
2、測試Logstash配置文件
[root@logstash conf.d]# logstash -f logstash-to-es.conf -t WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console [WARN ] 2020-03-16 19:09:09.448 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified Configuration OK # 顯示OK，說明配置文件沒問題
3、重啟Logstash服務
# systemctl restart logstash
3、在nginx服務器上進行測試
1、在nginx服務器上導入部分log。
[root@filebate src]# tail -n1000 access.log >> /var/log/nginx/access.log
2、在kibana刷新一下之前生成的索引文件信息，就會加載最新的索引信息
4、discover查看日志狀態
5、在可視化進行創建地圖，並保存
