日志寫入redis及讀取redis
用一台服務器按照部署redis服務，專門用於日志緩存使用，用於web服務器產生大量日志的場景，例如下面的服務器內存即將被使用完畢，查看是因為redis服務保存了大量的數據沒有被讀取而占用了大量的內存空間。
2.1.1 部署redis
[root@study63 src]# tar xf redis-5.0.8.tar.gz
[root@study63 src]# cd redis-5.0.8/
[root@study63 redis-5.0.8]# ls
00-RELEASENOTES COPYING Makefile redis.conf runtest-moduleapi src
BUGS deps MANIFESTO runtest runtest-sentinel tests
CONTRIBUTING INSTALL README.md runtest-cluster sentinel.conf utils
[root@study63 redis-5.0.8]# make -j2
[root@study63 redis-5.0.8]# ln -sv /opt/src/redis-5.0.8 /opt/redis
[root@study63 redis-5.0.8]# cd /opt/redis/
修改配置
69 bind 10.0.0.63
136 daemonize yes
216 save ""
217
218 #save 900 1
219 #save 300 10
220 #save 60 10000
507 requirepass 123456
啟動服務
[root@study63 redis]# cp src/redis-server /usr/bin
[root@study63 redis]# cp src/redis-cli /usr/bin
[root@study63 redis]# redis-server /opt/redis/redis.conf
驗證
[root@study63 opt]# redis-cli -h 10.0.0.63
10.0.0.63:6379> KEYS *
(error) NOAUTH Authentication required.
10.0.0.63:6379> AUTH 123456
OK
10.0.0.63:6379> KEYS *
(empty list or set)
2.1.2 配置logstash將日志寫入redis
[root@study62 ~]# vim /etc/logstash/conf.d/rsyslog.conf
input {
syslog {
type => "rsyslog-haproxy063"
port => "5160"
}
}
output {
if [type] == "rsyslog-haproxy063" {
redis {
data_type => "list"
host => "10.0.0.63"
db => "1"
port => "6379"
key => "rsyslog-haproxy063"
password => "123456"
}}
}
2.1.3 檢測配置語法
[root@study62 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/rsyslog.conf -t
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs to console
Configuration OK
11:01:36.649 [LogStash::Runner] INFO logstash.runner - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
[root@study62 ~]# systemctl restart logstash.service
2.1.4 登錄redis中查看
[root@study63 opt]# redis-cli -h 10.0.0.63 -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.0.0.63:6379> SELECT 1
10.0.0.63:6379[1]> keys *
1) "rsyslog-haproxy063"
10.0.0.63:6379[1]> LLEN rsyslog-haproxy063 # 查看key的長度
(integer) 12
10.0.0.63:6379[1]> LPOP rsyslog-haproxy063 #展示一條記錄會減少一條
"{\"severity\":6,\"pid\":\"14556\",\"program\":\"haproxy\",\"message\":\"Connect from 10.0.0.1:52144 to 10.0.0.63:9999 (stats/HTTP)\\n\",\"type\":\"rsyslog-haproxy063\",\"priority\":182,\"logsource\":\"localhost\",\"@timestamp\":\"2020-04-15T03:09:00.000Z\",\"@version\":\"1\",\"host\":\"10.0.0.63\",\"facility\":22,\"severity_label\":\"Informational\",\"timestamp\":\"Apr 15 11:09:00\",\"facility_label\":\"local6\"}"
10.0.0.63:6379[1]> LLEN rsyslog-haproxy063
(integer) 11
2.2.5 配置logstash從reids中取出數據到elasticsearch
2.2.5.1 使用study63上的logstash從redis取數據
[root@study63 ~]# vim /etc/logstash/conf.d/redis-es.conf
input {
redis {
data_type => "list"
host => "10.0.0.63"
db => "1"
port => "6379"
key => "rsyslog-haproxy063"
password => "123456"
}
}
output {
elasticsearch {
hosts => ["10.0.0.63:9200"]
index => "redis-rsyslog-haproxy063-%{+YYYY.MM.dd}"
}
}
[root@study63 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-es.conf -t
[root@study63 ~]# systemctl restart logstash.service
2.2.5.2 從study63上寫入數據查看
[root@study63 opt]# redis-cli -h 10.0.0.63 -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.0.0.63:6379> SELECT 1
10.0.0.63:6379[1]> keys *
10.0.0.63:6379[1]> LLEN rsyslog-haproxy063
(integer) 11
10.0.0.63:6379[1]> LLEN rsyslog-haproxy063
(integer) 0
2.2.5.3 head插件和Kibana添加索引查看
