CentOS7安裝elk

 

一、安裝Elasticsearch

1.1 安裝Java

# 系統版本
 cat /etc/redhat-release 
CentOS Linux release 7.5.1804 (Core) 

  uname -r
3.10.0-862.el7.x86_64


  yum -y install java
  java -version
openjdk version "1.8.0_212"
OpenJDK Runtime Environment (build 1.8.0_212-b04)
OpenJDK 64-Bit Server VM (build 25.212-b04, mixed mode)

 

1.2 安裝Elasticsearch

• 關閉防火牆

systemctl disable firewalld

• 關閉selinux

vim /etc/sysconfig/selinux 
SELINUX= Enforcing # <== 修改為 disabled, 重啟系統生效

• 官方文檔

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/rpm.html#install-rpm

• 下載並安裝公共簽名秘鑰

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 從RPM存儲庫安裝

在/etc/yum.repos.d/創建一個elasticsearch.repo文件，文件內容如下：

vim /etc/yum.repos.d/elasticsearch.repo

[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

• yum安裝ElasticSearch

yum install elasticsearch -y

• 啟動Elasticsearch

systemctl daemon-reload
systemctl enable elasticsearch.service

• 啟動/停止Elasticsearch

systemctl start elasticsearch.service
systemctl stop elasticsearch.service

• Elasticsearch服務運行狀態

systemctl status elasticsearch.service

Elasticsearch日志存儲在/var/log/elasticsearch/目錄下

•  檢查Elasticsearch是否正在運行

[root@localhost ~]# curl http://localhost:9200
{
  "name" : "O2pObfg",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "uXvFw1fgR1iTduPf1d-MAw",
  "version" : {
    "number" : "6.7.1",
    "build_flavor" : "default",
    "build_type" : "rpm",
    "build_hash" : "2f32220",
    "build_date" : "2019-04-02T15:59:27.961366Z",
    "build_snapshot" : false,
    "lucene_version" : "7.7.0",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}

1.3 配置Elasticsearch

Elasticsearch默認使用/etc/elasticsearch運行時配置

默認Elasticsearch從文件/etc/elasticsearch/elastisearch.yml加載配置。

[root@localhost ~]# ll -lrht /etc/elasticsearch/
total 36K
-rw-rw----. 1 root elasticsearch    0 Apr  3 00:07 users_roles
-rw-rw----. 1 root elasticsearch    0 Apr  3 00:07 users
-rw-rw----. 1 root elasticsearch  197 Apr  3 00:07 roles.yml
-rw-rw----. 1 root elasticsearch  473 Apr  3 00:07 role_mapping.yml
-rw-rw----. 1 root elasticsearch  13K Apr  3 00:07 log4j2.properties
-rw-rw----. 1 root elasticsearch 3.6K Apr  3 00:07 jvm.options
-rw-rw----. 1 root elasticsearch 2.9K Apr  3 00:07 elasticsearch.yml
-rw-rw----. 1 root elasticsearch  207 Apr 28 14:16 elasticsearch.keystore

• 配置Elasticsearch數據路徑和日志目錄：

官方文檔：

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/settings.html

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/important-settings.html

# 創建Elasticsearch數據目錄及日志目錄
mkdir /opt/elasticsearch/{data,log}/ -pv
cd /opt/
chown -R elasticsearch:elasticsearch elasticsearch/

cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml_`date +%Y%m%d_bak`

vim /etc/elasticsearch/elasticsearch.yml
# Elasticsearch 節點名稱
node.name: node-1
#數據
path.data: /opt/elasticsearch/data/
# 日志
path.logs: /opt/elasticsearch/log/

# 內存鎖定,將進程地址空間鎖定在RAM中，防止任何Elasticsearch內存被換出 
bootstrap.memory_lock: true

# 監聽地址
network.host: 192.168.198.130
# 監聽端口
http.port: 9200

# 查看修改了那些配置 grep '^[a-z]' /etc/elasticsearch/elasticsearch.yml
node.name: node-1
path.data: /opt/elasticsearch/data/
path.logs: /opt/elasticsearch/log/
bootstrap.memory_lock: true
network.host: 192.168.198.130
http.port: 9200

• 配置JVM

Jvm配置文件/etc/elasticsearch/jvm.options

-Xms2g
-Xmx2g

• 文件句柄配置

官方文檔：https://www.elastic.co/guide/en/elasticsearch/reference/6.5/setting-system-settings.html

ulimit -n 65536
tail -1 /etc/security/limits.conf 

elasticsearch  -  nofile  65536

• 系統配置

官方文檔：https://www.elastic.co/guide/en/elasticsearch/reference/6.5/setting-system-settings.html

vim /usr/lib/systemd/system/elasticsearch.service


[Service]        # <== 在此標簽下添加
LimitMEMLOCK=infinity

# 重新加載
systemctl daemon-reload

• 虛擬內存

官方文檔：

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/vm-max-map-count.html

臨時配置：

sysctl -w vm.max_map_count=262144

永久配置：/etc/sysctl.conf

tail -1 /etc/sysctl.conf
vm.max_map_count=262144
sysctl -p

二、安裝Logstash

• 官方文檔

https://www.elastic.co/guide/en/logstash/6.5/installing-logstash.html

• 下載並安裝公共簽名和密鑰

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目錄下新建logstash.repo文件，文件內容如下：

[logstash-6.x]
name=Elastic repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

• yum安裝logstash

yum -y install logstash

三、安裝Kibana

• 官方文檔

https://www.elastic.co/guide/en/kibana/6.5/rpm.html

• 下載並安裝公共簽名和密鑰

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目錄下，創建kibana.repo，文件內容如下：

[kibana-6.x]
name=Kibana repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

• yum 安裝kibana

yum install kibana

• 啟動/停止kibana

systemctl daemon-reload
systemctl enable kibana.service
systemctl start kibana.service
systemctl stop kibana.service

• 配置kibana

Kibana配置文件在/etc/kibana/目錄下的kibana.yml。默認kibana運行localhost:5601

[root@localhost ~]# grep '^[a-z]' /etc/kibana/kibana.yml 
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://192.168.198.130:9200"]
kibana.index: ".kibana"

 

四、安裝Filebeat

• 官方文檔

https://www.elastic.co/guide/en/beats/filebeat/5.5/setup-repositories.html

https://www.elastic.co/guide/en/beats/filebeat/6.5/configuring-output.html

• 下載並安裝公共簽名和密鑰

rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目錄下創建filebeat.repo文件，文件內容如下：

[elastic-6.x]
name=Elastic repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

•  yum安裝filebeat

yum -y install filebeat

#  filebeat記錄讀取文件位偏移 /var/lib/filebeat/registry

五、安裝Redis

參考：https://www.cnblogs.com/hwlong/p/9330191.html 

https://www.cnblogs.com/hwlong/p/6101019.html

wget http://download.redis.io/releases/redis-3.2.12.tar.gz
上傳至 /usr/local
tar xzf redis-3.2.12.tar.gz
mv redis-3.2.12 redis
cd redis
make
src/redis-server &

 

六、Filebeat+Redis+Logstash+Elasticsearch+Kibana

6.1 配置Filebeat

filebeat.prospectors:


- input_type: log
  paths:
    - /data/logs/localhost.localdomain.*.log
  exclude_files: [".gz$"]
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  tags: ["localhost-localdomain"]
  document_type: localhost-localdomain


- input_type: log
  paths:
    - /data/logs/localhost.localdomain-error.*.log
  exclude_files: [".gz$"]
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  tags: ["localhost-localdomain-error"]
  document_type: localhost-localdomain-error




fields:
  ipaddr: '192.168.198.131'   # <== filebeat本機IP
fields_under_root: true


output.redis:
  hosts: ["192.168.198.130"]        # <== redis地址
  port: 6379
  key: "default_list"
  keys:
    - key: "%{[type]}"                # <== 根據document_type類型寫入不同的key中
      mapping:
      "localhost-localdomain": "localhost-localdomain"
      "localhost-localdomain-error": "localhost-localdomain-error"
 
  db: 0
  datatype: list
  timeout: 5
  


processors:
  - drop_fields:
       fields: ["offset","beat","source","input"]

• 啟動filebeat

systemctl status filebeat

 

6.2 配置logstash.conf

input {

  redis {
     host => "192.168.198.130"
     port => "6379"
     db => "0"
     data_type => "list"
     key => "default_list"
     type => "default_list"
  }
  
  redis {
     host => "192.168.198.130"
     port => "6379"
     db => "0"
     data_type => "list"
     key => "localhost-localdomain"
     type => "localhost-localdomain"
  }
 
  redis {
     host => "192.168.198.130"
     port => "6379"
     db => "0"
     data_type => "list"
     key => "localhost-localdomain-error"
     type => "localhost-localdomain-error"
  }



}


filter {

}


output{
   if [type] == "localhost-localdomain" {

       elasticsearch {
           document_type => "localhost-localdomain"
           hosts => ["192.168.198.130:9200"]
           index => "localhost-localdomain.log"
       }
   }
   
   if [type] == "localhost-localdomain-error" {

       elasticsearch {
           document_type => "localhost-localdomain-error"
           hosts => ["192.168.198.130:9200"]
           index => "localhost-localdomain-error.log"
       }
   }


  

}

• 啟動logstash

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-logstash-es.conf  &