碼上歡樂
相關內容    簡體    繁體

centos7部署elk7.4

本文轉載自   查看原文   2020-03-17 09:55   960    elastic

  • filebeat安裝

    # 下載filebeat包
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.4.2-linux-x86_64.tar.gz

# 解壓
tar -xvf filebeat-7.4.2-linux-x86_64.tar.gz -C /data
mv filebeat-7.4.2-linux-x86_64.tar.gz filebeat

cd filebeat

# 采集mysql慢日志及錯誤日志
vim filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /data/mysql-log/mysqld.log
  tags: ["mysql_error"]
  tail_files: true
  fields:
  # 根據不同環境配置自定義名
    server: test-mysql_error

- type: log
  enabled: true
  paths:
    - /data/mysql-log/mysql_slow.log
  tags: ["mysql_slow"]
  tail_files: true
  fields:
    server: test-mysql_slow
#----------------------------- Logstash output --------------------------------
output.logstash:
  hosts: ["192.168.23.96:5044"]
  


# 采集nginx服務、spring cloud微服務
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /data/app/nginx/logs/access.log
  tags: ["nginx_access"]
  tail_files: true
  fields:
    server: test-nginx_access

- type: log
  enabled: true
  paths:
    - /data/app/nginx/logs/error.log
  tags: ["nginx_error"]
  tail_files: true
  fields:
    server: test-nginx_error
- type: log
  enabled: true
  paths:
    - /data/app/logs/*.log
  tailf_files: true
  fields:
    server: app1
  # 合並jvm堆棧報錯，多行合並一行
  multiline:
    pattern: '^\s*("{)'
    negate: true
    match: after
    max_lines: 100
#----------------------------- Logstash output --------------------------------

output.logstash:
  hosts: ["192.168.23.96:5044"]

  • 安裝supervisor並配置監控filebeat

    # easy_install的方式
yum install -y python-setuptools
easy_install supervisor
echo_supervisord_conf >/etc/supervisord.conf

# 配置supervisord.conf,修改supervisord.conf行尾
vim /etc/supervisord.conf
[include]
files = /etc/supervisor/*.conf

# web界面顯示（可選，如果想通過瀏覽器管理）
[inet_http_server]
port=IP:9001

# 啟動
supervisord -c supervisord.conf

# 創建目錄
mkdir -p /etc/supervisor

# 創建job服務
vim /etc/supervisor/filebeat.conf
[program:filebeat]
directory=/data/filebeat
command=/data/filebeat/filebeat -e -c filebeat.yml
stdout_logfile = /data/log/supervisor/filebets_out.log
user=root
autostart=true
autorestart=true
startsecs=10

#啟動前需先創建log目錄，不然會報錯（****）
mkdir /data/log/supervisor

# 通過supervisorctl管理啟動服務

# 重新加載
supervisorctl -c supervisord.conf reload

# 使用supervisorctl啟動filebeat服務
supervisorctl start filebeat

# 查看狀態
supervisorctl status

# 自行驗證,kill掉服務filebeat看看服務是否會自動重啟
ps -ef | grep filebeat

  • 部署ElaticSearch

    # 系統配置
swapoff -a

cat >> /etc/sysctl.conf <<EOF
fs.file-max=655360
vm.max_map_count = 262144
EOF

vim /etc/security/limits.conf
* soft nproc 20480
* hard nproc 20480
* soft nofile 65536
* hard nofile 65536
* soft memlock unlimited
* hard memlock unlimited

vim /etc/sysctl.conf
新增vm.max_map_count = 655360
執行sysctl –p

vim /etc/security/limits.d/20-nproc.conf
* soft nproc 20480

# ELK無法用root用戶啟動，創建新用戶並授權

groupadd elk

useradd -g elk elk

passwd elk  # 這里密碼: 12345678

# 創建運行目錄
mkdir /data/elk

chown -R elk:elk /data/elk

# 下載安裝包
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.4.2-linux-x86_64.tar.gz

# 解壓
tar -xvf elasticsearch-7.4.2-linux-x86_64.tar.gz -C /data/elk

# 切換用戶
su elk

cd /data/elk

# 重命名安裝目錄
mv elasticsearch-7.4.2-linux-x86_64.tar.gz elasticsearch

# 進入目錄
cd elasticsearch

# 修改jvm （配置8g）
vim config/jvm.options
-Xms8g
-Xmx8g

vim config/elasticsearch.yml
node.name: node-1
# ----------------------------------- Paths ------------------------------------
path.data: /data/elk/data
#
path.logs: /data/elk/logs
# ----------------------------------- Memory -----------------------------------
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
# ---------------------------------- Network -----------------------------------
network.host: 0.0.0.0
http.cors.enabled: true
http.cors.allow-origin: "*"
http.port: 9200
cluster.initial_master_nodes: ["node-1"]

# 參數優化
discovery.zen.fd.ping_interval: 120s
indices.requests.cache.size: 2%
 
# refresh_interval不能在配置文件需在命令行執行
curl -H "Content-Type: application/json" -XPUT 'http://localhost:9200/_all/_settings?preserve_existing=true' -d '{
  "index.merge.scheduler.max_thread_count" : "1",
  "index.refresh_interval" : "300s",
  "index.translog.durability" : "async"
}'

#啟動
./bin/elasticsearch -d 

# 測試是否啟動
瀏覽器訪問192.168.23.96:9200

  • logstash的安裝

    wget https://artifacts.elastic.co/downloads/logstash/logstash-7.4.2.tar.gz
su elk
cd /data/elk/logstash
tar -xvf logstash-7.4.2.tar.gz
mv logstash-7.4.2 logstash
# 進入到配置文件目錄
cd ./config

# 修改jvm （配置8g）
vim config/jvm.options
-Xms8g
-Xmx8g

# 修改配置文件logstash.yml
input {
    beats {
        port => 5044
    }
}

output {
    elasticsearch {
        hosts => ["localhost:9200"]
        index => ["%{[fields][server]}-%{+YYYY-MM-dd}"]
        manage_template => false
        }
}

# 執行啟動命令
./bin/logstash -f ./config/logstash.conf

#啟動成功后可以看到日志
Successfully started Logstash API endpoint {port=>9600}

  • kibana安裝

    # 下載
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.4.2-linux-x86_64.tar.gz

su elk
tar -xvf kibana-7.4.2-linux-x86_64.tar.gz -C /data/elk
mv kibana-7.4.2-linux-x86_64 kibana
cd /data/elk/kibana

# 配置信息
server.host: "0.0.0.0"
server.port: 5601
elasticsearch.hosts: ["http://192.168.23.96:9200"]

# 啟動
./bin/kibana

# 訪問驗證，瀏覽器訪問
http://192.168.23.96:5601

這里注意一個問題，我這里logstash由於以來jdk8，而elasticsearch7以后自帶jdk不用額外安裝所以啟動順序不要變，elastic -> logstash

  • logstash的安裝

    wget https://artifacts.elastic.co/downloads/logstash/logstash-7.4.2.tar.gz
su elk
cd /data/elk/logstash
tar -xvf logstash-7.4.2.tar.gz
mv logstash-7.4.2 logstash
# 進入到配置文件目錄
cd ./config

# 修改jvm （配置8g）
vim config/jvm.options
-Xms8g
-Xmx8g

# 修改配置文件logstash.yml
input {
    beats {
        port => 5044
    }
}

output {
    elasticsearch {
        hosts => ["localhost:9200"]
        index => ["%{[fields][server]}-%{+YYYY-MM-dd}"]
        manage_template => false
        }
}

# 執行啟動命令
./bin/logstash -f ./config/logstash.conf

#啟動成功后可以看到日志
Successfully started Logstash API endpoint {port=>9600}

  • kibana安裝

    # 下載
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.4.2-linux-x86_64.tar.gz

su elk
tar -xvf kibana-7.4.2-linux-x86_64.tar.gz -C /data/elk
mv kibana-7.4.2-linux-x86_64 kibana
cd /data/elk/kibana

# 配置信息
server.host: "0.0.0.0"
server.port: 5601
elasticsearch.hosts: ["http://192.168.23.96:9200"]

# 啟動
./bin/kibana

# 訪問驗證，瀏覽器訪問
http://192.168.23.96:5601

這里注意一個問題，我這里logstash由於以來jdk8，而elasticsearch自帶jdk不用額外安裝所以啟動順序最好不變，elastic -> logstash -> filebeat -> kibana

權限控制x-pack
#啟用elasticsearch xpack安全驗證

vim elasticsearch.yml
xpack.security.enabled: true
# 單個節點
discovery.type: single-node

# 重新啟動
./bin/elasticsearch -d 

#設置密碼,運行elasticsearch-setup-passwords設置密碼（賬號默認為elastic）：
./elasticsearch-setup-passwords interactive

# 密碼要記住，后面有需要
Changed password for user [apm_system] ： apm2020
Changed password for user [kibana]: kibana2020
Changed password for user [logstash_system]: logstash2020
Changed password for user [beats_system]: beats2020
Changed password for user [remote_monitoring_user]: remote2002
Changed password for user [elastic]: elastic2020

# logstash配置x-pack:

vim logstash.conf
input {
    beats {
        port => 5044
    }
}
output {
    elasticsearch {
        hosts => ["localhost:9200"]
        user => "elastic"
        password => "elastic2020"
        index => ["%{[fields][server]}-%{+YYYY-MM-dd}"]
        manage_template => false
        }
}

# Kibana的配置x-pack:

vim kibana.yml
elasticsearch.username: "kibana"
elasticsearch.password: "kibana2020"
# 隨便輸入不少於32位字符
xpack.security.encryptionKey: "rcrafrgraffbdsacdefghigklmnopqvvrsvrsrtfdfavfjkadfakfacjdiaofoidaui3cjda"
xpack.security.sessionTimeout: 600000
創建只讀賬戶

x-pack配置成功后就可以看到security，創建一個應用系統角色，選擇對應的索引文件，分配對應的權限read
在Management下面的Kibana有一個Security，有User和Role點擊Role


免責聲明！

本站轉載的文章為個人學習借鑒使用，本站對版權不負任何法律責任。如果侵犯了您的隱私權益，請聯系本站郵箱yoyou2525@163.com刪除。



猜您在找 centos7部署elk日志系統 centos7.5部署ELk centos7部署ELK(看這一篇就夠了) centos7部署elk集群(filebeat+kafka+logstash+elasticsearch+kibana) CentOS7部署GeoServer Centos7部署Zabbix Centos7部署Kubernetes集群 centos7部署Django Centos7部署PaddleOCR環境 CentOS7部署ToughRadius
 
粵ICP備18138465號   © 2018-2025 CODEPRJ.COM