mysqld_exporter 的使用（五）

1、安裝mysql_exporter

[root@mysqld ~]# wget https://github.com/prometheus/mysqld_exporter/releases/download/v0.12.1/mysqld_exporter-0.12.1.linux-amd64.tar.gz
[root@mysqld ~]# tar xf mysqld_exporter-0.12.1.linux-amd64.tar.gz -C /usr/local/
[root@mysqld ~]# ln -sv /usr/local/mysqld_exporter-0.12.1.linux-amd64/ /usr/local/prometheus_mysqld

2、在MySQL中創建用戶並授權

mysql> CREATE USER  'mysqld_exporter'@'%'  IDENTIFIED BY  'mysqld_exporter';
mysql> GRANT REPLICATION CLIENT,PROCESS ON *.* TO 'mysqld_exporter'@'%' identified by 'mysqld_exporter';
mysql> GRANT SELECT ON *.* TO 'mysqld_exporter'@'%';

3、寫一個mysqld_exporter的配置文件

[root@mysqld ~]# cat /usr/local/prometheus_mysqld/.my.cnf 
[client]
user=mysqld_exporter
password=mysqld_exporter
port=3306

4、配置systemd啟動mysqld_exporter

[root@mysqld ~]# cat /lib/systemd/system/mysqld_exporter.service
[Unit]
Description=Mysqld_exporter
After=network.target
[Service]
ExecStart=/usr/local/prometheus_mysqld/mysqld_exporter --config.my-cnf=/usr/local/prometheus_mysqld/.my.cnf
[Install]
WantedBy=multi-user.target
systemctl daemon-reload
systemctl start mysqld_exporter
systemctl enable mysqld_exporter
ss -lnpt|grep 9104
LISTEN     0      128       [::]:9104                  [::]:*                   users:(("mysqld_exporter",pid=1853,fd=3))

5、確認監控指標正常

[root@mysqld ~]# curl http://localhost:9104/metrics|grep mysql_up
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  130k    0  130k    0     0  6920k      0 --:--:-- --:--:-- --:--:-- 7269k
# HELP mysql_up Whether the MySQL server is up.
# TYPE mysql_up gauge
mysql_up 1

6、在Prometheus的server端添加job任務

[root@prometheus prometheus]# vim /usr/local/prometheus/prometheus.yml 
  - job_name: 'mysql_db'
    static_configs:
    - targets: ['10.0.0.52:9104']
      labels:
        group: mysql_db
[root@prometheus prometheus]# systemctl restart prometheus

在瀏覽器中訪問 http://10.0.0.51:9090/targets

7、在grafana中導入MySQL監控圖表
點擊import，在彈出界面中輸入7362，數據源選擇Prometheus
在grafana中看到采集到的MySQL數據庫的相關參數

8、配置mysql_exporter告警規則

[root@prometheus rules]# pwd
/usr/local/prometheus/rules
[root@prometheus rules]# cat mysql_rules.yml
groups:
- name: mysql.rules
  rules:
  - alert: MysqlDown
    expr: up == 0
    for: 0m
    labels:
      severity: critical
    annotations:
      title: 'MySQL down'
      description: "Mysql實例: 【{{ $labels.instance }}】, MySQL instance is down"

  - alert: MysqlRestarted
    expr: mysql_global_status_uptime < 60
    for: 0m
    labels:
      severity: info
    annotations:
      title: 'MySQL Restarted'
      description: "Mysql實例: 【{{ $labels.instance }}】, MySQL has just been restarted, less than one minute ago"

  - alert: MysqlTooManyConnections(>80%)
    expr: avg by (instance) (rate(mysql_global_status_threads_connected[1m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      title: 'MySQL too many connections (> 80%)'
      description: "Mysql實例: 【{{ $labels.instance }}】, More than 80% of MySQL connections are in use, Current Value: {{ $value }}%"

  - alert: MysqlThreadsRunningHigh
    expr: mysql_global_status_threads_running > 40
    for: 2m
    labels:
      severity: warning
    annotations:
      title: 'MySQL Threads_Running High'
      description: "Mysql實例: 【{{ $labels.instance }}】, Threads_Running above the threshold(40), Current Value: {{ $value }}"

  - alert: MysqlQpsHigh
    expr: sum by (instance) (rate(mysql_global_status_queries[2m])) > 500
    for: 2m
    labels:
      severity: warning
    annotations:
      title: 'MySQL QPS High'
      description: "Mysql實例: 【{{ $labels.instance }}】, MySQL QPS above 500"

  - alert: MysqlSlowQueries
    expr: increase(mysql_global_status_slow_queries[1m]) > 0
    for: 2m
    labels:
      severity: warning
    annotations:
      title: 'MySQL slow queries'
      description: "Mysql實例: 【{{ $labels.instance }}】, has some new slow query."

  - alert: MysqlTooManyAbortedConnections
    expr: round(increase(mysql_global_status_aborted_connects[5m])) > 20
    for: 2m
    labels:
      severity: warning
    annotations:
      title: 'MySQL too many Aborted connections in 2 minutes'
      description: "Mysql實例: 【{{ $labels.instance }}】, {{ $value }} Aborted connections within 2 minutes"

  - alert: MysqlTooManyAbortedClients
    expr: round(increase(mysql_global_status_aborted_clients[120m])) > 10
    for: 2m
    labels:
      severity: warning
    annotations:
      title: 'MySQL too many Aborted connections in 2 hours'
      description: "Mysql實例: 【{{ $labels.instance }}】, {{ $value }} Aborted Clients within 2 hours"

  - alert: MysqlSlaveIoThreadNotRunning
    expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0
    for: 0m
    labels:
      severity: critical
    annotations:
      title: 'MySQL Slave IO thread not running'
      description: "Mysql實例: 【{{ $labels.instance }}】, MySQL Slave IO thread not running"

  - alert: MysqlSlaveSqlThreadNotRunning
    expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0
    for: 0m
    labels:
      severity: critical
    annotations:
      title: 'MySQL Slave SQL thread not running'
      description: "Mysql實例: 【{{ $labels.instance }}】, MySQL Slave SQL thread not running"

  - alert: MysqlSlaveReplicationLag
    expr: mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 30
    for: 1m
    labels:
      severity: critical
    annotations:
      title: 'MySQL Slave replication lag'
      description: "Mysql實例: 【{{ $labels.instance }}】, MySQL replication lag"

  - alert: MysqlInnodbLogWaits
    expr: rate(mysql_global_status_innodb_log_waits[15m]) > 10
    for: 0m
    labels:
      severity: warning
    annotations:
      title: 'MySQL InnoDB log waits'
      description: "Mysql實例: 【{{ $labels.instance }}】, innodb log writes stalling"

9、將告警規則集成到Prometheus

[root@prometheus prometheus]# vim /usr/local/prometheus/prometheus.yml 
rule_files:
  - "/usr/local/prometheus/rules/*.yml"
[root@prometheus prometheus]# systemctl restart prometheus