Prometheus之Alertmanager郵件報警配置

一 Alertmanager配置

1.1 編輯Alertmanager配置文件

點擊查看代碼

root@node-02:~# cat /usr/local/alertmanager/alertmanager.yml 
global:
  smtp_from: '[e-mail]' 
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_auth_username: '[e-mail]'
  smtp_auth_password: 'xxxxxxxxx'
  smtp_hello: '@qq.com'
  smtp_require_tls: false 
route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 5s
  repeat_interval: 1m
  receiver: 'web.hook'
receivers:
- name: 'web.hook'
  email_configs:
    - to: '[e-mail]'
inhibit_rules:
  - source_match:
      alertname: InstanceDown
      severity: 'critical'
    target_match:
      alertname: InstanceDown
      severity: 'critical'
    equal: ['alertname', 'dev', 'instance']

1.2 重啟Alertmanager服務

root@node-02:~# systemctl restart alertmanager

二 Prometheus報警設置

2.1 修改Prometheus配置文件

root@prometheus-01:~# cat /usr/local/prometheus/prometheus.yml 
alerting:
  alertmanagers:
    - static_configs:
        - targets:
           - 192.168.174.104:9093
rule_files:

 
 
 
         

  
  
  
           
"rules/*.yaml"
 
  
  
  
          
"alert_rules/*.yaml"
 
 
 
 
         

2.2 創建告警規則文件

root@prometheus-01:~# cat /usr/local/prometheus/alert_rules/instance_down.yaml 
groups:
- name: ALLInstances
  rules:
    - alert: InstanceDown
      expr: up == 0
      for: 1m
      annotations:
        title: 'Instance down'
        description: 'Instance has been down for more than 1 munute.'
      labels:
        severity: 'critical'

2.3 驗證規則

root@prometheus-01:~# /usr/local/prometheus/promtool check rules /usr/local/prometheus/alert_rules/instance_down.yaml 
Checking /usr/local/prometheus/alert_rules/instance_down.yaml
  SUCCESS: 1 rules found

2.4 重啟Prometheus服務

root@prometheus-01:~# systemctl restart prometheus.service 

2.5 停止node_exporter服務

root@k8s-master-01:~# systemctl stop node-exporter

2.6 Prometheus web界面

2.7 驗證Alertmanager界面

2.8  驗證郵件告警信息

 

三 告警模板

3.1 修改Prometheus規則文件

root@prometheus-01:~# cat /usr/local/prometheus/alert_rules/pod.yaml 
groups:
  - name: alert_pod.rules
    rules:
    - alert: pod_all_cpu_usage
      expr: (sum by(name)(rate(container_cpu_user_seconds_total{image!=""}[5m]))* 100) > 1
      for: 2m
      labels:
        serverity: critical
        service: pod
      annotations:
        description: 容器 {{$labels.name}} CPU 資源利用率大於10%, (current value is {{$value}})
        summary: dev CPU 負載告警
- alert: pod_all_memory_usage
  expr: sort_desc(avg by(name)(irate(node_memory_MemFree_bytes{name!=""}[5m]))) > 2 #內存大於2G 
  for: 2m
  labels:
    serverity: critical
    service: pod
  annotations:
    description: 容器 {{$labels.name}} 內存 資源利用率大於2G, (current value is {{$value}})
    summary: dev memory 負載告警

3.2 重啟Prometheus服務

root@prometheus-01:~# systemctl restart prometheus.service 

3.3  Prometheus web界面驗證

3.4 查看郵件告警信息