一、環境介紹
[root@master test]# uname -r
4.4.223-1.el7.elrepo.x86_64
[root@master test]# kubectl get node
NAME STATUS ROLES AGE VERSION
master Ready master 27d v1.18.0
node Ready <none> 27d v1.18.0
[root@master test]# helm version
version.BuildInfo{Version:"v3.2.0", GitCommit:"e11b7ce3b12db2941e90399e874513fbd24bcb71", GitTreeState:"clean", GoVersion:"go1.13.10"}
二、helm添加倉庫
#阿里雲 helm repo add aliyuncs https://apphub.aliyuncs.com #官方 helm repo add stable https://kubernetes-charts.storage.googleapis.com
三、helm search prometheus-operator
[root@master test]# helm search repo prometheus-operator NAME CHART VERSION APP VERSION DESCRIPTION aliyuncs/prometheus-operator 8.7.0 0.35.0 Provides easy monitoring definitions for Kubern... stable/prometheus-operator 8.13.7 0.38.1 Provides easy monitoring definitions for Kubern...
四、安裝
helm install mypro aliyuncs/prometheus-operator
五、查看
[root@master test]# helm list NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION mypro default 1 2020-06-09 09:32:37.091220013 +0800 CST deployed prometheus-operator-8.7.0 0.35.0 [root@master test]# helm status mypro NAME: mypro LAST DEPLOYED: Tue Jun 9 09:32:37 2020 NAMESPACE: default STATUS: deployed REVISION: 1 NOTES: The Prometheus Operator has been installed. Check its status by running: kubectl --namespace default get pods -l "release=mypro" Visit https://github.com/coreos/prometheus-operator for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. [root@master test]# kubectl --namespace default get pods -l "release=mypro" NAME READY STATUS RESTARTS AGE mypro-grafana-f5b868868-8ckgs 2/2 Running 0 55m mypro-prometheus-node-exporter-dg6w4 1/1 Running 0 55m mypro-prometheus-node-exporter-x9l4b 1/1 Running 0 55m mypro-prometheus-operator-operator-5b458d4659-p7t4l 2/2 Running 0 55m
六、配置ingress瀏覽器訪問
[root@master test]# cat grafana-ingress.yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: name: ingress-grafana spec: rules: - host: grafana.zhang.com http: paths: - backend: serviceName: mypro-grafana servicePort: 80 [root@master test]# cat prometheus-ingress.yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: name: ingress-prometheus spec: rules: - host: prometheus.zhang.com http: paths: - backend: serviceName: mypro-prometheus-operator-prometheus servicePort: 9090 [root@master test]# cat alertmanager-ingress.yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: name: ingress-alertmanager spec: rules: - host: alertmanager.zhang.com http: paths: - backend: serviceName: mypro-prometheus-operator-alertmanager servicePort: 9093
七、查看
[root@master test]# kubectl get ingress NAME CLASS HOSTS ADDRESS PORTS AGE ingress-alertmanager <none> alertmanager.zhang.com 10.111.12.239 80 29m ingress-grafana <none> grafana.zhang.com 10.111.12.239 80 32m ingress-prometheus <none> prometheus.zhang.com 10.111.12.239 80 30m
八、瀏覽器訪問
九、修改alertmanager的報警配置
1、創建alertmanger的配置文件
[root@master test]# cat alertmanger_config.yaml global: resolve_timeout: 5m #處理超時時間，默認為5min smtp_smarthost: 'smtp.163.com:465' # 郵箱smtp服務器代理 smtp_from: 'xxxx@163.com' # 發送郵箱名稱 smtp_auth_username: 'xxxx@163.com' # 郵箱名稱 smtp_auth_password: 'xxxxxxxxx' #郵箱密碼 smtp_require_tls: false route: group_by: ['alertname'] # 報警分組名稱 group_wait: 10s # 最初即第一次等待多久時間發送一組警報的通知 group_interval: 10s # 在發送新警報前的等待時間 repeat_interval: 1m # 發送重復警報的周期 receiver: 'email' # 發送警報的接收者的名稱，以下receivers name的名稱 receivers: - name: 'email' # 警報 email_configs: # 郵箱配置 - to: 'xxxxxx@163.com' # 接收警報的email配置 inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'dev', 'instance']
2、base64編碼(要使用xargs設置為一行，再用sed刪掉空格)
cat alertmanger_config.yaml |base64
3、替換secret中的alertmanager.yaml配置（用上面base64編碼的替換）
kubectl edit secret alertmanager-mypro-prometheus-operator-alertmanager
4、查看郵箱
十、自定義監控報警項
1、介紹
prometheus-operator可以使用PrometheusRule來動態的添加自定義監控項
2、查看prometheus-operator項目中Prometheus的標簽選擇器
kubectl get prometheus mypro-prometheus-operator-prometheus -o jsonpath={".spec.ruleSelector"};echo
3、創建自定義的PrometheusRule監控
[root@master ~]# cat test-PrometheusRule.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
labels:
app: prometheus-operator #和Prometheus中的標簽選擇器中的標簽，如果要自己創建一個Prometheus的配置關聯到PrometheusRule的labels
release: mypro #和Prometheus中的標簽選擇器中的標簽，如果要自己創建一個Prometheus的配置關聯到PrometheusRule的labels
prometheus: test-example
name: test-load1-prometheusrule
spec:
groups:
- name: test-load-1
rules:
- alert: test-load-1
expr: node_load1 > 1
for: 2m
labels:
team: node
annotations:
summary: "{{$labels.instance}}: load 1 >1"
description: "{{$labels.instance}}: job {{$labels.job}} 測試測試 負載大於1"
#導入
kubectl apply -f test-PrometheusRule.yaml
4、登錄pod查看
[root@master ~]# kubectl exec -it prometheus-mypro-prometheus-operator-prometheus-0 sh kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl kubectl exec [POD] -- [COMMAND] instead. Defaulting container name to prometheus. Use 'kubectl describe pod/prometheus-mypro-prometheus-operator-prometheus-0 -n default' to see all of the containers in this pod. /prometheus $ ls /etc/prometheus/rules/prometheus-mypro-prometheus-operator-prometheus-rulefiles-0/default-test-load1-prometheusrule.yaml /etc/prometheus/rules/prometheus-mypro-prometheus-operator-prometheus-rulefiles-0/default-test-load1-prometheusrule.yaml /prometheus $ cat /etc/prometheus/rules/prometheus-mypro-prometheus-operator-prometheus-rulefiles-0/default-test-load1-prometheusrule.yaml groups: - name: test-load-1 rules: - alert: test-load-1 annotations: description: '{{$labels.instance}}: job {{$labels.job}} 測試測試 負載大於1' summary: '{{$labels.instance}}: load 1 >1' expr: node_load1 > 1 for: 2m labels: team: node
5、瀏覽器查看prometheus
