自動化監控kubernetes_sd_configs

參考鏈接

- 架構圖

- 監控K8s集群Pod（kubelet集成了cadvisor，暴露接口）

promethues -> apiserver(192.168.2.60:6443) -> kubelet(cadvisor)

創建rbac

[root@master k8s-ftp]# cat rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - services
  - endpoints
  - pods
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - "extensions"
  resources:
    - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - configmaps
  - nodes/metrics
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: kube-system 
[root@master k8s-ftp]# kubectl apply -f rbac.yaml
serviceaccount/prometheus created

獲取token

[root@master k8s-ftp]# kubectl get sa prometheus -n kube-system -o yaml|tail -2
secrets:
- name: prometheus-token-hx5h8
[root@master k8s-ftp]# kubectl describe secret prometheus-token-hx5h8 -n kube-system
Name:         prometheus-token-hx5h8
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: prometheus
              kubernetes.io/service-account.uid: 74882727-0808-43bb-ac3a-7b813af7c3ee

Type:  kubernetes.io/service-account-token

Data
====
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IjBfd1JIa0ItdTZnaTZONUxFc192dTBFc2VWYjh3TV9zMmxIeU1zYWQtSUUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLXRva2VuLWh4NWg4Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6InByb21ldGhldXMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI3NDg4MjcyNy0wODA4LTQzYmItYWMzYS03YjgxM2FmN2MzZWUiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06cHJvbWV0aGV1cyJ9.VTBej_PRKHRjMK4yI_JKm2dWb_s_ndN4NQ08k22Pl7yLilj62iZYoE0hywzpMLL149gHQmLyITmFODyJz98WfFeJS3h6RKsolNyBxE_3zvvKAqHG-RzI-LSrqBYFexfEilKwuQZ6K8cmjlJjxq1Gya3vE1MFeOT3d51tzV15hn-WtxNiOlEbwZno5hhfSLazS9seLjpnYrv02lUk-tZ5Fxv5E0XaEf6PbXRVYfn42d105_5wMvkA3lrqe3IK-u14awoKgH8MbqsDgqTCp0l8iePwc-s_zVL6FCeQSTnBZc0j9SWoUdIJIbAxhRbpwimmqeBomwFEGkSK-aGn82khJw
ca.crt:     1066 bytes
namespace:  11 bytes

新增prometheus配置項

[root@slave-2 prometheus]# cat k8s.token
eyJhbGciOiJSUzI1NiIsImtpZCI6IjBfd1JIa0ItdTZnaTZONUxFc192dTBFc2VWYjh3TV9zMmxIeU1zYWQtSUUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLXRva2VuLWh4NWg4Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6InByb21ldGhldXMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI3NDg4MjcyNy0wODA4LTQzYmItYWMzYS03YjgxM2FmN2MzZWUiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06cHJvbWV0aGV1cyJ9.VTBej_PRKHRjMK4yI_JKm2dWb_s_ndN4NQ08k22Pl7yLilj62iZYoE0hywzpMLL149gHQmLyITmFODyJz98WfFeJS3h6RKsolNyBxE_3zvvKAqHG-RzI-LSrqBYFexfEilKwuQZ6K8cmjlJjxq1Gya3vE1MFeOT3d51tzV15hn-WtxNiOlEbwZno5hhfSLazS9seLjpnYrv02lUk-tZ5Fxv5E0XaEf6PbXRVYfn42d105_5wMvkA3lrqe3IK-u14awoKgH8MbqsDgqTCp0l8iePwc-s_zVL6FCeQSTnBZc0j9SWoUdIJIbAxhRbpwimmqeBomwFEGkSK-aGn82khJw

- job_name: kubernetes-nodes-cadvisor
  metrics_path: /metrics
  scheme: https
  kubernetes_sd_configs:
  - role: node
    api_server: https://192.168.2.60:6443
    bearer_token_file: /opt/monitor/prometheus/token.k8s 
    tls_config:
      insecure_skip_verify: true
  bearer_token_file: /opt/monitor/prometheus/token.k8s 
  tls_config:
    insecure_skip_verify: true
  relabel_configs:
  # 將標簽(.*)作為新標簽名，原有值不變
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.*)
  # 修改NodeIP:10250為APIServerIP:6443
  - action: replace
    regex: (.*)
    source_labels: ["__address__"]
    target_label: __address__
    replacement: 192.168.2.60:6443
  # 實際訪問指標接口 https://NodeIP:10250/metrics/cadvisor 這個接口只能APISERVER訪問，故此重新標記標簽使用APISERVER代理訪問
  - action: replace
    source_labels: [__meta_kubernetes_node_name]
    target_label: __metrics_path__
    regex: (.*)
    replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

- 監控K8s資源對象狀態