K8S之監控etcd集群（自帶metrics接口）

一、監控etcd集群

1.1、查看接口信息

[root@k8s-master01 ~]# curl --cert /etc/etcd/ssl/etcd.pem --key /etc/etcd/ssl/etcd-key.pem  https://192.168.1.201:2379/metrics -k 
# 這樣也行
curl -L http://localhost:2379/metrics

1.2、創建service和Endpoints

# 創建ep和svc代理外部的etcd服務，其他自帶metrics接口的服務也是如此！
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    app: etcd-k8s
  name: etcd-k8s
  namespace: kube-system
subsets:
- addresses:     # etcd節點對應的主機ip，有幾台就寫幾台
  - ip: 192.168.1.201
  - ip: 192.168.1.202
  - ip: 192.168.1.203
  ports:
  - name: etcd-port
    port: 2379   # etcd端口
    protocol: TCP
---
apiVersion: v1
kind: Service 
metadata:
  labels:
    app: etcd-k8s
  name: etcd-k8s
  namespace: kube-system
spec:
  ports:
  - name: etcd-port
    port: 2379
    protocol: TCP
    targetPort: 2379
  type: ClusterIP

1.3、測試是否代理成功

#再次curl，把IP換成svc的IP測試，輸出相同內容即創建成功
[root@k8s-master01 ~]# kubectl get svc -n kube-system etcd-k8s
NAME      TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
etcd-ep   ClusterIP   10.103.53.103   <none>        2379/TCP   8m54s

# 再次請求接口
[root@k8s-master01 ~]# curl --cert /etc/etcd/ssl/etcd.pem --key /etc/etcd/ssl/etcd-key.pem  https://10.111.200.116:2379/metrics -k

1.4、創建secret

# 1、這里我們k8s-master01節點進行創建,ca為k8sca證書，剩下2個為etcd證書，這是我證書所在位置
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  
# 2、接下來我們需要創建一個secret，讓prometheus pod節點掛載
kubectl create secret generic etcd-ssl --from-file=/etc/kubernetes/pki/etcd/etcd-ca.pem --from-file=/etc/kubernetes/pki/etcd/etcd.pem --from-file=/etc/kubernetes/pki/etcd/etcd-key.pem -n monitoring

# 3、創建完成后可以檢查一下
[root@k8s-master01 prometheus-down]# kubectl describe secrets -n monitoring etcd-ssl
Name:         etcd-ssl
Namespace:    monitoring
Labels:       <none>
Annotations:  <none>

Type:  Opaque

Data
====
etcd-ca.pem:   1367 bytes
etcd-key.pem:  1679 bytes
etcd.pem:      1509 bytes

1.5、編輯prometheus，把證書掛載進去

# 1、通過edit直接編輯prometheus
[root@k8s-master01 ~]# kubectl edit prometheus k8s -n monitoring
# 在replicas底下加上secret名稱
replicas:2
secrets:
- etcd-ssl #添加secret名稱

# 進入容器查看，就可以看到證書掛載進去了
[root@k8s-master01 prometheus-down]# kubectl exec -it -n monitoring prometheus-k8s-0 /bin/sh

# 查看文件是否存在
/prometheus $ ls /etc/prometheus/secrets/etcd-ssl/
etcd-ca.pem   etcd-key.pem  etcd.pem

1.6、創建ServiceMonitor

[root@k8s-master01 ~]# cat etcd-servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: etcd-k8s
  namespace: monitoring
  labels:
    app: etcd-k8s
spec:
  jobLabel: app
  endpoints:
    - interval: 30s
      port: etcd-port  # 這個port對應 Service.spec.ports.name
      scheme: https
      tlsConfig:
        caFile: /etc/prometheus/secrets/etcd-ssl/etcd-ca.pem #證書路徑 (在prometheus pod里路徑)
        certFile: /etc/prometheus/secrets/etcd-ssl/etcd.pem
        keyFile: /etc/prometheus/secrets/etcd-ssl/etcd-key.pem
        insecureSkipVerify: true  # 關閉證書校驗
  selector:
    matchLabels:
      app: etcd-k8s  # 跟scv的lables保持一致
  namespaceSelector:
    matchNames:
    - kube-system    # 跟svc所在namespace保持一致
# 匹配Kube-system這個命名空間下面具有app=etcd-k8s這個label標簽的Serve，job label用於檢索job任務名稱的標簽。由於證書serverName和etcd中簽發的證書可能不匹配，所以添加了insecureSkipVerify=true將不再對服務端的證書進行校驗

1.7、頁面查看三個etcd節點都獲取到數據

此處數據獲取有點慢，需要等待一下

1.8、grafana模板導入

數據采集完成后，接下來可以在grafana中導入dashboard

# 打開官網來的如下圖所示，點擊下載JSO文件
grafana官網：https://grafana.com/grafana/dashboards/3070
中文版ETCD集群插件：https://grafana.com/grafana/dashboards/9733

點擊HOME–>導入模板

導入后頁面展示