监控页面出现看到有运行失败的pod
1) 查看有哪些不运行的pod
custom-metrics-apiserver日志占满空间被驱逐
[root@hadoop03 ~]# kubectl get pod -A
[root@hadoop03 ~]# kubectl get pod -n prom | grep Evicted custom-metrics-apiserver-694844dfbc-27jcs 0/1 Evicted 0 6d7h custom-metrics-apiserver-694844dfbc-2qqhq 0/1 Evicted 0 21h custom-metrics-apiserver-694844dfbc-2r9xd 0/1 Evicted 0 4d9h custom-metrics-apiserver-694844dfbc-44zlc 0/1 Evicted 0 4d23h custom-metrics-apiserver-694844dfbc-48knc 0/1 Evicted 0 3d20h custom-metrics-apiserver-694844dfbc-49j9n 0/1 Evicted 0 9d custom-metrics-apiserver-694844dfbc-4crrm 0/1 Evicted 0 4d23h custom-metrics-apiserver-694844dfbc-4kfpg 0/1 Evicted 0 4d23h
2) 获取所有Evicted pod名称
[root@hadoop03 ~]# kubectl get pod -n prom | grep Evicted | awk '{print $1}' custom-metrics-apiserver-694844dfbc-27jcs custom-metrics-apiserver-694844dfbc-2qqhq custom-metrics-apiserver-694844dfbc-2r9xd custom-metrics-apiserver-694844dfbc-44zlc custom-metrics-apiserver-694844dfbc-48knc custom-metrics-apiserver-694844dfbc-49j9n custom-metrics-apiserver-694844dfbc-4crrm custom-metrics-apiserver-694844dfbc-4kfpg
3) 删除pod
[root@hadoop03 ~]# kubectl get pod -n prom | grep Evicted | awk '{print $1}' | xargs kubectl delete pod -n prom pod "custom-metrics-apiserver-694844dfbc-27jcs" deleted pod "custom-metrics-apiserver-694844dfbc-2qqhq" deleted pod "custom-metrics-apiserver-694844dfbc-2r9xd" deleted pod "custom-metrics-apiserver-694844dfbc-44zlc" deleted pod "custom-metrics-apiserver-694844dfbc-48knc" deleted pod "custom-metrics-apiserver-694844dfbc-49j9n" deleted pod "custom-metrics-apiserver-694844dfbc-4crrm" deleted pod "custom-metrics-apiserver-694844dfbc-4kfpg" deleted
3) 查看pod
[root@hadoop03 ~]# kubectl get pod -n prom NAME READY STATUS RESTARTS AGE custom-metrics-apiserver-694844dfbc-szcbg 1/1 Running 0 5h14m grafana-core-5d657547bf-xwpgp 1/1 Running 0 12d kube-state-metrics-6c66fc445f-fjz54 1/1 Running 0 11d prometheus-node-exporter-dqtxd 1/1 Running 0 15d prometheus-node-exporter-nlbz9 1/1 Running 0 15d prometheus-node-exporter-p7z47 1/1 Running 0 15d prometheus-node-exporter-szck4 1/1 Running 0 15d prometheus-server-59b598cc9d-tsmqd 1/1 Running 0 15d