二進制部署k8s（1.18版本）+高可用版本測試

二進制部署k8s（1.18版本）

部署說明

部署說明

軟件名稱	下載地址	備注
centos7.7+	https://mirrors.aliyun.com/centos/7.7.1908/isos/x86_64/CentOS-7-x86_64-Minimal-1908.iso	宿主機操作系統
kubernetes-server	https://dl.k8s.io/v1.18.4/kubernetes-server-linux-amd64.tar.gz
etcd	https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz	k8s數據存儲
cfssl	https://pkg.cfssl.org/R1.2/cfssl_linux-amd64%3Cbr/%3Ehttps://pkg.cfssl.org/R1.2/cfssljson_linux-amd64%3Cbr/%3Ehttps://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64	證書簽發工具
docker	https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz	CNR運行引擎
cni	https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz	網絡環境

部署規划

主機	ip	作用	部署的軟件	備注
centos7-node4	172.17.17.85	master	kube-apiserver,kube-controller-manager,kube-scheduler,docker,etcd	后期介紹master擴容
centos7-node5	172.17.17.86	node	kubelet,kube-proxy,docker,etcd	后期介紹master擴容
centos7-node6	172.17.17.87	node	kubelet,kube-proxy,docker,etcd	后期介紹master擴容

 

 

 

 

 

系統初始化（所有節點執行）

• 軟件安裝路徑默認路徑為/data

# 更新yum源
yum -y install wget && wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo && yum -y install epel-release 
# 關閉selinux，firewalld，swap
sed -i 's/enforcing/disabled/' /etc/selinux/config
systemctl disable firewalld && systemctl stop firewalld
sed -ri 's/.*swap.*/#&/' /etc/fstab && swapoff -a
# 設置好主機名與主機名解析
cat >> /etc/hosts << EOF
172.17.17.85 k8s-master-1
172.17.17.89 k8s-master-2
172.17.17.86 k8s-node5
172.17.17.87 k8s-node6
EOF
# 將橋接的IPv4流量傳遞到iptables的鏈
modprobe br_netfilter
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
sysctl -p /etc/sysctl.d/k8s.conf #配置生效
#時間同步
yum install chrony -y && systemctl enable chronyd && systemctl start chronyd
timedatectl set-timezone Asia/Shanghai && timedatectl set-ntp yes

生成etcd證書配置

准備cfssl證書管理工具，使用json文件生成證書，相比openssl更方便使用

# 軟件安裝
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl*

准備ca與證書配置

mkdir -p ~/TLS/{etcd,k8s} && cd ~/TLS/etcd

# 自簽CA配置文件

cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
   },
    "profiles": {
      "www": {
        "expiry": "87600h",        
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
       ]
     }
   }
 }
}
EOF

# 自簽csr配置文件

cat > ca-csr.json << EOF
{
  "CN": "etcd CA",
  "key": {
    "algo": "rsa",
    "size": 2048
 },
  "ca": {
     "expiry": "87600h"
 },
  "names": [
   {
      "C": "CN",
      "L": "BJ",
      "ST": "BeiJing"
   }
 ]
}
EOF

# 生成CA證書

[root@k8s-master-1 etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
[root@k8s-master-1 etcd]# ls *pem

# 簽發ETCD https證書

[root@k8s-master-1 etcd]# cat > server-csr.json << EOF

{
    "CN": "etcd",
    "hosts": [
        "172.17.17.85",
        "172.17.17.86",
        "172.17.17.87",
        "172.17.17.84",
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}

EOF

 

注:上述文件hosts字段中IP為所有etcd節點的集群內部通信IP，一個都不能少!為了方便后期擴 容可以多寫幾個預留的IP。

簽發證書

[root@k8s-master-1 etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
[root@k8s-master-1 etcd]# ls server*pem #簽發證書文件

部署ETCD集群

單個節點配置

# 安裝路徑准備
mkdir /data/etcd/{bin,cfg,ssl,data} -p
# 二進制文件准備
wget https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz && tar xf etcd-v3.4.9-linux-amd64.tar.gz
mv etcd-v3.4.9-linux-amd64/etcd* /data/etcd/bin/
# 當前節點172.17.17.85配置文件
cat > /data/etcd/cfg/etcd.conf <<EOF
#[Member]
ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/data/etcd/data/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.17.17.85:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.17.17.85:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.17.17.85:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://172.17.17.85:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://172.17.17.85:2380,etcd-2=https://172.17.17.86:2380,etcd-3=https://172.17.17.87:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
# systemctl 啟動管理文件配置(所有節點配置是一致的)
cat > /usr/lib/systemd/system/etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/data/etcd/cfg/etcd.conf 
ExecStart=/data/etcd/bin/etcd \
   --cert-file=/data/etcd/ssl/server.pem \
   --key-file=/data/etcd/ssl/server-key.pem \
   --peer-cert-file=/data/etcd/ssl/server.pem \
   --peer-key-file=/data/etcd/ssl/server-key.pem \
   --trusted-ca-file=/data/etcd/ssl/ca.pem \
   --peer-trusted-ca-file=/data/etcd/ssl/ca.pem \
   --logger=zap
Restart=on-failure
LimitNOFILE=65536
​
[Install]
WantedBy=multi-user.target
EOF
# 拷貝證書文件
mv ~/TLS/etcd/*pem /data/etcd/ssl
# 啟動當前節點
systemctl daemon-reload && systemctl enable etcd && systemctl start etcd
​

配置文件字段介紹

ETCD_NAME:節點名稱，集群中唯一 ETCD_DATA_DIR:數據目錄 ETCD_LISTEN_PEER_URLS:集群通信監聽地址 ETCD_LISTEN_CLIENT_URLS:客戶端訪問監聽地址 ETCD_INITIAL_ADVERTISE_PEER_URLS:集群通告地址 ETCD_ADVERTISE_CLIENT_URLS:客戶端通告地址 ETCD_INITIAL_CLUSTER:集群節點地址 ETCD_INITIAL_CLUSTER_TOKEN:集群Token ETCD_INITIAL_CLUSTER_STATE:加入集群的當前狀態，new是新集群，existing表示加入已有集 群

其余兩個節點配置

1. 數據分發（從172.17.17.85分發到86，87兩個節點）

   scp -r /data/etcd/ root@172.17.17.86:/data/      scp -r /data/etcd/ root@172.17.17.87:/data/
   

   scp /usr/lib/systemd/system/etcd.service root@172.17.17.86:/usr/lib/systemd/system/

   scp /usr/lib/systemd/system/etcd.service root@172.17.17.87:/usr/lib/systemd/system/

然后在節點2和節點3分別修改etcd.conf配置文件中的節點名稱和當前服務器IP：

vi /data/etcd/cfg/etcd.conf

#[Member]

ETCD_NAME="etcd-1"# 修改此處，節點2改為etcd-2，節點3改為etcd-3

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"

ETCD_LISTEN_PEER_URLS="https://172.17.17.86:2380"# 修改此處為當前服務器

IPETCD_LISTEN_CLIENT_URLS="https://172.17.17.86:2379"# 修改此處為當前服務器IP

#[Clustering]

ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.17.17.86:2380"# 修改此處為當前服務器

IPETCD_ADVERTISE_CLIENT_URLS="https://172.17.17.86:2379"# 修改此處為當前服務器

IPETCD_INITIAL_CLUSTER="etcd-1=https://172.17.17.85:2380,etcd-2=https://172.17.17.86:2380,etcd-3=https://172.17.17.87:2380"

ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"

ETCD_INITIAL_CLUSTER_STATE="new"

最后啟動etcd並設置開機啟動，同上。

systemctl daemon-reload && systemctl enable etcd && systemctl start etcd

驗證Etcd部署狀態是否成功

[root@k8s-master-1 default.etcd]# ETCDCTL_API=3 /data/etcd/bin/etcdctl --cacert=/data/etcd/ssl/ca.pem --cert=/data/etcd/ssl/server.pem --key=/data/etcd/ssl/server-key.pem --endpoints="https://172.17.17.85:2379,https://172.17.17.86:2379,https://172.17.17.87:2379" endpoint health

正常返回結果

https://172.17.17.87:2379 is healthy: successfully committed proposal: took = 32.593008ms https://172.17.17.85:2379 is healthy: successfully committed proposal: took = 39.390838ms https://172.17.17.86:2379 is healthy: successfully committed proposal: took = 38.243128ms

集群異常排查

1. 查看/var/log/message日志或者journalctl -xe -f -uetcd
2. 一般配置文件沒問題的話就ok,最大的問題還有一點就是網絡通信和防火牆，注意響應的策略放開即可
3.如果集群ID重復錯誤   刪除這個目錄/data/etcd/data/default.etcd

所有節點安裝docker

# 下載和解壓docker二進制文件
wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz && tar xf docker-19.03.9.tgz 
# 轉移可執行文件
mv docker/* /usr/bin/
​
# 配置systemd管理docker (其余的兩個節點也需要安裝)
cat > /usr/lib/systemd/system/docker.service << EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
​
[Service]
Type=notify
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target
EOF
​

docker配置與啟動

# 配置docker阿里雲鏡像加速和存儲路徑（graph）
mkdir /etc/docker
cat > /etc/docker/daemon.json << EOF 
{
  "graph": "/data/docker",
  "registry-mirrors": ["https://b9pmyelo.mirror.aliyuncs.com"]
}
EOF
# 服務啟動
systemctl daemon-reload && systemctl restart docker && systemctl enable docker

k8s master節點的安裝與部署

當前部署master節點 ip: 172.17.17.85

生成k8s證書配置 自簽證書頒發機構（CA）

cd ~/TLS/k8s
cat > ca-config.json <<EOF
{
    "signing":{
        "default":{
            "expiry":"87600h"
       },
        "profiles":{
            "kubernetes":{
                "expiry":"87600h",
                "usages":[
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
               ]
           }
       }
   }
}
EOF
​
cat > ca-csr.json <<EOF
{
    "CN":"kubernetes",
    "key":{
        "algo":"rsa",
        "size":2048
   },
    "names":[
       {
            "C":"CN",
            "L":"Beijing",
            "ST":"Beijing",
            "O":"k8s",
            "OU":"System"
       }
   ]
}
EOF
​
# 生成CA證書
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

使用自簽CA簽發kube-apiserver HTTPS證書

cd ~/TLS/k8s
cat > server-csr.json << EOF
{
    "CN":"kubernetes",
    "hosts":[
        "10.0.0.1",
        "172.0.0.1",
        "127.0.0.1",
        "172.17.17.85",
        "172.17.17.86",
        "172.17.17.87",
        "172.17.17.89",
        "172.17.17.84",
  "172.17.17.83",
        "kubernetes",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local"
   ],
    "key":{
        "algo":"rsa",
        "size":2048
   },
    "names":[
       {
            "C":"CN",
            "L":"BeiJing",
            "ST":"BeiJing",
            "O":"k8s",
            "OU":"System"
       }
   ]
}
EOF

注:上述文件hosts字段中IP為所有Master/LB/VIP IP，一個都不能少!為了方便后期擴容可以多 寫幾個預留的IP。

生成apiserver證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
ls *pem

從Github下載二進制文件

# 創建軟件目錄
mkdir -p /data/kubernetes/{cfg,bin,ssl,logs}
# 文件下載與拷貝
wget https://dl.k8s.io/v1.18.4/kubernetes-server-linux-amd64.tar.gz && tar xf kubernetes-server-linux-amd64.tar.gz 
cp kubernetes/server/bin/kube-apiserver /data/kubernetes/bin/
cp kubernetes/server/bin/kube-controller-manager /data/kubernetes/bin/
cp kubernetes/server/bin/kube-scheduler /data/kubernetes/bin/ 
cp kubernetes/server/bin/kubectl /usr/bin/

部署kube-apiserver

# 創建配置文件
cat > /data/kubernetes/cfg/kube-apiserver.conf << EOF 
KUBE_APISERVER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/data/kubernetes/logs \
--etcd-servers=https://172.17.17.85:2379,https://172.17.17.86:2379,https://172.17.17.87:2379 \
--bind-address=172.17.17.85 \
--secure-port=6443 \
--advertise-address=172.17.17.85 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth=true \
--token-auth-file=/data/kubernetes/cfg/token.csv \
--service-node-port-range=30000-32767 \
--kubelet-client-certificate=/data/kubernetes/ssl/server.pem \
--kubelet-client-key=/data/kubernetes/ssl/server-key.pem \
--tls-cert-file=/data/kubernetes/ssl/server.pem \
--tls-private-key-file=/data/kubernetes/ssl/server-key.pem \
--client-ca-file=/data/kubernetes/ssl/ca.pem \
--service-account-key-file=/data/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/data/etcd/ssl/ca.pem \
--etcd-certfile=/data/etcd/ssl/server.pem \
--etcd-keyfile=/data/etcd/ssl/server-key.pem \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/data/kubernetes/logs/k8s-audit.log"
EOF
​
# 拷貝證書
mv ~/TLS/k8s/*pem /data/kubernetes/ssl/

注意事項：

--logtostderr:啟用日志 ---v:日志等級 --log-dir:日志目錄 --etcd-servers:etcd集群地址 --bind-address:監聽地址 --secure-port:https安全端口 --advertise-address:集群通告地址 --allow-privileged:啟用授權 --service-cluster-ip-range:Service虛擬IP地址段

--enable-admission-plugins:准入控制模塊

--authorization-mode:認證授權，啟用RBAC授權和節點自管理

--enable-bootstrap-token-auth:啟用TLS bootstrap機制

--token-auth-file:bootstrap token文件

--service-node-port-range:Service nodeport類型默認分配端口范圍

--kubelet-client-xxx:apiserver訪問kubelet客戶端證書

--tls-xxx-file:apiserver https證書

--etcd-xxxfile:連接Etcd集群證書 --audit-log-xxx:審計日志

啟用TLS Bootstrap機制

TLS Bootstraping：Master apiserver啟用TLS認證后，Node節點kubelet和kube-proxy要與kube-apiserver進行通信，必須使用CA簽發的有效證書才可以，當Node節點很多時，這種客戶端證書頒發需要大量工作，同樣也會增加集群擴展復雜度。為了簡化流程，Kubernetes引入了TLS bootstraping機制來自動頒發客戶端證書，kubelet會以一個低權限用戶自動向apiserver申請證書，kubelet的證書由apiserver動態簽署。所以強烈建議在Node上使用這種方式，目前主要用於kubelet，kube-proxy還是由我們統一頒發一個證書。TLS bootstraping 工作流程：

• 根據上述配置token文件

cat > /data/kubernetes/cfg/token.csv << EOF
2b4b65d2e33e24dc0beafddda6dd4b23,kubelet-bootstrap,10001,"system:node-bootstrapper"
EOF

格式:token，用戶名，UID，用戶組 token也可自行生成替換:

head -c 16 /dev/urandom | od -An -t x | tr -d ' '

使用systemctl管理apiserver

生成配置文件

cat > /usr/lib/systemd/system/kube-apiserver.service << EOF 
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/data/kubernetes/cfg/kube-apiserver.conf
ExecStart=/data/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF

啟動並設置開機自啟動

systemctl daemon-reload && systemctl start kube-apiserver && systemctl enable kube-apiserver

• 授權kubelet-bootstrap用戶允許請求證書

• kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

部署kube-controller-manager

創建配置文件

cat > /data/kubernetes/cfg/kube-controller-manager.conf << EOF KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \ --v=2 \ --log-dir=/data/kubernetes/logs \ --leader-elect=true \ --master=127.0.0.1:8080 \ --bind-address=127.0.0.1 \ --allocate-node-cidrs=true \ --cluster-cidr=10.244.0.0/16 \ --service-cluster-ip-range=10.0.0.0/24 \ --cluster-signing-cert-file=/data/kubernetes/ssl/ca.pem \ --cluster-signing-key-file=/data/kubernetes/ssl/ca-key.pem \ --root-ca-file=/data/kubernetes/ssl/ca.pem \ --service-account-private-key-file=/data/kubernetes/ssl/ca-key.pem \ --experimental-cluster-signing-duration=87600h0m0s" EOF

--master:通過本地非安全本地端口8080連接apiserver。

--leader-elect:當該組件啟動多個時，自動選舉(HA)

--cluster-signing-cert-file/--cluster-signing-key-file：自動為kubelet頒發證書的CA，與apiserver保持一致

systemctl管理controller-manager

vim  /data/kubernetes/cfg/kube-controller-manager.conf

KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \ 
--v=2 \
--log-dir=/data/kubernetes/logs \
--leader-elect=true \
--master=127.0.0.1:8080 \
--bind-address=127.0.0.1 \
--allocate-node-cidrs=true \ 
--cluster-cidr=10.244.0.0/16 \
--service-cluster-ip-range=10.0.0.0/24 \ 
--cluster-signing-cert-file=/data/kubernetes/ssl/ca.pem \ 
--cluster-signing-key-file=/data/kubernetes/ssl/ca-key.pem \
 --root-ca-file=/data/kubernetes/ssl/ca.pem \ 
--service-account-private-key-file=/data/kubernetes/ssl/ca-key.pem \ 
--experimental-cluster-signing-duration=87600h0m0s" 

 vim   /usr/lib/systemd/system/kube-controller-manager.service

[Unit] 
Description=Kubernetes Controller Manager Documentation=https://github.com/kubernetes/kubernetes 
[Service] 
EnvironmentFile=/data/kubernetes/cfg/kube-controller-manager.conf 
ExecStart=/data/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS 
Restart=on-failure 

[Install] 
WantedBy=multi-user.target

 

 

啟動&開機自啟

systemctl daemon-reload && systemctl start kube-controller-manager && systemctl enable kube-controller-manager

部署kube-scheduler

創建配置文件

cat > /data/kubernetes/cfg/kube-scheduler.conf << EOF

KUBE_SCHEDULER_OPTS="--logtostderr=false \\

--v=2 \\

--log-dir=/data/kubernetes/logs \\

--leader-elect \\

--master=127.0.0.1:8080 \\

--bind-address=127.0.0.1"

EOF

--master：通過本地非安全本地端口8080連接apiserver。

--leader-elect：當該組件啟動多個時，自動選舉（HA）

systemctl管理kube-scheduler

cat > /usr/lib/systemd/system/kube-scheduler.service << EOF

[Unit]

Description=Kubernetes Scheduler

Documentation=https://github.com/kubernetes/kubernetes

[Service]

EnvironmentFile=/data/kubernetes/cfg/kube-scheduler.conf

ExecStart=/data/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS

Restart=on-failure

[Install]

WantedBy=multi-user.target EOF

• 啟動&開機自啟

  systemctl daemon-reload && systemctl start kube-scheduler && systemctl enable kube-scheduler

至此master部署完成，集群狀態查看

kubectl get cs               

返回如下結果，證明mater部署ok

NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler           Healthy   ok                  
etcd-2               Healthy   {"health":"true"}   
etcd-0               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}  

部署WorkNode

當前操作節點是： 172.17.17.85(將這個master也作為node)

• 需要的軟件：

  • kubelet

  • kube-proxy

基礎軟件包准備

# 創建軟件目錄
mkdir -p /data/kubernetes/{cfg,bin,ssl,logs}
# 文件下載與拷貝
wget https://dl.k8s.io/v1.18.4/kubernetes-server-linux-amd64.tar.gz && tar xf kubernetes-server-linux-amd64.tar.gz 
cp kubernetes/server/bin/kube-proxy /data/kubernetes/bin/
cp kubernetes/server/bin/kubelet /data/kubernetes/bin/

部署kubelet

創建kubelet配置文件

cat > /data/kubernetes/cfg/kubelet.conf << EOF

KUBELET_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/data/kubernetes/logs \\
--hostname-override=k8s-master \\
--network-plugin=cni \\
--kubeconfig=/data/kubernetes/cfg/kubelet.kubeconfig \\
--bootstrap-kubeconfig=/data/kubernetes/cfg/bootstrap.kubeconfig \\
--config=/data/kubernetes/cfg/kubelet-config.yml \\
--cert-dir=/data/kubernetes/ssl \\
--pod-infra-container-image=lizhenliang/pause-amd64:3.0"

　　

--hostname-override:顯示名稱，集群中唯一

--network-plugin:啟用CNI

--kubeconfig:空路徑，會自動生成，后面用於連接apiserver

--bootstrap-kubeconfig:首次啟動向apiserver申請證書

--config:配置參數文件

--cert-dir:kubelet證書生成目錄

--pod-infra-container-image:管理Pod網絡容器的鏡像

創建參數配置文件

cat > /data/kubernetes/cfg/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 10.0.0.2
clusterDomain: cluster.local
failSwapOn: false
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /data/kubernetes/ssl/ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
evictionHard:
  imagefs.available: 15%
  memory.available: 100M
  nodefs.available: 10%
  nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110

生成bootstrap.kubeconfig文件

[root@k8s-master-1 ~]# KUBE_APISERVER="https://172.17.17.85:6443"

[root@k8s-master-1 ~]# TOKEN="2b4b65d2e33e24dc0beafddda6dd4b23"

[root@k8s-master-1 ~]# kubectl config set-cluster kubernetes --certificate-authority=/data/kubernetes/ssl/ca.pem --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=bootstrap.kubeconfig

[root@k8s-master-1 ~]# kubectl config set-credentials "kubelet-bootstrap" --token=${TOKEN} --kubeconfig=bootstrap.kubeconfig

[root@k8s-master-1 ~]# kubectl config set-context default --cluster=kubernetes --user="kubelet-bootstrap" --kubeconfig=bootstrap.kubeconfig

[root@k8s-master-1 ~]# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

拷貝到配置文件路徑：

[root@k8s-master-1 ~]# cp bootstrap.kubeconfig /data/kubernetes/cfg

systemctl管理kubelet

• 創建啟動文件

[root@k8s-master-1 ~]# cat > /usr/lib/systemd/system/kubelet.service << EOF

[Unit]

Description=Kubernetes Kubelet

After=docker.service

[Service]

EnvironmentFile=/data/kubernetes/cfg/kubelet.conf

ExecStart=/data/kubernetes/bin/kubelet \$KUBELET_OPTS

Restart=on-failure

LimitNOFILE=65536

[Install]

WantedBy=multi-user.target EOF

• 啟動kubelet&開機自啟動

  systemctl daemon-reload && systemctl start kubelet && systemctl enable kubelet

  如有異常，及時查看日志解決，大多數問題會出現在/data/kubernetes/cfg/kubelet-config.yml格式上

批准kubelet證書申請並加入集群

查看kubelet證書請求

[root@k8s-master-1 ~]# kubectl get csr NAME AGE SIGNERNAME REQUESTOR CONDITION node-csr-cUACWJSaXTJHpc1XIscmgbvarl1aZ3Hk79d7qjkII_A 68s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending

批准kubelet證書申請，並加入集群

[root@k8s-master-1 ~]# kubectl certificate approve node-csr-cUACWJSaXTJHpc1XIscmgbvarl1aZ3Hk79d7qjkII_A

查看節點 [root@k8s-master-1 ~]# kubectl get node NAME STATUS ROLES AGE VERSION k8s-master-1 NotReady <none> 4s v1.18.4

注：由於網絡插件還沒有部署，節點會沒有准備就緒 NotReady

部署kube-proxy

創建配置文件

[root@k8s-master-1 ~]# cat > /data/kubernetes/cfg/kube-proxy.conf << EOF KUBE_PROXY_OPTS="--logtostderr=false \ --v=2 \ --log-dir=/data/kubernetes/logs \ --config=/data/kubernetes/cfg/kube-proxy-config.yml" EOF

配置參數文件

[root@k8s-master-1 ~]# cat > /data/kubernetes/cfg/kube-proxy-config.yml << EOF kind: KubeProxyConfiguration apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 metricsBindAddress: 0.0.0.0:10249 clientConnection: kubeconfig: /data/kubernetes/cfg/kube-proxy.kubeconfig hostnameOverride: k8s-master-1 clusterCIDR: 10.0.0.0/24 EOF

生成kube-proxy.kubeconfig文件

生成kube-proxy證書

cd ~/TLS/k8s/
cat > kube-proxy-csr.json << EOF
{
    "CN":"system:kube-proxy",
    "hosts":[
​
   ],
    "key":{
        "algo":"rsa",
        "size":2048
   },
    "names":[
       {
            "C":"CN",
            "L":"BeiJing",
            "ST":"BeiJing",
            "O":"k8s",
            "OU":"System"
       }
   ]
}

# 生成證書
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
​
ls kube-proxy*pem
kube-proxy-key.pem kube-proxy.pem

生成kubeconfig配置文件

KUBE_APISERVER="https://172.17.17.85:6443"

kubectl config set-cluster kubernetes --certificate-authority=/data/kubernetes/ssl/ca.pem --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=kube-proxy.kubeconfig

kubectl config set-credentials kube-proxy --client-certificate=./kube-proxy.pem --client-key=./kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig

kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

# 拷貝配置到cfg下

cp kube-proxy.kubeconfig /data/kubernetes/cfg/

systemctl管理kube-proxy

• 創建啟動文件

cat > /usr/lib/systemd/system/kube-proxy.service << EOF [Unit] Description=Kubernetes Proxy After=network.target [Service] EnvironmentFile=/data/kubernetes/cfg/kube-proxy.conf ExecStart=/data/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS Restart=on-failure LimitNOFILE=65536

[Install] WantedBy=multi-user.target EOF

• 設置開機自啟動

systemctl daemon-reload && systemctl start kube-proxy && systemctl enable kube-proxy

部署CNI網絡

先准備好CNI二進制文件：下載地址：https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz

wget https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz &&  mkdir /opt/cni/bin -p
tar zxvf cni-plugins-linux-amd64-v0.8.6.tgz -C /opt/cni/bin/

• 部署flannel

  wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
  sed -i -r "s#quay.io/coreos/flannel:.*-amd64#lizhenliang/flannel:v0.12.0-amd64#g" kube-flannel.yml
  kubectl apply -f kube-flannel.yml

查看部署狀態

kubectl get nodes
NAME         STATUS   ROLES   AGE   VERSION
k8s-master-1   Ready   <none>   79m   v1.18.4

部署好網絡插件，Node准備就緒

授權apiserver訪問kubelet

[root@k8s-master-1 k8s]# cat > apiserver-to-kubelet-rbac.yaml << EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
 annotations:
   rbac.authorization.kubernetes.io/autoupdate: "true"
 labels:
   kubernetes.io/bootstrapping: rbac-defaults
 name: system:kube-apiserver-to-kubelet
rules:
 - apiGroups:
       - ""
   resources:
     - nodes/proxy
     - nodes/stats
     - nodes/log
     - nodes/spec
     - nodes/metrics
     - pods/log
   verbs:
     - "*"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata: kubernetesuthorization.k8s.iot
 name: system:kube-apiserver
 namespace: ""
roleRef:
 apiGroup: rbac.authorization.k8s.io
 kind: ClusterRole
 name: system:kube-apiserver-to-kubelet
subjects:
 - apiGroup: rbac.authorization.k8s.io
   kind: User
   name: kubernetes
EOF

## 創建授權

kubectl apply -f apiserver-to-kubelet-rbac.yaml

新增workNode

同步文件和配置

• 將172.17.17.85上的node相關的配置拷貝到172.17.17.86，172.17.17.87上

mkdir -p /data/kubernetes/{cfg,bin,ssl,logs} #在86 87上先創建目錄

# kubelet,kube-proxy拷貝
scp /data/kubernetes/bin/kubelet 172.17.17.86:/data/kubernetes/bin/
scp /data/kubernetes/bin/kubelet 172.17.17.87:/data/kubernetes/bin/
scp /data/kubernetes/bin/kube-proxy root@172.17.17.86:/data/kubernetes/bin/
scp /data/kubernetes/bin/kube-proxy root@172.17.17.87:/data/kubernetes/bin/
# cni插件拷貝
scp -rp /opt/cni/ root@172.17.17.86:/opt
scp -rp /opt/cni/ root@172.17.17.87:/opt
# 證書拷貝
scp /data/kubernetes/ssl/ca.pem 172.17.17.86:/data/kubernetes/ssl/
scp /data/kubernetes/ssl/ca.pem 172.17.17.87:/data/kubernetes/ssl/
​
# 配置文件拷貝
scp /data/kubernetes/cfg/kube-proxy* 172.17.17.86:/data/kubernetes/cfg/
scp /data/kubernetes/cfg/kube-proxy* 172.17.17.87:/data/kubernetes/cfg/
scp /data/kubernetes/cfg/kubelet* 172.17.17.86:/data/kubernetes/cfg/
scp /data/kubernetes/cfg/kubelet* 172.17.17.87:/data/kubernetes/cfg/
scp /data/kubernetes/cfg/bootstrap.kubeconfig 172.17.17.86:/data/kubernetes/cfg/
scp /data/kubernetes/cfg/bootstrap.kubeconfig 172.17.17.87:/data/kubernetes/cfg/
# 啟動文件拷貝
scp /usr/lib/systemd/system/kubelet.service 172.17.17.86:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/kubelet.service 172.17.17.87:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/kube-proxy.service 172.17.17.86:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/kube-proxy.service 172.17.17.87:/usr/lib/systemd/system/
​

刪除證書和配置文件 86 87 上操作

rm -rf /data/kubernetes/cfg/kubelet.kubeconfig
rm -f /data/kubernetes/ssl/kubelet*

注：這幾個文件是證書申請審批后自動生成的，每個

配置新的Node節點 86 87 上操作

• 修改kubelet和kube-proxy配置文件

vi /data/kubernetes/cfg/kubelet.conf 
--hostname-override=k8s-node5
vim /data/kubernetes/cfg/kube-proxy-config.yml 
hostnameOverride: k8s-node5
​
vi /data/kubernetes/cfg/kubelet.conf 
--hostname-override=k8s-node6
vi /data/kubernetes/cfg/kube-proxy-config.yml 
hostnameOverride: k8s-node6
​

• 配置kubectl和kube-proxy開機啟動

• systemctl daemon-reload && systemctl start kubelet && systemctl start kube-proxy
  systemctl enable kubelet && systemctl enable kube-proxy

在master節點上准許node加入

• 獲取准入的node信息

[root@k8s-master-1 k8s]# kubectl get csr NAME AGE SIGNERNAME REQUESTOR CONDITION node-csr-BKIF1VFOrkRKlbKeJc8CbEe22JLgsMwMU6aJ4KCjeMg 2m42s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending node-csr-MAr2KDH13mbed2dLM4QEmZGuikl1ooXW5ksLlkCVtOA 2m45s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending

• 准入node

kubectl certificate approve node-csr-BKIF1VFOrkRKlbKeJc8CbEe22JLgsMwMU6aJ4KCjeMg kubectl certificate approve node-csr-MAr2KDH13mbed2dLM4QEmZGuikl1ooXW5ksLlkCVtOA

查看狀態

[root@k8s-master-1 k8s]# kubectl get node NAME STATUS ROLES AGE VERSION k8s-master-1 Ready <none> 72m v1.18.4 k8s-node5 Ready <none> 60s v1.18.4 k8s-node6 Ready <none> 50s v1.18.4

部署Dashboard和CoreDNS

部署Dashboard

• git地址： https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.3/aio/deploy/recommended.yaml

wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta8/aio/deploy/recommended.yaml

默認Dashboard只能集群內部訪問，修改Service為NodePort類型，暴露到外部

vim recommended.yaml
​
kind: Service
apiVersion: v1
metadata:
 labels:
   k8s-app: kubernetes-dashboard
 name: kubernetes-dashboard
 namespace: kubernetes-dashboard
spec:
 ports:
   - port: 443
     targetPort: 8443
     nodePort: 30001
 type: NodePort
 selector:
   k8s-app: kubernetes-dashboard
# 部署dashboard
kubectl apply -f recommended.yml  
# 查看狀態
[root@k8s-master-1 k8s]# kubectl get pods,svc -n kubernetes-dashboard
NAME                                             READY   STATUS             RESTARTS   AGE
pod/dashboard-metrics-scraper-694557449d-82n2p   0/1     ContainerCreating   0         2m2s
pod/kubernetes-dashboard-9774cc786-2fcvk         0/1     ContainerCreating   0         2m2s
​
NAME                               TYPE       CLUSTER-IP   EXTERNAL-IP   PORT(S)         AGE
service/dashboard-metrics-scraper   ClusterIP   10.0.0.107   <none>       8000/TCP       2m2s
service/kubernetes-dashboard       NodePort   10.0.0.64   <none>       443:31614/TCP   2m3s

此時可以直接訪問https://NodeIP:30001 NodeIP也就是宿主機的IP

• 創建dashboard訪問token

• kubectl create serviceaccount dashboard-admin -n kube-system
  kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin
  kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')     #獲取token

部署coreDNS

上傳vim coredns.yaml

用於集群內部Service名稱解析

kubectl apply -f coredns.yaml
[root@k8s-master-1 k8s]# kubectl get pods -n kube-system
NAME                     READY   STATUS   RESTARTS   AGE
coredns-5675d5b4d-njxc4   1/1     Running   0         30m
kube-flannel-ds-69w4v     1/1     Running   0         84m
kube-flannel-ds-6twp8     1/1     Running   5         84m
kube-flannel-ds-t6sqv     1/1     Running   0         84m
​
如果創建不成功報錯 failed to set bridge addr: \"cni0\" already has an IP address different from 10.244.3.1/24"
ifconfig cni0 down    
ip link delete cni0
網卡刪除會自動重建

DNS解析測試：

[root@k8s-master-1 k8s]# kubectl run -it --rm dns-test --image=busybox:1.28.4 sh If you don't see a command prompt, try pressing enter. / # nslookup kubernetes Server: 10.0.0.2 Address 1: 10.0.0.2 kube-dns.kube-system.svc.cluster.local

Name: kubernetes Address 1: 10.0.0.1 kubernetes.default.svc.cluster.local

高可用架構（擴容多Master架構）

Kubernetes作為容器集群系統，通過健康檢查+重啟策略實現了Pod故障自我修復能力，通過調度算法實現將Pod分布式部署，並保持預期副本數，根據Node失效狀態自動在其他Node拉起Pod，實現了應用層的高可用性。

針對Kubernetes集群，高可用性還應包含以下兩個層面的考慮：Etcd數據庫的高可用性和KubernetesMaster組件的高可用性。而Etcd我們已經采用3個節點組建集群實現高可用，本節將對Master節點高可用進行說明和實施。

Master節點扮演着總控中心的角色，通過不斷與工作節點上的Kubelet和kube-proxy進行通信來維護整個集群的健康工作狀態。如果Master節點故障，將無法使用kubectl工具或者API做任何集群管理。

Master節點主要有三個服務kube-apiserver、kube-controller-manager和kube-scheduler，其中kube-controller-manager和kube-scheduler組件自身通過選擇機制已經實現了高可用，所以Master高可用主要針對kube-apiserver組件，而該組件是以HTTP API提供服務，因此對他高可用與Web服務器類似，增加負載均衡器對其負載均衡即可，並且可水平擴容。

多Master架構圖：

擴容流程

新增主機：centos7-node, 角色k8s-master2

• 系統初始化

• 安裝docker

• wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz
  tar xf docker-19.03.9.tgz
  mv docker/* /usr/bin/
  mkdir /data/docker
  mkdir /etc/docker

創建ETCD證書目錄（mkdir /data/etcd/ssl -p）

拷貝文件，從master-1拷貝到新機器

​
scp -rp /opt/cni/ 172.17.17.84:/opt/
scp -r /data/kubernetes root@172.17.17.84:/data
scp -r /data/etcd/ssl root@172.17.17.84:/data/etcd
scp /usr/lib/systemd/system/kube* root@172.17.17.84:/usr/lib/systemd/system
scp /usr/bin/kubectl root@172.17.17.84:/usr/bin
scp /usr/lib/systemd/system/docker.service root@172.17.17.84:/usr/lib/systemd/system

刪除證書文件

刪除kubelet證書和kubeconfig文件

rm -f /data/kubernetes/cfg/kubelet.kubeconfig

rm-f /data/kubernetes/ssl/kubelet*

修改配置文件IP和主機名

$ vim /data/kubernetes/cfg/kube-apiserver.conf
--bind-address=172.17.17.84 \
--advertise-address=172.17.17.84 \
$ vim /data/kubernetes/cfg/kubelet.conf
--hostname-override=k8s-master-2
$ vim /data/kubernetes/cfg/kube-proxy-config.yml 
hostnameOverride: k8s-master-2

啟動服務

systemctl daemon-reload && systemctl start docker && systemctl start kube-apiserver && systemctl start kube-controller-manager && systemctl start kube-scheduler && systemctl start kubelet && systemctl start kube-proxy
​
systemctl enable kube-apiserver && systemctl enable docker && systemctl enable kube-controller-manager && systemctl enable kube-scheduler && systemctl enable kubelet && systemctl enable kube-proxy

查看集群狀態

[root@k8s-master-1 data]# kubectl get cs

NAME STATUS MESSAGE ERROR

scheduler Healthy ok

controller-manager Healthy ok

etcd-1 Healthy {"health":"true"}

etcd-2 Healthy {"health":"true"}

etcd-0 Healthy {"health":"true"}

k8s-mater1 准入新master節點

[root@k8s-master-1 data]# kubectl get csr

NAME AGE SIGNERNAME REQUESTOR CONDITION node-csr-KUYz4cMykgJVDNgnXt0okzZsJ22YjAuJ8qWnvJrnyXo 19m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued

[root@k8s-master-1 data]# kubectl certificate approve node-csr-KUYz4cMykgJVDNgnXt0okzZsJ22YjAuJ8qWnvJrnyXo^C

[root@k8s-master-1 data]# kubectl get node

NAME STATUS ROLES AGE VERSION

k8s-master-1 Ready <none> 3d4h v1.18.4

k8s-master-2 Ready <none> 9m14s v1.18.4

k8s-node5 Ready <none> 2d23h v1.18.4

k8s-node6 Ready <none> 2d23h v1.18.4

部署ngixn負載均衡

Nginx是一個主流Web服務和反向代理服務器，這里用四層實現對apiserver實現負載均衡。Keepalived是一個主流高可用軟件，基於VIP綁定實現服務器雙機熱備，在上述拓撲中，Keepalived主要根據Nginx運行狀態判斷是否需要故障轉移（偏移VIP），例如當Nginx主節點掛掉，VIP會自動綁定在Nginx備節點，從而保證VIP一直可用，實現Nginx高可用。

安裝軟件包（主/備）

172.17.17.81 和172.17.17.84

yum install epel-release -y

yum install nginx keepalived -y

Nginx配置文件（主/備一樣）

# nginx配置
$ vim /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
​
events {
   worker_connections 1024;
}
# 四層負載均衡，為兩台Master apiserver組件提供負載均衡 
stream {
   log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
   access_log /var/log/nginx/k8s-access.log main;
   upstream k8s-apiserver { 
       server 192.168.56.14:6443; 
       server 192.168.56.74:6443;
   }
​
   server {
       listen 6443;
       proxy_pass k8s-apiserver;
   }
}
​
http {
   log_format main '$remote_addr - $remote_user [$time_local] "$request" '
       '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"';
   access_log /var/log/nginx/access.log main;
   sendfile on;
   tcp_nopush on;
   tcp_nodelay on;
   keepalive_timeout 65;
   types_hash_max_size 2048;
​
   include /etc/nginx/mime.types;
   default_type application/octet-stream;
​
   server {
       listen 80;
       server_name _;
       location / {
       }
   }
}   

keepalived配置文件（Nginx Master)

global_defs {
   notification_email {
     [e-mail]
     [e-mail]
     [e-mail]
   
   }
   notification_email_from [e-mail]
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id NGINX_MASTER
}
​
vrrp_script check_nginx {
   script "/etc/keepalived/check_nginx.sh"
}
​
vrrp_instance VI_1 {
   state MASTER
   interface enp9s0
   virtual_router_id 51
   priority 100
   advert_int 1
   authentication {
       auth_type PASS
       auth_pass 1111
   }
   virtual_ipaddress {
       172.17.17.79/24
   }
   track_script {
       check_nginx
   }
}

vrrp_script：指定檢查nginx工作狀態腳本（根據nginx狀態判斷是否故障轉移）

virtual_ipaddress：虛擬IP（VIP）

檢查nginx狀態腳本

vim /etc/keepalived/check_nginx.sh

#!/bin/bash count=$(ps -ef |grep nginx |egrep -cv "grep|$$") if [ "$count" -eq 0 ];then exit 1 else exit 0 fi

chmod+x /etc/keepalived/check_nginx.sh

keepalived配置文件（Nginx Backup)

vim /etc/keepalived/keepalived.conf

global_defs {
   notification_email {
     [e-mail]
     [e-mail]
     [e-mail]
​
   }
   notification_email_from [e-mail]
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id NGINX_BACKUP
}
​
vrrp_script check_nginx {
   script "/etc/keepalived/check_nginx.sh"
}
​
vrrp_instance VI_1 {
   state BACKUP
   interface enp9s0
   virtual_router_id 51
   priority 90
   advert_int 1
   authentication {
       auth_type PASS
       auth_pass 1111
   }
   virtual_ipaddress {
       172.17.17.79/24
   }
   track_script {
       check_nginx
   }
}

啟動並設置開機啟動

systemctl daemon-reload
systemctl start nginx
systemctl start keepalived
systemctl enable nginx
systemctl enable keepalived

查看keepalived工作狀態

ip  add 
enp9s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
   link/ether 10:c3:7b:49:25:2f brd ff:ff:ff:ff:ff:ff
   inet 172.17.17.83/25 brd 172.17.17.127 scope global enp9s0
       valid_lft forever preferred_lft forever
   inet 172.17.17.79/24 scope global enp9s0
       valid_lft forever preferred_lft forever
   inet6 fe80::12c3:7bff:fe49:252f/64 scope link 
       valid_lft forever preferred_lft forever

可以看到，在enp9s0網卡綁定了192.168.31.88 虛擬IP，說明工作正常。

Nginx+Keepalived高可用測試關閉主節點Nginx，

測試VIP是否漂移到備節點服務器。

在Nginx Master執行 pkill nginx在Nginx Backup，

ip addr命令查看已成功綁定VIP。

訪問負載均衡器測試

修改k8s-node5 和k8s-node6配置

cd /data/kubernetes/cfg/
[root@k8s-node6 cfg]# sed -i 's/172.17.17.85/172.17.17.79/g' *
[root@k8s-node6 cfg]# systemctl restart kubelet
[root@k8s-node6 cfg]# systemctl restart kube-proxy

https://pkg.cfssl.org/R1.2/cfssl_linux-amd64

cat > /opt/kubernetes/cfg/kubelet.conf << EOF KUBELET_OPTS="--logtostderr=false \\ --v=2 \\ --log-dir=/opt/kubernetes/logs \\ --hostname-override=k8s-master03 \\ --network-plugin=cni \\ --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \\ --bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \\ --config=/opt/kubernetes/cfg/kubelet-config.yml \\ --cert-dir=/opt/kubernetes/ssl \\ --pod-infra-container-image=registry.aliyuncs.com/k8sxio/pause:3.2" EOF