1、基礎環境
1.安裝cfssl（只需在k8s-master01節點即可）
$ wget -O /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
$ wget -O /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
$ wget -O /bin/cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
$ for cfssl in `ls /bin/cfssl*`;do chmod +x $cfssl;done;
2.配置hosts文件
cat >>/etc/hosts<< EOF
k8s-master01 10.0.0.31
k8s-master02 10.0.0.32
k8s-master03 10.0.0.39
EOF
2、etcd證書生成
1.安裝etcd
$ yum install -y etcd
2.配置證書
vim etcd-csr.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"10.0.0.31",
"10.0.0.32",
"10.0.0.39"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GD",
"L": "shenzh",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
vim ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
vim ca-csr.json
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "GD",
"L": "shenzh",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
3.創建CA證書和私鑰
$ cfssl gencert -initca ca-csr.json | cfssljson -bare ca
4.生成etcd證書和私鑰
$ cfssl gencert -ca=ca.pem \
-ca-key=ca-key.pem \
-config=config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd
// 查看生成的證書
$ ls *.pem
ca-key.pem ca.pem etcd-key.pem etcd.pem
$ mkdir -pv /etc/etcd/ssl
$ cp -r ./{ca-key,ca,etcd-key,etcd}.pem /etc/etcd/ssl/
5.將復制證書到其他節點
scp -r ./ root@10.0.0.32:/etc/etcd/ssl
scp -r ./ root@10.0.0.39:/etc/etcd/ssl
6.修改etcd各個節點的etcd.conf配置文件
k8s-master01
$ vim /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.0.0.31:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.31:2379"
ETCD_NAME="k8s-master01"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.0.31:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.31:2379"
ETCD_INITIAL_CLUSTER="k8s-master01=https://10.0.0.31:2380,k8s-master02=https://10.0.0.32:2380,k8s-master03=https://10.0.0.39:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
k8s-master02
$ vim /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.0.0.32:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.32:2379"
ETCD_NAME="k8s-master02"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.0.32:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.32:2379"
ETCD_INITIAL_CLUSTER="k8s-master01=https://10.0.0.31:2380,k8s-master02=https://10.0.0.32:2380,k8s-master03=https://10.0.0.39:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
k8s-master03
$ vim /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.0.0.39:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.39:2379"
ETCD_NAME="k8s-master03"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.0.39:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://10.0.0.39:2379"
ETCD_INITIAL_CLUSTER="k8s-master01=https://10.0.0.31:2380,k8s-master02=https://10.0.0.32:2380,k8s-master03=https://10.0.0.39:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem"
配置參數含義
· ETCD_NAME 節點名稱
· ETCD_DATA_DIR 數據目錄
· ETCD_LISTEN_PEER_URLS 集群通信監聽地址
· ETCD_LISTEN_CLIENT_URLS 客戶端訪問監聽地址
· ETCD_INITIAL_ADVERTISE_PEER_URLS 集群通告地址
· ETCD_ADVERTISE_CLIENT_URLS 客戶端通告地址
· ETCD_INITIAL_CLUSTER 集群節點地址
· ETCD_INITIAL_CLUSTER_TOKEN 集群Token
· ETCD_INITIAL_CLUSTER_STATE 加入集群的當前狀態，new是新集群，existing表示加入已有集群
在各個節點運行如下命令啟動etcd及開機啟動
$ systemctl start etcd
$ systemctl enable etcd
查看etcd是否運行正常
$ etcdctl --endpoints "https://10.0.0.31:2379,https://10.0.0.32:2379,https://10.0.0.39:2379" --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem cluster-health
// 輸出如下
member 61105fb5ea81da2 is healthy: got healthy result from https://10.0.0.39:2379
member 1f46bee47a4f04aa is healthy: got healthy result from https://10.0.0.31:2379
member 6443b97f5544707b is healthy: got healthy result from https://10.0.0.32:2379
cluster is healthy
至此etcd集群搭建完畢
3、master高可用
1.兩台lb（負載均衡機器）安裝haproxy
$ yum install haproxy
$ cat /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 10000
user haproxy
group haproxy
daemon
stats socket /var/lib/haproxy/stats
defaults
mode tcp
log global
retries 3
timeout connect 10s
timeout client 1m
timeout server 1m
frontend kube-apiserver
bind *:6443 # 指定前端端口
mode tcp
default_backend master
backend master # 指定后端機器及端口，負載方式為輪詢
balance roundrobin
server k8s-master01 10.0.0.31:6443 check maxconn 2000
server k8s-master02 10.0.0.32:6443 check maxconn 2000
server k8s-master03 10.0.0.39:6443 check maxconn 2000
啟動haproxy
$ systemctl enable haproxy
$ systemctl start haproxy
2.兩台lb部署
$ yum install keepalived
主lb
$ vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email_from Alexandre.Cassen@firewall.loc
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.101 #漂移ip
}
}
備lb
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.101 #漂移ip
}
}
啟動keepalived
systemctl start keepalived
systemctl enable keepalived
3.將etcd的證書移動至/etc/kubernetes/pki/
$ mkdir -p /etc/kubernetes/pki/
$ cp -r {ca,etcd,etcd-key}.pem /etc/kubernetes/pki/
4.配置kubeadm-config.yaml文件
$ cat /root/kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.0.0.31
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8s-master01
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "10.0.0.101:6443" # 該地址和vip要一致
controllerManager: {}
dns:
type: CoreDNS
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.15.1
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
scheduler: {}
etcd:
external:
endpoints:
- https://10.0.0.31:2379
- https://10.0.0.32:2379
- https://10.0.0.39:2379
caFile: /etc/kubernetes/pki/ca.pem
certFile: /etc/kubernetes/pki/etcd.pem
keyFile: /etc/kubernetes/pki/etcd-key.pem
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
SupportIPVSProxyMode: true
mode: ipvs
初始化master
$ kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs
初始化完成后提示執行的操作
$ mkdir -p $HOME/.kube
$ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config
5.加入另外一個master節點
原來的kubeadm版本，join命令只用於工作節點的加入，而新版本加入了 --control-plane 參數后，控制平面（master）節點也可以通過kubeadm join命令加入集群了。
$ kubeadm join 10.0.0.101:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:699cdd59cfa20509cc25794c5b153678a8ff354c9401e215cb1e41d750cbeb54 \
--control-plane --certificate-key 87d0f654fd4d2d563969ce24fa226321a3fd098477e0528479476fce3bf404c3
6.加入node節點
$ kubeadm join 10.0.0.101:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:699cdd59cfa20509cc25794c5b153678a8ff354c9401e215cb1e41d750cbeb54