Kubeadm部署安裝k8s1.17版本
0、設置時區及同步時間
//設置系統時區為中國/上海
timedatectl set-timezone Asia/Shanghai
//將當前的 UTC 時間寫入硬件時鍾
timedatectl set-local-rtc 0
// 重啟依賴於系統時間的服務
systemctl restart rsyslog && systemctl restart crond
1、設置系統主機名以及host文件的相互解析
hostnamectl set-hostname xxxx
2、系統yum鏡像更換
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
3、安裝依賴包
yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git
4、設置防火牆為iptables並設置空規則
systemctl stop firewalld && systemctl disable firewalld
yum -y install iptables-services && systemctl start firewalld && systemctl enable iptables && iptables -F && service iptables save
5、關閉selinux及swap
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled' /etc/selinux/config
6、調整內核參數，對於k8s
cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 #禁止使用swap空間，只有當系統OOM時才運行使用它
vm.overcommit_memory=1 #不檢查物理內存是否夠用
vm.panic_on_oom=0 #開啟OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
其中必備參數：
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1 //開啟網橋模式
net.ipv6.conf.all.disable_ipv6=1//關閉ipv6的協議
其余為優化參數，可不設置
sysctl -p /etc/sysctl.d/kubernetes.conf
7、關閉系統不需要的服務
systemctl stop postfix && systemctl disable postfix
8、設置rsyslogd和systemd journald
原因：centos7以后，引導方式改為了systemd，所以會有兩個日志系統同時工作只保留一個日志（journald）的方法
mkdir /var/log/journal # 持久化保存日志的目錄
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal]
#持久化保存到磁盤
Storage=persistent
# 壓縮歷史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
# 最大占用空間10G
SystemMaxUse=10G
# 單日志文件最大200M
SystemMaxFileSize=200M
# 日志保存時間 2 周
MaxRetentionSec=2week
# 不將日志轉發到 syslog
ForwardToSyslog=no
EOF
#重啟journald配置
systemctl restart systemd-journald
9、升級內核為4.4版本
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
# 安裝完成后檢查 /boot/grub2/grub.cfg 中對應內核 menuentry 中是否包含 initrd16 配置，如果沒有，再安裝一次！
yum --enablerepo=elrepo-kernel install -y kernel-lt
# 設置開機從新內核啟動
grub2-set-default "CentOS Linux (4.4.217-1.el7.elrepo.x86_64) 7 (Core)"
# 重啟后安裝內存源文件
yum --enablerepo=elrepo-kernel install kernel-lt-devel-$(uname -r) kernel-lt-headers-$(uname -r)
10、關閉numa
cp /etc/default/grub{,.bak}
vim /etc/default/grub #在GRUB_CMDLINE_LINUX 一行添加"numa=off" 參數，如下所示：
# diff /etc/default/grub /etc/default/grub.bak
6c6
< GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rhgb quiet numa=off"
---
> GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rhgb quiet"
拷貝boot下的grub配置文件：
cp /boot/grub2/grub.cfg{,.bak}
生成文件：
grub2-mkconfig -o /boot/grub2/grub.cfg
重啟服務器。。。。。
11、kube-proxy開啟ipvs的前置條件
//1、加載netfilter模塊
modprobe br_netfilter
//2、添加配置文件
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
//3、賦予權限並引導
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules &&lsmod | grep -e ip_vs -e nf_conntrack_ipv4
12、安裝docker軟件
//1、docker依賴
yum install -y yum-utils device-mapper-persistent-data lvm2
//2、導入阿里雲的docker-ce倉庫
yum-config-manager \
--add-repo \http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
//3、更新系統安裝docker-ce
yum update -y && yum install -y docker-ce
//4、uname -r 檢測版本，再設置版本，后又重啟reboot
grub2-set-default "CentOS Linux (4.4.217-1.el7.elrepo.x86_64) 7 (Core)"
//5、啟動
systemctl start docker
//6、開機自啟
systemctl enable docker
//7、配置deamon
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors":["https://docker.mirrors.ustc.edu.cn"],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
}
}
EOF
//8、創建目錄存放docker配置文件
mkdir -p /etc/systemd/system/docker.service.d
//9、重啟docker
systemctl daemon-reload && systemctl restart docker && systemctl enable docker
13、在主節點啟動haproxy和keepalived容器
//1、導入haproxy和keepalived鏡像
docker load < haproxy.tar
docker load < keepalived.tar
//2、修改haproxy和keepalived配置文件
[root@k8s-master01 lb]# cat /data/lb/etc/haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
retries 3
option redispatch
timeout connect 5000
timeout client 50000
timeout server 50000
frontend stats-front
bind *:8081
mode http
default_backend stats-back
frontend fe_k8s_6444
bind *:6444
mode tcp
timeout client 1h
log global
option tcplog
default_backend be_k8s_6443
acl is_websocket hdr(Upgrade) -i WebSocket
acl is_websocket hdr_beg(Host) -i ws
backend stats-back
mode http
balance roundrobin
stats uri /haproxy/stats
stats auth pxcstats:secret
backend be_k8s_6443
mode tcp
timeout queue 1h
timeout server 1h
timeout connect 1h
log global
balance roundrobin
server rancher01 192.168.105.246:6443 #先暫時添加一個節點
修改啟動腳本內容：
[root@k8s-master01 lb]# ./start-haproxy.sh
f03cc66f3bf8644fca6aa8cf887eee66ad1990ded9625d01bf77a97479eb7547
[root@k8s-master01 lb]# cat start-keepalived.sh
#!/bin/bash
VIRTUAL_IP=192.168.105.251 #vip 地址
INTERFACE=ens160 #當前節點的網卡端口
NETMASK_BIT=24
CHECK_PORT=6444 #檢查端口為6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18
docker run -itd --restart=always --name=Keepalived-K8S \
--net=host --cap-add=NET_ADMIN \
-e VIRTUAL_IP=$VIRTUAL_IP \
-e INTERFACE=$INTERFACE \
-e CHECK_PORT=$CHECK_PORT \
-e RID=$RID \
-e VRID=$VRID \
-e NETMASK_BIT=$NETMASK_BIT \
-e MCAST_GROUP=$MCAST_GROUP \
wise2c/keepalived-k8s
[root@k8s-master01 lb]# ./start-keepalived.sh
5c37c56bae99541ca19bb023579adbe3fec64ac97f626d812eed68533b13fb40
安裝kubeadm, kubelet , kubectl(所有master/node節點)
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum install -y kubelet-1.17.4 kubeadm-1.17.4 kubectl-1.17.4
systemctl enable kubelet
使用kubeadm部署
在 m[1:3].k8s.com 配置：
建議先把需要的鏡像安裝好
#kubeadm config images list --kubernetes-version=v1.17.4
W0320 15:26:32.612945 123330 validation.go:28] Cannot validate kubelet config - no validator is available
W0320 15:26:32.612995 123330 validation.go:28] Cannot validate kube-proxy config - no validator is available
k8s.gcr.io/kube-apiserver:v1.17.4
k8s.gcr.io/kube-controller-manager:v1.17.4
k8s.gcr.io/kube-scheduler:v1.17.4
k8s.gcr.io/kube-proxy:v1.17.4
k8s.gcr.io/pause:3.1
k8s.gcr.io/etcd:3.4.3-0
k8s.gcr.io/coredns:1.6.5
執行kubeadm證書有效期修改：請查看《證書有效期修改》內容！！！！！！
【#####################開始安裝Ks8#################################】
安裝方式一：
使用azure提供的國內源加速:
kubeadm config images pull --image-repository gcr.azk8s.cn/google_containers --kubernetes-version=v1.17.4
對鏡像打上標簽：
docker tag gcr.azk8s.cn/google_containers/kube-proxy:v1.17.4 k8s.gcr.io/kube-proxy:v1.17.4
docker tag gcr.azk8s.cn/google_containers/kube-apiserver:v1.17.4 k8s.gcr.io/kube-apiserver:v1.17.4
docker tag gcr.azk8s.cn/google_containers/kube-controller-manager:v1.17.4 k8s.gcr.io/kube-controller-manager:v1.17.4
docker tag gcr.azk8s.cn/google_containers/kube-scheduler:v1.17.4 k8s.gcr.io/kube-scheduler:v1.17.4
docker tag gcr.azk8s.cn/google_containers/coredns:1.6.5 k8s.gcr.io/coredns:1.6.5
docker tag gcr.azk8s.cn/google_containers/etcd:3.4.3-0 k8s.gcr.io/etcd:3.4.3-0
docker tag gcr.azk8s.cn/google_containers/pause:3.1 k8s.gcr.io/pause:3.1
在192.168.105.247創建第一個master節點：
kubeadm init --kubernetes-version=v1.17.4 \
--apiserver-advertise-address=192.168.105.247 \
--control-plane-endpoint=192.168.105.253:6443 \
--pod-network-cidr=10.64.0.0/16 \
--service-cidr=10.32.0.0/16 \
--upload-certs
執行結果如下：
.......
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 192.168.105.253:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:05aaf179f9e9a1d0f843ddbf9f7655f0340e434826d3f8dec1316cda4747cc8c \
--control-plane --certificate-key 63af2fc27dd66dd51e1ef8c296253f945e95fc2caad3b963b7b6291d2aa6fd1c
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.105.253:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:05aaf179f9e9a1d0f843ddbf9f7655f0340e434826d3f8dec1316cda4747cc8c
安裝方式二：通過指定文件的方式
[root@k8s-master01 ~]# kubeadm init --config=kubeadm-config.yaml --upload-certs --v=6 |tee kubeadm-init.log
【如果執行這里報如下錯誤：
錯誤一：
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups
問題補充：
journalctl -u kubelet 查看到的錯誤日志
Dec 17 07:23:06 k8s-master0 kubelet[8677]: E1217 07:23:06.438404 8677 kubelet.go:2267] node "k8s-master0" not found
...
解決辦法：
通過 kubeadm 命令的 --v=6 參數開啟更靈敏的雷達找到了問題的線索
kubeadm init --kubernetes-version=v1.17.4 \
--apiserver-advertise-address=192.168.105.247 \
--control-plane-endpoint=192.168.105.252:6443 \
--pod-network-cidr=10.64.0.0/16 \
--service-cidr=10.32.0.0/16 \
--upload-certs --v=6
有如下的錯誤信息：
[kubelet-check] Initial timeout of 40s passed.
I1217 08:39:21.852678 20972 round_trippers.go:443] GET https://k8s.cnblogs.com:6443/healthz?timeout=32s in 30000 milliseconds
是健康檢查時連接 control-plane-endpoint 地址超時了。
參考網址：https://q.cnblogs.com/q/124859/
清除執行錯誤的命令：kubeadm reset
錯誤二：
查看k8s-master01節點的kubelet日志有如下錯誤：
Mar 25 16:13:34 k8s-master01 kubelet[29423]: Trace[2125110574]: [10.001458085s] [10.001458085s] END
Mar 25 16:13:34 k8s-master01 kubelet[29423]: E0325 16:13:34.801119 29423 reflector.go:153] k8s.io/kubernetes/pkg/kubelet/kubelet.go:449: Failed to list *v1.Service: Get https://192.168.105.253:6443/api/v1/services?limit=500&resourceVersion=0: net/http: TLS handshake timeout
解決辦法：清除k8s-master01的防火牆策略
】
其他master節點加入集群：
kubeadm join 192.168.105.253:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:05aaf179f9e9a1d0f843ddbf9f7655f0340e434826d3f8dec1316cda4747cc8c \
--control-plane --certificate-key 63af2fc27dd66dd51e1ef8c296253f945e95fc2caad3b963b7b6291d2aa6fd1c
配置kubectl：
# mkdir -p $HOME/.kube
# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
# sudo chown $(id -u):$(id -g) $HOME/.kube/config
如果是worker節點則使用：
kubeadm join 192.168.105.253:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:05aaf179f9e9a1d0f843ddbf9f7655f0340e434826d3f8dec1316cda4747cc8c
安裝網絡插件：
在master01配置：
安裝完成之后會發現節點的狀態是NotReady
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 NotReady master 3m24s v1.17.4
k8s-master02 NotReady master 17m v1.17.4
k8s-master03 NotReady master 7m48s v1.17.4
k8s-work01 NotReady <none> 46s v1.17.4
k8s-work02 NotReady <none> 88s v1.17.4
查看kubelet會發現是網絡插件沒裝：
Mar 20 16:00:37 m1.k8s.com kubelet[15808]: E0320 16:00:37.274005 15808 kubelet.go:2183] Container runtime network not ready: NetworkReady=false reason:NetworkPlu...initialized
Mar 20 16:00:40 m1.k8s.com kubelet[15808]: W0320 16:00:40.733305 15808 cni.go:237] Unable to update cni config: no networks found in /etc/cni/net.d
安裝flannel插件【可選】
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
sed -i 's/10.244.0.0/10.64.0.0/g' kube-flannel.yml
kubectl apply -f kube-flannel.yml
我們這里安裝calico插件：
kubectl apply -f calico.yaml
注意：calico文件有如下幾個地方修改：
- name: CALICO_IPV4POOL_CIDR
value: "10.64.0.0/16" ##修改為對應的pod地址段
- name: CLUSTER_TYPE
value: "k8s,bgp"
# Auto-detect the BGP IP address.
- name: IP
value: "autodetect"
# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
value: "off" ##關閉IPIP,使用bgp
再次查看節點情況：
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 33m v1.17.4
k8s-master02 Ready master 46m v1.17.4
k8s-master03 Ready master 37m v1.17.4
k8s-work01 Ready <none> 30m v1.17.4
k8s-work02 Ready <none> 31m v1.17.4
master節點默認是pod不被調度：
[root@k8s-master01 ~]# kubectl describe node k8s-master01 |grep Taints:
Taints: node-role.kubernetes.io/master:NoSchedule
[root@k8s-master01 ~]# kubectl describe node k8s-master02 |grep Taints:
Taints: node-role.kubernetes.io/master:NoSchedule
[root@k8s-master01 ~]# kubectl describe node k8s-master03 |grep Taints:
Taints: node-role.kubernetes.io/master:NoSchedule
對node節點配置角色：
[root@k8s-master01 ~]# kubectl label node k8s-work01 node-role.kubernetes.io/worker=worker
node/k8s-work01 labeled
[root@k8s-master01 ~]# kubectl label node k8s-work02 node-role.kubernetes.io/worker=worker
node/k8s-work02 labeled
查看：
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 37m v1.17.4
k8s-master02 Ready master 50m v1.17.4
k8s-master03 Ready master 41m v1.17.4
k8s-work01 Ready worker 34m v1.17.4
k8s-work02 Ready worker 35m v1.17.4
問題補充：
有時calico的pod重啟后，calico-kube-controllers會漂移到work節點，如果work界面沒有安裝kube-proxy的鏡像，會一直報如下的錯誤：
Warning Unhealthy 14m (x185 over 104m) kubelet, k8s-work02 Readiness probe failed: calico/node is not ready: felix is not ready: Get http://localhost:9099/readiness: dial tcp 127.0.0.1:9099: connect: connection refused
Warning BackOff 4m34s (x308 over 97m) kubelet, k8s-work02 Back-off restarting failed container
解決辦法：
在所有worker節點部署安裝kube-proxy對應的鏡像。【dns鏡像盡量在work節點也配置部署】
參考文檔：
https://www.cnblogs.com/37yan/p/12530520.html
https://www.cnblogs.com/LiuQizhong/p/11508145.html
補充：新節點加入k8s集群操作
一、首先在master上生成新的token
[root@k8s-master02 ~]# kubeadm token create --print-join-command
W0321 20:33:30.496764 2731 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0321 20:33:30.496869 2731 validation.go:28] Cannot validate kubelet config - no validator is available
kubeadm join 192.168.105.252:6443 --token ruxg7j.qwqagppstrvr2k71 --discovery-token-ca-cert-hash sha256:436cbe481610de8e2a3d8883723a8172f3d16a7839daf98c01db417e247b641a
二、在master上生成用於新master加入的證書
[root@k8s-master02 ~]# kubeadm init phase upload-certs --upload-certs
W0321 20:36:48.172090 5085 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0321 20:36:48.172239 5085 validation.go:28] Cannot validate kubelet config - no validator is available
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
8b112a097d536f06d6e3a67b43349db1ce74a787226926dad7e87c5a08865138
三、添加新master
kubeadm join 192.168.105.252:6443 --token ruxg7j.qwqagppstrvr2k71 \
--discovery-token-ca-cert-hash sha256:436cbe481610de8e2a3d8883723a8172f3d16a7839daf98c01db417e247b641a \
--control-plane --certificate-key 8b112a097d536f06d6e3a67b43349db1ce74a787226926dad7e87c5a08865138
四、添加新node
kubeadm join 192.168.105.252:6443 --token ruxg7j.qwqagppstrvr2k71 \
--discovery-token-ca-cert-hash sha256:436cbe481610de8e2a3d8883723a8172f3d16a7839daf98c01db417e247b641a
【補充】kubeadm-config.yaml文件內容
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.105.246
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8s-master01
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.105.253:6443"
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.17.4
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/16
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
SupportIPVSProxyMode: true
mode: ipvs