Kubeadm部署安装k8s1.17版本
0、设置时区及同步时间
//设置系统时区为中国/上海
timedatectl set-timezone Asia/Shanghai
//将当前的 UTC 时间写入硬件时钟
timedatectl set-local-rtc 0
// 重启依赖于系统时间的服务
systemctl restart rsyslog && systemctl restart crond
1、设置系统主机名以及host文件的相互解析
hostnamectl set-hostname xxxx
2、系统yum镜像更换
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
3、安装依赖包
yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git
4、设置防火墙为iptables并设置空规则
systemctl stop firewalld && systemctl disable firewalld
yum -y install iptables-services && systemctl start firewalld && systemctl enable iptables && iptables -F && service iptables save
5、关闭selinux及swap
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled' /etc/selinux/config
6、调整内核参数，对于k8s
cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 #禁止使用swap空间，只有当系统OOM时才运行使用它
vm.overcommit_memory=1 #不检查物理内存是否够用
vm.panic_on_oom=0 #开启OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
其中必备参数：
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1 //开启网桥模式
net.ipv6.conf.all.disable_ipv6=1//关闭ipv6的协议
其余为优化参数，可不设置
sysctl -p /etc/sysctl.d/kubernetes.conf
7、关闭系统不需要的服务
systemctl stop postfix && systemctl disable postfix
8、设置rsyslogd和systemd journald
原因：centos7以后，引导方式改为了systemd，所以会有两个日志系统同时工作只保留一个日志（journald）的方法
mkdir /var/log/journal # 持久化保存日志的目录
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal]
#持久化保存到磁盘
Storage=persistent
# 压缩历史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
# 最大占用空间10G
SystemMaxUse=10G
# 单日志文件最大200M
SystemMaxFileSize=200M
# 日志保存时间 2 周
MaxRetentionSec=2week
# 不将日志转发到 syslog
ForwardToSyslog=no
EOF
#重启journald配置
systemctl restart systemd-journald
9、升级内核为4.4版本
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
# 安装完成后检查 /boot/grub2/grub.cfg 中对应内核 menuentry 中是否包含 initrd16 配置，如果没有，再安装一次！
yum --enablerepo=elrepo-kernel install -y kernel-lt
# 设置开机从新内核启动
grub2-set-default "CentOS Linux (4.4.217-1.el7.elrepo.x86_64) 7 (Core)"
# 重启后安装内存源文件
yum --enablerepo=elrepo-kernel install kernel-lt-devel-$(uname -r) kernel-lt-headers-$(uname -r)
10、关闭numa
cp /etc/default/grub{,.bak}
vim /etc/default/grub #在GRUB_CMDLINE_LINUX 一行添加"numa=off" 参数，如下所示：
# diff /etc/default/grub /etc/default/grub.bak
6c6
< GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rhgb quiet numa=off"
---
> GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rhgb quiet"
拷贝boot下的grub配置文件：
cp /boot/grub2/grub.cfg{,.bak}
生成文件：
grub2-mkconfig -o /boot/grub2/grub.cfg
重启服务器。。。。。
11、kube-proxy开启ipvs的前置条件
//1、加载netfilter模块
modprobe br_netfilter
//2、添加配置文件
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
//3、赋予权限并引导
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules &&lsmod | grep -e ip_vs -e nf_conntrack_ipv4
12、安装docker软件
//1、docker依赖
yum install -y yum-utils device-mapper-persistent-data lvm2
//2、导入阿里云的docker-ce仓库
yum-config-manager \
--add-repo \http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
//3、更新系统安装docker-ce
yum update -y && yum install -y docker-ce
//4、uname -r 检测版本，再设置版本，后又重启reboot
grub2-set-default "CentOS Linux (4.4.217-1.el7.elrepo.x86_64) 7 (Core)"
//5、启动
systemctl start docker
//6、开机自启
systemctl enable docker
//7、配置deamon
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors":["https://docker.mirrors.ustc.edu.cn"],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
}
}
EOF
//8、创建目录存放docker配置文件
mkdir -p /etc/systemd/system/docker.service.d
//9、重启docker
systemctl daemon-reload && systemctl restart docker && systemctl enable docker
13、在主节点启动haproxy和keepalived容器
//1、导入haproxy和keepalived镜像
docker load < haproxy.tar
docker load < keepalived.tar
//2、修改haproxy和keepalived配置文件
[root@k8s-master01 lb]# cat /data/lb/etc/haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
retries 3
option redispatch
timeout connect 5000
timeout client 50000
timeout server 50000
frontend stats-front
bind *:8081
mode http
default_backend stats-back
frontend fe_k8s_6444
bind *:6444
mode tcp
timeout client 1h
log global
option tcplog
default_backend be_k8s_6443
acl is_websocket hdr(Upgrade) -i WebSocket
acl is_websocket hdr_beg(Host) -i ws
backend stats-back
mode http
balance roundrobin
stats uri /haproxy/stats
stats auth pxcstats:secret
backend be_k8s_6443
mode tcp
timeout queue 1h
timeout server 1h
timeout connect 1h
log global
balance roundrobin
server rancher01 192.168.105.246:6443 #先暂时添加一个节点
修改启动脚本内容：
[root@k8s-master01 lb]# ./start-haproxy.sh
f03cc66f3bf8644fca6aa8cf887eee66ad1990ded9625d01bf77a97479eb7547
[root@k8s-master01 lb]# cat start-keepalived.sh
#!/bin/bash
VIRTUAL_IP=192.168.105.251 #vip 地址
INTERFACE=ens160 #当前节点的网卡端口
NETMASK_BIT=24
CHECK_PORT=6444 #检查端口为6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18
docker run -itd --restart=always --name=Keepalived-K8S \
--net=host --cap-add=NET_ADMIN \
-e VIRTUAL_IP=$VIRTUAL_IP \
-e INTERFACE=$INTERFACE \
-e CHECK_PORT=$CHECK_PORT \
-e RID=$RID \
-e VRID=$VRID \
-e NETMASK_BIT=$NETMASK_BIT \
-e MCAST_GROUP=$MCAST_GROUP \
wise2c/keepalived-k8s
[root@k8s-master01 lb]# ./start-keepalived.sh
5c37c56bae99541ca19bb023579adbe3fec64ac97f626d812eed68533b13fb40
安装kubeadm, kubelet , kubectl(所有master/node节点)
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum install -y kubelet-1.17.4 kubeadm-1.17.4 kubectl-1.17.4
systemctl enable kubelet
使用kubeadm部署
在 m[1:3].k8s.com 配置：
建议先把需要的镜像安装好
#kubeadm config images list --kubernetes-version=v1.17.4
W0320 15:26:32.612945 123330 validation.go:28] Cannot validate kubelet config - no validator is available
W0320 15:26:32.612995 123330 validation.go:28] Cannot validate kube-proxy config - no validator is available
k8s.gcr.io/kube-apiserver:v1.17.4
k8s.gcr.io/kube-controller-manager:v1.17.4
k8s.gcr.io/kube-scheduler:v1.17.4
k8s.gcr.io/kube-proxy:v1.17.4
k8s.gcr.io/pause:3.1
k8s.gcr.io/etcd:3.4.3-0
k8s.gcr.io/coredns:1.6.5
执行kubeadm证书有效期修改：请查看《证书有效期修改》内容！！！！！！
【#####################开始安装Ks8#################################】
安装方式一：
使用azure提供的国内源加速:
kubeadm config images pull --image-repository gcr.azk8s.cn/google_containers --kubernetes-version=v1.17.4
对镜像打上标签：
docker tag gcr.azk8s.cn/google_containers/kube-proxy:v1.17.4 k8s.gcr.io/kube-proxy:v1.17.4
docker tag gcr.azk8s.cn/google_containers/kube-apiserver:v1.17.4 k8s.gcr.io/kube-apiserver:v1.17.4
docker tag gcr.azk8s.cn/google_containers/kube-controller-manager:v1.17.4 k8s.gcr.io/kube-controller-manager:v1.17.4
docker tag gcr.azk8s.cn/google_containers/kube-scheduler:v1.17.4 k8s.gcr.io/kube-scheduler:v1.17.4
docker tag gcr.azk8s.cn/google_containers/coredns:1.6.5 k8s.gcr.io/coredns:1.6.5
docker tag gcr.azk8s.cn/google_containers/etcd:3.4.3-0 k8s.gcr.io/etcd:3.4.3-0
docker tag gcr.azk8s.cn/google_containers/pause:3.1 k8s.gcr.io/pause:3.1
在192.168.105.247创建第一个master节点：
kubeadm init --kubernetes-version=v1.17.4 \
--apiserver-advertise-address=192.168.105.247 \
--control-plane-endpoint=192.168.105.253:6443 \
--pod-network-cidr=10.64.0.0/16 \
--service-cidr=10.32.0.0/16 \
--upload-certs
执行结果如下：
.......
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 192.168.105.253:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:05aaf179f9e9a1d0f843ddbf9f7655f0340e434826d3f8dec1316cda4747cc8c \
--control-plane --certificate-key 63af2fc27dd66dd51e1ef8c296253f945e95fc2caad3b963b7b6291d2aa6fd1c
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.105.253:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:05aaf179f9e9a1d0f843ddbf9f7655f0340e434826d3f8dec1316cda4747cc8c
安装方式二：通过指定文件的方式
[root@k8s-master01 ~]# kubeadm init --config=kubeadm-config.yaml --upload-certs --v=6 |tee kubeadm-init.log
【如果执行这里报如下错误：
错误一：
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups
问题补充：
journalctl -u kubelet 查看到的错误日志
Dec 17 07:23:06 k8s-master0 kubelet[8677]: E1217 07:23:06.438404 8677 kubelet.go:2267] node "k8s-master0" not found
...
解决办法：
通过 kubeadm 命令的 --v=6 参数开启更灵敏的雷达找到了问题的线索
kubeadm init --kubernetes-version=v1.17.4 \
--apiserver-advertise-address=192.168.105.247 \
--control-plane-endpoint=192.168.105.252:6443 \
--pod-network-cidr=10.64.0.0/16 \
--service-cidr=10.32.0.0/16 \
--upload-certs --v=6
有如下的错误信息：
[kubelet-check] Initial timeout of 40s passed.
I1217 08:39:21.852678 20972 round_trippers.go:443] GET https://k8s.cnblogs.com:6443/healthz?timeout=32s in 30000 milliseconds
是健康检查时连接 control-plane-endpoint 地址超时了。
参考网址：https://q.cnblogs.com/q/124859/
清除执行错误的命令：kubeadm reset
错误二：
查看k8s-master01节点的kubelet日志有如下错误：
Mar 25 16:13:34 k8s-master01 kubelet[29423]: Trace[2125110574]: [10.001458085s] [10.001458085s] END
Mar 25 16:13:34 k8s-master01 kubelet[29423]: E0325 16:13:34.801119 29423 reflector.go:153] k8s.io/kubernetes/pkg/kubelet/kubelet.go:449: Failed to list *v1.Service: Get https://192.168.105.253:6443/api/v1/services?limit=500&resourceVersion=0: net/http: TLS handshake timeout
解决办法：清除k8s-master01的防火墙策略
】
其他master节点加入集群：
kubeadm join 192.168.105.253:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:05aaf179f9e9a1d0f843ddbf9f7655f0340e434826d3f8dec1316cda4747cc8c \
--control-plane --certificate-key 63af2fc27dd66dd51e1ef8c296253f945e95fc2caad3b963b7b6291d2aa6fd1c
配置kubectl：
# mkdir -p $HOME/.kube
# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
# sudo chown $(id -u):$(id -g) $HOME/.kube/config
如果是worker节点则使用：
kubeadm join 192.168.105.253:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:05aaf179f9e9a1d0f843ddbf9f7655f0340e434826d3f8dec1316cda4747cc8c
安装网络插件：
在master01配置：
安装完成之后会发现节点的状态是NotReady
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 NotReady master 3m24s v1.17.4
k8s-master02 NotReady master 17m v1.17.4
k8s-master03 NotReady master 7m48s v1.17.4
k8s-work01 NotReady <none> 46s v1.17.4
k8s-work02 NotReady <none> 88s v1.17.4
查看kubelet会发现是网络插件没装：
Mar 20 16:00:37 m1.k8s.com kubelet[15808]: E0320 16:00:37.274005 15808 kubelet.go:2183] Container runtime network not ready: NetworkReady=false reason:NetworkPlu...initialized
Mar 20 16:00:40 m1.k8s.com kubelet[15808]: W0320 16:00:40.733305 15808 cni.go:237] Unable to update cni config: no networks found in /etc/cni/net.d
安装flannel插件【可选】
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
sed -i 's/10.244.0.0/10.64.0.0/g' kube-flannel.yml
kubectl apply -f kube-flannel.yml
我们这里安装calico插件：
kubectl apply -f calico.yaml
注意：calico文件有如下几个地方修改：
- name: CALICO_IPV4POOL_CIDR
value: "10.64.0.0/16" ##修改为对应的pod地址段
- name: CLUSTER_TYPE
value: "k8s,bgp"
# Auto-detect the BGP IP address.
- name: IP
value: "autodetect"
# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
value: "off" ##关闭IPIP,使用bgp
再次查看节点情况：
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 33m v1.17.4
k8s-master02 Ready master 46m v1.17.4
k8s-master03 Ready master 37m v1.17.4
k8s-work01 Ready <none> 30m v1.17.4
k8s-work02 Ready <none> 31m v1.17.4
master节点默认是pod不被调度：
[root@k8s-master01 ~]# kubectl describe node k8s-master01 |grep Taints:
Taints: node-role.kubernetes.io/master:NoSchedule
[root@k8s-master01 ~]# kubectl describe node k8s-master02 |grep Taints:
Taints: node-role.kubernetes.io/master:NoSchedule
[root@k8s-master01 ~]# kubectl describe node k8s-master03 |grep Taints:
Taints: node-role.kubernetes.io/master:NoSchedule
对node节点配置角色：
[root@k8s-master01 ~]# kubectl label node k8s-work01 node-role.kubernetes.io/worker=worker
node/k8s-work01 labeled
[root@k8s-master01 ~]# kubectl label node k8s-work02 node-role.kubernetes.io/worker=worker
node/k8s-work02 labeled
查看：
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 37m v1.17.4
k8s-master02 Ready master 50m v1.17.4
k8s-master03 Ready master 41m v1.17.4
k8s-work01 Ready worker 34m v1.17.4
k8s-work02 Ready worker 35m v1.17.4
问题补充：
有时calico的pod重启后，calico-kube-controllers会漂移到work节点，如果work界面没有安装kube-proxy的镜像，会一直报如下的错误：
Warning Unhealthy 14m (x185 over 104m) kubelet, k8s-work02 Readiness probe failed: calico/node is not ready: felix is not ready: Get http://localhost:9099/readiness: dial tcp 127.0.0.1:9099: connect: connection refused
Warning BackOff 4m34s (x308 over 97m) kubelet, k8s-work02 Back-off restarting failed container
解决办法：
在所有worker节点部署安装kube-proxy对应的镜像。【dns镜像尽量在work节点也配置部署】
参考文档：
https://www.cnblogs.com/37yan/p/12530520.html
https://www.cnblogs.com/LiuQizhong/p/11508145.html
补充：新节点加入k8s集群操作
一、首先在master上生成新的token
[root@k8s-master02 ~]# kubeadm token create --print-join-command
W0321 20:33:30.496764 2731 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0321 20:33:30.496869 2731 validation.go:28] Cannot validate kubelet config - no validator is available
kubeadm join 192.168.105.252:6443 --token ruxg7j.qwqagppstrvr2k71 --discovery-token-ca-cert-hash sha256:436cbe481610de8e2a3d8883723a8172f3d16a7839daf98c01db417e247b641a
二、在master上生成用于新master加入的证书
[root@k8s-master02 ~]# kubeadm init phase upload-certs --upload-certs
W0321 20:36:48.172090 5085 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0321 20:36:48.172239 5085 validation.go:28] Cannot validate kubelet config - no validator is available
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
8b112a097d536f06d6e3a67b43349db1ce74a787226926dad7e87c5a08865138
三、添加新master
kubeadm join 192.168.105.252:6443 --token ruxg7j.qwqagppstrvr2k71 \
--discovery-token-ca-cert-hash sha256:436cbe481610de8e2a3d8883723a8172f3d16a7839daf98c01db417e247b641a \
--control-plane --certificate-key 8b112a097d536f06d6e3a67b43349db1ce74a787226926dad7e87c5a08865138
四、添加新node
kubeadm join 192.168.105.252:6443 --token ruxg7j.qwqagppstrvr2k71 \
--discovery-token-ca-cert-hash sha256:436cbe481610de8e2a3d8883723a8172f3d16a7839daf98c01db417e247b641a
【补充】kubeadm-config.yaml文件内容
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.105.246
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8s-master01
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.105.253:6443"
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.17.4
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/16
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
SupportIPVSProxyMode: true
mode: ipvs