CentOS7 kubeadm 安装 k8s cri-o

背景

IP	角色	操作系统
172.16.16.110	k8s-master	CentOS Linux release 7.8.2003
172.16.16.111	k8s-node1	CentOS Linux release 7.8.2003
172.16.16.112	k8s-node2	CentOS Linux release 7.8.2003

3 台服务器都需要操作

# 关闭 selinux
getenforce 
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config

# 时间同步
yum install ntpdate -y
timedatectl set-timezone Asia/Shanghai 
/usr/sbin/ntpdate ntp1.aliyun.com

crontab -e
*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com > /dev/null 2>&1

# 关闭防火墙
systemctl status firewalld
systemctl disable firewalld
systemctl stop firewalld

# 禁止 swap
sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a 

vi /etc/hosts
# 添加
172.16.16.110 k8s-master
172.16.16.111 k8s-node1
172.16.16.112 k8s-node2

cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness=0
EOF

modprobe br_netfilter
sysctl -p /etc/sysctl.d/k8s.conf

cat > /etc/sysconfig/modules/ipvs.modules << EOF
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
modprobe -- br_netfilter
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod |grep -e ip_vs -e nf_conntrack_ipv4
yum install yum-utils

cat > /etc/modules-load.d/crio.conf << EOF
overlay
br_netfilter
EOF

modprobe overlay

VERSION=1.20
OS=CentOS_7
sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/devel:kubic:libcontainers:stable.repo
sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.repo https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:$VERSION/$OS/devel:kubic:libcontainers:stable:cri-o:$VERSION.repo

yum install -y cri-o

cd /etc/crio/
cp crio.conf crio.conf.20210923

vi crio.conf
# 修改
pause_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2"
registries = [
   "docker.mirrors.ustc.edu.cn","dockerhub.azk8s.cn","hub-mirror.c.163.com"
 ]

systemctl start crio

# 测试
curl -v --unix-socket /var/run/crio/crio.sock http://localhost/info

# 安装 kubeadm-1.20.5 kubectl-1.20.5 kubelet-1.20.5
cat << EOF > /etc/yum.repos.d/kubernetes.repo 
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum install -y kubeadm-1.20.5 kubectl-1.20.5 kubelet-1.20.5

在 master 上操作

kubeadm config print init-defaults > kubeadm.yaml 

# 修改
advertiseAddress: 172.16.16.110
criSocket: /var/run/crio/crio.sock
name: k8s-master

imageRepository: registry.aliyuncs.com/google_containers
kubernetesVersion: v1.20.5

# 在 dnsDomain: cluster.local 下添加
podSubnet: 10.244.0.0/16

# 最后添加
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

# 初始化 k8s master 
kubeadm init --config=kubeadm.yaml

# 配置config
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

# 安装 calico
yum install wget -y
mkdir -p /data/yaml/default/calico  
cd /data/yaml/default/calico
wget https://docs.projectcalico.org/v3.8/manifests/calico.yaml

vi calico.yaml
# 找到
 - name: calico-node
   image: calico/node:v3.8.9
   env:
     # Use Kubernetes API as the backing datastore.
     - name: DATASTORE_TYPE
       value: "kubernetes"
# 添加
     - name: IP_AUTODETECTION_METHOD              
       value: interface=ens33

# 修改，值改为上面 podSubnet 对应的值
- name: CALICO_IPV4POOL_CIDR
  value: "10.244.0.0/16"

kubectl apply -f calico.yaml 

node 加入 master

kubeadm join 172.16.16.110:6443 --token abcdef.0123456789abcdef     --discovery-token-ca-cert-hash sha256:69d6637127936ebfe8ac07caf2a2234dd43f5415816af259652b22e53ace3fc4

如果忘记了token

# 如果忘记了token，在Master节点上通过如下命令查看:
kubeadm token list

#如果 token 过期了，在Master节点上通过如下命令重新生成:
kubeadm token create

# --discovery-token-ca-cert-hash 参数的值，在管理节点上通过如下命令获取:
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

测试集群, 在master节点上操作

kubectl get nodes

# 测试DNS解析
kubectl run -it --rm dns-test --image=busybox:1.28.4 sh

# 执行下面，查看能否正常解析出 kubernetes
nslookup kubernetes

crictl 命令

crictl --runtime-endpoint /var/run/crio/crio.sock ps
crictl --runtime-endpoint /var/run/crio/crio.sock images
crictl --runtime-endpoint /var/run/crio/crio.sock pods
crictl --runtime-endpoint /var/run/crio/crio.sock pull calico/node:v3.8.9

可以使用 podman 来管理 cri-o 中的镜像

# 安装 
yum -y install podman

# 打tag 和 push
podman tag docker.io/library/nginx:1.14.2 harbor.junengcloud.com/tmp/nginx:1.14.2
podman push harbor.junengcloud.com/tmp/nginx:1.14.2

# 查看命令帮助
podman --help