環境說明
四台節點
123 nginx
124 node1
125 node2
126 node3
nginx節點實現自動ssh免密登錄
建立免密登錄連接不能使用root賬號 必須在所有節點上創建一個相同名稱的普通賬號,同時給這個普通賬號sudo權限同時設置執行sudo的時候不需要輸入密碼
在編寫腳本的時候用戶的家目錄最后不要~ 最好指定為絕對路徑 這樣可以避免一些錯誤
centos用戶建立ssh免密連接
不能直接root用戶創建公私鑰,必須要創建一個普通用戶
普通用戶執行docker命令
sudo groupadd docker #添加docker用戶組
sudo gpasswd - a $USER docker #將當前用戶添加至docker用戶組
執行 groupadd docker的時候提示已經存在,所以后面的兩條指令都不會執行導致后面在執行docker指令的時候出現異常
普通用戶沒有sudo權限
通過rke部署k8s集群
#!/usr/bin/bash echo "安裝shell模塊expect" sudo yum -y install expect echo "安裝expect模塊成功" echo "安裝ntpdate" sudo yum install -y ntpdate echo "開始安裝yml配置讀取模塊" function parse_yaml() { local yaml_file=$1 local prefix=$2 local s local w local fs s='[[:space:]]*' w='[a-zA-Z0-9_.-]*' fs="$(echo @|tr @ '\034')" ( sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \ -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \ -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" | awk -F"$fs" '{ indent = length($1)/2; if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";} vname[indent] = $2; for (i in vname) {if (i > indent) {delete vname[i]}} if (length($3) > 0) { vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3); } }' | sed -e 's/_=/+=/g' \ -e '/\..*=/s|\.|_|' \ -e '/\-.*=/s|\-|_|' ) < "$yaml_file" } function create_variables() { local yaml_file="$1" eval "$(parse_yaml "$yaml_file")" } create_variables config/system-init.yml echo "yml配置讀取模塊成功" echo "獲取yml的配置信息" echo $fqdn echo $nginxIp echo $rancher1Ip echo $rancher2Ip echo $rancher3Ip echo $userpasswd echo $sshuser #------------------------------------------# # 建立ssh連接互信 #------------------------------------------# echo '開始建立ssh互信連接' if [ ! -f ~/.ssh/id_rsa ];then ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa else echo "id_rsa has created ..." fi expect <<EOF set timeout 10 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher1Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher2Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher3Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } expect eof EOF echo '建立ssh互信連接成功' hosts=("$rancher1Ip" "$rancher2Ip" "$rancher3Ip") echo '同步節點時間' sudo ntpdate time1.aliyun.com for ip in ${hosts[@]};do ssh $sshuser@$ip "yum install -y ntpdate" ssh $sshuser@$ip "sudo ntpdate time1.aliyun.com" done echo '同步節點時間成功' echo '配置nameserver' cp -r config/resolv.conf /etc/resolv.conf for ip in ${hosts[@]};do scp -r config/resolv.conf $sshuser@$ip:~/resolv.conf ssh $sshuser@$ip "sudo cp ~/resolv.conf /etc/resolv.conf" done echo '配置nameserver成功' echo "設置開放端口" sudo firewall-cmd --permanent --add-port=80/tcp sudo firewall-cmd --permanent --add-port=443/tcp sudo firewall-cmd --reload for ip in ${hosts[@]};do ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=22/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=80/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=443/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2376/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2379/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=2380/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=6443/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=8472/udp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=8472/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=9099/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=10250/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=10254/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=30000-32767/tcp" ssh $sshuser@$ip "sudo firewall-cmd --permanent --add-port=30000-32767/udp" ssh $sshuser@$ip "sudo firewall-cmd --reload" done echo "設置端口結束" echo "禁用swap" for ip in ${hosts[@]};do ssh $sshuser@$ip "sudo swapoff -a" done echo "禁用swap結束" echo '開始給集群節點安裝docker' for ip in ${hosts[@]};do scp -r pkg/docker/docker-ce.repo $sshuser@$ip:~/ ssh $sshuser@$ip "sudo cp ~/docker-ce.repo /etc/yum.repos.d/docker-ce.repo && sudo yum install -y docker-ce-18.09.3-3.el7" scp -r pkg/docker/daemon.json $sshuser@$ip:~/ ssh $sshuser@$ip "sudo groupadd docker && sudo gpasswd -a $sshuser docker" ssh $sshuser@$ip "sudo mkdir -p /etc/docker/ && sudo cp ~/daemon.json /etc/docker/daemon.json && sudo systemctl start docker" done echo '安裝docker結束' echo '開始給集群節點加載鏡像' for ip in ${hosts[@]};do scp -r images/ $sshuser@$ip:~/ ssh $sshuser@$ip "sudo docker load -i ~/images/autoscaler.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/calico-node.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/cni.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/coredns.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/defaultbackend.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/etcd.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/flannel.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/flexvol.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/hyperkube.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/ingress-controller.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/metrics.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/metrics-server.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/nginx.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/pause.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/rancher.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/rancher-agent.tar" ssh $sshuser@$ip "sudo docker load -i ~/images/tools.tar" done echo '加載鏡像結束' echo "安裝kubectl" sudo cp pkg/kubectl/kubernets.repo /etc/yum.repos.d/kubernets.repo sudo yum install -y kubectl echo "kubectl安裝結束" echo "安裝rke" cp pkg/rke/rke ~/rke && sudo chmod +x ~/rke cp config/rancher-cluster.yml ~/rancher-cluster.yml sed -i "s/node1/$rancher1Ip/" ~/rancher-cluster.yml sed -i "s/node2/$rancher2Ip/" ~/rancher-cluster.yml sed -i "s/node3/$rancher3Ip/" ~/rancher-cluster.yml echo "rke安裝結束" echo "安裝k8s集群" ~/rke up --config=/home/admin/rancher-cluster.yml mkdir ~/.kube cp ~/kube_config_rancher-cluster.yml /home/admin/.kube/config echo "安裝k8s集群成功"
在k8s上部署rancher server
#!/usr/bin/bash echo "開始安裝yml配置讀取模塊" function parse_yaml() { local yaml_file=$1 local prefix=$2 local s local w local fs s='[[:space:]]*' w='[a-zA-Z0-9_.-]*' fs="$(echo @|tr @ '\034')" ( sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \ -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \ -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" | awk -F"$fs" '{ indent = length($1)/2; if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";} vname[indent] = $2; for (i in vname) {if (i > indent) {delete vname[i]}} if (length($3) > 0) { vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3); } }' | sed -e 's/_=/+=/g' \ -e '/\..*=/s|\.|_|' \ -e '/\-.*=/s|\-|_|' ) < "$yaml_file" } function create_variables() { local yaml_file="$1" eval "$(parse_yaml "$yaml_file")" } create_variables config/system-init.yml echo "yml配置讀取模塊成功" echo "獲取yml的配置信息" echo $fqdn echo $sshuser echo "開始生成自簽名證書" mkdir -p /home/$sshuser/certs/ cp pkg/certs/create_self-signed-cert.sh /home/$sshuser/certs/ sudo chmod +x /home/$sshuser/certs/create_self-signed-cert.sh cd /home/$sshuser/certs/ && ./create_self-signed-cert.sh --ssl-domain=$fqdn echo "生成證書完畢" echo "創建secret" kubectl delete secret tls-rancher-ingress -n cattle-system kubectl delete secret tls-ca -n cattle-system kubectl create ns cattle-system kubectl -n cattle-system create secret tls tls-rancher-ingress --cert=/home/$sshuser/certs/tls.crt --key=/home/$sshuser/certs/tls.key kubectl -n cattle-system create secret generic tls-ca --from-file=/home/$sshuser/certs/cacerts.pem echo "創建secret結束" echo "helm安裝rancher" sudo cp pkg/helm/helm /home/$sshuser/helm && sudo chmod +x /home/$sshuser/helm /home/$sshuser/helm repo add rancher-stable http://rancher-mirror.oss-cn-beijing.aliyuncs.com/server-charts/stable /home/$sshuser/helm repo update /home/$sshuser/helm install rancher rancher-stable/rancher --namespace cattle-system --set hostname=$fqdn --set ingress.tls.source=secret --set privateCA=true echo "helm安裝rancher成功" echo "配置nginx" sudo cp pkg/nginx/nginx.repo /etc/yum.repos.d/nginx.repo sudo yum install -y nginx sudo cp pkg/nginx/nginx.conf /etc/nginx/nginx.conf sudo sed -i "s/node1/$rancher1Ip/" /etc/nginx/nginx.conf sudo sed -i "s/node2/$rancher2Ip/" /etc/nginx/nginx.conf sudo sed -i "s/node3/$rancher3Ip/" /etc/nginx/nginx.conf sudo systemctl start nginx echo "配置nginx結束"
回滾卸載殘留文件
cat /proc/mounts | grep "kubelet" | awk '{print $2}' | xargs umount
for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{print $3}') /var/lib/kubelet /var/lib/rancher; do umount $mount; done
kubelet里面的文件必須先進行卸載 然后才能進行刪除
#!/bin/bash function parse_yaml() { local yaml_file=$1 local prefix=$2 local s local w local fs s='[[:space:]]*' w='[a-zA-Z0-9_.-]*' fs="$(echo @|tr @ '\034')" ( sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/\s*$//g;' \ -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \ -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" | awk -F"$fs" '{ indent = length($1)/2; if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";} vname[indent] = $2; for (i in vname) {if (i > indent) {delete vname[i]}} if (length($3) > 0) { vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} printf("%s%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, conj[indent-1],$3); } }' | sed -e 's/_=/+=/g' \ -e '/\..*=/s|\.|_|' \ -e '/\-.*=/s|\-|_|' ) < "$yaml_file" } function create_variables() { local yaml_file="$1" eval "$(parse_yaml "$yaml_file")" } create_variables config/system-init.yml echo "yml配置讀取模塊成功" echo "獲取yml的配置信息" echo $rancher1Ip echo $rancher2Ip echo $rancher3Ip echo $userpasswd echo $sshuser read -p "確定要對此集群進行回滾操作?[y/n] " input case $input in [yY]*) #------------------------------------------# # 建立ssh連接互信 #------------------------------------------# echo 'rke刪除k8s集群' /home/$sshuser/rke remove --config=/home/$sshuser/rancher-cluster.yml echo 'rke刪除k8s集群結束' echo '開始建立ssh互信連接' if [ ! -f ~/.ssh/id_rsa ];then ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa else echo "id_rsa has created ..." fi expect <<EOF set timeout 10 spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher1Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } spawn ssh-copy-id -i /home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher2Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } spawn ssh-copy-id -i/home/$sshuser/.ssh/id_rsa.pub $sshuser@$rancher3Ip expect { "yes/no" { send "yes\n";exp_continue} "password" { send "$userpasswd\n"} } expect eof EOF echo '建立ssh互信連接成功' hosts=("$rancher1Ip" "$rancher2Ip" "$rancher3Ip") echo '開始清空集群節點' for ip in ${hosts[@]};do ssh $sshuser@$ip "docker rm -f $(docker ps -qa)" ssh $sshuser@$ip "docker rmi `docker images -q`" ssh $sshuser@$ip "for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done" ssh $sshuser@$ip "sudo rm -rf /etc/ceph \ /etc/etcd \ /etc/kubernetes \ /etc/cni \ /opt/cni \ /run/secrets/kubernetes.io \ /run/calico \ /run/flannel \ /var/lib/calico \ /var/lib/cni \ /var/lib/kubelet \ /var/lib/etcd \ /var/log/containers \ /var/log/pods \ /var/run/calico" done echo '清空集群節點成功' echo '卸載docker' # for ip in ${hosts[@]};do # echo $ip # ssh $sshuser@$ip "sudo yum remove -y docker-ce.x86_64" # ssh $sshuser@$ip "sudo yum remove -y containerd.io.x86_64" # ssh $sshuser@$ip "sudo yum remove -y docker-ce-cli.x86_64" # ssh $sshuser@$ip "sudo rm -fr /var/lib/docker" # ssh $sshuser@$ip "sudo rm -fr /etc/docker" # done echo '卸載docker完成' ;; [nN]*) exit ;; *) echo "輸入非法,請輸入y或者n" exit ;; esac
測試結束
telnet 端口不通並且在防火牆中已經放開
端口必須處於監聽狀態才能被連接
連接端口成功
docker鏡像的導入導出
需要保存原有鏡像的標簽信息
docker save rancher/calico-cni:v3.13.4 -o cni.tar
docker save rancher/hyperkube:v1.18.3-rancher2 -o hyperkube.tar
批量導入鏡像
for i in $(ls /home/admin/images)
do
docker load -i $i
done