LVS實現Kubernetes集群高可用

服務器規划：

三台k8s-master                                                     兩台lvs                                      

k8s01：10.20.31.157                                            lb01：10.20.31.184

k8s02：10.20.31.167                                            lb02：10.20.31.185

k8s03：10.20.31.186                                            vipIP：10.20.31.187

從構圖中可以看到，所有節點都需要通過負載均衡器和API Server進行通信，負載均衡器就非常重要了。這里考慮負載均衡器的性能與高可用，我們選擇了LVS + keepalived（LVS當然也可以部署在k8s的節點機器上，為了保證集群高可用，建議還是部署在單獨的機器上。）。

lvs-master(10.20.31.184)

# 安裝依賴
$ yum install -y ipvsadm wget curl gcc openssl-devel libnl3-devel net-snmp-devel libnfnetlink-devel
# 安裝keepalived，centos7通過yum下載的版本有問題，會報一個叫【TCP socket bind failed. Rescheduling】的錯誤
$ wget http://www.keepalived.org/software/keepalived-1.4.5.tar.gz && tar -zxvf keepalived-1.4.5.tar.gz && cd keepalived-1.4.5 && ./configure && make && make install && cd .. && rm -f keepalived-1.4.5.tar.gz && rm -rf keepalived-1.4.5
################ keepalived負載均衡配置 ################
# 生成keepalived配置
$ cd /etc/keepalived && cat <<E0F > /etc/keepalived/keepalived.conf
global_defs {
   router_id keepalived-master
}

vrrp_instance vip_1 {
  state MASTER
  ! 注意這是網卡名稱，使用ip a命令查看自己的局域網網卡名稱
  interface ens192
  ! keepalived主備router_id必須一致
  virtual_router_id 88
  ! 優先級，keepalived主節點優先級要比備節點高
  priority 100
  advert_int 3
  ! 配置虛擬ip地址
  virtual_ipaddress {
    10.20.31.187
  }
}

virtual_server 10.20.31.187 6443 {
  delay_loop 6
  lb_algo rr
  lb_kind DR
  persistence_timeout 0
  protocol TCP
    
  real_server 10.20.31.157 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
  real_server 10.20.31.167 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
  real_server 10.20.31.186 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
}
E0F

# 啟動keepalived
$ systemctl enable keepalived && service keepalived start

# 檢查keepalived狀態
$ service keepalived status

# 查看日志
$ journalctl -f -u keepalived

# 查看虛擬ip
$ ip a

lvs-backup(10.20.31.185)

# 安裝依賴
$ yum install -y ipvsadm wget curl gcc openssl-devel libnl3-devel net-snmp-devel libnfnetlink-devel
# 安裝keepalived，centos7通過yum下載的版本有問題，會報一個叫【TCP socket bind failed. Rescheduling】的錯誤
$ wget http://www.keepalived.org/software/keepalived-1.4.5.tar.gz && tar -zxvf keepalived-1.4.5.tar.gz && cd keepalived-1.4.5 && ./configure && make && make install && cd .. && rm -f keepalived-1.4.5.tar.gz && rm -rf keepalived-1.4.5
################ keepalived負載均衡配置 ################
# 生成keepalived配置
$ mkdir -p /etc/keepalived && cd /etc/keepalived && cat <<E0F > /etc/keepalived/keepalived.conf
global_defs {
   router_id keepalived-backup
}

vrrp_instance vip_1 {
  state BACKUP
  ! 注意這是網卡名稱，使用ip a命令查看自己的局域網網卡名稱
  interface ens192
  ! keepalived主備router_id必須一致
  virtual_router_id 88
  ! 優先級，keepalived主節點優先級要比備節點高
  priority 99
  advert_int 3
  ! 配置虛擬ip地址
  virtual_ipaddress {
    10.20.31.187
  }
}

virtual_server 10.20.31.187 6443 {
  delay_loop 6
  lb_algo rr
  lb_kind DR
  persistence_timeout 0
  protocol TCP
    
  real_server 10.20.31.157 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
  real_server 10.20.31.167 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
  real_server 10.20.31.186 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
}
E0F

# 啟動keepalived
$ systemctl enable keepalived && service keepalived start

# 檢查keepalived狀態
$ service keepalived status

# 查看日志
$ journalctl -f -u keepalived

# 查看虛擬ip
$ ip a

　

real_server配置，也就是每個k8s Master節點機器

# 創建rs腳本
$ mkdir -p /opt/rs/ && cd /opt/rs && cat <<E0F > /opt/rs/rs.sh
#!/bin/bash
# 虛擬ip
vip=10.20.31.187
# 停止以前的lo:0
ifconfig lo:0 down
echo "1" > /proc/sys/net/ipv4/ip_forward
echo "0" > /proc/sys/net/ipv4/conf/all/arp_announce
# 啟動一個回環地址並綁定給vip
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
# ens33是主網卡名
echo "1" >/proc/sys/net/ipv4/conf/ens192/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/ens192/arp_announce
E0F

# 添加執行權限
$ chmod +x /opt/rs/rs.sh

# 執行rs腳本（如果出現錯誤，重新執行一遍即可）
$ ./rs.sh

# 添加到開機啟動
$ echo '/opt/rs/rs.sh'  >> /etc/rc.d/rc.local
# 在centos7中，/etc/rc.d/rc.local的權限被降低了，所以需要執行如下命令賦予其可執行權限
$ chmod +x /etc/rc.d/rc.local