容器化haproxy+keepalived

# 拉取haproxy鏡像
docker pull haproxy:1.7.8-alpine
mkdir /etc/haproxy
cat >/etc/haproxy/haproxy.cfg<<EOF
global
  log 127.0.0.1 local0 err
  maxconn 50000
  uid 99
  gid 99
  #daemon
  nbproc 1
  pidfile haproxy.pid

defaults
  mode http
  log 127.0.0.1 local0 err
  maxconn 50000
  retries 3
  timeout connect 5s
  timeout client 30s
  timeout server 30s
  timeout check 2s

listen admin_stats
  mode http
  bind 0.0.0.0:1080
  log 127.0.0.1 local0 err
  stats refresh 30s
  stats uri     /haproxy-status
  stats realm   Haproxy\ Statistics
  stats auth    will:will
  stats hide-version
  stats admin if TRUE

frontend k8s-https
  bind 0.0.0.0:8443
  mode tcp
  #maxconn 50000
  default_backend k8s-https

backend k8s-https
  mode tcp
  balance roundrobin
  server lab1 192.168.105.92:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
  server lab2 192.168.105.93:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
  server lab3 192.168.105.94:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
EOF

# 啟動haproxy
docker run -d --name my-haproxy \
-v /etc/haproxy:/usr/local/etc/haproxy:ro \
-p 8443:8443 \
-p 1080:1080 \
--restart always \
haproxy:1.7.8-alpine

# 查看日志
docker logs my-haproxy

# 瀏覽器查看狀態
http://192.168.105.92:1080/haproxy-status
http://192.168.105.93:1080/haproxy-status
http://192.168.105.94:1080/haproxy-status

# 拉取keepalived鏡像
docker pull osixia/keepalived:1.4.4

# 啟動
# 載入內核相關模塊
lsmod | grep ip_vs
modprobe ip_vs

# 啟動keepalived
# ens32為本次實驗192.168.105.0/24網段的所在網卡
docker run --net=host --cap-add=NET_ADMIN \
-e KEEPALIVED_INTERFACE=ens32 \
-e KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.105.99']" \
-e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['192.168.105.92','192.168.105.93','192.168.105.94']" \
-e KEEPALIVED_PASSWORD=hello \
--name k8s-keepalived \
--restart always \
-d osixia/keepalived:1.4.4

# 查看日志
# 會看到兩個成為backup 一個成為master
docker logs k8s-keepalived

# 此時會配置 192.168.105.99 到其中一台機器
# ping測試
ping -c4 192.168.105.99

# 如果失敗后清理后，重新實驗
#docker rm -f k8s-keepalived
#ip a del 192.168.105.99/32 dev ens32

測試Haproxy高可用

#抓個包看看現在VIP在哪台機器上，然后去關掉這台機器

tcpdump -nn host 224.0.100.100

[root@master2 ~]# tcpdump -nn host 224.0.100.100
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:45:59.768033 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:46:00.769503 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:46:01.771062 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
^C

#現在VIP是在192.168.1.4這台主機上，關掉這台機器
init 0

#關掉之后發現192.168.1.5立馬接管了。現在VIP是在1.5上
[root@node3 ~]# tcpdump -nn host 224.0.100.100
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:48:25.031679 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:26.033805 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:27.035313 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:28.036628 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:29.039011 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:30.041249 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:31.043065 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:32.045007 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:33.046781 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:34.048776 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:35.051280 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:35.929482 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 0, authtype simple, intvl 1s, length 20
17:48:36.618749 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:37.699849 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:38.700669 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:39.702840 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:40.704254 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:41.706221 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:42.706478 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
^C
19 packets captured
326 packets received by filter
172 packets dropped by kernel

#切換到master2，再創建個POD試試
kubectl run curl --image=radial/busyboxplus:curl -i --tty
date
exit
kubectl delete deployment curl

[root@master2 ~]# kubectl run curl --image=radial/busyboxplus:curl -i --tty
If you don't see a command prompt, try pressing enter.
[ root@curl-87b54756-xfgrn:/ ]$ date
Thu Sep 13 09:50:58 UTC 2018
[ root@curl-87b54756-xfgrn:/ ]$
[ root@curl-87b54756-xfgrn:/ ]$ exit
Session ended, resume using 'kubectl attach curl-87b54756-xfgrn -c curl -i -t' command when the pod is running
[root@master2 ~]# kubectl delete deployment curl
deployment.extensions "curl" deleted