一、實驗環境
主機：四台 CentOS7系統的虛擬機，node1(192.168.27.7)，node2(192.168.27.17)，VIP(192.168.27.100)，web1(192.168.27.27)，web2(192.168.27.37)
軟件：haproxy-1.8.20.tar.gz，keepalived（光盤yum源），httpd（光盤yum源）
二、實驗步驟
1、安裝haproxy
兩台主機都源碼安裝haproxy-1.8.20.tar.gz，都兩台主機操作一樣，以下為node1的操作
[root@node1 ~]# ll haproxy-1.8.20.tar.gz -rw-r--r-- 1 root root 2083917 Jan 10 20:39 haproxy-1.8.20.tar.gz #安裝依賴包 [root@node1 ~]# yum install -y gcc gcc-c++ pcre pcre-devel openssl openssl-devel systemd-devel #開始編譯安裝 [root@node1 ~]# tar xf haproxy-1.8.20.tar.gz [root@node1 ~]# cd haproxy-1.8.20/ [root@node1 haproxy-1.8.20]# make ARCH=x86_64 TARGET=linux2628 USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_CPU_AFFINITY=1 PREFIX=/usr/local/haproxy [root@node1 haproxy-1.8.20]# make install PREFIX=/usr/local/haproxy [root@node1 haproxy-1.8.20]# cp haproxy /usr/sbin/ #准備啟動文件 [root@node1 haproxy-1.8.20]# vim /usr/lib/systemd/system/haproxy.service [Unit] Description=HAProxy Load Balancer After=syslog.target network.target [Service] ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /usr/local/haproxy/run/haproxy.pid ExecReload=/bin/kill -USR2 $MAINPID [Install] WantedBy=multi-user.target #創建用戶與配置文件 [root@node1 haproxy-1.8.20]# useradd -r -s /sbin/nologin haproxy [root@node1 haproxy-1.8.20]# mkdir /etc/haproxy [root@node1 haproxy-1.8.20]# mkdir /usr/local/haproxy/run [root@node1 haproxy-1.8.20]# mkdir /var/lib/haproxy [root@node1 haproxy-1.8.20]# chown -R haproxy:haproxy /var/lib/haproxy/ [root@node1 haproxy-1.8.20]# vim /etc/haproxy/haproxy.cfg
[root@node1 haproxy-1.8.20]# cat /etc/haproxy/haproxy.cfg global maxconn 100000 chroot /usr/local/haproxy #stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin #uid 981 #gid 981 user haproxy group haproxy daemon #nbproc 4 #cpu-map 1 0 #cpu-map 2 1 #cpu-map 3 2 #cpu-map 4 3 pidfile /usr/local/haproxy/run/haproxy.pid log 127.0.0.1 local3 info defaults option http-keep-alive option forwardfor maxconn 100000 mode http timeout connect 300000ms timeout client 300000ms timeout server 300000ms listen stats mode http bind 192.168.27.7:9999 stats enable log global stats uri /haproxy-status stats auth haadmin:q1w2e3r4ys listen web_port bind 192.168.27.100:80 #此處為VIP mode http log global server web1 192.168.27.27:80 check inter 3000 fall 2 rise 5 #后端服務器 server web1 192.168.27.37:80 check inter 3000 fall 2 rise 5 #后端服務器
2、安裝keepalived
[root@node1 ~]# yum install -y keepalived [root@node2 ~]# yum install -y keepalived
3、配置keepalived
[root@node1 ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { root@localhost } notification_email_from root@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id node1 #node2節點此處改為node2 vrrp_skip_check_adv_addr vrrp_strict vrrp_garp_interval 0 vrrp_gna_interval 0 } vrrp_instance VI_1 { state MASTER #node2節點此處改為BACKUP interface eth0 virtual_router_id 51 priority 100 #node2節點此處改為80 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 192.168.27.100/24 dev eth0 label eth0:0 } }
4、修改內核參數
因haproxy配置文件中綁定的IP是一個虛擬IP，會haproxy服務導致啟動不了，此時需要修改內核參數
[root@node1 ~]# vim /etc/sysctl.conf #添加以下兩行 net.ipv4.ip_nonlocal_bind = 1 net.ipv4.ip_forward = 1 [root@node1 ~]# sysctl -p #使配置生效 #node2節點上一樣配置
5、配置后端服務器的web服務
[root@web1 ~]# yum install -y httpd [root@web2 ~]# yum install -y httpd #准備頁面 [root@web1 ~]# echo 'web page 192.168.17.27' > /var/www/html/index.html [root@web2 ~]# echo 'web page 192.168.17.37' > /var/www/html/index.html #啟動httpd服務 [root@web1 ~]# systemctl start httpd [root@web2 ~]# systemctl start httpd
6、修改VIP防火牆策略
keepalived會對VIP生成防火牆策略，導致訪問不到后端服務器資源，此時可在keepalived配置文件中加一個配置vrrp_iptables 使其不生成防火牆策略，或手動刪除防火牆規則 iptables -D INPUT -s 0.0.0.0/0 -d 192.168.27.100 -j DROP，本次就修改配置文件了
global_defs { notification_email { root@localhost } notification_email_from root@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id node2 vrrp_skip_check_adv_addr vrrp_strict vrrp_iptables #添加此項，記得兩台都要加 vrrp_garp_interval 0 vrrp_gna_interval 0 } ...以下省略
7、啟動服務，進行測試
[root@node1 ~]# systemctl start haproxy keepalived [root@node2 ~]# systemctl start haproxy keepalived #開始測試，當node1節點的keepalived掛了，VIP會轉移到node2上，並保證業務不中斷 [root@node1 ~]# ip a |grep 192.168.27.100 #vip在node1上 inet 192.168.27.100/24 scope global secondary eth0:0 [root@node1 ~]# systemctl stop keepalived #關掉node1的keepalived [root@node2 ~]# ip a |grep 192.168.27.100 #vip轉移到了node2上 inet 192.168.27.100/24 scope global secondary eth0:0 [root@web1 ~]# while true;do curl http://192.168.27.100; sleep 1;done #服務未中斷 web page 192.168.17.27 web page 192.168.17.37 web page 192.168.17.27 web page 192.168.17.37 ... [root@node1 ~]# systemctl start keepalived #恢復node1上的keepalived服務 [root@node1 ~]# ip a|grep 192.168.27.100 #vip又回到了node1上 inet 192.168.27.100/24 scope global secondary eth0:0
8、實現haproxy高可用
上述配置，只有前主節點故障時才會切換vip，當keepalived正常，但haproxy導常時並不會切換，這樣也會導致業務訪問出問題，此時，可以使用keepalived調用外部腳本進行資源監控，並根據監控的結果狀態實現動態調整。
vrrp_script <SCRIPT_NAME> { #定義一個檢測腳本，在global_defs 之外配置 script <STRING>|<QUOTED-STRING> #shell命令或腳本路徑 interval <INTEGER> #間隔時間，單位為秒，默認1秒 timeout <INTEGER> #超時時間 weight <INTEGER:-254..254> #權重，腳本監測失敗后會執行權重+/-操作 fall <INTEGER> #腳本幾次失敗轉換為失敗 rise <INTEGER> #腳本連續幾次監測成功后，把服務器從失敗標記為成功 user USERNAME [GROUPNAME] #執行監測的用戶或組 init_fail #設置默認標記為失敗狀態，監測成功之后再轉換為成功狀態 } vrrp_instance VI_1 { … track_script { #觸發腳本 chk_down } }
#操作步驟
#創建腳本 [root@node1 ~]# vim /etc/keepalived/chk_haproxy.sh #!/bin/bash /usr/bin/killall -0 haproxy #給腳本加執行權限 [root@node1 ~]# chmod +x /etc/keepalived/chk_haproxy.sh [root@node1 ~]# vim /etc/keepalived/keepalived.conf [root@node1 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { root@localhost } notification_email_from root@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id node1 vrrp_skip_check_adv_addr vrrp_strict vrrp_iptables vrrp_garp_interval 0 vrrp_gna_interval 0 } #添加以下段 vrrp_script chk_haproxy { script "/etc/keepalived/chk_haproxy.sh" interval 2 weight -50 fall 3 rise 5 timeout 3 } vrrp_instance VI_1 { state MASTER interface eth0 virtual_router_id 51 priority 100 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 192.168.27.100/24 dev eth0 label eth0:0 } #添加此段 track_script { chk_haproxy } } #腳本與配置文件傳到另一台主機 [root@node1 ~]# scp /etc/keepalived/chk_haproxy.sh 192.168.27.17:/etc/keepalived/ [root@node1 ~]# scp /etc/keepalived/keepalived.conf 192.168.27.17:/etc/keepalived/ #重啟服務 [root@node1 ~]# systemctl restart keepalived [root@node2 ~]# systemctl restart keepalived #測試 [root@node1 ~]# ip a|grep 192.168.27.100 #vip一開始在node1上 inet 192.168.27.100/24 scope global secondary eth0:0 [root@node1 ~]# systemctl stop haproxy #停掉node1上的haproxy服務 [root@node1 ~]# ip a|grep 192.168.27.100 #vip轉移了 [root@node2 ~]# ip a|grep 192.168.27.100 #vip轉移到了node2 inet 192.168.27.100/24 scope global secondary eth0:0
9、實現keepalived的郵件通知功能
#定義通知腳本： notify_master <STRING>|<QUOTED-STRING>： 當前節點成為主節點時觸發的腳本 notify_backup <STRING>|<QUOTED-STRING>： 當前節點轉為備節點時觸發的腳本 notify_fault <STRING>|<QUOTED-STRING>： 當前節點轉為“失敗”狀態時觸發的腳本 notify <STRING>|<QUOTED-STRING>： 通用格式的通知觸發機制，一個腳本可完成以上三種狀態的轉換時的通知
#安裝郵件服務 [root@node1 ~]# yum install -y postfix #發件人配置 [root@node1 ~]# vim /etc/mail.rc set from=1954938301@qq.com set smtp=smtp.qq.com set smtp-auth-user=1954938301@qq.com set smtp-auth-password=mfcjxxjezahijgddj #到QQ郵箱里配置生成 set smtp-auth=login set ssl-verify=ignore #准備通知腳本 [root@node1 ~]# cat /etc/keepalived/nodify.sh #!/bin/bash contact='1954938301@qq.com' notify() { mailsubject="$(hostname) to be $1, vip 轉移" mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1" echo "$mailbody" | mail -s "$mailsubject" $contact } case $1 in master) notify master ;; backup) notify backup ;; fault) notify fault ;; *) echo "Usage: $(basename $0) {master|backup|fault}" exit 1 ;; esac #腳本的調用方法，在vrrp_instance中配置即可 notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" #如下所示 vrrp_instance VI_1 { state MASTER interface eth0 virtual_router_id 51 priority 100 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 192.168.27.100/24 dev eth0 label eth0:0 } track_script { chk_haproxy } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" }