Haproxy+Keepalived實現Haproxy的高可用

一、實驗環境

　主機：四台 CentOS7系統的虛擬機，node1(192.168.27.7)，node2(192.168.27.17)，VIP(192.168.27.100)，web1(192.168.27.27)，web2(192.168.27.37)

　軟件：haproxy-1.8.20.tar.gz，keepalived（光盤yum源），httpd（光盤yum源）

二、實驗步驟

1、安裝haproxy

　兩台主機都源碼安裝haproxy-1.8.20.tar.gz，都兩台主機操作一樣，以下為node1的操作

[root@node1 ~]# ll haproxy-1.8.20.tar.gz 
-rw-r--r-- 1 root root 2083917 Jan 10 20:39 haproxy-1.8.20.tar.gz
#安裝依賴包
[root@node1 ~]# yum install -y gcc gcc-c++ pcre pcre-devel openssl openssl-devel systemd-devel
#開始編譯安裝
[root@node1 ~]# tar xf haproxy-1.8.20.tar.gz 
[root@node1 ~]# cd haproxy-1.8.20/
[root@node1 haproxy-1.8.20]# make ARCH=x86_64 TARGET=linux2628 USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_CPU_AFFINITY=1 PREFIX=/usr/local/haproxy
[root@node1 haproxy-1.8.20]# make install PREFIX=/usr/local/haproxy
[root@node1 haproxy-1.8.20]# cp haproxy /usr/sbin/
#准備啟動文件
[root@node1 haproxy-1.8.20]# vim /usr/lib/systemd/system/haproxy.service
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /usr/local/haproxy/run/haproxy.pid
ExecReload=/bin/kill -USR2 $MAINPID

[Install]
WantedBy=multi-user.target
#創建用戶與配置文件
[root@node1 haproxy-1.8.20]# useradd -r -s  /sbin/nologin haproxy
[root@node1 haproxy-1.8.20]# mkdir /etc/haproxy
[root@node1 haproxy-1.8.20]# mkdir /usr/local/haproxy/run
[root@node1 haproxy-1.8.20]# mkdir /var/lib/haproxy
[root@node1 haproxy-1.8.20]# chown -R haproxy:haproxy /var/lib/haproxy/
[root@node1 haproxy-1.8.20]# vim /etc/haproxy/haproxy.cfg

[root@node1 haproxy-1.8.20]# cat /etc/haproxy/haproxy.cfg 
global
maxconn 100000
chroot /usr/local/haproxy
#stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
#uid 981
#gid 981
user haproxy
group haproxy
daemon
#nbproc 4
#cpu-map 1 0
#cpu-map 2 1
#cpu-map 3 2
#cpu-map 4 3
pidfile /usr/local/haproxy/run/haproxy.pid
log 127.0.0.1 local3 info

defaults
option http-keep-alive
option  forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client  300000ms
timeout server  300000ms

listen stats
 mode http
 bind 192.168.27.7:9999
 stats enable
 log global
 stats uri     /haproxy-status
 stats auth    haadmin:q1w2e3r4ys

listen  web_port
 bind 192.168.27.100:80    #此處為VIP
 mode http
 log global
 server web1  192.168.27.27:80  check inter 3000 fall 2 rise 5    #后端服務器
 server web1  192.168.27.37:80  check inter 3000 fall 2 rise 5    #后端服務器

2、安裝keepalived

[root@node1 ~]# yum install -y keepalived

[root@node2 ~]# yum install -y keepalived

3、配置keepalived

[root@node1 ~]# vim /etc/keepalived/keepalived.conf 

! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from root@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1    #node2節點此處改為node2
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state MASTER    #node2節點此處改為BACKUP
    interface eth0
    virtual_router_id 51
    priority 100    #node2節點此處改為80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.27.100/24 dev eth0 label eth0:0
    }
}

4、修改內核參數

　因haproxy配置文件中綁定的IP是一個虛擬IP，會haproxy服務導致啟動不了，此時需要修改內核參數

[root@node1 ~]# vim /etc/sysctl.conf
#添加以下兩行
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
[root@node1 ~]# sysctl -p    #使配置生效
#node2節點上一樣配置

5、配置后端服務器的web服務

[root@web1 ~]# yum install -y httpd
[root@web2 ~]# yum install -y httpd
#准備頁面
[root@web1 ~]# echo 'web page 192.168.17.27' > /var/www/html/index.html
[root@web2 ~]# echo 'web page 192.168.17.37' > /var/www/html/index.html
#啟動httpd服務
[root@web1 ~]# systemctl start httpd
[root@web2 ~]# systemctl start httpd

6、修改VIP防火牆策略

　keepalived會對VIP生成防火牆策略，導致訪問不到后端服務器資源，此時可在keepalived配置文件中加一個配置vrrp_iptables 使其不生成防火牆策略，或手動刪除防火牆規則 iptables -D INPUT -s 0.0.0.0/0 -d 192.168.27.100 -j DROP，本次就修改配置文件了

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from root@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables    #添加此項，記得兩台都要加
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
...以下省略

7、啟動服務，進行測試

[root@node1 ~]# systemctl start haproxy keepalived
[root@node2 ~]# systemctl start haproxy keepalived
#開始測試，當node1節點的keepalived掛了，VIP會轉移到node2上，並保證業務不中斷
[root@node1 ~]# ip a |grep 192.168.27.100    #vip在node1上
    inet 192.168.27.100/24 scope global secondary eth0:0
[root@node1 ~]# systemctl stop keepalived    #關掉node1的keepalived
[root@node2 ~]# ip a |grep 192.168.27.100    #vip轉移到了node2上
    inet 192.168.27.100/24 scope global secondary eth0:0
[root@web1 ~]# while true;do curl http://192.168.27.100; sleep 1;done    #服務未中斷
web page 192.168.17.27
web page 192.168.17.37
web page 192.168.17.27
web page 192.168.17.37
...
[root@node1 ~]# systemctl start keepalived    #恢復node1上的keepalived服務
[root@node1 ~]# ip a|grep 192.168.27.100    #vip又回到了node1上
    inet 192.168.27.100/24 scope global secondary eth0:0

8、實現haproxy高可用

　上述配置，只有前主節點故障時才會切換vip，當keepalived正常，但haproxy導常時並不會切換，這樣也會導致業務訪問出問題，此時，可以使用keepalived調用外部腳本進行資源監控，並根據監控的結果狀態實現動態調整。

vrrp_script <SCRIPT_NAME> { #定義一個檢測腳本，在global_defs 之外配置
    script <STRING>|<QUOTED-STRING>     #shell命令或腳本路徑
    interval <INTEGER>     #間隔時間，單位為秒，默認1秒
    timeout <INTEGER>     #超時時間
    weight <INTEGER:-254..254>     #權重，腳本監測失敗后會執行權重+/-操作
    fall <INTEGER>     #腳本幾次失敗轉換為失敗
    rise <INTEGER>     #腳本連續幾次監測成功后，把服務器從失敗標記為成功
    user USERNAME [GROUPNAME]     #執行監測的用戶或組
    init_fail     #設置默認標記為失敗狀態，監測成功之后再轉換為成功狀態
}

vrrp_instance VI_1 {
    …
    track_script {    #觸發腳本
        chk_down
    } 
}

#操作步驟
#創建腳本
[root@node1 ~]# vim /etc/keepalived/chk_haproxy.sh
#!/bin/bash
/usr/bin/killall -0 haproxy
#給腳本加執行權限
[root@node1 ~]# chmod +x /etc/keepalived/chk_haproxy.sh
[root@node1 ~]# vim /etc/keepalived/keepalived.conf 
[root@node1 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from root@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
#添加以下段
vrrp_script chk_haproxy {
    script "/etc/keepalived/chk_haproxy.sh"
    interval 2
    weight -50
    fall 3
    rise 5 
    timeout 3
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.27.100/24 dev eth0 label eth0:0
    }
    #添加此段
    track_script {
    chk_haproxy
    }
}
#腳本與配置文件傳到另一台主機
[root@node1 ~]# scp /etc/keepalived/chk_haproxy.sh 192.168.27.17:/etc/keepalived/
[root@node1 ~]# scp /etc/keepalived/keepalived.conf 192.168.27.17:/etc/keepalived/  
#重啟服務
[root@node1 ~]# systemctl restart keepalived
[root@node2 ~]# systemctl restart keepalived
#測試
[root@node1 ~]# ip a|grep 192.168.27.100    #vip一開始在node1上
    inet 192.168.27.100/24 scope global secondary eth0:0
[root@node1 ~]# systemctl stop haproxy    #停掉node1上的haproxy服務
[root@node1 ~]# ip a|grep 192.168.27.100    #vip轉移了

[root@node2 ~]# ip a|grep 192.168.27.100    #vip轉移到了node2
    inet 192.168.27.100/24 scope global secondary eth0:0

9、實現keepalived的郵件通知功能

#定義通知腳本：
notify_master <STRING>|<QUOTED-STRING>：    當前節點成為主節點時觸發的腳本
notify_backup <STRING>|<QUOTED-STRING>：    當前節點轉為備節點時觸發的腳本
notify_fault <STRING>|<QUOTED-STRING>：        當前節點轉為“失敗”狀態時觸發的腳本
notify <STRING>|<QUOTED-STRING>：            通用格式的通知觸發機制，一個腳本可完成以上三種狀態的轉換時的通知

#安裝郵件服務
[root@node1 ~]# yum install -y postfix

#發件人配置
[root@node1 ~]# vim /etc/mail.rc
set from=[e-mail]
set smtp=smtp.qq.com
set smtp-auth-user=[e-mail]    
set smtp-auth-password=mfcjxxjezahijgddj    #到QQ郵箱里配置生成
set smtp-auth=login
set ssl-verify=ignore

#准備通知腳本
[root@node1 ~]# cat /etc/keepalived/nodify.sh
#!/bin/bash

contact='[e-mail]'

notify() {
    mailsubject="$(hostname) to be $1, vip 轉移"
    mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
    echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in

master)
    notify master
    ;;
backup)
    notify backup
    ;;
fault)
    notify fault
    ;;
*)
    echo "Usage: $(basename $0) {master|backup|fault}"
    exit 1
    ;;
esac

#腳本的調用方法，在vrrp_instance中配置即可
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"

#如下所示
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.27.100/24 dev eth0 label eth0:0
    }
    track_script {
        chk_haproxy
    }
 notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault"
}