jenkins的灰度發布

環境如圖：
簡單環境搭建：

keepalived

MASTER38

root@38:~# vim /etc/keepalived/keepalived.conf
...
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    garp_master_delay 10
    #smtp_alert
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.88 dev eth0 label eth0:1
    }
}

BACKUP

root@48:~# vim /etc/keepalived/keepalived.conf
...
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    garp_master_delay 10
    #smtp_alert
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.88 dev eth0 label eth0:1
    }
}

haproxy

38

root@38:~# echo >> "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf
root@38:~# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
root@38:~# sysctl -a | grep nonlocal
net.ipv4.ip_nonlocal_bind = 1

root@38:~# vim /etc/haproxy/haproxy.cfg
...
listen tomcat-web-80
  bind 10.0.0.88:80
  mode http
  balance roundrobin
  server tomcat-1 10.0.0.58:8080 check inter 3s fall 3 rise 3
  server tomcat-2 10.0.0.68:8080 check inter 3s fall 3 rise 3

listen status
  bind :9009
  stats enable
  stats uri /haproxy-status
  stats realm HAProxy\ Stats\ Page
  stats auth haadmin:lijie
  stats auth admin:lijie

48

root@48:~# echo >> "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf
root@48:~# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
root@48:~# sysctl -a | grep nonlocal
net.ipv4.ip_nonlocal_bind = 1

root@48:~# vim /etc/haproxy/haproxy.cfg
...
listen tomcat-web-80
  bind 10.0.0.88:80
  mode http
  server tomcat-1 10.0.0.58:8080 check inter 3s fall 3 rise 3
  server tomcat-2 10.0.0.68:8080 check inter 3s fall 3 rise 3

listen status
  bind :9009
  stats enable
  stats uri /haproxy-status
  stats realm HAProxy\ Stats\ Page
  stats auth haadmin:lijie
  stats auth admin:lijie

tomcat

略

appBase目錄如下
<Host name="localhost"  appBase="/data/tomcat/tomcat_webapps"
            unpackWARs="true" autoDeploy="true">

root@58:~# hostname -I > /data/tomcat/tomcat_webdir/myapp/index.jsp
root@58:~# tree /data/tomcat/
/data/tomcat/
├── tomcat_appdir
├── tomcat_webapps
│   └── myapp -> /data/tomcat/tomcat_webdir/myapp
└── tomcat_webdir
    └── myapp
        └── index.jsp

root@68:~# hostname -I > /data/tomcat/tomcat_webdir/myapp/index.jsp
root@68:~# tree /data/tomcat/
/data/tomcat/
├── tomcat_appdir
├── tomcat_webapps
│   └── myapp -> /data/tomcat/tomcat_webdir/myapp/
└── tomcat_webdir
    └── myapp
        └── index.jsp

5 directories, 1 file

訪問測試

root@18:~# curl http://10.0.0.88/myapp/
10.0.0.58 
root@18:~# curl http://10.0.0.88/myapp/
10.0.0.68 
root@18:~# curl http://10.0.0.88/myapp/
10.0.0.58 
root@18:~# curl http://10.0.0.88/myapp/
10.0.0.68 

jenkins的job腳本

代碼clone：

首先需要完成gitlab中添加jenkins的公鑰作為ssh-key實現非交互式代碼克隆

代碼部署：

需要將jenkins的公鑰推送到web服務器，實現代碼非交互式拷貝；

root@28:~# ssh-copy-id root@10.0.0.68
root@28:~# ssh-copy-id root@10.0.0.58
root@28:~# ssh root@10.0.0.68 'hostname -I'
10.0.0.68 
root@28:~# ssh root@10.0.0.58 'hostname -I'
10.0.0.58 

部署腳本

#!/bin/bash
DATE=`date +%Y-%m-%d_%H_%M_%S`
METHOD=$1
BRANCH=$2
GROUP_LIST=$3
HA1="10.0.0.38"
HA2="10.0.0.48"
OK_COLOR="echo -e \e[1;32m"
ERR_COLOR="echo -e \e[1;35m"
END="\e[0m"

host_list(){
  if [ ${GROUP_LIST} = group1 ];then
	  hosts="10.0.0.58"
  elif [ ${GROUP_LIST} = group2 ];then
	  hosts="10.0.0.68"
  else
	  hosts="10.0.0.58 10.0.0.68"
  fi
}

code_clone(){
  cd /data/gitdata/ && rm -rf ./*
  ${OK_COLOR} "開始clone ${BRANCH}分支代碼到`pwd`" ${END}
  git clone -b ${BRANCH} git@10.0.0.18:magedu/test.git && echo "代碼克隆完成" && echo "代碼克隆完成"
}

code_scanner(){
  echo "開始代碼掃描"
}

code_compress(){
  cd /data/gitdata/test && tar czvf code.tar.gz ./index.html && echo "代碼打包完成"
}

node_down(){
  ssh root@${HA1} "echo "disable server tomcat-web-80/${node}" | socat stdio /run/haproxy/admin.sock" && echo ${hosts}從HA1下線成功
  ssh root@${HA2} "echo "disable server tomcat-web-80/${node}" | socat stdio /run/haproxy/admin.sock" && echo ${hosts}從HA2下線成功
}

tomcat_down(){
  echo "${node}即將停止tomcat服務"
  ssh root@${node} "/apps/tomcat/bin/catalina.sh stop" && echo "tomcat服務已停止"
  sleep 1 && echo 3
  sleep 1 && echo 2
  sleep 1 && echo 1
}

code_deploy(){
  scp /data/gitdata/test/code.tar.gz root@${node}:/data/tomcat/tomcat_appdir/code-${DATE}.tar.gz && echo "${node}代碼拷貝完成"
  ssh root@${node} "mkdir /data/tomcat/tomcat_webdir/code-${DATE} && tar xzvf /data/tomcat/tomcat_appdir/code-${DATE}.tar.gz -C /data/tomcat/tomcat_webdir/code-${DATE} && rm -rf /data/tomcat/tomcat_webapps/myapp && ln -sv /data/tomcat/tomcat_webdir/code-${DATE} /data/tomcat/tomcat_webapps/myapp"
}

tomcat_up(){
  echo "${node}正在啟動tomcat"
  ssh root@${node} "/apps/tomcat/bin/catalina.sh start" && echo "tomcat服務已啟動"
  sleep 1 && echo 3
  sleep 1 && echo 2
  sleep 1 && echo 1
  curl http://${node}:8080/myapp && echo "${node}部署完成"
}

node_up(){
   ssh root@${HA1} "echo "enable server tomcat-web-80/${node}" | socat stdio /run/haproxy/admin.sock"\
   && echo ${node}從HA1上線成功 
   ssh root@${HA2} "echo "enable server tomcat-web-80/${node}" | socat stdio /run/haproxy/admin.sock"\
   && echo ${node}從HA2上線成功 
}

code_del(){
  ssh root@${node} "rm -rf /data/tomcat/tomcat_appdir/*"
  VER_NUM=`ssh root@${node} "/bin/ls -ldrt /data/tomcat/tomcat_webdir/code-* | wc -l"`
  if [ ${VER_NUM} -gt 3 ];then
    OLD_CODE_DIR=`ssh root@${node} ""/bin/ls -ldrt /data/tomcat/tomcat_webdir/code-* | head -n1 | awk '{print $NF}'""`
    ssh root@${node} "rm -rf ${OLD_CODE_DIR}" && "舊代碼${OLD_CODE_DIR}刪除成功"
  else
    echo "舊代碼未超過3個版本無需刪除"
  fi
}

code_rollback(){
  NOW_VER=`ssh root@${node} ""/bin/ls -ld /data/tomcat/tomcat_webapps/myapp | awk '{print $NF}'""`
  NOW_VER=`basename ${NOW_VER}`
  echo "當前代碼版本是: ${NOW_VER}"
  PRE_VER=`ssh root@${node} ""/bin/ls -lrt /data/tomcat/tomcat_webdir/ | grep -B 1 ${NOW_VER} | head -n1 | awk '{print $NF}'""`
  echo "前一版本: ${PRE_VER}"
  ssh root@${node} "rm -rf /data/tomcat/tomcat_webapps/myapp && ln -sv /data/tomcat/tomcat_webdir/${PRE_VER} /data/tomcat/tomcat_webapps/myapp"
}

case $METHOD in
deploy)
    host_list
    code_clone
    code_scanner
    code_compress
    #code_scanner
    for node in ${hosts};do
      node_down
      tomcat_down
      code_deploy
      tomcat_up
      node_up
      code_del
    done
    ;;
rollback)
    host_list
  for node in ${hosts};do
    node_down
    tomcat_down
    code_rollback
    tomcat_up
    node_up
  done
    ;;
esac