基於DNS(Consul)高可用

DNS

推薦從Bind-DLZ入手，資料多
可控制度更好(查詢DNS記錄SQL可定制)
據說性能差

Bind-DLZ
https://www.cnblogs.com/saneri/p/8178065.html
  
PowerDNS
SQL schema設置規范
性能比Bind-DLZ好

 

coredns 和k8s結合比較多

nacos 阿里開源，含DNS和服務發現

 

監控程序：
主從結構，支持GTID

 

監控邏輯：
按分組取出來機器節點

master:
嘗試連接成功 ok 保持
失敗  進行從庫選舉

slave：
檢查是不是在線online
在線的：
連接成功，復制是不是正常，不正常下線，檢驗延遲
下線更新cmdb,dns records
下線的：
連接成功，復制正常，不延遲 上線
上線更新cmdb,dns records

從庫選舉：
獲取從庫列表

 

故障切換：
確認所有節點都復制中斷，判斷復制完成
對比，所有節點是不是復制到一個位置
通過獲取的GTID對比，是不是所有節點同步到一個位置，如果不是，選舉出來最靠前的做master
如果同步位置都一樣，根據cmdb中定義的level選擇，最大的那個
新的主節點選舉成功后，其他節點change過來
更新cmdb中的角色,oldmaster->slave,選舉出來的master，更改新主節點的read_only
更新dns_records

 

在線切換：
oldmaster上執行super_read_only & read_only干掉業務連接
獲取oldmaster中的show master status信息
獲取從節點中的show slave status對比，確認都同步完成
按cmdb中的level或是指定的節點為新master
更新重做master/slave架構
更新cmdb
更新dns
記錄log

 

一定要提高英文閱讀能力

[root@mydb1 ~]# wget https://releases.hashicorp.com/consul/1.4.0/consul_1.4.0_linux_amd64.zip
[root@mydb1 ~]# mkdir -p /opt/consul /opt/consul/conf /data/consul /data/consul/shell/
[root@mydb2 ~]# mkdir -p /opt/consul /opt/consul/conf /data/consul /data/consul/shell/
[root@mydb3 ~]# mkdir -p /opt/consul /opt/consul/conf /data/consul /data/consul/shell/
[root@mydb1 ~]# unzip consul_1.4.0_linux_amd64.zip
將consul拷貝至/opt/consul目錄
[root@mydb1 ~]# cat /opt/consul/conf/server.json
{
  "data_dir": "/data/consul",
  "enable_script_checks": true,
  "datacenter": "dc1",
  "log_level": "INFO",          
  "server": true,              
  "bootstrap_expect": 3,          
  "ui":true
}
[root@mydb1 consul]# ./consul agent -config-dir=/opt/consul/conf > /data/consul/consul.log &
[root@mydb2 consul]# ./consul agent -config-dir=/opt/consul/conf > /data/consul/consul.log &
[root@mydb3 consul]# ./consul agent -config-dir=/opt/consul/conf > /data/consul/consul.log &
[root@mydb2 consul]# ./consul join 192.168.1.101
[root@mydb3 consul]# ./consul join 192.168.1.101
[root@mydb1 consul]# ./consul members
Node   Address             Status  Type    Build  Protocol  DC   Segment
mydb1  192.168.1.101:8301  alive   server  1.4.0  2         dc1  <all>
mydb2  192.168.1.102:8301  alive   server  1.4.0  2         dc1  <all>
mydb3  192.168.1.103:8301  alive   server  1.4.0  2         dc1  <all>

[root@mydb1 consul]# ./consul catalog nodes
Node   ID        Address        DC
mydb1  52514e74  192.168.1.101  dc1
mydb2  aebbf0b2  192.168.1.102  dc1
mydb3  0e179069  192.168.1.103  dc1

# dig @127.0.0.1 -p 8600 mydb1.node.consul
# dig @127.0.0.1 -p 8600 mydb2.node.consul
# dig @127.0.0.1 -p 8600 mydb3.node.consul

[root@mydb1 consul]# ./consul operator raft list-peers
Node   ID                                    Address             State     Voter  RaftProtocol
mydb1  52514e74-d063-cfe3-1d58-55fda9fc2451  192.168.1.101:8300  leader    true   3
mydb2  aebbf0b2-09ad-f396-4c21-3f9ee40a16da  192.168.1.102:8300  follower  true   3
mydb3  0e179069-7360-3866-d9a6-7ea60c540c04  192.168.1.103:8300  follower  true   3

[root@mydb1 consul]# ./consul kv put id 11
Success! Data written to: id
[root@mydb1 consul]# ./consul kv get id
11
[root@mydb2 consul]# ./consul kv get id
11
[root@mydb3 consul]# ./consul kv get id
11

consul是用Raft來實現分布式一致性的

 

 

[root@mydb1 ~]# cat /opt/consul/conf/r-test-mgr-ser.json
{
  "service": {
    "name": "r-test-3306-mydb-ser",
    "tags": ["測試-3306"],
    "address": "192.168.1.101",
    "meta": {
      "meta": "for my service"
    },
    "port": 3306,
    "enable_tag_override": false,
    "checks": [
      {
        "args": ["/data/consul/shell/check_mysql_mgr_slave.sh"],
        "interval": "1s"
      }
    ]
  }
}
[root@mydb1 ~]# cat /opt/consul/conf/w-test-mgr-ser.json
{
  "service": {
    "name": "w-test-3306-mydb-ser",
    "tags": ["測試-3306"],
    "address": "192.168.1.101",
    "meta": {
      "meta": "for my service"
    },
    "port": 3306,
    "enable_tag_override": false,
    "checks": [
      {
        "args": ["/data/consul/shell/check_mysql_mgr_master.sh"],
        "interval": "10s"
      }
    ]
  }
}
注意在mydb2,mydb3上調整ip
檢測腳本如下
[root@mydb1 ~]# cat /data/consul/shell/check_mysql_mgr_master.sh
#!/bin/bash
host="192.168.1.101"
port=3306
user="dba_user"
passwod="msds007"
comm="/usr/local/mysql/bin/mysql -u$user -h$host -P $port -p$passwod"
value=`$comm -Nse "select 1"`
primary_member=`$comm -Nse "select variable_value from performance_schema.global_status WHERE VARIABLE_NAME= 'group_replication_primary_member'"`
server_uuid=`$comm -Nse "select variable_value from performance_schema.global_variables where VARIABLE_NAME='server_uuid';"`
# 判斷MySQL是否存活
if [ -z $value ]
then
   echo "mysql $port is down....."
   exit 2
fi
# 判斷節點狀態,是否存活
node_state=`$comm -Nse "select MEMBER_STATE from performance_schema.replication_group_members where MEMBER_ID='$server_uuid'"`
if [ $node_state != "ONLINE" ]
then
   echo "MySQL $port state is not online...."
   exit 2
fi
# 判斷是不是主節點
if [[ $server_uuid == $primary_member ]]
then
   echo "MySQL $port Instance is master ........"
   exit 0
else
   echo "MySQL $port Instance is slave ........"
   exit 2
fi
[root@mydb1 ~]# cat /data/consul/shell/check_mysql_mgr_slave.sh
#!/bin/bash
host="192.168.1.101"
port=3306
user="dba_user"
passwod="msds007"
comm="/usr/local/mysql/bin/mysql -u$user -h$host -P $port -p$passwod"
value=`$comm -Nse "select 1"`
primary_member=`$comm -Nse "select variable_value from performance_schema.global_status WHERE VARIABLE_NAME= 'group_replication_primary_member'"`
server_uuid=`$comm -Nse "select variable_value from performance_schema.global_variables where VARIABLE_NAME='server_uuid';"`
# 判斷mysql是否存活
if [ -z $value ]
then
   echo "mysql $port is down....."
   exit 2
fi
# 判斷節點狀態
node_state=`$comm -Nse "select MEMBER_STATE from performance_schema.replication_group_members where MEMBER_ID='$server_uuid'"`
if [ $node_state != "ONLINE" ]
then
   echo "MySQL $port state is not online...."
   exit 2
fi
# 判斷是不是主節點
if [[ $server_uuid != $primary_member ]]
then
   echo "MySQL $port Instance is slave ........"
   exit 0
else
   node_num=`$comm -Nse "select count(*) from performance_schema.replication_group_members"`
# 判斷如果沒有任何從節點，主節點也注冊從角色服務。
   if [ $node_num -eq 1 ]
   then
       echo "MySQL $port Instance is slave ........"
       exit 0
   else
       echo "MySQL $port Instance is master ........"
       exit 2
   fi
fi
注意在mydb2,mydb3上調整ip

 

[root@mydb1 consul]# ./consul agent -config-dir=/opt/consul/conf > /data/consul/consul.log &
[root@mydb2 consul]# ./consul agent -config-dir=/opt/consul/conf > /data/consul/consul.log &
[root@mydb3 consul]# ./consul agent -config-dir=/opt/consul/conf > /data/consul/consul.log &
[root@mydb2 consul]# ./consul join 192.168.1.101
[root@mydb3 consul]# ./consul join 192.168.1.101
[root@mydb1 consul]# ./consul members

 

# dig @127.0.0.1 -p 8600 w-test-3306-mydb-ser.service.consul
# dig @127.0.0.1 -p 8600 r-test-3306-mydb-ser.service.consul

 

Consul使用手冊
http://www.liangxiansen.cn/2017/04/06/consul/