mysql MHA高可用故障恢復

、恢復MHA故障

1.手動修復

1）修復掛掉的數據庫

[root@db01 ~]# systemctl start mysqld

2）找到主從語句

[root@db03 ~]# grep 'CHANGE MASTER TO' /service/mha/manager 
Mon Nov  9 20:14:17 2020 - [info]  All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='172.16.1.52', MASTER_PORT=3306, MASTER_LOG_FILE='mysql-bin.000007', MASTER_LOG_POS=120, MASTER_USER='rep', MASTER_PASSWORD='xxx';

3）修復的數據庫執行change語句

#修改一下語句中的密碼，執行即可
mysql> CHANGE MASTER TO MASTER_HOST='172.16.1.52', MASTER_PORT=3306, MASTER_LOG_FILE='mysql-bin.000007', MASTER_LOG_POS=120, MASTER_USER='rep', MASTER_PASSWORD='123';
Query OK, 0 rows affected, 2 warnings (0.01 sec)

mysql> start slave;
Query OK, 0 rows affected (0.00 sec)

4）查看主從狀態

mysql> show slave status\G
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 172.16.1.52
                  Master_User: rep
                  Master_Port: 3306
                Connect_Retry: 60
              Master_Log_File: mysql-bin.000007
          Read_Master_Log_Pos: 211
               Relay_Log_File: db01-relay-bin.000002
                Relay_Log_Pos: 374
        Relay_Master_Log_File: mysql-bin.000007
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes

5）修復MHA配置

[root@db03 ~]# vim /service/mha/app1.cnf
[server default]
manager_log=/service/mha/manager
manager_workdir=/service/mha/app1
master_binlog_dir=/usr/local/mysql/data
password=mha
ping_interval=2
repl_password=123
repl_user=rep
ssh_user=root
user=mha

[server1]
hostname=172.16.1.51
port=3306

[server2]
hostname=172.16.1.52
port=3306

[server3]
hostname=172.16.1.53
port=3306

6）重新啟動MHA

[root@db03 ~]# nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &

2.腳本修復

1）創建腳本目錄

[root@db02 ~]# mkdir /scripts

2）寫腳本

[root@db02 ~]# vim /scripts/start_mha.sh
[root@db02 ~]# cat /scripts/start_mha.sh 
#!/bin/bash
#1.啟動數據庫
systemctl start mysqld
#2.獲取配置主從語句
change=`ssh 172.16.1.53 "grep 'CHANGE MASTER TO' /service/mha/manager | tail -1 | sed 's#xxx#123#g'" | awk -F: '{print $4}'`
#3.執行主從語句並啟動線程
mysql -e "$change; start slave"
#4.替換MHA配置文件
ssh 172.16.1.53 "\cp /service/mha/app1.bak /service/mha/app1.cnf"
#5.啟動MHA
ssh 172.16.1.53 "nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &"

3）加上判斷

[root@db02 ~]# cat /scripts/start_mha.sh 
#!/bin/bash
#1.mysql進程數賦值
mysqlpid=`ps -ef | grep [m]ysql | wc -l`
#2.判斷MySQL是否假死，如果假死殺掉重啟，如果關閉則啟動
if [ $mysqlpid -eq 0 ];then
    systemctl start mysqld
else
    pkill mysqld
    systemctl start mysqld
fi
#3.獲取配置主從語句
change=`ssh 172.16.1.53 "grep 'CHANGE MASTER TO' /service/mha/manager | tail -1 | sed 's#xxx#123#g'" | awk -F: '{print $4}'`
#4.執行主從語句並啟動線程
mysql -e "$change; start slave"
#5.替換MHA配置文件
ssh 172.16.1.53 "\cp /service/mha/app1.bak /service/mha/app1.cnf"
#6.啟動MHA
ssh 172.16.1.53 "nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &"

三、MHA切換機制

1.MHA切換機制

1.讀取配置中指定優先級的配置
	candidate_master=1
	check_repl_delay=0
2.如果沒有配置優先級，讀取數據最新的
3.如果數據量相同，讀取主機標簽，值越小越優先

2.測試標簽優先級

#1.停掉MHA
[root@db03 ~]# masterha_stop --conf=/service/mha/app1.cnf

#2.配置MHA
[root@db03 ~]# vim /service/mha/app1.cnf
... ...
[server1]
hostname=172.16.1.51
port=3306

[server2]
hostname=172.16.1.52
port=3306

[server3]
hostname=172.16.1.53
port=3306

#3.重啟MHA
[root@db03 ~]# nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &

#4.停掉主庫
[root@db02 ~]# systemctl stop mysqld

#5.查看主從
[root@db01 ~]# mysql
mysql> show slave status\G
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 172.16.1.53
                  Master_User: rep
                  Master_Port: 3306
                Connect_Retry: 60
              Master_Log_File: mysql-bin.000008
          Read_Master_Log_Pos: 120
               Relay_Log_File: db01-relay-bin.000002
                Relay_Log_Pos: 283
        Relay_Master_Log_File: mysql-bin.000008
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes

3.指定優先級測試

#配置優先級
[root@db03 ~]# vim /service/mha/app1.cnf
... ...
[server3]
candidate_master=1
check_repl_delay=0
hostname=172.16.1.53
port=3306

#重啟MHA
[root@db03 ~]# nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &

#停止主庫
[root@db01 ~]# systemctl stop mysqld

4.測試數據最新的優先級

1）去掉優先級配置

[root@db03 ~]# masterha_stop --conf=/service/mha/app1.cnf
#去掉優先級配置
#candidate_master=1
#check_repl_delay=0

#重啟mha
[root@db03 ~]# nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &

2）主庫建庫建表

mysql> create database youxianji;
Query OK, 1 row affected (0.00 sec)

mysql> use youxianji
Database changed

mysql> create table linux10(id int not null primary key auto_increment,name varchar(10));
Query OK, 0 rows affected (0.02 sec)

3）編寫腳本插入數據

[root@db03 ~]# mkdir /scripts
[root@db03 ~]# vim /scripts/insert.sh
#!/bin/bash
while true;do
    mysql -e "use youxianji;insert linux10(name) values('qiudao')"
done

4）停掉db01的IO線程

[root@db01 ~]# mysql
mysql> stop slave io_thread;

5）停掉主庫

[root@db03 ~]# systemctl stop mysqld

6）查看從庫狀態

[root@db01 ~]# mysql
mysql> show slave status\G
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 172.16.1.52
                  Master_User: rep
                  Master_Port: 3306
                Connect_Retry: 60
              Master_Log_File: mysql-bin.000010
          Read_Master_Log_Pos: 1524539
               Relay_Log_File: db01-relay-bin.000002
                Relay_Log_Pos: 283
        Relay_Master_Log_File: mysql-bin.000010
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes

四、主庫斷電，binlog如何保存

1.配置binlog-server

[root@db03 ~]# cat /service/mha/app1.cnf 
... ...
[server1]
hostname=172.16.1.51
port=3306

[server2]
hostname=172.16.1.52
port=3306

[server3]
hostname=172.16.1.53
port=3306

[binlog1]
no_master=1
hostname=172.16.1.53
master_binlog_dir=/data/mysql/binlog/

2.創建存放binlog的目錄

[root@db03 ~]# mkdir /data/mysql/binlog/ -p

3.手動執行實時備份binlog的命令

[root@db03 ~]# cd /data/mysql/binlog/

#備份binlog命令
[root@db03 /data/mysql/binlog]# mysqlbinlog -R --host=172.16.1.51 --user=mha --password=mha --raw --stop-never mysql-bin.000001 &

#參數詳解
-R				# 從遠端服務器獲取binlog
--host=172.16.1.51	        # 指定遠端的主機
--user=mha			# 數據庫mha用戶
--password=mha		        # 數據庫mha用戶的密碼
--raw				# binlog獲取時的一種格式
--stop-never mysql-bin.000001	# 從mysql-bin.000001開始不停的備份binlog

4.啟動mha

#啟動mha
nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &

5.加入恢復MHA進程腳本

[root@db02 ~]# cat /scripts/start_mha.sh
#!/bin/bash

#1.mysql進程數賦值
mysqlpid=`ps -ef | grep [m]ysql | wc -l`

#2.判斷MySQL是否假死，如果假死殺掉重啟，如果關閉則啟動
if [ $mysqlpid -eq 0 ];then
    systemctl start mysqld
else
    pkill mysqld
    systemctl start mysqld
fi

#3.獲取配置主從語句
change=`ssh 172.16.1.53 "grep 'CHANGE MASTER TO' /service/mha/manager | tail -1 | sed 's#xxx#123#g'" | awk -F: '{print $4}'`

#4.執行主從語句並啟動線程
mysql -e "$change; start slave"

#5.獲取主節點IP
master_ip=`ssh 172.16.1.53 "mysql -e 'show slave status\G'" | awk 'NR==3 {print $2}'`

#6.啟動實時獲取binlog進程
ssh 172.16.1.53 "cd /data/mysql/binlog && mysqlbinlog -R --host=$master_ip --user=mha --password=mha --raw --stop-never mysql-bin.000001 &> /dev/null &"

#7.替換MHA配置文件
ssh 172.16.1.53 "\cp /service/mha/app1.bak /service/mha/app1.cnf"

#8.啟動MHA
ssh 172.16.1.53 "nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &"

6.拆成兩個腳本

[root@db01 ~]# cat /scripts/start_mha.sh 
#!/bin/bash

#1.mysql進程數賦值
mysqlpid=`ps -ef | grep [m]ysql | wc -l`

#2.判斷MySQL是否假死，如果假死殺掉重啟，如果關閉則啟動
if [ $mysqlpid -eq 0 ];then
    systemctl start mysqld
else
    pkill mysqld
    systemctl start mysqld
fi

sleep 3

#3.獲取配置主從語句
change=`ssh 172.16.1.53 "grep 'CHANGE MASTER TO' /service/mha/manager | tail -1 | sed 's#xxx#123#g'" | awk -F: '{print $4}'`

#4.執行主從語句並啟動線程
mysql -e "$change; start slave" 2>&1

#5.遠程執行啟動mha腳本
ssh 172.16.1.53 "sh /scripts/recovery.sh"

[root@db03 ~]# cat /scripts/recovery.sh 
#1.獲取主節點IP
master_ip=`mysql -e 'show slave status\G' | awk 'NR==3 {print $2}'`

#2.進入保存binlog目錄
cd /data/mysql/binlog 

#3.啟動實時獲取binlog進程
mysqlbinlog -R --host=$master_ip --user=mha --password=mha --raw --stop-never mysql-bin.000001&> /dev/null &

#4.替換MHA配置文件
/usr/bin/cp /service/mha/app1.bak /service/mha/app1.cnf

#8.啟動MHA
nohup masterha_manager --conf=/service/mha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /service/mha/app1/manager.log 2>&1 &