騰訊redis2.3集群搭建

環境規划：

IP	主機名	組件	配置	備注
192.168.3.81	node1		8核16G 磁盤X2-50G	沒有8核16G無法添加服務器
192.168.3.82	node2	cache，proxy	8核16G 磁盤X2-50G
192.168.3.83	node3c	cache，proxy	8核16G 磁盤X2-50G
192.168.3.84	node4	cache	8核16G 磁盤X2-50G
192.168.3.85	node5	cache	8核16G 磁盤X2-50G

安裝包與配套文檔

#官方安裝包redis2.3
http://deliver-service-packet-1258877907.cos.ap-guangzhou.myqcloud.com/redis/V2.3/redis_for_tstack_x86-2.3-install.tar.gz?sign=q-sign-algorithm%3Dsha1%26q-ak%3DAKID54tUofahnurY7VvU8UjoZ4MN7aoxac2F%26q-sign-time%3D1622115037%3B1622374297%26q-key-time%3D1622115037%3B1622374297%26q-header-list%3Dhost%26q-url-param-list%3D%26q-signature%3Dd9463fdbd23851d8a7199289d1d620c62e60c811

#官方文檔
http://deliver-service-packet-1258877907.cos.ap-guangzhou.myqcloud.com/redis/V2.3/Redis%E7%8B%AC%E7%AB%8B%E8%BE%93%E5%87%BA%E7%89%88%E4%B8%80%E9%94%AE%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3%282.3%E7%89%88%E6%9C%AC%29.docx?sign=q-sign-algorithm%3Dsha1%26q-ak%3DAKID54tUofahnurY7VvU8UjoZ4MN7aoxac2F%26q-sign-time%3D1622115040%3B1622374300%26q-key-time%3D1622115040%3B1622374300%26q-header-list%3Dhost%26q-url-param-list%3D%26q-signature%3D45337c72f36e1c1b73e630d84e05a86fe9aebcfd

基礎優化

# --所有機器
#追加以下參數到/etc/sysctl.conf
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_fin_timeout = 30
最后生效以上設置：
sysctl -p

# --cache_agent機器額外兩條優化
#追加以下參數到/etc/sysctl.conf
net.core.somaxconn = 511
vm.swappiness = 1
最后生效以上設置：
sysctl -p

#--cache/Proxy機器配置以下設置：
echo never > /sys/kernel/mm/transparent_hugepage/enabled 

#如果需要重啟生效，需要追加命令到/etc/rc.local
echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.local


#--所有Cache/Proxy機器
 ulimit -SHn 65535
 
#配置重啟生效
cat >>/etc/security/limits.conf<<eof
* soft nofile 65535
* hard nofile 65535
eof



#掛載磁盤

#每台服務器新增一塊磁盤
echo "- - -" > /sys/class/scsi_host/host0/scan
echo "- - -" > /sys/class/scsi_host/host1/scan
echo "- - -" > /sys/class/scsi_host/host2/scan
lsblk

#掛載新磁盤
umount /dev/sdb
mkdir -p /data
mkfs.xfs -f /dev/sdb
echo '/dev/sdb   /data    xfs  defaults   0 0'  >>/etc/fstab
mount -a
df -h

中控機操作[第一台服務器]

#中控機服務器安裝mariadb
yum install -y mariadb-server 
systemctl start mariadb
systemctl enable mariadb

#mariadb初始化[可以不用初始化]：
[root@server1 ~]# mysql_secure_installation
Enter current password for root (enter for none):  <–初次運行直接回車
Set root password? [Y/n]   <– 是否設置root用戶密碼，輸入y並回車或直接回車
New password:          <– 設置root用戶的密碼
Re-enter new password: <– 再輸入一次你設置的密碼
Password updated successfully!
… Success!
Remove anonymous users? [Y/n] <– 是否刪除匿名用戶,生產環境建議刪除，所以直接回車
… Success!
Disallow root login remotely? [Y/n] n  <–是否禁止root遠程登錄,根據自己的需求選擇Y/n並回車,建議禁止，這里不用禁止
… Success!
Remove test database and access to it? [Y/n] y <– 是否刪除test數據庫,直接回車
Reload privilege tables now? [Y/n] <– 是否重新加載權限表，直接回車
… Success!


連接數據庫：
mysql -uroot -p123456 


#創建用戶並授權
GRANT ALL PRIVILEGES ON *.* to 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;
FLUSH privileges;


GRANT ALL PRIVILEGES ON *.* TO 'mysql'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;
flush privileges;


#刪除空賬號密碼的條目
use mysql;
delete from user where user='';
delete from user where password='';
select user,host,password from mysql.user;


#測試連接：
其他服務器安裝一個mariadb，這里我直接用 node2 服務器測試連接node1的mysql
測試連接
yum install -y mariadb

#測試：
[root@node2 ~]#  mysql -umysql -p123456 -h 192.168.3.81
Welcome to the MariaDB monitor.  Commands end with ; or \g.
Your MariaDB connection id is 17
Server version: 5.5.68-MariaDB MariaDB Server
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> 


#注意
1.	MySQL密碼說明
MySQL密碼中不可出現"@"、"$"特殊符號


#創建redis使用的數據庫：
mysql -uroot -p123456

安裝redis[中控機操作]

mkdir -p /root
tar xf redis_for_tstack_x86-2.3-install.tar.gz
cd redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-sql-oper/TCE/


#去新增用戶
mysql
GRANT ALL PRIVILEGES ON *.* TO 'mysql'@'%' IDENTIFIED BY '123456';
flush privileges;

#創建庫用於導入數據
mysql 
create database newcc_1001;


#測試鏈接
mysql -h192.168.3.81 -umysql -p123456

#登錄mysql導入數據
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-sql-oper/TCE
./python/bin/python3 sqlimport.py -u "mysql@192.168.3.81:3306:root:123456:newcc_1001" -d common -b control_center -v 2.3

#導入結果：
[root@node1 TCE]# pwd
/root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-sql-oper/TCE
[root@node1 TCE]# ./python/bin/python3 sqlimport.py -u "mysql@192.168.3.81:3306:root:123456:newcc_1001" -d common -b control_center -v 2.3
Execute sql_file/control_center/common/V2.1.0_0__create_table.sql success!
Execute sql_file/control_center/common/V2.1.0_1__insert_data.sql success!
Execute sql_file/control_center/common/V2.2.0_3__create_table.sql success!
Execute sql_file/control_center/common/V2.2.0_4__insert_data.sql success!
Execute sql_file/control_center/common/V2.2.0_5__alter_table.sql success!
Execute sql_file/control_center/common/V2.3.0_11__create_table.sql success!
Execute sql_file/control_center/common/V2.3.0_12__insert_data.sql success!
_SQL:
        INSERT INTO flyway_schema_history(installed_rank, version, description, 
        type, script, checksum, installed_by, installed_on, execution_time, success) 
        VALUES("1", "2.1.0.0", "create table", "SQL", "V2.1.0_0__create_table.sql", "31594625acf5ef500f738254d609a6fe", "root", "2021-05-29 21:24:14", "0", 1),("2", "2.1.0.1", "insert data", "SQL", "V2.1.0_1__insert_data.sql", "688485c275ae37ecc3c42be43f2f5aae", "root", "2021-05-29 21:24:14", "0", 1),("3", "2.2.0.3", "create table", "SQL", "V2.2.0_3__create_table.sql", "5a252a96a892417f0c686b388b0aa41b", "root", "2021-05-29 21:24:14", "0", 1),("4", "2.2.0.4", "insert data", "SQL", "V2.2.0_4__insert_data.sql", "310ce80193368c95dd56c886b042e748", "root", "2021-05-29 21:24:14", "0", 1),("5", "2.2.0.5", "alter table", "SQL", "V2.2.0_5__alter_table.sql", "738a9d41e31c0ddce0a444ef1ef77ab2", "root", "2021-05-29 21:24:14", "0", 1),("6", "2.3.0.11", "create table", "SQL", "V2.3.0_11__create_table.sql", "228dbcb8764022cfe2eb43b0926e092f", "root", "2021-05-29 21:24:14", "0", 1),("7", "2.3.0.12", "insert data", "SQL", "V2.3.0_12__insert_data.sql", "019e0d99abfab2a3b002319a1ff42d73", "root", "2021-05-29 21:24:14", "0", 1)


#導入數據[value="17179869184" 就是內存16G服務器配置需要大於等於16G才可以添加"是這個意思？不知道"]：
use newcc_1001;
update sys_misc_config_t set value="http://192.168.3.81:8086/query" where type_id=5 and name="influxdb_url";
update sys_misc_config_t set value="http://192.168.3.81:9090/writer" where name="ctsdb_receivers" and type_id=0;
update sys_misc_config_t set value="http://192.168.3.81:9090/cc" where name="ctsdbserver_cc_url" and type_id=5;
insert into sys_misc_config_t values(5,"any","0.0.0.0");
update sys_misc_config_t set value="17179869184" where type_id=5 and name="mixed_min_memory";




#創建monitor數據庫並導入相關數據：
mysql -h192.168.3.81 -umysql -p123456
create database monitor_1001;

#導入
./python/bin/python3 sqlimport.py -u "mysql@192.168.3.81:3306:root:123456:monitor_1001" -d common -b monitor_center -v 2.3


#創建
create database access_auth;
#導入
./python/bin/python3 sqlimport.py -u "mysql@192.168.3.81:3306:root:123456:access_auth" -d common -b access_auth -v 2.3


#創建
create database access_gateway;
#導入
./python/bin/python3 sqlimport.py -u "mysql@192.168.3.81:3306:root:123456:access_gateway" -d common -b access_gateway -v 2.3


#創建
create database ctsdb_server;
#導入
./python/bin/python3 sqlimport.py -u "mysql@192.168.3.81:3306:root:123456:ctsdb_server" -d common -b ctsdb_server -v 2.3


#到此為止數據庫安裝完畢





#安裝 module-redis-control-center [安裝到中控機]
#修改db_config.py
ip_a=$(hostname -I|awk '{print $1}')
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-control-center/config
sed -i  "s#9.134.241.18#$(hostname -I|awk '{print $1}')#g" db_config.py
sed -i "s#myPassword#123456#g" db_config.py


#修改cluster_watcher.properties
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-control-center/config
sed -i  "s#9.134.241.18#$(hostname -I|awk '{print $1}')#g" cluster_watcher.properties
sed -i "s#myPassword#123456#g" cluster_watcher.properties


#修改 MulServer.conf
InfluxIp=127.0.0.1
由於InfluxIp就在本機部署，所以不需要修改



#修改redis_prober.conf
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-control-center/config
sed -i "s#myPassword#123456#g" redis_prober.conf
sed -i  "s#9.134.241.18#$(hostname -I|awk '{print $1}')#g" redis_prober.conf
sed -i  "s#9.134.240.167#$(hostname -I|awk '{print $1}')#g" redis_prober.conf



#修改key_analyzer.conf
sed -i  "s#9.134.240.167#$(hostname -I|awk '{print $1}')#g"  key_analyzer.conf



#修改patrol_redis.yaml
sed -i  "s#9.134.241.18#$(hostname -I|awk '{print $1}')#g"  patrol_redis.yaml
sed -i "s#myPassword#123456#g" patrol_redis.yaml


#以上修改無誤后返回TCE目錄執行安裝操作：
cd ../TCE/
pwd
/root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-control-center/TCE

#執行安裝操作
sh -x install.sh all

#檢查安裝結果
sh check.sh
提示：
control_center is ok.
代表安裝完成 redis-control





#安裝influxdb
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-influxdb
sed -i  "s#9.134.241.18#$(hostname -I|awk '{print $1}')#g" config/ctsdbserver.conf
sed -i "s#myPassword#123456#g" config/ctsdbserver.conf

cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-influxdb/TCE
sh -x install.sh all

檢查安裝結果：
sh check.sh

提示：
influxdb is ok. 
代表安裝完成 influxdb




#安裝lvs_keepalived
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-lvs-keepalived
sed -i "s#eth1#eth0#g" config/redis_lvs.conf

#sed只是把網卡替換成了真實網卡名 eth0
[root@node1 module-redis-lvs-keepalived]# cat config/redis_lvs.conf
! Configuration File for keepalived
global_defs {               ##全局配置部分
    router_id LVS_MASTER       ##運行keepalived機器的一個標識
}
vrrp_instance VI_1 {       ##設置vrrp組，唯一且同一LVS服務器組要相同
    state MASTER               ##備份LVS服務器設置為BACKUP
    interface eth0       # #設置對外服務的接口(根據實際情況修改)
    virtual_router_id 51       ##設置虛擬路由標識
    priority 100               #設置優先級，數值越大，優先級越高，backup設置小於100，當master宕機后自動將backup高的變為master。
    advert_int 1               ##設置同步時間間隔
    authentication {           ##設置驗證類型和密碼，master和buckup一定要設置一樣
        auth_type PASS
        auth_pass 1111
    }
virtual_ipaddress {        ##設置VIP，可以多個，每個占一行
       100.000.100.182
    }
}
include /data/redis/keepalived/etc/redis_lvs/*.conf


#修改lvsmanger.conf
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-lvs-keepalived
sed -i "s#9.134.241.18#$(hostname -I|awk '{print $1}')#g"  config/lvsmanger.conf
sed -i "s#myPassword#123456#g" config/lvsmanger.conf
sed -i "s#:9010#$(hostname -I|awk '{print $1}'):9010#g" config/lvsmanger.conf
sed -i "s#106.55.61.42#$(hostname -I|awk '{print $1}')#g" config/lvsmanger.conf
cd ../TCE
[root@node1 TCE]# pwd
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-lvs-keepalived/TCE
sh -x install.sh all

#檢查：
sh -x check.sh
輸出信息：
lvs_keepalived is ok.

安裝 cache服務和proxy

看環境規划中有兩台是作為cache服務的

分別是 
192.168.3.84
192.168.3.85


需要把安裝包傳到這兩台服務器進行安裝：
[可以全部安裝 然后界面選擇就好，我這里全部安裝了，演示只裝了2台]

# 192.168.3.84安裝cache_agent服務：
scp -r root@192.168.3.81:/root/redis_for_tstack_x86-2.3-install.tar.gz ./
tar xf redis_for_tstack_x86-2.3-install.tar.gz
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-cache-agent/

#IP改為管控機的ip，而不是本機IP
sed -i "s#9.134.240.167#192.168.3.81#g" config/mul_monitor_agent.conf


#結果如下：
[root@node4 module-redis-cache-agent]# cat config/mul_monitor_agent.conf
waterLogLevel 2
coloeLogLevel 2
ShmChangeIntval 600
reportIntval 60
LOG_FILE ../log

zkhost 10.24.0.10:2181,10.24.0.11:2181,10.24.0.12:2181
sysId 1001
cityid 1001

mulservers 192.168.3.81:55100


#安裝cache-agent
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-cache-agent/TCE
sh -x install.sh all

#檢查cache-anget是否正常安裝：
[root@node4 TCE]# sh  check.sh
cache_agent is ok.



--------------------------------------
# 192.168.3.85安裝cache_agent服務：
scp -r root@192.168.3.81:/root/redis_for_tstack_x86-2.3-install.tar.gz ./
tar xf redis_for_tstack_x86-2.3-install.tar.gz
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-cache-agent/


#IP改為管控機的ip，而不是本機IP
sed -i "s#9.134.240.167#192.168.3.81#g" config/mul_monitor_agent.conf


#結果如下：
[root@node4 module-redis-cache-agent]# cat config/mul_monitor_agent.conf
waterLogLevel 2
coloeLogLevel 2
ShmChangeIntval 600
reportIntval 60
LOG_FILE ../log

zkhost 10.24.0.10:2181,10.24.0.11:2181,10.24.0.12:2181
sysId 1001
cityid 1001

mulservers 192.168.3.81:55100



#安裝cache-agent 【這里安裝的服務器可以被動選擇為proxy服務器或者cache服務器 由自己決定，只要安裝了就可以把這個服務器加入到集群】
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-cache-agent/TCE
sh -x install.sh all

#檢查cache-anget是否正常安裝：
[root@node4 TCE]# sh  check.sh
cache_agent is ok.


------------------------------------------------------------------------------------------

#安裝web頁面【中控機】
# 修改web地址
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-user-interface/config
sed -i "s#9.134.241.18#192.168.3.81#g" application.yml

# 密鑰生成工具添加執行權限：
chmod +x aestool 

#2. 生成密文的連接服務器的密碼[123456] 如果服務器root密碼不是這個就需要改成別的：
[root@node1 config]# ./aestool -e 123456
18hvlaMM7yvK6GKFsYbdFx31O8IaNw==

#3. 密碼配置到 application.yml 文件中的password項中
url: jdbc:mysql://192.168.3.81:3306/access_auth?useSSL=false&characterEncoding=utf8
username: root
password: ENC(18hvlaMM7yvK6GKFsYbdFx31O8IaNw==)    # 生成服務器密碼123456寫在這里


#4.  找到： cluster-nodes: 127.0.0.1:6379 這行下方的密碼應該是Tcdn@2007加密后的密碼
./aestool -e Tcdn@2007
spk4z9YIKWTnYSZMwl+FG0zVLN0bDlselg==

vim application.yml：
--------------------------------------------------
redis:
  cluster-nodes: 127.0.0.1:6379
  mode: single
  password: ENC(spk4z9YIKWTnYSZMwl+FG0zVLN0bDlselg==)    #這個密碼是Tcdn@2007加密后的密碼，寫入application.yml配置文件中
--------------------------------------------------


# application_gateway.yml 文件修改
修改：
url: jdbc:mysql://9.134.241.18:3306/access_gateway?useSSL=false&characterEncoding=utf8
username: root
password: ENC(i4MMwOVJbjuiAnY04AQAGLWgym+KFbZrftY=)

1. 修改IP
sed -i "s#9.134.241.18#192.168.3.81#g" application_gateway.yml

2. 修改密碼
password: ENC(i4MMwOVJbjuiAnY04AQAGLWgym+KFbZrftY=)
其中密碼改為： 18hvlaMM7yvK6GKFsYbdFx31O8IaNw==



#8. 安裝redis-user-interface [中控機]
cd /root/redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-user-interface/TCE
sh -x install.sh all

#安裝完成后通過check.sh 腳本來檢測成功
sh check.sh

正確輸出：
user_interface is ok.


此時打開web頁面檢查：
默認賬號密碼：
賬號：qcloudAdmin
密碼：qcloudAdmin




#添加proxy服務：
本質上proxy服務與cache服務配置相同
本文規划的proxy
192.168.3.81
192.168.3.82
192.168.3.83



192.168.3.81 第一台服務器啥也不用裝

192.168.3.82安裝proxy
# cd /root
scp  -r root@192.168.3.85:/root/redis_for_tstack_x86-2.3-install.tar.gz ./
tar xf redis_for_tstack_x86-2.3-install.tar.gz 
cd redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-cache-agent/
sed -i "s#9.134.240.167#192.168.3.81#g" config/mul_monitor_agent.conf
cd TCE/
sh -x install.sh all
sh check.sh


192.168.3.83安裝proxy
# cd /root
scp  -r root@192.168.3.85:/root/redis_for_tstack_x86-2.3-install.tar.gz ./
tar xf redis_for_tstack_x86-2.3-install.tar.gz 
cd redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-cache-agent/
sed -i "s#9.134.240.167#192.168.3.81#g" config/mul_monitor_agent.conf
cd TCE/
sh -x install.sh all
sh check.sh



192.168.3.84安裝cache
# cd /root
scp  -r root@192.168.3.85:/root/redis_for_tstack_x86-2.3-install.tar.gz ./
tar xf redis_for_tstack_x86-2.3-install.tar.gz 
cd redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-cache-agent/
sed -i "s#9.134.240.167#192.168.3.81#g" config/mul_monitor_agent.conf
cd TCE/
sh -x install.sh all
sh check.sh



192.168.3.85安裝cache
# cd /root
scp  -r root@192.168.3.85:/root/redis_for_tstack_x86-2.3-install.tar.gz ./
tar xf redis_for_tstack_x86-2.3-install.tar.gz 
cd redis_for_tstack_x86-2.3-install/ted.redis/product/product-redis-cc_ccagent/module-redis-cache-agent/
sed -i "s#9.134.240.167#192.168.3.81#g" config/mul_monitor_agent.conf
cd TCE/
sh -x install.sh all
sh check.sh

上架機器

#proxy 2台 [規划 192.168.3.82，192.168.3.83]
連接到數據庫i：
mysql -h192.168.3.81 -umysql -p123456

#首先把所有cacahe都上架完成
cache 4台 [規划 192.168.3.82，192.168.3.83,192.168.3.84,192.168.3.85]
MariaDB [newcc_1001]> select * from redis_machine_t;
+--------------+-------+--------+
| ip           | state | idc_id |
+--------------+-------+--------+
| 192.168.3.82 |     1 |      0 |
| 192.168.3.83 |     1 |      0 |
| 192.168.3.84 |     1 |      0 |
| 192.168.3.85 |     1 |      0 |
+--------------+-------+--------+
4 rows in set (0.01 sec)




#然后通過數據庫上架兩台proxy
MariaDB [(none)]> use newcc_1001;
MariaDB [newcc_1001]> insert into interface_machine_t values("192.168.3.82",1,0);
MariaDB [newcc_1001]> insert into interface_machine_t values("192.168.3.83",1,0);
MariaDB [newcc_1001]> select * from interface_machine_t;
+--------------+-------+--------+
| ip           | state | idc_id |
+--------------+-------+--------+
| 192.168.3.82 |     1 |      0 |
| 192.168.3.83 |     1 |      0 |
+--------------+-------+--------+
2 rows in set (0.00 sec)



#然后開始進行web頁面創建redis實例

創建實例測試