一．EMQ集群搭建實現高可用和負載均衡
架構服務器規划
| 服務器IP |
部署業務 |
作用 |
| 192.168.81.13 |
EMQTTD |
EMQ集群 |
| 192.168.81.22 |
EMQTTD |
EMQ集群 |
| 192.168.81.23 |
EMQTTD |
EMQ集群 |
| 192.168.81.12(VIP:192.168.81.101) |
haproxy、keepalived |
HA和LB |
| 192.168.81.21(VIP:192.168.81.101) |
haproxy、keepalived |
HA和LB |
二．架構圖
三．EMQ集群搭建
192.168.81.13 , 192.168.81.22 , 192.168.81.23 三台服務器作為emq集成服務器，三台都部署emqttd服務
3.1 環境安裝
yum -y remove erlang
rpm -qa | grep erlang | xargs -I {} rpm -e {}
rpm -ivh erlang-21.3.8.1-1.el7.x86_64.rpm
socat安裝
yum -y install socat
nginx安裝
yum -y install nginx
3.2 SSL證書准備
mkdir –p /etc/nginx/ssl
mkdir –p /etc/nginx/ssl/ca
mkdir –p /etc/nginx/ssl/server
mkdir –p /etc/nginx/ssl/client
mkdir –p /etc/nginx/ssl/certs
mkdir –p /etc/nginx/ssl/crl
touch /etc/nginx/ssl/index.txt
cat <<EOF >/etc/nginx/ssl/serial
0
EOF
cat <<EOF >/etc/nginx/ssl/crlnumber
01
EOF
mkdir -p /etc/nginx/ssl/newcerts/server
生成ca證書:
openssl genrsa -out /etc/nginx/ssl/ca/ca.key 1024
openssl req -out /etc/nginx/ssl/ca/ca.req -key /etc/nginx/ssl/ca/ca.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Xxxxxx/CN=xxxxxx/email=info@xxxxxx.com"
openssl x509 -req -in /etc/nginx/ssl/ca/ca.req -out /etc/nginx/ssl/ca/ca.crt -sha1 -days 5000 -signkey /etc/nginx/ssl/ca/ca.key
rm -f /etc/nginx/ssl/ca/ca.req
生成server服務端證書:
openssl genrsa -out /etc/nginx/ssl/server/dev.xxxxxx.com.key 1024
openssl req -out /etc/nginx/ssl/server/dev.xxxxxx.com.req -key /etc/nginx/ssl/server/dev.xxxxxx.com.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Medc IoT/CN=dev.xxxxxx.com/Email=info@xxxxxx.com"
openssl x509 -req -in /etc/nginx/ssl/server/dev.xxxxxx.com.req -out /etc/nginx/ssl/server/dev.xxxxxx.com.crt -sha1 -CAcreateserial -days 5000 -CA /etc/nginx/ssl/ca/ca.crt -CAkey /etc/nginx/ssl/ca/ca.key
rm -f /etc/nginx/ssl/server/dev.xxxxxx.com.req
生成client客戶端證書（若有先吊銷然后再生成 ）:
openssl genrsa -out /etc/nginx/ssl/client/client.key 1024
openssl req -out /etc/nginx/ssl/client/client.req -key /etc/nginx/ssl/client/client.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Medc IoT/Email=info@xxxxxx.com"
openssl x509 -req -in /etc/nginx/ssl/client/client.req -out /etc/nginx/ssl/client/client.crt -sha1 -CAcreateserial -days 5000 -CA /etc/nginx/ssl/ca/ca.crt -CAkey /etc/nginx/ssl/ca/ca.key
rm -f /etc/nginx/ssl/client/client.req
3.3 修改openssl配置文件(centos7)
sed -i "s/\/etc\/pki\/CA/\/etc\/nginx\/ssl/g" /etc/pki/tls/openssl.cnf
sed -i "s/cacert.pem/ca\/ca.crt/g" /etc/pki/tls/openssl.cnf
sed -i "s/private\/cakey.pem/ca\/ca.key/g" /etc/pki/tls/openssl.cnf
sed -i "s/private\/.rand/ca\/.rand/g" /etc/pki/tls/openssl.cnf
3.4 准備nginx的配置文件
配置nginx
cat <<EOF >/etc/nginx/nginx.conf
# For more information on configuration, see:
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
# Load dynamic modules. See /usr/share/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format main '\$remote_addr - \$remote_user [\$time_local] "\$request" '
'\$status \$body_bytes_sent "\$http_referer" '
'"\$http_user_agent" "\$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
include /etc/nginx/conf.d/http/*.conf;
}
stream {
include /etc/nginx/conf.d/tcp/*.conf;
}
EOF
新建相關文件夾
mkdir -p /etc/nginx/conf.d/{http,tcp}
cat <<EOF >/etc/nginx/conf.d/http/80.conf
upstream xxxxxx{
ip_hash;
server ${gateway_ip_address}:8800; #網關微服務地址和端口號
}
server {
error_log /var/log/nginx/80_error.log;
listen 80;
server_name xxxxxx.com;
location / {
index index.html index.htm;
root /usr/share/nginx/html/dist;
try_files \$uri \$uri/ /index.html;
}
location /api/ {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Credentials' 'true';
add_header 'Access-Control-Allow-Methods' '*';
add_header 'Access-Control-Max-Age' '18000L';
add_header 'Access-Control-Allow-Headers' '*';
client_max_body_size 1000m;
proxy_http_version 1.1;
proxy_set_header Upgrade \$http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host \$host;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_pass http://xxxxxx/;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
EOF
cat <<EOF >/etc/nginx/conf.d/tcp/1882.conf
server {
error_log /var/log/nginx/1882_error.log;
listen 1882;
proxy_connect_timeout 600s;
proxy_pass xxxxxx;
}
EOF
cat <<EOF >/etc/nginx/conf.d/tcp/1883.conf
upstream xxxxxx{
server ${emq_ip_address}:1885; #這個配置無論寫哪個tcp的配置文件里都可以，emq的端口必須寫無鑒權的
}
server {
error_log /var/log/nginx/1883_error.log;
listen 1883 ssl;
proxy_connect_timeout 600s;
#proxy_timeout 3s;
proxy_pass xxxxxx;
ssl_certificate /etc/nginx/ssl/server/dev.xxxxxx.com.crt;
ssl_certificate_key /etc/nginx/ssl/server/dev.xxxxxx.com.key;
ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 4h;
ssl_handshake_timeout 30s;
}
EOF
cat <<EOF >/etc/nginx/conf.d/tcp/1884.conf
server {
error_log /var/log/nginx/1884_error.log;
listen 1884 ssl;
proxy_connect_timeout 600s;
#proxy_timeout 3s;
proxy_pass xxxxxx;
ssl_verify_client on;
ssl_client_certificate /etc/nginx/ssl/ca/ca.crt;
ssl_certificate /etc/nginx/ssl/server/dev.xxxxxx.com.crt;
ssl_certificate_key /etc/nginx/ssl/server/dev.xxxxxx.com.key;
ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 4h;
ssl_handshake_timeout 30s;
}
EOF
3.5 修改emq的配置文件
修改插件配置文件，更改域名地址
sed -i "s/test.xxxxxx.com/dev.xxxxxx.com/g" /usr/local/src/emqx-rel/deps/emqx_plugin_template/src/emqx_plugin_template.erl
cd /usr/local/src/emqx-rel/
重新編譯，此操作會重新生成配置文件
make -C /usr/local/src/emqx-rel
修改emq配置文件，開啟雙向認證
sed -i "/^node.name = emqx@/cnode.name = emqx@192.168.81.XX" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf
配置端口號和證書路徑
sed -i "/^listener.tcp.external =/clistener.tcp.external = 0.0.0.0:1885" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf
sed -i "/^{deny, all, subscribe, /c%%{deny, all, subscribe, \[\"\$SYS\/#\", {eq, \"#\"}\]}." /usr/local/src/emqx-rel/_rel/emqx/etc/acl.conf
sed -i"/^cluster.discovery=/ccluster.discovery=static"/usr/local/src/emqx-rel/emqx/etc/emqx.conf
sed-i"/^##cluster.static.seeds=/ccluster.static.seeds=emqx@192.168.81.13,emqx@192.168.81.22,emqx@192.168.81.23" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf
啟動emq
/usr/local/src/emqx-rel/_rel/emqx/bin/emqx start
啟動nginx
/usr/sbin/nginx -c /etc/nginx/nginx.conf
查看集群狀態：
[root@dev-13 bin]# ./emqx_ctl status
Node 'emqx@192.168.81.13' is started
emqx a2a8e428 is running
[root@dev-13 bin]# ./emqx_ctl cluster status
Cluster status: [{running_nodes,['emqx@192.168.81.22','emqx@192.168.81.23',
'emqx@192.168.81.13']}]
3.6 Liunx和Erlang虛擬機調優
Linux 系統參數優化
修改系統所有進程可打開的文件數量:
vim /etc/sysctl.conf
fs.file-max = 2097152
fs.nr_open = 2097152
### backlog - Socket 監聽隊列長度:
net.core.somaxconn=32768
net.ipv4.tcp_max_syn_backlog=16384
net.core.netdev_max_backlog=16384
## 可用知名端口范圍:
net.ipv4.ip_local_port_range='1000 65535'
## TCP Socket 讀寫 Buffer 設置:
net.core.rmem_default=262144
net.core.wmem_default=262144
net.core.rmem_max=16777216
net.core.wmem_max=16777216
net.core.optmem_max=16777216
#sysctl -w net.ipv4.tcp_mem='16777216 16777216 16777216'
net.ipv4.tcp_rmem='1024 4096 16777216'
net.ipv4.tcp_wmem='1024 4096 16777216'
## TCP 連接追蹤設置:
net.nf_conntrack_max=1000000
net.netfilter.nf_conntrack_max=1000000
net.netfilter.nf_conntrack_tcp_timeout_time_wait=30
## FIN-WAIT-2 Socket 超時設置:
net.ipv4.tcp_fin_timeout = 15
## TIME-WAIT Socket 最大數量、回收與重用設置:
net.ipv4.tcp_max_tw_buckets=1048576
# 注意: 不建議開啟該設置，NAT模式下可能引起連接RST
# net.ipv4.tcp_tw_recycle = 1
# net.ipv4.tcp_tw_reuse = 1
設置服務最大文件句柄數:
vim /etc/systemd/system.conf
DefaultLimitNOFILE=1048576
持久化設置允許用戶/進程打開文件句柄數:
vim /etc/security/limits.conf
* soft nofile 1048576
* hard nofile 1048576
Erlang 虛擬機參數:
vim /usr/local/emqttd/etc/emq.conf
## Erlang Process Limit
node.process_limit = 2097152
## Sets the maximum number of simultaneously existing ports for this system
node.max_ports = 1048576
## EMQ 最大允許連接數
## Size of acceptor pool
listener.tcp.external.acceptors = 64
## Maximum number of concurrent clients(以1G內存比5W進行配置)
listener.tcp.external.max_clients = 1000000
四．haproxy部署
192.168.81.12和192.168.81.21服務器部署和配置一樣
4.1 haproxy安裝
yum install -y pcre-devel bzip2-devel gcc gcc-c++ make
tar –zxvf haproxy-1.8.26.tar.gz
cd haproxy-1.8.26
make TARGET=linux2628 PREFIX=/usr/local/haproxy
make install PREFIX=/usr/local/haproxy
/usr/local/haproxy/sbin/haproxy -v
HA-Proxy version 1.8.26 2020/08/03
Copyright 2000-2020 Willy Tarreau willy@haproxy.org
mkdir /etc/haproxy
groupadd haproxy
useradd -s /sbin/nologin -M -g haproxy haproxy //添加haproxy運行haproxy賬號並設置及屬主與屬組
cp examples/haproxy.init /etc/init.d/haproxy
chmod 755 /etc/init.d/haproxy
chkconfig --add haproxy
cp /usr/local/haproxy/sbin/haproxy /usr/sbin/
4.2 增加配置文件
cat <<EOF>>/etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
log global
option dontlognull
option http-server-close
# option forwardfor
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 60s
timeout client 2m
timeout server 2m
timeout http-keep-alive 10s
timeout check 10s
frontend emqtt-front
bind *:1885
maxconn 1000000
mode tcp
default_backend emqtt-backend
backend emqtt-backend
balance roundrobin
# balance source
server emq1 192.168.81.13:1885 check inter 100000 fall 2 rise 5 weight 1
server emq2 192.168.81.22:1885 check inter 100000 fall 2 rise 5 weight 1
server emq3 192.168.81.23:1885 check inter 100000 fall 2 rise 5 weight 1
# source 0.0.0.0 usesrc clientip
frontend emqtt-admin-front
bind *:18083
mode http
default_backend emqtt-admin-backend
backend emqtt-admin-backend
mode http
balance roundrobin
server emq1 192.168.81.13:18083 check
server emq2 192.168.81.22:18083 check
server emq3 192.168.81.23:18083 check
listen admin_stats
stats enable
bind *:8080
mode http
option httplog
log global
maxconn 10
stats refresh 30s
stats uri /admin
stats realm haproxy
stats auth admin:admin
stats hide-version
stats admin if TRUE
4.3 啟動haproxy
systemctl start haproxy
chkconfig haproxy on
五．keepalived部署
192.168.81.12和192.168.81.21服務器部署和配置略有差別
5.1 keepalived安裝
yum –y Install keepalived
5.2 增加配置文件
! Configuration File for keepalived
global_defs {
router_id mqtt81
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP
interface ens192
virtual_router_id 81
nopreempt
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
mcast_src_ip 192.168.81.12
virtual_ipaddress {
192.168.81.101/24
}
}
5.3 啟動keepalived並設置開機自啟動
systemctl start keepalived
systemctl enable keepalived
六．架構測試
6.1 keepalived架構測試
關閉192.168.81.12或者192.168.81.21其中一台，查看服務是否還可以正常訪問
6.2 haproxy負載均衡
客戶端工具發起pub，看是否按照輪詢方式發送到后端3台EMQ服務上面。
七．結果展示
7.1 emq集群展示
7.2 客戶端發起6個連接測試
7.3 haproxy統計顯示
