MinIO+Keepalived+Nginx

摘要：

使用4台服務器搭建minIO集群，前段2台服務器使用keepalived+nginx實現高可用和負載均衡。

6台服務器使用系統CentOS07.

搭建minIO集群：

以下操作要在minio的4台上都執行一遍

#在4台minio上下載minioserver程序
wget https://dl.minio.io/server/minio/release/linux-amd64/minio

#創建minio啟動腳本,內容如下
vim miniostart.sh 

#!/bin/bash

export MINIO_ACCESS_KEY=minio
export MINIO_SECRET_KEY=Mima+123456   #密碼要包含大小寫，數字
/root/minio server --config-dir /etc/minio --address :9000 http://172.17.170.230/data/minio/data http://172.17.170.231/data/minio/data http://172.17.170.232/data/minio/data  http://172.17.170.233/data/minio/data

#給腳本和minio程序添加執行權限
 chmod +x minio miniostart.sh 

#將minio添加到systemctl中管理，添加腳本如下
vim /usr/lib/systemd/system/minio.service

[Unit]
Description=Minio service
Documentation=https://docs.minio.io/

[Service]
WorkingDirectory=/root/
ExecStart=/root/miniostart.sh

Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

#設置minio開機啟動
systemctl enable minio
#關閉防火牆
systemctl disable firewalld
#重啟
reboot

重啟后輸入任意ip:9000就可看到minio頁面，輸入剛才設置的賬號密碼就可登錄

Keepalived+Nginx

在前段的2台上執行以下操作。

Keepalived

#安裝nginx
yum -y install nginx.x86_64 
#安裝keepalived
yum -y install keepalived.x86_64 

#配置keepalived，2台一個為master一個為backup，主從模式，主的配置如下
vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     [e-mail]   #通知收件人
   }
   notification_email_from [e-mail]
   smtp_server 172.17.168.106  #smtpip
   smtp_connect_timeout 30
   router_id mm01   #主機名
}

vrrp_instance VI_1 {
    state MASTER    #主的一台為master
    interface eth0  #主機的網卡
    virtual_router_id 60 #2台一樣就可
    priority 100    #主的100
    advert_int 1
        nopreempt    #非搶占模式
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        172.17.170.236   #虛擬IP
    }
}
#監聽虛擬ip和端口
virtual_server 172.17.170.236 80  {
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    nat_mask 255.255.255.0
    persistence_timeout 50
    protocol TCP
    #這台服務上的ip和服務端口
    real_server 172.17.170.234 80 {
                weight 1
                notify_down /etc/keepalived/killkeep.sh  #如果nginx故障的話就殺掉這台的keepalived，
                TCP_CHECK {                              #虛擬ip就到了另外那台，此腳本就是殺keepalived
                  connect_port 80
                  connect_timeout 3
                  nb_get_retry 2
                  delay_before_retry 1
                } 
  }             
} 

從的配置如下：

! Configuration File for keepalived

global_defs {
   notification_email {
     [e-mail]   #通知收件人
   }
   notification_email_from [e-mail]
   smtp_server 172.17.168.106  #smtpip
   smtp_connect_timeout 30
   router_id mm01   #主機名
}

vrrp_instance VI_1 {
    state BACKUP    #從的一台為BACKUP
    interface eth0  #主機的網卡
    virtual_router_id 60 #2台一樣就可
    priority 50    #從的50
    advert_int 1
        nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        172.17.170.236   #虛擬IP
    }
}
#監聽虛擬ip和端口
virtual_server 172.17.170.236 80  {
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    nat_mask 255.255.255.0
    persistence_timeout 50
    protocol TCP
    #這台服務上的ip和服務端口
    real_server 172.17.170.235 80 {
                weight 1
                notify_down /etc/keepalived/killkeep.sh
                TCP_CHECK { 
                  connect_port 80
                  connect_timeout 3
                  nb_get_retry 2
                  delay_before_retry 1
                } 
  }             
} 

#殺keepalived腳本
vim /etc/keepalived/killkeep.sh

#!/bin/bash
pkill keepalived

#添加執行權限
chmod +x killkeep.sh

Nginx

#安裝Nginx
yum -y install nginx.x86_64
#配置Nginx負載均衡模式
vim /etc/nginx/nginx.conf

events {
    worker_connections 1024;
}

http {
    upstream myserver{
        server 172.17.170.230:9000 weight=1; #轉發給4台Minio
        server 172.17.170.231:9000 weight=1;
        server 172.17.170.232:9000 weight=1;
        server 172.17.170.233:9000 weight=1;
    }
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   3;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    # Load modular configuration files from the /etc/nginx/conf.d directory.
    # See http://nginx.org/en/docs/ngx_core_module.html#include
    # for more information.
    include /etc/nginx/conf.d/*.conf;

    server {
        listen       80 ;
        listen       [::]:80 default_server;
        server_name  172.17.170.236;           #虛擬IP
        root         /usr/share/nginx/html;
        client_max_body_size 0;
        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;

        location / {
                proxy_set_header Host $http_host;  #將客戶端請求原樣轉發
                proxy_pass http://myserver;        #轉發給上面配置的4台minio

        }

        error_page 404 /404.html;
        location = /404.html {
        }

#關閉防火牆，設置keepalived和nginx開機啟動

systemctl disable firewalld

systemctl enable keepalived
systemctl enable nginx

#然后重啟
reboot

重啟后使用虛擬IP即可訪問

各語言鏈接minio SDK參考

https://docs.min.io/cn/dotnet-client-quickstart-guide.html