openresty(nginx) 配置 stream 轉發

nginx從1.9.0開始，新增加了一個stream模塊，用來實現四層協議的轉發、代理或者負載均衡等。

（1）關於stream域的模塊有哪些？

目前官網上列出的第三方模塊、簡直就是http模塊的鏡像、比如access模塊訪問控制ip和ip段，map模塊實現映射、 geo模塊實現地理位置映射、等等。使用這些模塊的時候一定要看是哪個版本才支持的、比如log模塊，只有在nginx-1.11.4才支持。

NGINX的stream相關模塊有如下（有些模塊特定版本才有，才支持，比如，log模塊是NGINX的1.11.4版本）：

ngx_stream_core_module
ngx_stream_access_module
ngx_stream_geo_module
ngx_stream_geoip_module
ngx_stream_js_module
ngx_stream_limit_conn_module
ngx_stream_log_module
ngx_stream_map_module
ngx_stream_proxy_module
ngx_stream_realip_module
ngx_stream_return_module
ngx_stream_split_clients_module
ngx_stream_ssl_module
ngx_stream_ssl_preread_module
ngx_stream_upstream_module
ngx_stream_upstream_hc_module

注意：如果使用 nginx 的 stream 功能，在編譯時一定要加上 “--with-stream”

這里使用官方提供的方式在線yum安裝openesty，默認已經加上stream 功能了

[root@test sbin]# ./nginx -V
nginx version: openresty/1.19.3.2
built by gcc 9.3.1 20200408 (Red Hat 9.3.1-2) (GCC) 
built with OpenSSL 1.1.1k  25 Mar 2021 (running with OpenSSL 1.1.1i  8 Dec 2020)
TLS SNI support enabled
configure arguments: --prefix=/usr/local/openresty/nginx --with-cc-opt='-O2 -DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/zlib/include -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl111/include' --add-module=../ngx_devel_kit-0.3.1 --add-module=../echo-nginx-module-0.62 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2 --add-module=../set-misc-nginx-module-0.32 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.08 --add-module=../srcache-nginx-module-0.32 --add-module=../ngx_lua-0.10.19 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.33 --add-module=../array-var-nginx-module-0.05 --add-module=../memc-nginx-module-0.19 --add-module=../redis2-nginx-module-0.15 --add-module=../redis-nginx-module-0.3.7 --add-module=../ngx_stream_lua-0.0.9 --with-ld-opt='-Wl,-rpath,/usr/local/openresty/luajit/lib -L/usr/local/openresty/zlib/lib -L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl111/lib -Wl,-rpath,/usr/local/openresty/zlib/lib:/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl111/lib' --with-cc='ccache gcc -fdiagnostics-color=always' --with-pcre-jit --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-http_v2_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-http_stub_status_module --with-http_realip_module --with-http_addition_module --with-http_auth_request_module --with-http_secure_link_module --with-http_random_index_module --with-http_gzip_static_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-threads --with-compat --with-stream --with-http_ssl_module

wget https://openresty.org/package/centos/openresty.repo
mv openresty.repo /etc/yum.repos.d/
yum check-update
yum -y install openresty

systemctl start openresty.service

stream模塊通常寫在events模塊下面,與http同一級別.

如下的配置，是監聽本機的5678端口轉發給stream中upstream的rabbitmq

worker_processes  1;
error_log  logs/error.log;
pid        logs/nginx.pid;

events {
    worker_connections  1024;
}

stream{
    upstream rabbitmq{
        server 192.168.20.100:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.101:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.102:5672 max_fails=2 fail_timeout=5s weight=2;
    }
    server{
        listen 5678; # 任意不占用的端口
        proxy_connect_timeout 10s;
        proxy_timeout 300s;
        proxy_pass rabbitmq; # 注意寫法，不帶http://
    }
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  logs/access.log  main;

    sendfile        on;
    tcp_nopush      on;

    keepalive_timeout  65;

    gzip  on;
    gzip_min_length 1k;
    gzip_comp_level 7;
    gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
    gzip_vary on;
    gzip_disable "MSIE [1-6]\.";

    upstream mqweb {
        ip_hash;
        server 192.168.20.100:15672 weight=1 max_fails=2 fail_timeout=30s;
        server 192.168.20.101:15672 weight=1 max_fails=2 fail_timeout=30s;
        server 192.168.20.102:15672 weight=1 max_fails=2 fail_timeout=30s;
    }

    server {
        listen       80;
        server_name  localhost;
        charset      utf-8;
        location / {
            proxy_pass http://mqweb;
            proxy_redirect off;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            client_max_body_size 100m;
            client_body_buffer_size 256k;
        
            proxy_connect_timeout 60;
            proxy_send_timeout 60;
            proxy_read_timeout 60;
        
            proxy_buffer_size 256k;
            proxy_buffers 8 256k;
            proxy_busy_buffers_size 512k;
            proxy_temp_file_write_size 512k;
        }
        access_log  logs/host.access.log  main;
    }
}

改變負載均衡的方法
a）least-connected ：對於每個請求，nginx plus選擇當前連接數最少的server來處理:

stream{
    upstream rabbitmq{
        least_conn;
        server 192.168.20.100:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.101:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.102:5672 max_fails=2 fail_timeout=5s weight=2;
    }
    server{
        listen 5672;
        proxy_connect_timeout 10s;
        proxy_timeout 300s;
        proxy_pass rabbitmq;
    }
}

b）ip_hash ：客戶機的IP地址用作散列鍵，用於確定應該為客戶機的請求選擇服務器組中的哪個服務器

stream{
    upstream rabbitmq{
        ip_hash;
        server 192.168.20.100:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.101:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.102:5672 max_fails=2 fail_timeout=5s weight=2;
    }
    server{
        listen 5672;
        proxy_connect_timeout 10s;
        proxy_timeout 300s;
        proxy_pass rabbitmq;
    }
}

注：這個least time均衡方法沒有

c）普通的hash算法：nginx plus選擇這個server是通過user_defined 關鍵字，就是IP地址：$remote_addr;

stream{
    upstream rabbitmq{
        hash $remote_addr consistent;
        server 192.168.20.100:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.101:5672 max_fails=2 fail_timeout=5s weight=2;
        server 192.168.20.102:5672 max_fails=2 fail_timeout=5s weight=2 max_conns=3;;
    }
    server{
        listen 5672;
        proxy_connect_timeout 10s;
        proxy_timeout 300s;
        proxy_pass rabbitmq;
    }
}