haproxy透明代理验证

一、代理协议简介

代理协议即 PROXY protocol, 是 haproxy 的作者 Willy Tarreau 于 2010 年开发和设计的一个 Internet 协议，通过为 tcp 添加一个很小的头信息，来方便的传递客户端信息（协议栈、源 IP、目的 IP、源端口、目的端口等)，在网络情况复杂又需要获取客户 IP 时非常有用。如：

• 多层 NAT 网络
• TCP 代理（四层）或多层 tcp 代理
• https 反向代理 http (某些情况下由于 Keep-alive 导致不是每次请求都传递 x-forword-for)

代理协议分为 v1 和 v2 两个版本，v1 人类易读，v2 是二进制格式，方便程序处理。Proxy protocol 是比较新的协议，但目前已经有很多软件支持，如 haproxy、nginx、apache、squid、mysql 等等，要使用 proxy protocol 需要两个角色 sender 和 receiver,sender 在与 receiver 之间建立连接后，会先发送一个带有客户信息的 tcp header, 因为更改了 tcp 协议，需 receiver 也支持 proxy protocol，否则不能识别 tcp 包头，导致无法成功建立连接。

https://www.haproxy.com/blog/haproxy/proxy-protocol/

二、proxy protocol 格式

version 1 header

• 每个字段中间为空格(\x20)

// PROXY AF L3_SADDR L3_DADDR L4_SADDR L4_DADDR\r\n 
PROXY TCP4 202.112.144.236 10.210.12.10 5678 80\r\n
PROXY TCP6 2001:da8:205::100 2400:89c0:2110:1::21 6324 80\r\n
PROXY UKNOWN\r\n

version 2 header

• 12字节的固定signature

\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A

• 4bits 协议版本号

\x2 // v2

• 4bits cmd

\x0 // LOCAL
\x1 // PROXY

• 4bits 地址族

\x0 // AF_UNSPEC
\x1 // AF_INET
\x2 // AF_INET6
\x3 // AF_UNIX

• 4bits transport protocol

\x0 // UNSPEC
\x1 // STREAM
\x2 // DGRAM

• 2字节地址长度字段(网络字节序)，指接下来剩余的报头长度
• L3 S_ADDR
• L3 D_ADDR
• L4 S_ADDR (如果有)
• L4 D_ADDR (如果有)

Tips

• 目前nginx的v2版本的读取只支持 cmd 为 PROXY，transport protocol 为 STREAM
• v2版本支持了很多新特性，详情可参考官方文档(见reference)

三、nginx反向代理mysql验证tcp协议的透明代理

haproxy配置

# Configuration for wuzstest
global
    daemon
    user nobody
    group haproxy
    log /dev/log local0 debug alert
    log /dev/log local1 notice alert
    stats socket /var/lib/neutron/lbaas/v2/0908ffd2-23ed-408b-ab30-46b3cb548785/haproxy_stats.sock mode 0666 level user
defaults
    log global
    retries 3
    option redispatch
    timeout connect 5000
    timeout client 50000
    timeout server 50000
frontend 85e95497-3708-4712-bb50-265eb0a794b3
    option tcplog
    maxconn 2000
    bind 192.168.88.16:3307
    mode tcp
    default_backend 8081356a-1048-4b30-b1a9-81b29898a363
backend 8081356a-1048-4b30-b1a9-81b29898a363
    mode tcp
    balance roundrobin
    server 3b5c2019-d2a1-4b7d-b2db-a591b110c21e 192.168.88.11:3307 weight 1 send-proxy
    server 8dfb5733-3336-466c-a92d-827b703daf50 192.168.88.15:3307 weight 1 send-proxy
    server ffbd967e-eaaa-4be8-9f31-e6d052c8794a 192.168.88.5:3307 weight 1 send-proxy

nginx配置

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
    worker_connections 1024;
}
http {
    log_format  main  '$proxy_protocol_addr - $remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
    include /etc/nginx/conf.d/*.conf;

    server {
        listen       80 default_server proxy_protocol;
        server_name  _;
        root         /usr/share/nginx/html;
        include /etc/nginx/default.d/*.conf;

        location / {
        }
        error_page 404 /404.html;
        location = /404.html {
        }
        error_page 500 502 503 504 /50x.html;
        location = /50x.html {
        }
    }
}
stream {
log_format proxy '$proxy_protocol_addr -- $remote_addr [$time_local] '
                 '$protocol $status $bytes_sent $bytes_received '
                 '$session_time "$upstream_addr" '
                 '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
    access_log /var/log/nginx/tcp-access.log proxy ;
server {
       listen 3307   proxy_protocol;
       proxy_pass db;
    }
    upstream db {
       server localhost:3306;
    }
}

配置文件主要注意2部分

1.日志格式log_format proxy，添加$proxy_protocol_addr

2.监听添加代理协议支持,listen 3307 proxy_protocol;

测试结果：

四、http协议监听的透明代理

haproxy配置

# Configuration for 测试http
global
    daemon
    user nobody
    group haproxy
    log /dev/log local0 debug alert
    log /dev/log local1 notice alert
    stats socket /var/lib/neutron/lbaas/v2/9863eccc-93f9-4cdd-99f8-60add1b7b0c1/haproxy_stats.sock mode 0666 level user
defaults
    log global
    retries 3
    option redispatch
    timeout connect 5000
    timeout client 50000
    timeout server 50000
frontend 38e3ce31-2c64-4d78-bbc7-f4319198fc5a
    option tcplog
    maxconn 2000
    option forwardfor
    bind 192.168.88.19:80
    mode http
    default_backend 67c28ff7-7782-484f-a5ca-84800c8a3837
backend 67c28ff7-7782-484f-a5ca-84800c8a3837
    mode http
    balance roundrobin
    cookie SRV insert indirect nocache
    timeout check 5s
    server 07d9f74e-1f19-4ee3-a441-6adff505b09d 192.168.88.15:80 weight 1 check inter 10s fall 3 cookie 07d9f74e-1f19-4ee3-a441-6adff505b09d send-proxy
    server 6ab8aac8-5bea-4a0d-b73a-1007935da3d6 192.168.88.5:80 weight 1 check inter 10s fall 3 cookie 6ab8aac8-5bea-4a0d-b73a-1007935da3d6 send-proxy
    server dd5d3b4b-763a-49cd-8a5c-40ff19d69dcf 192.168.88.11:80 weight 1 check inter 10s fall 3 cookie dd5d3b4b-763a-49cd-8a5c-40ff19d69dcf send-proxy

nginx配置：

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
    worker_connections 1024;
}
http {
    log_format  main  '$proxy_protocol_addr - $remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
    include /etc/nginx/conf.d/*.conf;

    server {
        listen       80 default_server proxy_protocol;
        server_name  _;
        root         /usr/share/nginx/html;
        include /etc/nginx/default.d/*.conf;

        location / {
        }
        error_page 404 /404.html;
        location = /404.html {
        }
        error_page 500 502 503 504 /50x.html;
        location = /50x.html {
        }
    }
}

验证结果

参考：https://www.nixops.me/articles/PROXY_protocol_pass_client_ip.html
https://www.jianshu.com/p/cc8d592582c9