http://sxca.miit.gov.cn/txxh/xujfil/4749.hrh
root@ # iptables -vnL cali-PREROUTING -t nat Chain cali-PREROUTING (1 references) pkts bytes target prot opt in out source destination 105K 15M cali-fip-dnat all -- * * 0.0.0.0/0 0.0.0.0/0 /* cali:r6XmIziWUJsdOK6Z */ root@ # iptables -vnL PREROUTING -t nat Chain PREROUTING (policy ACCEPT 33 packets, 2938 bytes) pkts bytes target prot opt in out source destination 588 49101 cali-PREROUTING all -- * * 0.0.0.0/0 0.0.0.0/0 /* cali:6gwbT8clXdHdC1b1 */ 578 48301 KUBE-SERVICES all -- * * 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ 7144 430K CNI-HOSTPORT-DNAT all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL 5258 316K DOCKER all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL root@: #
root # iptables -t nat -L PREROUTING -n --line-number Chain PREROUTING (policy ACCEPT) num target prot opt source destination 1 cali-PREROUTING all -- 0.0.0.0/0 0.0.0.0/0 /* cali:6gwbT8clXdHdC1b1 */ 2 DNAT tcp -- 0.0.0.0/0 10.10.16.48 tcp dpt:6000 to:172.17.0.4:6000 3 KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ 4 CNI-HOSTPORT-DNAT all -- 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL 5 DOCKER all -- 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL root # iptables -t nat -D PREROUTING 2 roo # iptables -t nat -L cali-PREROUTING -n --line-number Chain cali-PREROUTING (1 references) num target prot opt source destination 1 cali-fip-dnat all -- 0.0.0.0/0 0.0.0.0/0 /* cali:r6XmIziWUJsdOK6Z */ root # iptables -t nat -L cali-fip-dnat -n --line-number Chain cali-fip-dnat (2 references) num target prot opt source destination root # netstat -lpn | grep 6000 root #
root@ # iptables -t nat -L PREROUTING -n --line-number Chain PREROUTING (policy ACCEPT) num target prot opt source destination 1 cali-PREROUTING all -- 0.0.0.0/0 0.0.0.0/0 /* cali:6gwbT8clXdHdC1b1 */ 2 KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ 3 CNI-HOSTPORT-DNAT all -- 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL 4 DOCKER all -- 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL root@ # iptables -vnL cali-PREROUTING -t nat Chain cali-PREROUTING (1 references) pkts bytes target prot opt in out source destination 105K 15M cali-fip-dnat all -- * * 0.0.0.0/0 0.0.0.0/0 /* cali:r6XmIziWUJsdOK6Z */ root@ # iptables -vnL PREROUTING -t nat Chain PREROUTING (policy ACCEPT 33 packets, 2938 bytes) pkts bytes target prot opt in out source destination 588 49101 cali-PREROUTING all -- * * 0.0.0.0/0 0.0.0.0/0 /* cali:6gwbT8clXdHdC1b1 */ 578 48301 KUBE-SERVICES all -- * * 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ 7144 430K CNI-HOSTPORT-DNAT all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL 5258 316K DOCKER all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL root # iptables -t nat -L KUBE-SERVICES -n --line-number Chain KUBE-SERVICES (2 references) num target prot opt source destination 1 KUBE-MARK-MASQ tcp -- !10.244.0.0/16 10.96.0.1 /* default/kubernetes:https cluster IP */ tcp dpt:443 2 KUBE-SVC-NPX46M4PTMTKRN6Y tcp -- 0.0.0.0/0 10.96.0.1 /* default/kubernetes:https cluster IP */ tcp dpt:443 3 KUBE-MARK-MASQ udp -- !10.244.0.0/16 10.96.0.10 /* kube-system/kube-dns:dns cluster IP */ udp dpt:53 4 KUBE-SVC-TCOU7JCQXEZGVUNU udp -- 0.0.0.0/0 10.96.0.10 /* kube-system/kube-dns:dns cluster IP */ udp dpt:53 5 KUBE-MARK-MASQ tcp -- !10.244.0.0/16 10.96.0.10 /* kube-system/kube-dns:dns-tcp cluster IP */ tcp dpt:53 6 KUBE-SVC-ERIFXISQEP7F7OF4 tcp -- 0.0.0.0/0 10.96.0.10 /* kube-system/kube-dns:dns-tcp cluster IP */ tcp dpt:53 7 KUBE-MARK-MASQ tcp -- !10.244.0.0/16 10.96.0.10 /* kube-system/kube-dns:metrics cluster IP */ tcp dpt:9153 8 KUBE-SVC-JD5MR3NA4I4DYORP tcp -- 0.0.0.0/0 10.96.0.10 /* kube-system/kube-dns:metrics cluster IP */ tcp dpt:9153 9 KUBE-NODEPORTS all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes service nodeports; NOTE: this must be the last rule in this chain */ ADDRTYPE match dst-type LOCAL root@pcl-01:/home/pcl# iptables -t nat -L CNI-HOSTPORT-DNAT -n --line-number Chain CNI-HOSTPORT-DNAT (2 references) num target prot opt source destination root@pcl-01:/home/pcl# iptables -t nat -L DOCKER -n --line-number Chain DOCKER (2 references) num target prot opt source destination 1 RETURN all -- 0.0.0.0/0 0.0.0.0/0 root #
方說我們使用docker容器，一定會在iptables的NAT表中看到下在這樣的一條配置規則：
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER從整體上看，這條規則是要把符合什么匹配規則的數據包，在數據包進入NAT表PREROUTING鏈時，讓它直接jump到一個名為DOCKER的鏈。至於在這個DOCKER的鏈中有哪些繼續生效的NAT規則，不是我們要討論的。
我們主要是分析一下“-m addrtype --dst-type”的數據包匹配規則該做怎樣的理解。
首先是，-m addrtype。
iptables提供了眾多的擴展模塊，以支持更多的功能。addrtype就是這樣的一個擴展模塊，提供的是Address type match的功能。引用的方式就是 -m 模塊名。
對於iptables擴展模塊應用的最多的莫過於在INPUT表中的類似下面這個規則了：
-A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT注：至於conntrack和state兩個模塊的異同，其實沒有什么差別，效果一致。
其次是，--dst-type
我們可以通過查看addrtype模塊的使用說明來了解--dst-type選項的使用方法以及取值范圍。
從上面的內容可以看到該模塊支持按源地址或目標地址類型去做匹配，支持的地址類型有很多種，比如LOCAL表示是本地網絡地址，BROADCAST表示匹配廣播地址，以及其它各種特殊用途的地址類型。
所以回到開頭的那條規則上，其作用就是：把目標地址類型屬於主機系統的本地網絡地址的數據包，在數據包進入NAT表PREROUTING鏈時，都讓它們直接jump到一個名為DOCKER的鏈。
root # iptables -S -P INPUT ACCEPT -P FORWARD DROP --默認是Drop -P OUTPUT ACCEPT -N DOCKER -N DOCKER-ISOLATION-STAGE-1 -N DOCKER-ISOLATION-STAGE-2 -N DOCKER-USER -N KUBE-EXTERNAL-SERVICES -N KUBE-FIREWALL -N KUBE-FORWARD -N KUBE-SERVICES -N cali-FORWARD -N cali-INPUT -N cali-OUTPUT -N cali-failsafe-in -N cali-failsafe-out -N cali-from-hep-forward -N cali-from-host-endpoint -N cali-from-wl-dispatch -N cali-fw-cali264323708e2 -N cali-fw-calic6133e3c424 -N cali-pri-_u2Tn2rSoAPffvE7JO6 -N cali-pri-kns.kube-system -N cali-pro-_u2Tn2rSoAPffvE7JO6 -N cali-pro-kns.kube-system -N cali-to-hep-forward -N cali-to-host-endpoint -N cali-to-wl-dispatch -N cali-tw-cali264323708e2 -N cali-tw-calic6133e3c424 -N cali-wl-to-host -A INPUT -m comment --comment "cali:Cz_u1IQiXIMmKD4c" -j cali-INPUT -A INPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES -A INPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES -A INPUT -j KUBE-FIREWALL -A FORWARD -m comment --comment "cali:wUHhoiAYhphO9Mso" -j cali-FORWARD -A FORWARD -j DOCKER-ISOLATION-STAGE-1 -A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD -A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES -A FORWARD -j DOCKER-USER -A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -A FORWARD -o docker0 -j DOCKER -A FORWARD -i docker0 ! -o docker0 -j ACCEPT -A FORWARD -i docker0 -o docker0 -j ACCEPT -A FORWARD -s 10.244.0.0/16 -j ACCEPT -A FORWARD -d 10.244.0.0/16 -j ACCEPT -A OUTPUT -m comment --comment "cali:tVnHkvAo15HuiPy0" -j cali-OUTPUT -A OUTPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES -A OUTPUT -j KUBE-FIREWALL -A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2 -A DOCKER-ISOLATION-STAGE-1 -j RETURN -A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP -A DOCKER-ISOLATION-STAGE-2 -j RETURN -A DOCKER-USER -j RETURN
新增一條規則
iptables -A DOCKER -d 172.17.0.4/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6000 -j ACCEPT
root # iptables -A DOCKER -d 172.17.0.4/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6000 -j ACCEPT root : # iptables -vnL PREROUTING -t nat Chain PREROUTING (policy ACCEPT 4 packets, 160 bytes) pkts bytes target prot opt in out source destination 3169 228K cali-PREROUTING all -- * * 0.0.0.0/0 0.0.0.0/0 /* cali:6gwbT8clXdHdC1b1 */ 27 1620 DNAT tcp -- * * 0.0.0.0/0 10.10.16.48 tcp dpt:6000 to:172.17.0.4:6000 5286 378K KUBE-SERVICES all -- * * 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ 7147 430K CNI-HOSTPORT-DNAT all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL 5261 316K DOCKER all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL root : # iptables -vnL DOCKER Chain DOCKER (1 references) pkts bytes target prot opt in out source destination 1 60 ACCEPT tcp -- !docker0 docker0 0.0.0.0/0 172.17.0.4 tcp dpt:6000 root@pcl-01:/home/pcl#
client
conntrack表
conntrack -L | grep 10.10.16.81 tcp 6 86259 ESTABLISHED src=10.10.16.81 dst=10.10.16.48 sport=51030 dport=6000 src=172.17.0.4 dst=10.10.16.81 sport=6000 dport=51030 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 206 flow entries have been shown.
server
問題 docker固定容器ip前提是設置net為none，此情景下所有的網絡配置都失效，包括-p端口映射。 目的 使用其他的方法做端口映射，繞過net為none 方法 docker的端口映射並不是在docker技術中實現的，而是通過宿主機的iptables來實現；通過控制網橋來做端口映射，類似路由器中設置路由端口映射。 先檢查配置端口映射，iptable設置了什么 執行：docker run -d -p 9000:9000 redis_cluster 9000 root@ubuntu:~# iptables -t nat -L -n Chain PREROUTING (policy ACCEPT) target prot opt source destination DOCKER all -- 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL Chain INPUT (policy ACCEPT) target prot opt source destination Chain OUTPUT (policy ACCEPT) target prot opt source destination DOCKER all -- 0.0.0.0/0 !127.0.0.0/8 ADDRTYPE match dst-type LOCAL Chain POSTROUTING (policy ACCEPT) target prot opt source destination MASQUERADE all -- 172.17.0.0/16 0.0.0.0/0 MASQUERADE tcp -- 172.17.0.1 172.17.0.1 tcp dpt:9000 Chain DOCKER (2 references) target prot opt source destination DNAT tcp -- 0.0.0.0/0 0.0.0.0/0 tcp dpt:9000 to:172.17.0.1:9000 那么我們就可以自己寫DNAT的命令，讓外部的端口進行轉換… docker創建了一個名為DOKCER的自定義的鏈條Chain … … iptables自定義鏈條的好處就是可以讓防火牆的策略更加的層次化… … 查看命令 root@ubuntu:~# iptables -S -P INPUT ACCEPT -P FORWARD ACCEPT -P OUTPUT ACCEPT -N DOCKER -A FORWARD -o docker0 -j DOCKER -A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -A FORWARD -i docker0 ! -o docker0 -j ACCEPT -A FORWARD -i docker0 -o docker0 -j ACCEPT -A DOCKER -d 172.17.0.1/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 9000 -j ACCEPT root@ubuntu:~# iptables -t nat -S -P PREROUTING ACCEPT -P INPUT ACCEPT -P OUTPUT ACCEPT -P POSTROUTING ACCEPT -N DOCKER -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER -A PREROUTING -p tcp -m tcp --dport 80 -j DNAT --to-destination 172.17.0.1:80 -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER -A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE -A POSTROUTING -s 172.17.0.1/32 -d 172.17.0.1/32 -p tcp -m tcp --dport 9000 -j MASQUERADE -A DOCKER -p tcp -m tcp --dport 9000 -j DNAT --to-destination 172.17.0.1:9000 執行DNAT命令iptables -t nat -A PREROUTING -p tcp –dport 80 -j DNAT –to 172.17.0.1:80 將宿主的80端口映射到docker容器的172.17.0.1的80端口
root # conntrack -L | grep 10.10.16.81 tcp 6 86259 ESTABLISHED src=10.10.16.81 dst=10.10.16.48 sport=51030 dport=6000 src=172.17.0.4 dst=10.10.16.81 sport=6000 dport=51030 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 206 flow entries have been shown. root # conntrack -L | grep 172.17.0.4 tcp 6 83 TIME_WAIT src=10.10.16.1 dst=10.10.16.48 sport=7471 dport=6000 src=172.17.0.4 dst=10.10.16.1 sport=6000 dport=7471 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 206 flow entries have been shown. root # conntrack -L | grep 172.17.0.4 tcp 6 69 TIME_WAIT src=10.10.16.1 dst=10.10.16.48 sport=7471 dport=6000 src=172.17.0.4 dst=10.10.16.1 sport=6000 dport=7471 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 206 flow entries have been shown. root l# conntrack -L | grep 172.17.0.4 tcp 6 67 TIME_WAIT src=10.10.16.1 dst=10.10.16.48 sport=7471 dport=6000 src=172.17.0.4 dst=10.10.16.1 sport=6000 dport=7471 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 206 flow entries have been shown. root # conntrack -L | grep 172.17.0.4 tcp 6 55 TIME_WAIT src=10.10.16.1 dst=10.10.16.48 sport=7471 dport=6000 src=172.17.0.4 dst=10.10.16.1 sport=6000 dport=7471 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 206 flow entries have been shown.