部署OVN網絡拓撲
OVN-安裝軟件包
/etc/yum.repos.d/CentOS-OpenStack-ocata.repo
# yum list installed | grep openvswitch
openvswitch.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-devel.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-ovn-central.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-ovn-common.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-ovn-docker.x86_64 1:2.6.1-10.1.git20161206.el7
openvswitch-ovn-host.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-ovn-vtep.x86_64 1:2.9.0-3.el7 @centos-openstack-ocata
openvswitch-test.noarch 1:2.9.0-3.el7 @centos-openstack-ocata
python2-openvswitch.noarch 1:2.9.0-3.el7 @centos-openstack-ocata
### 關閉防火牆和SELINUX
# systemctl stop firewalld
# systemctl disable firewalld
### Central節點
# yum install -y openvswitch-ovn-central.x86_64 openvswitch-ovn-host.x86_64
# systemctl enable ovn-northd openvswitch ovn-controller
# systemctl start ovn-northd ovn-controller
### Node節點
# yum install -y openvswitch.x86_64 openvswitch-ovn-host.x86_64
# systemctl enable openvswitch ovn-controller
# systemctl start ovn-controller
配置OVN
export Centralip=10.33.46.182
export Nodeip=10.33.46.68
### Central節點
# ovn-nbctl set-connection ptcp:6641:$Centralip
# ovn-sbctl set-connection ptcp:6642:$Centralip
# ovs-vsctl set open . external-ids:ovn-remote=tcp:$Centralip:6642
# ovs-vsctl set open . external-ids:ovn-encap-type=geneve
# ovs-vsctl set open . external-ids:ovn-encap-ip=$Centralip
### Node節點
# ovs-vsctl set open . external-ids:ovn-remote=tcp:$Centralip:6642
# ovs-vsctl set open . external-ids:ovn-encap-type=geneve
# ovs-vsctl set open . external-ids:ovn-encap-ip=$Nodeip
tcp 0 1 10.33.46.68:43132 10.33.46.68:6642 SYN_SENT 0 20089825 44936/ovn-controlle
#reboot后可以建立連接
system-id是ovn-sbctl中的Chassis的ID
OVN-L2網絡
定義邏輯網絡:創建一個邏輯交換機，然后添加兩個交換機端口，並為端口設置物理地址
# $FQDN|md5sum|sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'
export vm1mac=02:d4:1d:8c:d9:8f
export vm2mac=02:d4:1d:8c:d9:8e
export vm1ip=172.16.255.11
export vm2ip=172.16.255.22
### Central節點
# ovn-nbctl ls-add ls1
# ovn-nbctl lsp-add ls1 ls1-vm1
# ovn-nbctl lsp-set-addresses ls1-vm1 $vm1mac
# ovn-nbctl lsp-set-port-security ls1-vm1 $vm1mac
# ovn-nbctl lsp-add ls1 ls1-vm2
# ovn-nbctl lsp-set-addresses ls1-vm2 $vm2mac
# ovn-nbctl lsp-set-port-security ls1-vm2 $vm2mac
偽造虛擬機:創建網絡命名空間，並在br-int上添加端口，然后將端口添加到命名空間，最后通過設置端口的MAC地址和網卡名完成和交換機端口的映射
### Central節點
# ip netns add vm1
# ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
# ip link set vm1 netns vm1
# ip netns exec vm1 ip link set vm1 address $vm1mac
# ip netns exec vm1 ip addr add $vm1ip/24 dev vm1
# ip netns exec vm1 ip link set vm1 up
# ovs-vsctl set Interface vm1 external_ids:iface-id=ls1-vm1
# ip netns exec vm1 ip addr show
### Node節點
# ip netns add vm2
# ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
# ip link set vm2 netns vm2
# ip netns exec vm2 ip link set vm2 address $vm2mac
# ip netns exec vm2 ip addr add $vm2ip/24 dev vm2
# ip netns exec vm2 ip link set vm2 up
# ovs-vsctl set Interface vm2 external_ids:iface-id=ls1-vm2
# ip netns exec vm2 ip addr show
vm互ping，可ping通
OVN- L3網絡
添加L3網關
### 創建邏輯路由
# ovn-nbctl lr-add edge1
### 創建邏輯交換機用於連接edge1和tenant1
# ovn-nbctl ls-add transit
### 連接edge1到邏輯交換機上
# ovn-nbctl lrp-add edge1 edge1-transit 02:d4:1d:8c:d9:ae 192.168.0.1/24
# ovn-nbctl lsp-add transit transit-edge1
# ovn-nbctl lsp-set-type transit-edge1 router
# ovn-nbctl lsp-set-addresses transit-edge1 02:d4:1d:8c:d9:ae
# ovn-nbctl lsp-set-options transit-edge1 router-port=edge1-transit
### 連接tenant1到邏輯交換機上
# ovn-nbctl lrp-add tenant1 tenant1-transit 02:d4:1d:8c:d9:af 192.168.0.2/24
# ovn-nbctl lsp-add transit transit-tenant1
# ovn-nbctl lsp-set-type transit-tenant1 router
# ovn-nbctl lsp-set-addresses transit-tenant1 02:d4:1d:8c:d9:af
# ovn-nbctl lsp-set-options transit-tenant1 router-port=tenant1-transit
### 添加靜態路由
ovn-nbctl lr-route-add edge1 "20.0.0.0/24" 192.168.0.2
ovn-nbctl lr-route-add edge1 "10.0.0.0/24" 192.168.0.2
ovn-nbctl lr-route-add tenant1 "0.0.0.0/0" 192.168.0.1
ovn-nbctl lr-route-list edge1
ovn-nbctl lr-route-list tenant1
### 測試連通性
ip netns exec vm21 ping -c 2 192.168.0.1
網關與外網連接
### Central節點
### 創建外網邏輯交換機，並配置網關到叫交換機的連接
ovn-nbctl ls-add outside
ovn-nbctl lrp-add edge1 edge1-outside 02:d4:1d:8c:d9:be 192.168.233.177/24
ovn-nbctl lsp-add outside outside-edge1
ovn-nbctl lsp-set-type outside-edge1 router
ovn-nbctl lsp-set-addresses outside-edge1 02:d4:1d:8c:d9:be
ovn-nbctl lsp-set-options outside-edge1 router-port=edge1-outside
### 為外網網卡ens4創建網橋
ovs-vsctl add-br br-ex
### 為外網網卡ens4創建網橋到網絡的映射
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=dataNet:br-ex
### 在邏輯交換機outside上添加本地網絡端口，並且本地網絡的名字為dataNet
ovn-nbctl lsp-add outside outside-localnet
ovn-nbctl lsp-set-addresses outside-localnet unknown
ovn-nbctl lsp-set-type outside-localnet localnet
ovn-nbctl lsp-set-options outside-localnet network_name=dataNet
### 關聯外網網卡到網橋上
# ovs-vsctl add-port br-ex eth1
### 測試連通性（需要注意vm2的ip地址是不是沒了，dhclient好像有些問題）
ip netns exec vm22 ping -c 2 192.168.233.177
### 設置網橋地址
ip addr add 192.168.233.7/24 dev br-ex
ip link set br-ex up
### 重置下路由
# ip route
# ip route del default via 192.168.233.1
# ip route del 192.168.233.0/24 dev eth1
設置SNAT
### Central節點
### 設置網關chassis
ovn-nbctl lrp-set-gateway-chassis edge1-outside 35a10447-0513-4f8f-a340-33220258b9d9 #為ovn-snctl show對應的Chassis ID
### 配置SNAT規則
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=20.0.0.0/24 external_ip=192.168.233.177 -- add logical_router edge1 nat @nat
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=10.0.0.0/24 external_ip=192.168.233.177 -- add logical_router edge1 nat @nat
/var/log/openvswitch/ovn-controller.log：
2019-07-09T00:37:14.205Z|00265|ofctrl|INFO|OpenFlow error: OFPT_ERROR (OF1.3) (xid=0x9f7): NXBAC_CT_DATAPATH_SUPPORT
OFPT_FLOW_MOD (OF1.3) (xid=0x9f7): ADD table:41 priority=25,ip,reg15=0x2,metadata=0x5,nw_src=20.0.0.0/24 cookie:0x71db37d actions=ct(commit,table=42,zone=NXM_NX_REG12[0..15],nat(src=192.168.233.177))
2019-07-09T00:37:14.205Z|00266|ofctrl|INFO|OpenFlow error: OFPT_ERROR (OF1.3) (xid=0x9fa): NXBAC_CT_DATAPATH_SUPPORT
OFPT_FLOW_MOD (OF1.3) (xid=0x9fa): ADD table:11 priority=100,ip,reg14=0x2,metadata=0x5,nw_dst=192.168.233.177 cookie:0x236f944b actions=ct(table=12,zone=NXM_NX_REG12[0..15],nat)
對應的命令：
ovs-ofctl add-flow br-int "table=41,priority=25,ip,reg15=0x2,metadata=0x5,nw_src=20.0.0.0/24 cookie:0x71db37d actions=ct(commit,table=42,zone=NXM_NX_REG12[0..15],nat(src=192.168.233.177))"
詳細信息：
ovs-ofctl add-flow br-int "table=41,priority=25,ip,reg15=0x2,metadata=0x5,nw_src=20.0.0.0/24 cookie:0x71db37d actions=ct(commit,table=42,zone=NXM_NX_REG12[0..15],nat(src=192.168.233.177))" -v
失敗原因：OFPT_FLOW_MOD
內核datapath不支持該特性，環境部署問題
查看發現：
# kmod list | grep openvs
openvswitch 84535 1 vport_geneve
libcrc32c 12644 2 xfs,openvswitch
正常情況下：（原因是我用的是虛機搭建的環境，centos7.2版本，7.4版本ok）
[root@Images ~]# kmod list | grep openvs
[root@Images ~]# modprobe openvswitch
[root@Images ~]# kmod list | grep openvs
openvswitch 106739 0
nf_nat_ipv6 14131 1 openvswitch
nf_defrag_ipv6 35104 2 openvswitch,nf_conntrack_ipv6
nf_nat_ipv4 14115 2 openvswitch,iptable_nat
nf_nat 26147 4 openvswitch,nf_nat_ipv4,nf_nat_ipv6,nf_nat_masquerade_ipv4
nf_conntrack 111302 8 openvswitch,nf_nat,nf_nat_ipv4,nf_nat_ipv6,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_ipv4,nf_conntrack_ipv6
libcrc32c 12644 2 xfs,openvswitch
# ovs-dpctl show -v
02:d4:1d:8c:d9:be > c2:af:5a:9e:73:47, ethertype IPv4 (0x0800), length 98: 20.0.0.20 > 192.168.233.7: ICMP echo request, id 24614, seq 508, length 64
### 測試連通性
ip netns exec vm22 ping -c 2 192.168.233.7
簡化環境后也可以：
流表分析
table 0 主要工作如下：
l 完成物理到邏輯的翻譯，將邏輯信息，比如上面提到的信息記錄到寄存器中。
l VM中的容器的報文用VLAN進行區分
l 別的chassis過來的報文，根據入端口和tunnel_id進行區分，然后獲取出端口，這個在封裝的時候已經有了
table 16-31 主要是將邏輯流表ingress pipeline 0-15 的操作部分轉換為openflow流表，主要工作如下：
l 每個邏輯流表會映射一個或者多個openflow流表，通常報文只是匹配其中一條流表。
l ovn-controller使用邏輯流表的UUID的前32位作為openflow流表的cookie值。查看邏輯流表的UUID使用ovn-sbctl list Logical_Flow，對應上面cookie的邏輯流表的UUID的信息在這里。
l 一些邏輯流表可以映射到ovs的”conjunctive match”擴展名(參見這里)，這時候因為一條openflow流表對應了多條邏輯流表，所以cookie為0。這里的”conjunctive match”表示一個集合的匹配，比如tcp_src ∈ {80, 443, 8080} and tcp_dst ∈ {80, 443, 8080}。
l 一些邏輯流表可能不會轉換成openflow流表，如果交換機上虛擬接口沒有添加到ovs中，添加命令ovs-vsctl set Interface veth2_b external_ids:iface-id=ls2-vm4，那么相應的openflow流表將不會生成。
l 最后就是有一些邏輯流表和openflow流表很明顯的對應操作關系，我們列一下
l next對應resubmit
l field = constant對應set_field
l output，將報文resubmit到表32，如果邏輯流表有多個output操作，那么每個都要resubmit到表32。
l get_arp(P, A)和get_nd(P, A)，通過講參數存儲在openflow字段中(上面例子中存儲在NXM_NX_REG0，流表cookie=0x5dbc664)，然后resubmit到表66，然后ovn-controller從MAC_Binding表生成流填充，如果表66中有匹配項，其action將綁定的MAC存儲在目的MAC地址字段中
l put_arp(P, A, E)和put_nd(P, A, E)講參數存儲到openflow的字段中(字段太多，查看上面流表cookie=0x92af5d1c)，然后更新MAC_Binding表中。
table 32-47 主要是將邏輯流表ingress pipeline的output action轉換為openflow流表。以下詳細介紹下：
表32主要是處理到其他宿主機中虛擬機的報文，講VNI設置到metadata，然后resubmit到表33
表33主要是將報文resubmit到表34，對於多個邏輯output端口的時候，需要改為每個邏輯端口P，然后resubmit到表34
表34檢查報文的邏輯ingress和egress的端口是否一致，一致則丟棄。剩下的resubmit到表48
table 48-63 主要是講邏輯流表的egress pipeline部分轉換成openflow流表，這塊屬於報文發送之前的最后驗證，最終resubmit到表64，最終沒有執行output的報文將被丟棄。
table 64 貌似和loopback有關，修改邏輯入端口。
table 65 邏輯到物理的轉換，和表0相反，主要是將找到邏輯端口對應的物理端口，然后發送，如果虛擬機中還有容器的話，需要添加vlan頭。
table 66 主要是對應MAC_Binding中的數據，來修改目的IP對應的目的MAC，功能類似arp。
# ovs-ofctl dump-flows br-int
//cookie沒有值表示不是直接從邏輯流表轉換而來的
//兩個虛擬機進來的報文進行一些寄存器的操作，這個不是根據邏輯流表來的，但是和邏輯拓撲還是有關系的，具體這些寄存器的意義和獲取我們下面介紹
cookie=0x0, table=0, priority=100,in_port=4 actions=load:0x1->NXM_NX_REG13[],load:0x6->NXM_NX_REG11[],load:0x8->NXM_NX_REG12[],load:0x3->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,16)
cookie=0x0, table=0, priority=100,in_port=3 actions=load:0x2->NXM_NX_REG13[],load:0x7->NXM_NX_REG11[],load:0x5->NXM_NX_REG12[],load:0x2->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,16)
//表示從其他宿主機發送過來的報文應該如何處理，這里的tun_id分別表示從兩個邏輯交換中的哪一個發送過來的
cookie=0x0, table=0, priority=100,tun_id=0x3,in_port=7 actions=move:NXM_NX_TUN_ID[0..23]->OXM_OF_METADATA[0..23],load:0x3->NXM_NX_REG14[0..14],load:0x1->NXM_NX_REG10[1],resubmit(,16)
cookie=0x0, table=0, priority=100,tun_id=0x2,in_port=7 actions=move:NXM_NX_TUN_ID[0..23]->OXM_OF_METADATA[0..23],load:0x3->NXM_NX_REG14[0..14],load:0x1->NXM_NX_REG10[1],resubmit(,16)
//一些我們不關注的流表主要是一些錯誤報文的丟棄操作，相關流表已經刪除了
//以下metadata不是1表示從邏輯交換發過來的報文怎么處理，前面的reg14表示從哪個邏輯端口發送過來的
cookie=0xa7c014e8, table=16, priority=50,reg14=0x2,metadata=0x3,dl_src=52:54:00:c1:68:71 actions=resubmit(,17)
cookie=0x3ed26758, table=16, priority=50,reg14=0x2,metadata=0x2,dl_src=52:54:00:c1:68:70 actions=resubmit(,17)
cookie=0x11dd5c04, table=16, priority=50,reg14=0x3,metadata=0x2,dl_src=52:54:00:c1:68:72 actions=resubmit(,17)
cookie=0x6126e3c1, table=16, priority=50,reg14=0x3,metadata=0x3,dl_src=52:54:00:c1:68:73 actions=resubmit(,17)
cookie=0x75e7ab7b, table=16, priority=50,reg14=0x1,metadata=0x2 actions=resubmit(,17)
cookie=0x8c78254f, table=16, priority=50,reg14=0x1,metadata=0x3 actions=resubmit(,17)
//以下metadata為1表示從邏輯路由過來的報文，需要進行怎樣的操作
cookie=0xd9caf1fd, table=16, priority=50,reg14=0x1,metadata=0x1,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,17)
cookie=0xeac605df, table=16, priority=50,reg14=0x2,metadata=0x1,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,17)
cookie=0x819b5118, table=16, priority=50,reg14=0x1,metadata=0x1,dl_dst=52:54:00:c1:68:50 actions=resubmit(,17)
cookie=0xbe725a2b, table=16, priority=50,reg14=0x2,metadata=0x1,dl_dst=52:54:00:c1:68:60 actions=resubmit(,17)
//arp代答的流表
cookie=0xf4ca156, table=17, priority=90,arp,reg14=0x2,metadata=0x1,arp_tpa=192.168.2.1,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:52:54:00:c1:68:60,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x525400c16860->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xc0a80201->NXM_OF_ARP_SPA[],load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0xb5d8c2e4, table=17, priority=90,arp,reg14=0x1,metadata=0x1,arp_tpa=192.168.1.1,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:52:54:00:c1:68:50,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x525400c16850->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xc0a80101->NXM_OF_ARP_SPA[],load:0x1->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
//arp回復報文的信息存入MAC_Binding
cookie=0x92af5d1c, table=17, priority=90,arp,metadata=0x1,arp_op=2 actions=push:NXM_NX_REG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ARP_SHA[],push:NXM_OF_ARP_SPA[],pop:NXM_NX_REG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.01.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_REG0[]
//icmp代答
cookie=0x815a3063, table=17, priority=90,icmp,metadata=0x1,nw_dst=192.168.1.1,icmp_type=8,icmp_code=0 actions=push:NXM_OF_IP_SRC[],push:NXM_OF_IP_DST[],pop:NXM_OF_IP_SRC[],pop:NXM_OF_IP_DST[],load:0xff->NXM_NX_IP_TTL[],load:0->NXM_OF_ICMP_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,18)
cookie=0xf3d609b1, table=17, priority=90,icmp,metadata=0x1,nw_dst=192.168.2.1,icmp_type=8,icmp_code=0 actions=push:NXM_OF_IP_SRC[],push:NXM_OF_IP_DST[],pop:NXM_OF_IP_SRC[],pop:NXM_OF_IP_DST[],load:0xff->NXM_NX_IP_TTL[],load:0->NXM_OF_ICMP_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,18)
//三個邏輯設備的流量繼續往下走
cookie=0x56295f89, table=17, priority=0,metadata=0x1 actions=resubmit(,18)
cookie=0x791195e0, table=17, priority=0,metadata=0x3 actions=resubmit(,18)
cookie=0x4b1c93d4, table=17, priority=0,metadata=0x2 actions=resubmit(,18)
//arp通過
cookie=0x4a80a501, table=18, priority=90,arp,reg14=0x3,metadata=0x3,dl_src=52:54:00:c1:68:73,arp_sha=52:54:00:c1:68:73 actions=resubmit(,19)
cookie=0xc6c881ee, table=18, priority=90,arp,reg14=0x3,metadata=0x2,dl_src=52:54:00:c1:68:72,arp_sha=52:54:00:c1:68:72 actions=resubmit(,19)
cookie=0x9e2a7562, table=18, priority=90,arp,reg14=0x2,metadata=0x2,dl_src=52:54:00:c1:68:70,arp_sha=52:54:00:c1:68:70 actions=resubmit(,19)
cookie=0x686267fe, table=18, priority=90,arp,reg14=0x2,metadata=0x3,dl_src=52:54:00:c1:68:71,arp_sha=52:54:00:c1:68:71 actions=resubmit(,19)
//繼續
cookie=0xb76a420f, table=18, priority=0,metadata=0x2 actions=resubmit(,19)
cookie=0x3ecbeeec, table=18, priority=0,metadata=0x1 actions=resubmit(,19)
cookie=0x78c16fb8, table=18, priority=0,metadata=0x3 actions=resubmit(,19)
//繼續
cookie=0x76f9414c, table=19, priority=0,metadata=0x3 actions=resubmit(,20)
cookie=0xff75779d, table=19, priority=0,metadata=0x2 actions=resubmit(,20)
cookie=0xa4a71b19, table=19, priority=0,metadata=0x1 actions=resubmit(,20)
//繼續
cookie=0x4c209f08, table=20, priority=0,metadata=0x3 actions=resubmit(,21)
cookie=0xc99c5154, table=20, priority=0,metadata=0x1 actions=resubmit(,21)
cookie=0xe187a6b4, table=20, priority=0,metadata=0x2 actions=resubmit(,21)
//conntrack記錄
cookie=0x5c49d2d2, table=21, priority=100,ip,reg0=0x1/0x1,metadata=0x3 actions=ct(table=22,zone=NXM_NX_REG13[0..15])
cookie=0x596e0c95, table=21, priority=100,ip,reg0=0x1/0x1,metadata=0x2 actions=ct(table=22,zone=NXM_NX_REG13[0..15])
//模擬過網關時的操作
cookie=0xaea49216, table=21, priority=49,ip,metadata=0x1,nw_dst=192.168.1.0/24 actions=dec_ttl(),move:NXM_OF_IP_DST[]->NXM_NX_XXREG0[96..127],load:0xc0a80101->NXM_NX_XXREG0[64..95],mod_dl_src:52:54:00:c1:68:50,load:0x1->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,22)
cookie=0x3ebae949, table=21, priority=49,ip,metadata=0x1,nw_dst=192.168.2.0/24 actions=dec_ttl(),move:NXM_OF_IP_DST[]->NXM_NX_XXREG0[96..127],load:0xc0a80201->NXM_NX_XXREG0[64..95],mod_dl_src:52:54:00:c1:68:60,load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,22)
//繼續
cookie=0xe3a08e2b, table=21, priority=0,metadata=0x3 actions=resubmit(,22)
cookie=0x80407476, table=21, priority=0,metadata=0x2 actions=resubmit(,22)
//獲取MAC_Binding表里的數據，回復arp
cookie=0x5dbc664, table=22, priority=0,ip,metadata=0x1 actions=push:NXM_NX_REG0[],push:NXM_NX_XXREG0[96..127],pop:NXM_NX_REG0[],mod_dl_dst:00:00:00:00:00:00,resubmit(,66),pop:NXM_NX_REG0[],resubmit(,23)
//繼續
cookie=0x66236a1, table=22, priority=0,metadata=0x2 actions=resubmit(,23)
cookie=0xefaed143, table=22, priority=0,metadata=0x3 actions=resubmit(,23)
//繼續
cookie=0x3998ed82, table=23, priority=0,metadata=0x1 actions=resubmit(,24)
cookie=0xc475a7b3, table=23, priority=0,metadata=0x3 actions=resubmit(,24)
cookie=0xacda159d, table=23, priority=0,metadata=0x2 actions=resubmit(,24)
//????發送arp？
cookie=0xe51fffad, table=24, priority=100,ip,metadata=0x1,dl_dst=00:00:00:00:00:00 actions=controller(userdata=00.00.00.00.00.00.00.00.00.19.00.10.80.00.06.06.ff.ff.ff.ff.ff.ff.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.40.00.00.00.01.de.10.00.00.20.04.ff.ff.00.18.00.00.23.20.00.06.00.20.00.60.00.00.00.01.de.10.00.00.22.04.00.19.00.10.80.00.2a.02.00.01.00.00.00.00.00.00.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
//繼續
cookie=0xd9c9912b, table=24, priority=0,metadata=0x1 actions=resubmit(,32)
cookie=0x9b703aff, table=24, priority=0,metadata=0x2 actions=resubmit(,25)
cookie=0xd44f4b41, table=24, priority=0,metadata=0x3 actions=resubmit(,25)
//conntrack lb
cookie=0xed10c525, table=25, priority=100,ip,reg0=0x4/0x4,metadata=0x3 actions=ct(table=26,zone=NXM_NX_REG13[0..15],nat)
cookie=0xb0869023, table=25, priority=100,ip,reg0=0x4/0x4,metadata=0x2 actions=ct(table=26,zone=NXM_NX_REG13[0..15],nat)
//conntrack
cookie=0xc8dfda6d, table=25, priority=100,ip,reg0=0x2/0x2,metadata=0x2 actions=ct(commit,zone=NXM_NX_REG13[0..15],exec(load:0->NXM_NX_CT_LABEL[0])),resubmit(,26)
cookie=0xf71a37ba, table=25, priority=100,ip,reg0=0x2/0x2,metadata=0x3 actions=ct(commit,zone=NXM_NX_REG13[0..15],exec(load:0->NXM_NX_CT_LABEL[0])),resubmit(,26)
//繼續
cookie=0x3c4b37a7, table=25, priority=0,metadata=0x2 actions=resubmit(,26)
cookie=0x315f30b3, table=25, priority=0,metadata=0x3 actions=resubmit(,26)
//繼續
cookie=0x4368d2e8, table=26, priority=0,metadata=0x3 actions=resubmit(,27)
cookie=0xf906a487, table=26, priority=0,metadata=0x2 actions=resubmit(,27)
cookie=0x1ab8df97, table=27, priority=0,metadata=0x3 actions=resubmit(,28)
cookie=0x8592b902, table=27, priority=0,metadata=0x2 actions=resubmit(,28)
cookie=0xe3f59b41, table=28, priority=0,metadata=0x3 actions=resubmit(,29)
cookie=0xba22fb48, table=28, priority=0,metadata=0x2 actions=resubmit(,29)
//泛洪
cookie=0x159f7998, table=29, priority=100,metadata=0x3,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=load:0xffff->NXM_NX_REG15[],resubmit(,32)
cookie=0xcbb8e72a, table=29, priority=100,metadata=0x2,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=load:0xffff->NXM_NX_REG15[],resubmit(,32)
//出口流量
cookie=0xc0e4e6a6, table=29, priority=50,metadata=0x2,dl_dst=52:54:00:c1:68:72 actions=load:0x3->NXM_NX_REG15[],resubmit(,32)
cookie=0x13381c84, table=29, priority=50,metadata=0x3,dl_dst=52:54:00:c1:68:73 actions=load:0x3->NXM_NX_REG15[],resubmit(,32)
cookie=0x23555b13, table=29, priority=50,metadata=0x2,dl_dst=52:54:00:c1:68:50 actions=load:0x1->NXM_NX_REG15[],resubmit(,32)
cookie=0x3f8b4ff9, table=29, priority=50,metadata=0x2,dl_dst=52:54:00:c1:68:70 actions=load:0x2->NXM_NX_REG15[],resubmit(,32)
cookie=0x615dbb2a, table=29, priority=50,metadata=0x3,dl_dst=52:54:00:c1:68:71 actions=load:0x2->NXM_NX_REG15[],resubmit(,32)
cookie=0xb88437bc, table=29, priority=50,metadata=0x3,dl_dst=52:54:00:c1:68:60 actions=load:0x1->NXM_NX_REG15[],resubmit(,32)
//????沒有flags為2的標志
cookie=0x0, table=32, priority=150,reg10=0x2/0x2 actions=resubmit(,33)
//到邏輯路由的流量
cookie=0x0, table=32, priority=100,reg15=0xffff,metadata=0x3 actions=load:0x1->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[],load:0x3->NXM_NX_TUN_ID[0..23],output:7,resubmit(,33)
cookie=0x0, table=32, priority=100,reg15=0xffff,metadata=0x2 actions=load:0x1->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[],load:0x2->NXM_NX_TUN_ID[0..23],output:7,resubmit(,33)
//到邏輯交換的流量
cookie=0x0, table=32, priority=100,reg15=0x3,metadata=0x2 actions=load:0x2->NXM_NX_TUN_ID[0..23],output:7
cookie=0x0, table=32, priority=100,reg15=0x3,metadata=0x3 actions=load:0x3->NXM_NX_TUN_ID[0..23],output:7
//繼續
cookie=0x0, table=32, priority=0 actions=resubmit(,33)
//????到網絡節點需要NAT的流量，可是我們沒有相應的配置
cookie=0x0, table=33, priority=100,reg15=0x1,metadata=0x3 actions=load:0x6->NXM_NX_REG11[],load:0x8->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x2,metadata=0x1 actions=load:0x3->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x2,metadata=0x2 actions=load:0x2->NXM_NX_REG13[],load:0x7->NXM_NX_REG11[],load:0x5->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x1,metadata=0x2 actions=load:0x7->NXM_NX_REG11[],load:0x5->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x1,metadata=0x1 actions=load:0x3->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, table=33, priority=100,reg15=0x2,metadata=0x3 actions=load:0x1->NXM_NX_REG13[],load:0x6->NXM_NX_REG11[],load:0x8->NXM_NX_REG12[],resubmit(,34)
//繼續
cookie=0x0, table=33, priority=100,reg15=0xffff,metadata=0x2 actions=load:0x2->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[]
cookie=0x0, table=33, priority=100,reg15=0xffff,metadata=0x3 actions=load:0x1->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[]
//繼續
cookie=0x0, table=34, priority=0 actions=load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],resubmit(,48)
//繼續
cookie=0x38579acc, table=48, priority=0,metadata=0x1 actions=resubmit(,49)
cookie=0x402567e, table=48, priority=0,metadata=0x3 actions=resubmit(,49)
cookie=0x7e6e093d, table=48, priority=0,metadata=0x2 actions=resubmit(,49)
//繼續
cookie=0xbce65dae, table=49, priority=0,metadata=0x2 actions=resubmit(,50)
cookie=0xf6e47c0e, table=49, priority=0,metadata=0x1 actions=resubmit(,50)
cookie=0xa630e910, table=49, priority=0,metadata=0x3 actions=resubmit(,50)
//conntrack
cookie=0xe6e35197, table=50, priority=100,ipv6,reg0=0x1/0x1,metadata=0x3 actions=ct(table=51,zone=NXM_NX_REG13[0..15])
cookie=0xa7a5e5f3, table=50, priority=100,ipv6,reg0=0x1/0x1,metadata=0x2 actions=ct(table=51,zone=NXM_NX_REG13[0..15])
cookie=0xa7a5e5f3, table=50, priority=100,ip,reg0=0x1/0x1,metadata=0x2 actions=ct(table=51,zone=NXM_NX_REG13[0..15])
cookie=0xe6e35197, table=50, priority=100,ip,reg0=0x1/0x1,metadata=0x3 actions=ct(table=51,zone=NXM_NX_REG13[0..15])
//繼續
cookie=0x4e268323, table=50, priority=0,metadata=0x1 actions=resubmit(,51)
cookie=0x2e28bd0c, table=50, priority=0,metadata=0x2 actions=resubmit(,51)
cookie=0x7cca0b71, table=50, priority=0,metadata=0x3 actions=resubmit(,51)
//需要輸出到邏輯路由的流量
cookie=0x1c84ef4, table=51, priority=100,reg15=0x2,metadata=0x1 actions=resubmit(,64)
cookie=0x83ce9e62, table=51, priority=100,reg15=0x1,metadata=0x1 actions=resubmit(,64)
//繼續
cookie=0x51c9cccf, table=51, priority=0,metadata=0x2 actions=resubmit(,52)
cookie=0x7778d918, table=51, priority=0,metadata=0x3 actions=resubmit(,52)
//繼續
cookie=0xa9ae4aaa, table=52, priority=0,metadata=0x2 actions=resubmit(,53)
cookie=0xe190604a, table=52, priority=0,metadata=0x3 actions=resubmit(,53)
cookie=0x934c95d9, table=53, priority=0,metadata=0x3 actions=resubmit(,54)
cookie=0x828e0c10, table=53, priority=0,metadata=0x2 actions=resubmit(,54)
//conntrack lb
cookie=0xb1d05c18, table=54, priority=100,ip,reg0=0x4/0x4,metadata=0x3 actions=ct(table=55,zone=NXM_NX_REG13[0..15],nat)
cookie=0x4b8234d9, table=54, priority=100,ip,reg0=0x4/0x4,metadata=0x2 actions=ct(table=55,zone=NXM_NX_REG13[0..15],nat)
//conntrack
cookie=0x6027420b, table=54, priority=100,ip,reg0=0x2/0x2,metadata=0x3 actions=ct(commit,zone=NXM_NX_REG13[0..15],exec(load:0->NXM_NX_CT_LABEL[0])),resubmit(,55)
cookie=0x76bd97bd, table=54, priority=100,ip,reg0=0x2/0x2,metadata=0x2 actions=ct(commit,zone=NXM_NX_REG13[0..15],exec(load:0->NXM_NX_CT_LABEL[0])),resubmit(,55)
//繼續
cookie=0x390ebf5f, table=54, priority=0,metadata=0x2 actions=resubmit(,55)
cookie=0x6537ab93, table=54, priority=0,metadata=0x3 actions=resubmit(,55)
cookie=0x13159847, table=55, priority=0,metadata=0x3 actions=resubmit(,56)
cookie=0x439f6726, table=55, priority=0,metadata=0x2 actions=resubmit(,56)
//多播流量
cookie=0xb5641b45, table=56, priority=100,metadata=0x2,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,64)
cookie=0x7b1296c4, table=56, priority=100,metadata=0x3,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,64)
//到某個虛擬機的流量
cookie=0xcfbbf747, table=56, priority=50,reg15=0x3,metadata=0x2,dl_dst=52:54:00:c1:68:72 actions=resubmit(,64)
cookie=0xd39cd78f, table=56, priority=50,reg15=0x3,metadata=0x3,dl_dst=52:54:00:c1:68:73 actions=resubmit(,64)
cookie=0x46f7518d, table=56, priority=50,reg15=0x2,metadata=0x3,dl_dst=52:54:00:c1:68:71 actions=resubmit(,64)
cookie=0x10683faf, table=56, priority=50,reg15=0x2,metadata=0x2,dl_dst=52:54:00:c1:68:70 actions=resubmit(,64)
//繼續
cookie=0xdf1a835, table=56, priority=50,reg15=0x1,metadata=0x3 actions=resubmit(,64)
cookie=0x69d25440, table=56, priority=50,reg15=0x1,metadata=0x2 actions=resubmit(,64)
//修改入端口，為重新循環做准備
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x1,metadata=0x1 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x3 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x2 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x1,metadata=0x3 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x1 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=100,reg10=0x1/0x1,reg15=0x1,metadata=0x2 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, table=64, priority=0 actions=resubmit(,65)
//將報文重新resubmit到表16，表示過完一個邏輯網元，需要進入下一個邏輯網元了
cookie=0x0, table=65, priority=100,reg15=0x2,metadata=0x1 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x6->NXM_NX_REG11[],load:0x8->NXM_NX_REG12[],load:0x3->OXM_OF_METADATA[],load:0x1->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,16))
cookie=0x0, table=65, priority=100,reg15=0x1,metadata=0x2 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x3->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x1->OXM_OF_METADATA[],load:0x1->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,16))
cookie=0x0, table=65, priority=100,reg15=0x1,metadata=0x1 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x7->NXM_NX_REG11[],load:0x5->NXM_NX_REG12[],load:0x2->OXM_OF_METADATA[],load:0x1->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,16))
cookie=0x0, table=65, priority=100,reg15=0x1,metadata=0x3 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x3->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x1->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,16))
//到本地某個虛擬機的直接發送
cookie=0x0, table=65, priority=100,reg15=0x2,metadata=0x2 actions=output:3
cookie=0x0, table=65, priority=100,reg15=0x2,metadata=0x3 actions=output:4
//通過MAC_Binding修改IP對應的MAC
cookie=0x0, table=66, priority=100,reg0=0xc0a8025c,reg15=0x2,metadata=0x1 actions=mod_dl_dst:52:54:00:c1:68:73
cookie=0x0, table=66, priority=100,reg0=0xc0a8025b,reg15=0x2,metadata=0x1 actions=mod_dl_dst:52:54:00:c1:68:71
cookie=0x0, table=66, priority=100,reg0=0xc0a8015b,reg15=0x1,metadata=0x1 actions=mod_dl_dst:52:54:00:c1:68:70
cookie=0x0, table=66, priority=100,reg0=0,reg1=0,reg2=0,reg3=0,reg15=0x2,metadata=0x1 actions=mod_dl_dst:00:00:00:00:00:00
cookie=0x0, table=66, priority=100,reg0=0,reg1=0,reg2=0,reg3=0,reg15=0x1,metadata=0x1 actions=mod_dl_dst:00:00:00:00:00:00
寄存器意義
| 寄存器 |
功能 |
詳解 |
| metadata |
作為vni使用 |
是ovn的Logical Datapath Field，命令ovn-sbctl list Datapath_Binding查看tunnel_key，封裝到geneve或者stt中 |
| reg14 |
記錄邏輯入端口 |
是ovn的Logical InputPort Field，命令ovn-sbctl list Port_Binding查看tunnel_key，封裝到geneve或者stt中 |
| reg15 |
記錄邏輯出端口 |
是ovn的Logical OutputPort Field，命令ovn-sbctl list Port_Binding查看tunnel_key，封裝到geneve或者stt中 |
| reg13 |
邏輯端口的conntrack zone |
chassis內部有用，出了chassis無用 |
| reg12 |
SNAT的conntrack zone |
也是chassis內部使用 |
| reg11 |
DNAT的conntrack zone |
也是chassis內部使用 |
| reg10 |
邏輯流表標志 |
可能是邏輯流表中的flags.loopback之類的標志 |
# ovs-ofctl dump-flows br-int
cookie=0x0, duration=7415.683s, table=0, n_packets=7599, n_bytes=735898, priority=100,in_port=vm21 actions=load:0x6->NXM_NX_REG13[],load:0x5->NXM_NX_REG11[],load:0x2->NXM_NX_REG12[],load:0x3->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,8)
cookie=0x0, duration=7415.684s, table=0, n_packets=156585, n_bytes=9893858, priority=100,in_port="patch-br-int-to",vlan_tci=0x0000/0x1000 actions=load:0x8->NXM_NX_REG13[],load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x7->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,8)
cookie=0x0, duration=7415.683s, table=0, n_packets=0, n_bytes=0, priority=100,in_port="patch-br-int-to",dl_vlan=0 actions=strip_vlan,load:0x8->NXM_NX_REG13[],load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x7->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],resubmit(,8)
cookie=0xded39c69, duration=7415.685s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x7,vlan_tci=0x1000/0x1000 actions=drop
cookie=0x1b7ef5c3, duration=7415.684s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x5,vlan_tci=0x1000/0x1000 actions=drop
cookie=0x4433dfa7, duration=7415.683s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x3,vlan_tci=0x1000/0x1000 actions=drop
cookie=0xc08e126a, duration=7415.685s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x3,dl_src=01:00:00:00:00:00/01:00:00:00:00:00 actions=drop
cookie=0x1b7ef5c3, duration=7415.684s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x5,dl_src=01:00:00:00:00:00/01:00:00:00:00:00 actions=drop
cookie=0x299365e2, duration=7415.683s, table=8, n_packets=0, n_bytes=0, priority=100,metadata=0x7,dl_src=01:00:00:00:00:00/01:00:00:00:00:00 actions=drop
cookie=0x23f5facd, duration=7415.685s, table=8, n_packets=7584, n_bytes=733768, priority=50,reg14=0x5,metadata=0x5,dl_dst=02:d4:1d:8c:d9:af actions=resubmit(,9)
cookie=0x754b0851, duration=7415.683s, table=8, n_packets=0, n_bytes=0, priority=50,reg14=0x2,metadata=0x5,dl_dst=02:d4:1d:8c:d9:be actions=resubmit(,9)
cookie=0x36ac109e, duration=7415.685s, table=8, n_packets=7415, n_bytes=726670, priority=50,reg14=0x3,metadata=0x7 actions=resubmit(,9)
cookie=0x27036c09, duration=7415.683s, table=8, n_packets=156585, n_bytes=9893858, priority=50,reg14=0x2,metadata=0x7 actions=resubmit(,9)
cookie=0x737e698b, duration=7415.683s, table=8, n_packets=169, n_bytes=7098, priority=50,reg14=0x3,metadata=0x3 actions=resubmit(,9)
cookie=0xdc12a385, duration=7415.684s, table=8, n_packets=0, n_bytes=0, priority=50,reg14=0x5,metadata=0x5,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,9)
cookie=0x5cc8cc08, duration=7415.683s, table=8, n_packets=151968, n_bytes=9382386, priority=50,reg14=0x2,metadata=0x5,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,9)
cookie=0xa1f465ca, duration=7415.683s, table=8, n_packets=7599, n_bytes=735898, priority=50,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d actions=resubmit(,9)
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=103, n_bytes=35226, priority=100,ip,metadata=0x5,nw_src=0.0.0.0/8 actions=drop
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,nw_src=127.0.0.0/8 actions=drop
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,nw_dst=0.0.0.0/8 actions=drop
cookie=0x5d8a6979, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,nw_dst=127.0.0.0/8 actions=drop
cookie=0x4c80ad5c, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=100,ip,reg9=0/0x2,metadata=0x5,nw_src=192.168.233.177 actions=drop
cookie=0x863aa1e1, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=100,ip,reg9=0/0x2,metadata=0x5,nw_src=20.0.0.1 actions=drop
cookie=0x4c80ad5c, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=100,ip,reg9=0/0x2,metadata=0x5,nw_src=192.168.233.255 actions=drop
cookie=0x863aa1e1, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=100,ip,reg9=0/0x2,metadata=0x5,nw_src=20.0.0.255 actions=drop
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=108412, n_bytes=6497220, priority=100,ip,metadata=0x5,nw_dst=224.0.0.0/4 actions=drop
cookie=0x5d8a6979, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,nw_src=255.255.255.255 actions=drop
cookie=0x98e795ee, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=100,ipv6,metadata=0x5,ipv6_src=fe80::d4:1dff:fe8c:d9be actions=drop
cookie=0xc7e185f8, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=100,ipv6,metadata=0x5,ipv6_src=fe80::d4:1dff:fe8c:d9af actions=drop
cookie=0x944a74e0, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,icmp,metadata=0x5,nw_dst=20.0.0.1,icmp_type=8,icmp_code=0 actions=push:NXM_OF_IP_SRC[],push:NXM_OF_IP_DST[],pop:NXM_OF_IP_SRC[],pop:NXM_OF_IP_DST[],load:0xff->NXM_NX_IP_TTL[],load:0->NXM_OF_ICMP_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,10)
cookie=0xde91ac7f, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=90,icmp,metadata=0x5,nw_dst=192.168.233.177,icmp_type=8,icmp_code=0 actions=push:NXM_OF_IP_SRC[],push:NXM_OF_IP_DST[],pop:NXM_OF_IP_SRC[],pop:NXM_OF_IP_DST[],load:0xff->NXM_NX_IP_TTL[],load:0->NXM_OF_ICMP_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,10)
cookie=0x77f75a25, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,reg14=0x5,metadata=0x5,ipv6_dst=ff02::1:ff8c:d9af,nw_ttl=255,icmp_type=135,icmp_code=0,nd_target=fe80::d4:1dff:fe8c:d9af actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[],controller(userdata=00.00.00.03.00.00.00.00.00.19.00.10.80.00.08.06.02.d4.1d.8c.d9.af.00.00.00.19.00.18.80.00.34.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.af.00.19.00.18.80.00.3e.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.af.00.19.00.10.80.00.42.06.02.d4.1d.8c.d9.af.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.00.00.00.00.01.1c.04.00.01.1e.04.ff.ff.00.18.00.00.23.20.00.07.00.00.00.01.14.04.00.00.00.00.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0x77f75a25, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,reg14=0x5,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9af,nw_ttl=255,icmp_type=135,icmp_code=0,nd_target=fe80::d4:1dff:fe8c:d9af actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[],controller(userdata=00.00.00.03.00.00.00.00.00.19.00.10.80.00.08.06.02.d4.1d.8c.d9.af.00.00.00.19.00.18.80.00.34.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.af.00.19.00.18.80.00.3e.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.af.00.19.00.10.80.00.42.06.02.d4.1d.8c.d9.af.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.00.00.00.00.01.1c.04.00.01.1e.04.ff.ff.00.18.00.00.23.20.00.07.00.00.00.01.14.04.00.00.00.00.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0x7542fd0b, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,reg14=0x2,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9be,nw_ttl=255,icmp_type=135,icmp_code=0,nd_target=fe80::d4:1dff:fe8c:d9be actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[],controller(userdata=00.00.00.03.00.00.00.00.00.19.00.10.80.00.08.06.02.d4.1d.8c.d9.be.00.00.00.19.00.18.80.00.34.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.be.00.19.00.18.80.00.3e.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.be.00.19.00.10.80.00.42.06.02.d4.1d.8c.d9.be.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.00.00.00.00.01.1c.04.00.01.1e.04.ff.ff.00.18.00.00.23.20.00.07.00.00.00.01.14.04.00.00.00.00.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0x7542fd0b, duration=7415.683s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,reg14=0x2,metadata=0x5,ipv6_dst=ff02::1:ff8c:d9be,nw_ttl=255,icmp_type=135,icmp_code=0,nd_target=fe80::d4:1dff:fe8c:d9be actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[],controller(userdata=00.00.00.03.00.00.00.00.00.19.00.10.80.00.08.06.02.d4.1d.8c.d9.be.00.00.00.19.00.18.80.00.34.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.be.00.19.00.18.80.00.3e.10.fe.80.00.00.00.00.00.00.00.d4.1d.ff.fe.8c.d9.be.00.19.00.10.80.00.42.06.02.d4.1d.8c.d9.be.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.00.00.00.00.01.1c.04.00.01.1e.04.ff.ff.00.18.00.00.23.20.00.07.00.00.00.01.14.04.00.00.00.00.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0xf0a998ea, duration=7415.686s, table=9, n_packets=0, n_bytes=0, priority=90,arp,reg14=0x2,metadata=0x5,arp_tpa=192.168.233.177,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:02:d4:1d:8c:d9:be,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x2d41d8cd9be->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xc0a8e9b1->NXM_OF_ARP_SPA[],load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0xbc00a65b, duration=7415.685s, table=9, n_packets=169, n_bytes=7098, priority=90,arp,reg14=0x5,metadata=0x5,arp_tpa=20.0.0.1,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:02:d4:1d:8c:d9:af,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x2d41d8cd9af->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0x14000001->NXM_OF_ARP_SPA[],load:0x5->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0x1f495ec9, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9af,icmp_type=128,icmp_code=0 actions=push:NXM_NX_IPV6_SRC[],push:NXM_NX_IPV6_DST[],pop:NXM_NX_IPV6_SRC[],pop:NXM_NX_IPV6_DST[],load:0xff->NXM_NX_IP_TTL[],load:0x81->NXM_NX_ICMPV6_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,10)
cookie=0x89f21ed8, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9be,icmp_type=128,icmp_code=0 actions=push:NXM_NX_IPV6_SRC[],push:NXM_NX_IPV6_DST[],pop:NXM_NX_IPV6_SRC[],pop:NXM_NX_IPV6_DST[],load:0xff->NXM_NX_IP_TTL[],load:0x81->NXM_NX_ICMPV6_TYPE[],load:0x1->NXM_NX_REG10[0],resubmit(,10)
cookie=0x40c4377d, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,arp,metadata=0x5,arp_op=2 actions=push:NXM_NX_REG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ARP_SHA[],push:NXM_OF_ARP_SPA[],pop:NXM_NX_REG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.01.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_REG0[]
cookie=0x6636d7dd, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=90,udp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=0.0.0.0,nw_dst=255.255.255.255,tp_src=68,tp_dst=67 actions=resubmit(,10)
cookie=0x8095971d, duration=7415.684s, table=9, n_packets=7420, n_bytes=728380, priority=90,ip,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=20.0.0.10 actions=resubmit(,10)
cookie=0xf90b1a37, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=90,icmp6,metadata=0x5,nw_ttl=255,icmp_type=136,icmp_code=0 actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_TLL[],push:NXM_NX_ND_TARGET[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[]
cookie=0x89fa3891, duration=7415.684s, table=9, n_packets=194, n_bytes=16684, priority=80,icmp6,metadata=0x5,nw_ttl=255,icmp_type=135,icmp_code=0 actions=push:NXM_NX_XXREG0[],push:NXM_OF_ETH_SRC[],push:NXM_NX_ND_SLL[],push:NXM_NX_IPV6_SRC[],pop:NXM_NX_XXREG0[],pop:NXM_OF_ETH_SRC[],controller(userdata=00.00.00.04.00.00.00.00),pop:NXM_OF_ETH_SRC[],pop:NXM_NX_XXREG0[]
cookie=0x612ff906, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=80,ip,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d actions=drop
cookie=0x612ff906, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=80,ipv6,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d actions=drop
cookie=0xb7bd9e1d, duration=7415.686s, table=9, n_packets=0, n_bytes=0, priority=60,ipv6,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9af actions=drop
cookie=0x52b8a0bd, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=60,ipv6,metadata=0x5,ipv6_dst=fe80::d4:1dff:fe8c:d9be actions=drop
cookie=0x28d382fa, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=60,ip,metadata=0x5,nw_dst=20.0.0.1 actions=drop
cookie=0xf16a2d3e, duration=7415.686s, table=9, n_packets=39348, n_bytes=2364630, priority=50,metadata=0x5,dl_dst=ff:ff:ff:ff:ff:ff actions=drop
cookie=0xedb0015b, duration=7415.685s, table=9, n_packets=0, n_bytes=0, priority=30,ip,metadata=0x5,nw_ttl=1 actions=drop
cookie=0xedb0015b, duration=7415.684s, table=9, n_packets=0, n_bytes=0, priority=30,ip,metadata=0x5,nw_ttl=0 actions=drop
cookie=0x5557ea52, duration=7415.686s, table=9, n_packets=348, n_bytes=14616, priority=0,metadata=0x3 actions=resubmit(,10)
cookie=0x2b40cc48, duration=7415.685s, table=9, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,10)
cookie=0x376d13ff, duration=7415.685s, table=9, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,10)
cookie=0xb57f57f7, duration=7415.686s, table=10, n_packets=169, n_bytes=7098, priority=90,arp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,arp_spa=20.0.0.10,arp_sha=02:d4:1d:8c:d9:9d actions=resubmit(,11)
cookie=0xb34d78e8, duration=7415.685s, table=10, n_packets=10, n_bytes=420, priority=80,arp,reg14=0x2,metadata=0x3 actions=drop
cookie=0xb34d78e8, duration=7415.684s, table=10, n_packets=0, n_bytes=0, priority=80,icmp6,reg14=0x2,metadata=0x3,nw_ttl=255,icmp_type=135,icmp_code=0 actions=drop
cookie=0xb34d78e8, duration=7415.684s, table=10, n_packets=0, n_bytes=0, priority=80,icmp6,reg14=0x2,metadata=0x3,nw_ttl=255,icmp_type=136,icmp_code=0 actions=drop
cookie=0x5ab65a62, duration=7415.686s, table=10, n_packets=7589, n_bytes=735478, priority=0,metadata=0x3 actions=resubmit(,11)
cookie=0x30ab50cf, duration=7415.685s, table=10, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,11)
cookie=0x5efd761e, duration=7415.685s, table=10, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,11)
cookie=0xae6dae29, duration=7415.684s, table=11, n_packets=0, n_bytes=0, priority=50,ip,metadata=0x5,nw_dst=192.168.233.177 actions=load:0x1->OXM_OF_PKT_REG4[0],resubmit(,12)
cookie=0x93f6b9c, duration=7415.685s, table=11, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,12)
cookie=0xa99bdefc, duration=7415.685s, table=11, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,12)
cookie=0xd5587e18, duration=7415.684s, table=11, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,12)
cookie=0x6b45a8de, duration=7415.685s, table=12, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,13)
cookie=0x91478c72, duration=7415.685s, table=12, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,13)
cookie=0xa9e8bb31, duration=7415.685s, table=12, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,13)
cookie=0xaae4d01f, duration=7415.686s, table=13, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,14)
cookie=0x32f0ba0b, duration=7415.686s, table=13, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,14)
cookie=0xa02c3ecf, duration=7415.684s, table=13, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,14)
cookie=0x5b9d743a, duration=7415.686s, table=14, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,15)
cookie=0xcf2678fc, duration=7415.684s, table=14, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,15)
cookie=0x9a0b181d, duration=7415.684s, table=14, n_packets=11326, n_bytes=1195296, priority=0,metadata=0x5 actions=resubmit(,15)
cookie=0x8c7fbe9f, duration=7415.686s, table=15, n_packets=0, n_bytes=0, priority=300,ipv6,reg9=0x1/0x1,metadata=0x5 actions=dec_ttl(),resubmit(,16)
cookie=0x8c7fbe9f, duration=7415.685s, table=15, n_packets=0, n_bytes=0, priority=300,ip,reg9=0x1/0x1,metadata=0x5 actions=dec_ttl(),resubmit(,16)
cookie=0xa5be5a62, duration=7415.686s, table=15, n_packets=0, n_bytes=0, priority=129,ipv6,reg14=0x5,metadata=0x5,ipv6_dst=fe80::/64 actions=dec_ttl(),move:NXM_NX_IPV6_DST[]->NXM_NX_XXREG0[],load:0xd41dfffe8cd9af->NXM_NX_XXREG1[0..63],load:0xfe80000000000000->NXM_NX_XXREG1[64..127],mod_dl_src:02:d4:1d:8c:d9:af,load:0x5->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0x23104177, duration=7415.685s, table=15, n_packets=0, n_bytes=0, priority=129,ipv6,reg14=0x2,metadata=0x5,ipv6_dst=fe80::/64 actions=dec_ttl(),move:NXM_NX_IPV6_DST[]->NXM_NX_XXREG0[],load:0xd41dfffe8cd9be->NXM_NX_XXREG1[0..63],load:0xfe80000000000000->NXM_NX_XXREG1[64..127],mod_dl_src:02:d4:1d:8c:d9:be,load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0xf97c81e7, duration=7415.686s, table=15, n_packets=7415, n_bytes=726670, priority=49,ip,metadata=0x5,nw_dst=192.168.233.0/24 actions=dec_ttl(),move:NXM_OF_IP_DST[]->NXM_NX_XXREG0[96..127],load:0xc0a8e9b1->NXM_NX_XXREG0[64..95],mod_dl_src:02:d4:1d:8c:d9:be,load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0x1526dfe6, duration=7415.686s, table=15, n_packets=0, n_bytes=0, priority=49,ip,metadata=0x5,nw_dst=20.0.0.0/24 actions=dec_ttl(),move:NXM_OF_IP_DST[]->NXM_NX_XXREG0[96..127],load:0x14000001->NXM_NX_XXREG0[64..95],mod_dl_src:02:d4:1d:8c:d9:af,load:0x5->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0x1479186f, duration=7415.684s, table=15, n_packets=0, n_bytes=0, priority=1,ip,metadata=0x5 actions=dec_ttl(),load:0xc0a8e9b1->NXM_NX_XXREG0[96..127],load:0xc0a8e9b1->NXM_NX_XXREG0[64..95],mod_dl_src:02:d4:1d:8c:d9:be,load:0x2->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,16)
cookie=0xff85f9d6, duration=7415.685s, table=15, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,16)
cookie=0x766070a, duration=7415.685s, table=15, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,16)
cookie=0xf2f4b9da, duration=7415.685s, table=16, n_packets=0, n_bytes=0, priority=200,reg9=0x1/0x1,metadata=0x5 actions=mod_dl_dst:02:d4:1d:8c:d9:be,resubmit(,17)
cookie=0xa12b693, duration=7415.684s, table=16, n_packets=0, n_bytes=0, priority=100,reg0=0x1400000a,reg15=0x5,metadata=0x5 actions=mod_dl_dst:02:d4:1d:8c:d9:9d,resubmit(,17)
cookie=0xdd6bdca0, duration=7415.685s, table=16, n_packets=0, n_bytes=0, priority=0,ipv6,metadata=0x5 actions=mod_dl_dst:00:00:00:00:00:00,resubmit(,66),resubmit(,17)
cookie=0x9e7a5a88, duration=7415.684s, table=16, n_packets=7415, n_bytes=726670, priority=0,ip,metadata=0x5 actions=push:NXM_NX_REG0[],push:NXM_NX_XXREG0[96..127],pop:NXM_NX_REG0[],mod_dl_dst:00:00:00:00:00:00,resubmit(,66),pop:NXM_NX_REG0[],resubmit(,17)
cookie=0xdf9fa20, duration=7415.685s, table=16, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,17)
cookie=0x853ab9e6, duration=7415.684s, table=16, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,17)
cookie=0x5d82efa, duration=7415.685s, table=17, n_packets=0, n_bytes=0, priority=200,reg9=0x1/0x1,metadata=0x5 actions=load:0x6->NXM_NX_REG15[],resubmit(,18)
cookie=0x8832e20e, duration=7415.686s, table=17, n_packets=0, n_bytes=0, priority=150,reg15=0x2,metadata=0x5,dl_dst=00:00:00:00:00:00 actions=load:0x6->NXM_NX_REG15[],resubmit(,18)
cookie=0x65331fc0, duration=7415.685s, table=17, n_packets=7415, n_bytes=726670, priority=50,reg15=0x2,metadata=0x5 actions=load:0x6->NXM_NX_REG15[],resubmit(,18)
cookie=0x617dd0d3, duration=7415.686s, table=17, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,18)
cookie=0xa8c020ff, duration=7415.685s, table=17, n_packets=0, n_bytes=0, priority=0,metadata=0x5 actions=resubmit(,18)
cookie=0xbe7a33b7, duration=7415.684s, table=17, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,18)
cookie=0xb0a974d1, duration=7415.685s, table=18, n_packets=0, n_bytes=0, priority=100,ipv6,metadata=0x5,dl_dst=00:00:00:00:00:00 actions=controller(userdata=00.00.00.09.00.00.00.00.ff.ff.00.18.00.00.23.20.00.06.00.80.00.00.00.00.00.01.de.10.00.01.2e.10.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0x947d2134, duration=7415.684s, table=18, n_packets=0, n_bytes=0, priority=100,ip,metadata=0x5,dl_dst=00:00:00:00:00:00 actions=controller(userdata=00.00.00.00.00.00.00.00.00.19.00.10.80.00.06.06.ff.ff.ff.ff.ff.ff.00.00.ff.ff.00.18.00.00.23.20.00.06.00.20.00.40.00.00.00.01.de.10.00.00.20.04.ff.ff.00.18.00.00.23.20.00.06.00.20.00.60.00.00.00.01.de.10.00.00.22.04.00.19.00.10.80.00.2a.02.00.01.00.00.00.00.00.00.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.20.00.00.00)
cookie=0xe7029d42, duration=7415.685s, table=18, n_packets=7415, n_bytes=726670, priority=0,metadata=0x5 actions=resubmit(,32)
cookie=0x354130d2, duration=7415.684s, table=18, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,19)
cookie=0xa0e9ffc, duration=7415.684s, table=18, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,19)
cookie=0x160901ae, duration=7415.684s, table=19, n_packets=156585, n_bytes=9893858, priority=100,reg14=0x2,metadata=0x7 actions=resubmit(,20)
cookie=0x6f016248, duration=7415.684s, table=19, n_packets=0, n_bytes=0, priority=100,arp,reg14=0x2,metadata=0x3,arp_tpa=20.0.0.10,arp_op=1 actions=resubmit(,20)
cookie=0x1c76bc17, duration=7415.685s, table=19, n_packets=0, n_bytes=0, priority=50,arp,metadata=0x3,arp_tpa=20.0.0.10,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:02:d4:1d:8c:d9:9d,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x2d41d8cd99d->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0x1400000a->NXM_OF_ARP_SPA[],move:NXM_NX_REG14[]->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0x5a75b5c5, duration=7415.686s, table=19, n_packets=7415, n_bytes=726670, priority=0,metadata=0x7 actions=resubmit(,20)
cookie=0x25090509, duration=7415.684s, table=19, n_packets=7758, n_bytes=742576, priority=0,metadata=0x3 actions=resubmit(,20)
cookie=0x43536dbb, duration=7415.685s, table=20, n_packets=0, n_bytes=0, priority=100,udp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=20.0.0.10,nw_dst=255.255.255.255,tp_src=68,tp_dst=67 actions=controller(userdata=00.00.00.02.00.00.00.00.00.01.de.10.00.00.00.63.14.00.00.0a.33.04.00.00.0e.10.01.04.ff.ff.ff.00.03.04.14.00.00.01.36.04.14.00.00.01,pause),resubmit(,21)
cookie=0x43536dbb, duration=7415.685s, table=20, n_packets=5, n_bytes=1710, priority=100,udp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=20.0.0.10,nw_dst=20.0.0.1,tp_src=68,tp_dst=67 actions=controller(userdata=00.00.00.02.00.00.00.00.00.01.de.10.00.00.00.63.14.00.00.0a.33.04.00.00.0e.10.01.04.ff.ff.ff.00.03.04.14.00.00.01.36.04.14.00.00.01,pause),resubmit(,21)
cookie=0x4f868dc8, duration=7415.684s, table=20, n_packets=0, n_bytes=0, priority=100,udp,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,nw_src=0.0.0.0,nw_dst=255.255.255.255,tp_src=68,tp_dst=67 actions=controller(userdata=00.00.00.02.00.00.00.00.00.01.de.10.00.00.00.63.14.00.00.0a.33.04.00.00.0e.10.01.04.ff.ff.ff.00.03.04.14.00.00.01.36.04.14.00.00.01,pause),resubmit(,21)
cookie=0x6829477e, duration=7415.684s, table=20, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,21)
cookie=0x9c47b2ee, duration=7415.684s, table=20, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,21)
cookie=0xfec4dff5, duration=7415.685s, table=21, n_packets=0, n_bytes=0, priority=100,udp,reg0=0x8/0x8,reg14=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9d,tp_src=68,tp_dst=67 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:02:d4:1d:8c:d9:9f,mod_nw_dst:20.0.0.10,mod_nw_src:20.0.0.1,mod_tp_src:67,mod_tp_dst:68,move:NXM_NX_REG14[]->NXM_NX_REG15[],load:0x1->NXM_NX_REG10[0],resubmit(,32)
cookie=0x883f0b8e, duration=7415.685s, table=21, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,22)
cookie=0xba05df68, duration=7415.684s, table=21, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,22)
cookie=0x2b1cd3f2, duration=7415.686s, table=22, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,23)
cookie=0x518bc984, duration=7415.685s, table=22, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,23)
cookie=0x1416a4cf, duration=7415.685s, table=23, n_packets=164000, n_bytes=10620528, priority=0,metadata=0x7 actions=resubmit(,24)
cookie=0x2545137, duration=7415.684s, table=23, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,24)
cookie=0x24578bfb, duration=7415.685s, table=24, n_packets=0, n_bytes=0, priority=100,metadata=0x3,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=load:0xffff->NXM_NX_REG15[],resubmit(,32)
cookie=0x40ea8458, duration=7415.684s, table=24, n_packets=151968, n_bytes=9382386, priority=100,metadata=0x7,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=load:0xffff->NXM_NX_REG15[],resubmit(,32)
cookie=0xf1103160, duration=7415.686s, table=24, n_packets=7584, n_bytes=733768, priority=50,metadata=0x3,dl_dst=02:d4:1d:8c:d9:af actions=load:0x3->NXM_NX_REG15[],resubmit(,32)
cookie=0x3efaae35, duration=7415.684s, table=24, n_packets=169, n_bytes=7098, priority=50,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d actions=load:0x2->NXM_NX_REG15[],resubmit(,32)
cookie=0xdf7b6600, duration=7415.684s, table=24, n_packets=0, n_bytes=0, priority=50,metadata=0x7,dl_dst=02:d4:1d:8c:d9:be actions=load:0x3->NXM_NX_REG15[],resubmit(,32)
cookie=0x711af6a2, duration=7415.685s, table=24, n_packets=12032, n_bytes=1238142, priority=0,metadata=0x7 actions=load:0xfffe->NXM_NX_REG15[],resubmit(,32)
cookie=0x0, duration=7415.685s, table=32, n_packets=0, n_bytes=0, priority=150,reg10=0x10/0x10,reg15=0xffff,metadata=0x7 actions=resubmit(,33)
cookie=0x0, duration=7415.685s, table=32, n_packets=0, n_bytes=0, priority=150,reg10=0x10/0x10,reg15=0xffff,metadata=0x3 actions=resubmit(,33)
cookie=0x0, duration=7415.684s, table=32, n_packets=0, n_bytes=0, priority=150,reg10=0x10/0x10,reg15=0xfffe,metadata=0x7 actions=resubmit(,33)
cookie=0x0, duration=7415.684s, table=32, n_packets=0, n_bytes=0, priority=150,reg10=0x2/0x2 actions=resubmit(,33)
cookie=0x0, duration=7415.685s, table=32, n_packets=151968, n_bytes=9382386, priority=100,reg15=0xffff,metadata=0x7 actions=load:0x3->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[],resubmit(,33)
cookie=0x0, duration=7415.684s, table=32, n_packets=0, n_bytes=0, priority=100,reg15=0xffff,metadata=0x3 actions=load:0x3->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[],resubmit(,33)
cookie=0x0, duration=7415.686s, table=32, n_packets=27369, n_bytes=2712776, priority=0 actions=resubmit(,33)
cookie=0x0, duration=7415.686s, table=33, n_packets=7584, n_bytes=733768, priority=100,reg15=0x3,metadata=0x3 actions=load:0x5->NXM_NX_REG11[],load:0x2->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.686s, table=33, n_packets=151968, n_bytes=9382386, priority=100,reg15=0xffff,metadata=0x7 actions=load:0x8->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[]
cookie=0x0, duration=7415.685s, table=33, n_packets=0, n_bytes=0, priority=100,reg15=0x2,metadata=0x7 actions=load:0x8->NXM_NX_REG13[],load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.685s, table=33, n_packets=0, n_bytes=0, priority=100,reg15=0x2,metadata=0x5 actions=load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.685s, table=33, n_packets=12032, n_bytes=1238142, priority=100,reg15=0xfffe,metadata=0x7 actions=load:0x8->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xfffe->NXM_NX_REG15[]
cookie=0x0, duration=7415.685s, table=33, n_packets=0, n_bytes=0, priority=100,reg15=0xffff,metadata=0x3 actions=load:0x6->NXM_NX_REG13[],load:0x2->NXM_NX_REG15[],resubmit(,34),load:0xffff->NXM_NX_REG15[]
cookie=0x0, duration=7415.685s, table=33, n_packets=7415, n_bytes=726670, priority=100,reg15=0x6,metadata=0x5 actions=load:0x2->NXM_NX_REG15[],load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.684s, table=33, n_packets=169, n_bytes=7098, priority=100,reg15=0x2,metadata=0x3 actions=load:0x6->NXM_NX_REG13[],load:0x5->NXM_NX_REG11[],load:0x2->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.684s, table=33, n_packets=169, n_bytes=7098, priority=100,reg15=0x5,metadata=0x5 actions=load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.684s, table=33, n_packets=0, n_bytes=0, priority=100,reg15=0x3,metadata=0x7 actions=load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],resubmit(,34)
cookie=0x0, duration=7415.686s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x3,reg15=0x3,metadata=0x7 actions=drop
cookie=0x0, duration=7415.686s, table=34, n_packets=156585, n_bytes=9893858, priority=100,reg10=0/0x1,reg14=0x2,reg15=0x2,metadata=0x7 actions=drop
cookie=0x0, duration=7415.685s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x2,reg15=0x2,metadata=0x5 actions=drop
cookie=0x0, duration=7415.685s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x2,reg15=0x2,metadata=0x3 actions=drop
cookie=0x0, duration=7415.685s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x3,reg15=0x3,metadata=0x3 actions=drop
cookie=0x0, duration=7415.684s, table=34, n_packets=0, n_bytes=0, priority=100,reg10=0/0x1,reg14=0x5,reg15=0x5,metadata=0x5 actions=drop
cookie=0x0, duration=7415.686s, table=34, n_packets=174720, n_bytes=11583690, priority=0 actions=load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],resubmit(,40)
cookie=0xab47d381, duration=7415.686s, table=40, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,41)
cookie=0xe30c8cbc, duration=7415.685s, table=40, n_packets=7584, n_bytes=733768, priority=0,metadata=0x5 actions=resubmit(,41)
cookie=0x64b884b4, duration=7415.685s, table=40, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,41)
cookie=0xf2d36f82, duration=7415.686s, table=41, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,42)
cookie=0xbb7b8f92, duration=7415.685s, table=41, n_packets=7584, n_bytes=733768, priority=0,metadata=0x5 actions=resubmit(,42)
cookie=0xafac9b88, duration=7415.684s, table=41, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,42)
cookie=0xd6d23b6d, duration=7415.684s, table=42, n_packets=0, n_bytes=0, priority=100,ip,reg15=0x2,metadata=0x5,nw_dst=192.168.233.177 actions=clone(ct_clear,move:NXM_NX_REG15[]->NXM_NX_REG14[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG10[],load:0x1->NXM_NX_REG10[0],load:0->NXM_NX_XXREG0[96..127],load:0->NXM_NX_XXREG0[64..95],load:0->NXM_NX_XXREG0[32..63],load:0->NXM_NX_XXREG0[0..31],load:0->NXM_NX_XXREG1[96..127],load:0->NXM_NX_XXREG1[64..95],load:0->NXM_NX_XXREG1[32..63],load:0->NXM_NX_XXREG1[0..31],load:0->OXM_OF_PKT_REG4[32..63],load:0->OXM_OF_PKT_REG4[0..31],load:0x1->OXM_OF_PKT_REG4[1],resubmit(,8))
cookie=0x3025b51b, duration=7415.686s, table=42, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,43)
cookie=0x8ef5f82b, duration=7415.685s, table=42, n_packets=7584, n_bytes=733768, priority=0,metadata=0x5 actions=resubmit(,43)
cookie=0xab15d779, duration=7415.684s, table=42, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,43)
cookie=0x4deb265c, duration=7415.685s, table=43, n_packets=169, n_bytes=7098, priority=100,reg15=0x5,metadata=0x5 actions=resubmit(,64)
cookie=0xd0d02e2e, duration=7415.684s, table=43, n_packets=7415, n_bytes=726670, priority=100,reg15=0x2,metadata=0x5 actions=resubmit(,64)
cookie=0x8a5f0c7c, duration=7415.686s, table=43, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,44)
cookie=0xabb769a7, duration=7415.684s, table=43, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,44)
cookie=0x7745fee3, duration=7415.684s, table=44, n_packets=0, n_bytes=0, priority=34000,udp,reg15=0x2,metadata=0x3,dl_src=02:d4:1d:8c:d9:9f,nw_src=20.0.0.1,tp_src=67,tp_dst=68 actions=resubmit(,45)
cookie=0xa677c8f, duration=7415.686s, table=44, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,45)
cookie=0xf9a88ae5, duration=7415.684s, table=44, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,45)
cookie=0x7cc31a30, duration=7415.686s, table=45, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,46)
cookie=0xe485076a, duration=7415.684s, table=45, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,46)
cookie=0xa2db634c, duration=7415.686s, table=46, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,47)
cookie=0x2eb88901, duration=7415.684s, table=46, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,47)
cookie=0xa47461af, duration=7415.685s, table=47, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,48)
cookie=0xf9838769, duration=7415.685s, table=47, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,48)
cookie=0x8b1f4a36, duration=7415.685s, table=48, n_packets=0, n_bytes=0, priority=90,ip,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d,nw_dst=255.255.255.255 actions=resubmit(,49)
cookie=0x8b1f4a36, duration=7415.684s, table=48, n_packets=0, n_bytes=0, priority=90,ip,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d,nw_dst=20.0.0.10 actions=resubmit(,49)
cookie=0x8b1f4a36, duration=7415.684s, table=48, n_packets=0, n_bytes=0, priority=90,ip,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d,nw_dst=224.0.0.0/4 actions=resubmit(,49)
cookie=0x6335fd40, duration=7415.685s, table=48, n_packets=0, n_bytes=0, priority=80,ipv6,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d actions=drop
cookie=0x6335fd40, duration=7415.684s, table=48, n_packets=0, n_bytes=0, priority=80,ip,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d actions=drop
cookie=0x1e836ae8, duration=7415.685s, table=48, n_packets=7753, n_bytes=740866, priority=0,metadata=0x3 actions=resubmit(,49)
cookie=0x3106bcc, duration=7415.684s, table=48, n_packets=159383, n_bytes=10109056, priority=0,metadata=0x7 actions=resubmit(,49)
cookie=0xd0a2471c, duration=7415.684s, table=49, n_packets=0, n_bytes=0, priority=100,metadata=0x3,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,64)
cookie=0x22816064, duration=7415.684s, table=49, n_packets=151968, n_bytes=9382386, priority=100,metadata=0x7,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,64)
cookie=0x23119cfc, duration=7415.685s, table=49, n_packets=7415, n_bytes=726670, priority=50,reg15=0x2,metadata=0x7 actions=resubmit(,64)
cookie=0x87806516, duration=7415.684s, table=49, n_packets=7584, n_bytes=733768, priority=50,reg15=0x3,metadata=0x3 actions=resubmit(,64)
cookie=0xf9e7a385, duration=7415.684s, table=49, n_packets=0, n_bytes=0, priority=50,reg15=0x3,metadata=0x7 actions=resubmit(,64)
cookie=0x5555f23f, duration=7415.684s, table=49, n_packets=169, n_bytes=7098, priority=50,reg15=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9d actions=resubmit(,64)
cookie=0x0, duration=7415.686s, table=64, n_packets=0, n_bytes=0, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x7 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.686s, table=64, n_packets=169, n_bytes=7098, priority=100,reg10=0x1/0x1,reg15=0x5,metadata=0x5 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.685s, table=64, n_packets=0, n_bytes=0, priority=100,reg10=0x1/0x1,reg15=0x3,metadata=0x3 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.685s, table=64, n_packets=0, n_bytes=0, priority=100,reg10=0x1/0x1,reg15=0x3,metadata=0x7 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.684s, table=64, n_packets=7415, n_bytes=726670, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x5 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.684s, table=64, n_packets=0, n_bytes=0, priority=100,reg10=0x1/0x1,reg15=0x2,metadata=0x3 actions=push:NXM_OF_IN_PORT[],load:0->NXM_OF_IN_PORT[],resubmit(,65),pop:NXM_OF_IN_PORT[]
cookie=0x0, duration=7415.684s, table=64, n_packets=167136, n_bytes=10849922, priority=0 actions=resubmit(,65)
cookie=0x0, duration=7415.686s, table=65, n_packets=169, n_bytes=7098, priority=100,reg15=0x5,metadata=0x5 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x5->NXM_NX_REG11[],load:0x2->NXM_NX_REG12[],load:0x3->OXM_OF_METADATA[],load:0x3->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,8))
cookie=0x0, duration=7415.685s, table=65, n_packets=169, n_bytes=7098, priority=100,reg15=0x2,metadata=0x3 actions=output:vm21
cookie=0x0, duration=7415.684s, table=65, n_packets=7415, n_bytes=726670, priority=100,reg15=0x2,metadata=0x7 actions=output:"patch-br-int-to"
cookie=0x0, duration=7415.684s, table=65, n_packets=7415, n_bytes=726670, priority=100,reg15=0x2,metadata=0x5 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x1->NXM_NX_REG11[],load:0x4->NXM_NX_REG12[],load:0x7->OXM_OF_METADATA[],load:0x3->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,8))
cookie=0x0, duration=7415.684s, table=65, n_packets=7584, n_bytes=733768, priority=100,reg15=0x3,metadata=0x3 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],load:0x5->OXM_OF_METADATA[],load:0x5->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,8))
cookie=0x0, duration=7415.684s, table=65, n_packets=151968, n_bytes=9382386, priority=100,reg15=0x3,metadata=0x7 actions=clone(ct_clear,load:0->NXM_NX_REG11[],load:0->NXM_NX_REG12[],load:0->NXM_NX_REG13[],load:0x3->NXM_NX_REG11[],load:0x7->NXM_NX_REG12[],load:0x5->OXM_OF_METADATA[],load:0x2->NXM_NX_REG14[],load:0->NXM_NX_REG10[],load:0->NXM_NX_REG15[],load:0->NXM_NX_REG0[],load:0->NXM_NX_REG1[],load:0->NXM_NX_REG2[],load:0->NXM_NX_REG3[],load:0->NXM_NX_REG4[],load:0->NXM_NX_REG5[],load:0->NXM_NX_REG6[],load:0->NXM_NX_REG7[],load:0->NXM_NX_REG8[],load:0->NXM_NX_REG9[],load:0->NXM_OF_IN_PORT[],resubmit(,8))
cookie=0x0, duration=7415.686s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xc0a8e9b2,reg15=0x2,metadata=0x5 actions=mod_dl_dst:c2:af:5a:9e:73:47
cookie=0x0, duration=7415.685s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xc0a80fc8,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:f9:91:7a
cookie=0x0, duration=7415.685s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xc0a8e901,reg15=0x2,metadata=0x5 actions=mod_dl_dst:e8:61:1f:15:2b:94
cookie=0x0, duration=7415.685s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xac1e0a07,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:2a:b9:c8
cookie=0x0, duration=7415.685s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xac1e0c07,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:f7:00:67
cookie=0x0, duration=7415.685s, table=66, n_packets=7415, n_bytes=726670, priority=100,reg0=0xc0a8e907,reg15=0x2,metadata=0x5 actions=mod_dl_dst:c2:af:5a:9e:73:47
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xac1e0d07,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:18:1f:b9
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xac1e0b07,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:d8:1d:25
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xc0a8e903,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:92:59:af
cookie=0x0, duration=7415.686s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0,reg1=0,reg2=0,reg3=0,reg15=0x2,metadata=0x5 actions=mod_dl_dst:00:00:00:00:00:00
cookie=0x0, duration=7415.686s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xfe800000,reg1=0,reg2=0x25056ff,reg3=0xfe883878,reg15=0x2,metadata=0x5 actions=mod_dl_dst:00:50:56:88:38:78
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xfe800000,reg1=0,reg2=0xf8163eff,reg3=0xfef9917a,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:f9:91:7a
cookie=0x0, duration=7415.684s, table=66, n_packets=0, n_bytes=0, priority=100,reg0=0xfe800000,reg1=0,reg2=0xf8163eff,reg3=0xfeb00567,reg15=0x2,metadata=0x5 actions=mod_dl_dst:fa:16:3e:b0:05:67
OVN-LB
lb-add LB VIP[:PORT] IP[:PORT]... [PROTOCOL]
lb-del LB [VIP] remove a load-balancer or just the VIP from
lb-list [LB] print load-balancers
lr-lb-add ROUTER LB add a load-balancer to ROUTER
lr-lb-del ROUTER [LB] remove load-balancers from ROUTER
lr-lb-list ROUTER print load-balancers
ls-lb-add SWITCH LB add a load-balancer to SWITCH
ls-lb-del SWITCH [LB] remove load-balancers from SWITCH
ls-lb-list SWITCH print load-balancers
配置負載均衡器規則
定義我們的負載均衡規則，即VIP和后端服務器IP池。 這里涉及的是在OVN北向數據庫中創建一個條目，並捕獲生成的UUID。 在的這次實驗中，我們將使用位於實驗室“數據”網絡中的VIP 10.127.0.254。 我們將使用vm1/vm2的地址作為池IP。
uuid=`ovn-nbctl create load_balancer vips:10.127.0.254="172.16.255.130,172.16.255.131"`
echo $uuid
上述命令在北向數據庫的load_balancer表中創建一個條目，並將生成的UUID存儲到變量“uuid”。 我們將在后面的命令中引用這個變量。
在網關路由器上配置負載均衡
在OVN網關路由器“edge1”上開啟負載均衡器功能。
ovn-nbctl set logical_router edge1 load_balancer=$uuid
ovn-nbctl set logical_router edge1 load_balancer=$uuid
您可以通過檢查edge1的數據庫條目來驗證是否成功開啟負載均衡器功能。
ovn-nbctl get logical_router edge1 load_balancer
ovn-nbctl get logical_router edge1 load_balancer
現在，我們可以從任何Ubuntu主機的全局命名空間連接到VIP。
# curl 10.127.0.254:8000
i am vm2
測試多次之后，可以確認負載平衡是相當隨機的。
讓我們看看禁用一個Web服務器會發生什么。 嘗試停止在vm1命名空間中運行的python進程。 這是我得到的輸出結果：
# curl 10.127.0.254:8000
curl: (7) Failed to connect to 10.127.0.254 port 8000: Connection refused
# curl 10.127.0.254:8000
i am vm2
# curl 10.127.0.254:8000
curl: (7) Failed to connect to 10.127.0.254 port 8000: Connection refused
# curl 10.127.0.254:8000
i am vm2
負載均衡器未執行任何類型的運行狀態檢查。 目前的計划是，運行狀態檢查將由協調解決方案（如Kubernetes）執行，該功能將在未來某個時間點被加入。
在進行下一個測試之前，在vm1上重新啟動python Web服務器。
負載均衡器在虛擬機外部運行着，讓我們來看看從內部虛擬機訪問VIP時會發生什么。
# ip netns exec vm3 curl 10.127.0.254:8000
i am vm1
# ip netns exec vm3 curl 10.127.0.254:8000
i am vm2
# ip netns exec vm3 curl 10.127.0.254:8000
i am vm1
# ip netns exec vm3 curl 10.127.0.254:8000
i am vm2
注意日志中的客戶端IP地址。第一個IP是上一輪測試的ubuntu1。第二個IP是edge1（來自vm3的請求）。為什么請求來自edge1而不是直接來自vm3？答案是，實現負載平衡的OVN開發人員使用了一種稱為“代理模式”的方法，其中負載均衡器在某些情況下隱藏了客戶端IP。為什么這是必要的？想想如果Web服務器看到vm3的真實IP會發生什么。來自服務器的響應將直接路由回到vm3，繞過edge1上的負載均衡器。從vm3的角度來看，它看起來像是向VIP發出請求，但收到了來自其中一個Web服務器的真實IP的回復。（如果不使用代理模式）負載均衡器就不工作了，這就是為什么代理模式功能很重要。
為了進行第二輪測試，先刪除負載均衡器配置
ovn-nbctl clear logical_router edge1 load_balancer
ovn-nbctl destroy load_balancer $uuid
ovn-nbctl clear logical_router edge1 load_balancer
ovn-nbctl destroy load_balancer $uuid
在邏輯交換機上配置負載均衡
接下來的實驗將負載均衡規則應用到邏輯交換機，會發生什么呢？ 由於我們將負載均衡從邊緣移開，第一步需要創建一個帶有內部VIP的新的負載均衡器。 我們將使用172.16.255.62作為VIP。
uuid=`ovn-nbctl create load_balancer vips:172.16.255.62="172.16.255.130,172.16.255.131"`
echo $uuid
uuid=`ovn-nbctl create load_balancer vips:172.16.255.62="172.16.255.130,172.16.255.131"`
echo $uuid
第一個測試:將負載均衡器應用於“內部”邏輯交換機。
# apply and verify
ovn-nbctl set logical_switch inside load_balancer=$uuid
ovn-nbctl get logical_switch inside load_balancer
# apply and verify
ovn-nbctl set logical_switch inside load_balancer=$uuid
ovn-nbctl get logical_switch inside load_balancer
然后從vm3測試（位於“inside”）:
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm1
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm1
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm2
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm1
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm1
# ip netns exec vm3 curl 172.16.255.62:8000
i am vm2
ovn-nbctl clear logical_switch inside load_balancer
ovn-nbctl set logical_switch dmz load_balancer=$uuid
ovn-nbctl get logical_switch dmz load_balancer
ovn-nbctl clear logical_switch inside load_balancer
ovn-nbctl set logical_switch dmz load_balancer=$uuid
ovn-nbctl get logical_switch dmz load_balancer
然后再次從 vm3測試:
# ip netns exec vm3 curl 172.16.255.62:8000
# ip netns exec vm3 curl 172.16.255.62:8000
# ip netns exec vm1 curl 172.16.255.62:8000
# ip netns exec vm1 curl 172.16.255.62:8000
ovn-nbctl clear logical_switch dmz load_balancer
ovn-nbctl destroy load_balancer $uuid
ovn-nbctl clear logical_switch dmz load_balancer
ovn-nbctl destroy load_balancer $uuid
uuid=`ovn-nbctl create load_balancer vips:20.0.0.100="20.0.0.10,20.0.0.20"`
ovn-nbctl ls-lb-add inside $uuid
流表差異
table=12, priority=100,ip,metadata=0x1,nw_dst=20.0.0.100 actions=load:0x1->NXM_NX_XXREG0[96],resubmit(,13)
table=17, priority=65535,ct_state=-new+est-rel-inv+trk,metadata=0x1 actions=load:0x1->NXM_NX_XXREG0[98],resubmit(,18)
table=18, priority=110,ct_state=+new+trk,ip,metadata=0x1,nw_dst=20.0.0.100 actions=group:1
table=40, priority=100,ip,metadata=0x1 actions=load:0x1->NXM_NX_XXREG0[96],resubmit(,41)
table=40, priority=100,ipv6,metadata=0x1 actions=load:0x1->NXM_NX_XXREG0[96],resubmit(,41)
table=43, priority=65535,ct_state=-new+est-rel-inv+trk,metadata=0x1 actions=load:0x1->NXM_NX_XXREG0[98],resubmit(,44)
流量追蹤
ping
[root@HikvisionOS ~]# ovs-appctl dpif/dump-flows br-int
recirc_id(0),in_port(3),ct_state(-new-est-rel-inv-trk),eth(src=02:d4:1d:8c:d9:9d,dst=02:d4:1d:8c:d9:9f),eth_type(0x0806),arp(sip=20.0.0.10,tip=20.0.0.1,op=1/0xff,sha=02:d4:1d:8c:d9:9d,tha=00:00:00:00:00:00), packets:0, bytes:0, used:never, actions:userspace(pid=4294963168,slow_path(action))
recirc_id(0x11d),in_port(5),ct_state(-new+est-rel-inv+trk),eth(src=02:d4:1d:8c:d9:9b,dst=02:d4:1d:8c:d9:9e),eth_type(0x0800),ipv4(src=10.0.0.10,dst=20.0.0.10,ttl=64,frag=no), packets:11, bytes:1078, used:0.296s, actions:ct_clear,ct_clear,set(eth(src=02:d4:1d:8c:d9:9f,dst=02:d4:1d:8c:d9:9d)),set(ipv4(src=10.0.0.10,dst=20.0.0.10,ttl=63)),3
recirc_id(0x11a),in_port(3),ct_state(-new+est-rel-inv+trk),eth(src=02:d4:1d:8c:d9:9e),eth_type(0x0800),ipv4(src=20.0.0.8/255.255.255.248,frag=no), packets:12, bytes:1176, used:0.296s, actions:ct(zone=9,nat),recirc(0x11b)
recirc_id(0),in_port(5),eth(src=02:d4:1d:8c:d9:9b),eth_type(0x0800),ipv4(src=10.0.0.10,dst=20.0.0.100,frag=no), packets:12, bytes:1176, used:0.296s, actions:ct(zone=9),recirc(0x117)
recirc_id(0x11b),in_port(3),eth(dst=02:d4:1d:8c:d9:9b),eth_type(0x0800),ipv4(dst=10.0.0.10,frag=no), packets:12, bytes:1176, used:0.296s, actions:5
recirc_id(0x11c),in_port(5),eth(src=02:d4:1d:8c:d9:9b,dst=02:d4:1d:8c:d9:9e),eth_type(0x0800),ipv4(dst=20.0.0.8/255.255.255.248,frag=no), packets:11, bytes:1078, used:0.296s, actions:ct(zone=9),recirc(0x119)
recirc_id(0),in_port(5),ct_state(-new-est-rel-inv-trk),eth(src=02:d4:1d:8c:d9:9b,dst=02:d4:1d:8c:d9:9e),eth_type(0x0806),arp(sip=10.0.0.10,tip=10.0.0.1,op=1/0xff,sha=02:d4:1d:8c:d9:9b,tha=00:00:00:00:00:00), packets:0, bytes:0, used:never, actions:userspace(pid=4294963166,slow_path(action))
recirc_id(0x119),in_port(5),ct_state(-new+est-rel-inv+trk),eth_type(0x0800),ipv4(frag=no), packets:11, bytes:1078, used:0.296s, actions:ct(zone=9,nat),recirc(0x11d)
recirc_id(0x117),in_port(5),ct_state(-new+est-rel-inv+trk),eth_type(0x0800),ipv4(frag=no), packets:11, bytes:1078, used:0.296s, actions:ct(zone=9,nat),recirc(0x11c)
recirc_id(0),in_port(3),ct_state(-new-est-rel-inv-trk),eth(src=02:d4:1d:8c:d9:9d,dst=02:d4:1d:8c:d9:9f),eth_type(0x0800),ipv4(src=20.0.0.10,dst=10.0.0.10,ttl=64,frag=no), packets:12, bytes:1176, used:0.296s, actions:ct_clear,ct_clear,set(eth(src=02:d4:1d:8c:d9:9e,dst=02:d4:1d:8c:d9:9b)),set(ipv4(src=20.0.0.10,dst=10.0.0.10,ttl=63)),ct(zone=9),recirc(0x11a)
ovs-appctl ofproto/trace br-int in_port=4,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,ipv4,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_proto=1,icmp_type=0,icmp_code=0 -generate
# ovs-appctl ofproto/trace br-int in_port=4,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,ipv4,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_proto=1,icmp_type=0,icmp_code=0 -generate
Flow: icmp,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-int")
----------------
0. in_port=4, priority 100
set_field:0x9->reg13
set_field:0x7->reg11
set_field:0xd->reg12
set_field:0x1->metadata
set_field:0x2->reg14
resubmit(,8)
8. reg14=0x2,metadata=0x1,dl_src=02:d4:1d:8c:d9:9b, priority 50, cookie 0x6047969c
resubmit(,9)
9. ip,reg14=0x2,metadata=0x1,dl_src=02:d4:1d:8c:d9:9b,nw_src=10.0.0.10, priority 90, cookie 0xb948ce75
resubmit(,10)
10. metadata=0x1, priority 0, cookie 0x5b23fa1b
resubmit(,11)
11. metadata=0x1, priority 0, cookie 0x85c5c31e
resubmit(,12)
12. ip,metadata=0x1,nw_dst=20.0.0.100, priority 100, cookie 0xdfbc9cba
load:0x1->NXM_NX_XXREG0[96]
resubmit(,13)
13. ip,reg0=0x1/0x1,metadata=0x1, priority 100, cookie 0xa5b7b054
ct(table=14,zone=NXM_NX_REG13[0..15])
drop
-> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 14.
Final flow: icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
Megaflow: recirc_id=0,eth,ip,in_port=4,vlan_tci=0x0000/0x1000,dl_src=02:d4:1d:8c:d9:9b,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_frag=no
Datapath actions: ct(zone=9),recirc(0x1cb)
===============================================================================
recirc(0x1cb) - resume conntrack with default ct_state=trk|new (use --ct-next to customize)
===============================================================================
Flow: recirc_id=0x1cb,ct_state=new|trk,ct_zone=9,eth,icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-int")
----------------
thaw
Resuming from table 14
14. metadata=0x1, priority 0, cookie 0x50063cdd
resubmit(,15)
15. metadata=0x1, priority 0, cookie 0xf31c70df
resubmit(,16)
16. metadata=0x1, priority 0, cookie 0x13c4db5f
resubmit(,17)
17. metadata=0x1, priority 0, cookie 0x78c30bb9
resubmit(,18)
18. ct_state=+new+trk,ip,metadata=0x1,nw_dst=20.0.0.100, priority 110, cookie 0x752ce65e
group:1
ct(commit,table=19,zone=NXM_NX_REG13[0..15],nat(dst=20.0.0.10))
nat(dst=20.0.0.10)
-> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 19.
Final flow: unchanged
Megaflow: recirc_id=0x1cb,ct_state=+new-est-rel-inv+trk,eth,icmp,in_port=4,vlan_tci=0x0000/0x1fff,vlan_tci1=0x0000/0x1fff,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_frag=no,icmp_type=0x0/0xff,icmp_code=0x0/0xff
Datapath actions: ct(commit,zone=9,nat(dst=20.0.0.10)),recirc(0x1cc)
===============================================================================
recirc(0x1cc) - resume conntrack with default ct_state=trk|new (use --ct-next to customize)
===============================================================================
Flow: recirc_id=0x1cc,ct_state=new|trk,ct_zone=9,eth,icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-int")
----------------
thaw
Resuming from table 19
19. metadata=0x1, priority 0, cookie 0x5dc957e1
resubmit(,20)
20. metadata=0x1, priority 0, cookie 0x75f5bdfa
resubmit(,21)
21. metadata=0x1, priority 0, cookie 0xa21b1697
resubmit(,22)
22. metadata=0x1, priority 0, cookie 0x31cb2e34
resubmit(,23)
23. metadata=0x1, priority 0, cookie 0x3626ad6f
resubmit(,24)
24. metadata=0x1,dl_dst=02:d4:1d:8c:d9:9e, priority 50, cookie 0x502275b8
set_field:0x1->reg15
resubmit(,32)
32. priority 0
resubmit(,33)
33. reg15=0x1,metadata=0x1, priority 100
set_field:0x7->reg11
set_field:0xd->reg12
resubmit(,34)
34. priority 0
set_field:0->reg0
set_field:0->reg1
set_field:0->reg2
set_field:0->reg3
set_field:0->reg4
set_field:0->reg5
set_field:0->reg6
set_field:0->reg7
set_field:0->reg8
set_field:0->reg9
resubmit(,40)
40. ip,metadata=0x1, priority 100, cookie 0x14cc5da4
load:0x1->NXM_NX_XXREG0[96]
resubmit(,41)
41. metadata=0x1, priority 0, cookie 0x65381f07
resubmit(,42)
42. ip,reg0=0x1/0x1,metadata=0x1, priority 100, cookie 0x65dbb075
ct(table=43,zone=NXM_NX_REG13[0..15])
drop
-> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 43.
Final flow: recirc_id=0x1cc,eth,icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,reg15=0x1,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
Megaflow: recirc_id=0x1cc,eth,ip,in_port=4,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_dst=20.0.0.64/26,nw_frag=no
Datapath actions: ct(zone=9),recirc(0x1cd)
===============================================================================
recirc(0x1cd) - resume conntrack with default ct_state=trk|new (use --ct-next to customize)
===============================================================================
Flow: recirc_id=0x1cd,ct_state=new|trk,ct_zone=9,eth,icmp,reg0=0x1,reg11=0x7,reg12=0xd,reg13=0x9,reg14=0x2,reg15=0x1,metadata=0x1,in_port=4,vlan_tci=0x0000,dl_src=02:d4:1d:8c:d9:9b,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.100,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-int")
----------------
thaw
Resuming from table 43
43. metadata=0x1, priority 0, cookie 0x441f8496
resubmit(,44)
44. metadata=0x1, priority 0, cookie 0x10069659
resubmit(,45)
45. metadata=0x1, priority 0, cookie 0xe5a2272f
resubmit(,46)
46. metadata=0x1, priority 0, cookie 0xdfdd721e
resubmit(,47)
47. metadata=0x1, priority 0, cookie 0x103a342b
resubmit(,48)
48. metadata=0x1, priority 0, cookie 0x49deb0bb
resubmit(,49)
49. reg15=0x1,metadata=0x1, priority 50, cookie 0x74ad6dec
resubmit(,64)
64. priority 0
resubmit(,65)
65. reg15=0x1,metadata=0x1, priority 100
clone(ct_clear,set_field:0->reg11,set_field:0->reg12,set_field:0->reg13,set_field:0x5->reg11,set_field:0xb->reg12,set_field:0x3->metadata,set_field:0x2->reg14,set_field:0->reg10,set_field:0->reg15,set_field:0->reg0,set_field:0->reg1,set_field:0->reg2,set_field:0->reg3,set_field:0->reg4,set_field:0->reg5,set_field:0->reg6,set_field:0->reg7,set_field:0->reg8,set_field:0->reg9,set_field:0->in_port,resubmit(,8))
ct_clear
set_field:0->reg11
set_field:0->reg12
set_field:0->reg13
set_field:0x5->reg11
set_field:0xb->reg12
set_field:0x3->metadata
set_field:0x2->reg14
set_field:0->reg10
set_field:0->reg15
set_field:0->reg0
set_field:0->reg1
set_field:0->reg2
set_field:0->reg3
set_field:0->reg4
set_field:0->reg5
set_field:0->reg6
set_field:0->reg7
set_field:0->reg8
set_field:0->reg9
set_field:0->in_port
resubmit(,8)
8. reg14=0x2,metadata=0x3,dl_dst=02:d4:1d:8c:d9:9e, priority 50, cookie 0x4a6a2617
resubmit(,9)
9. ip,metadata=0x3,nw_ttl=0, priority 30, cookie 0xcb4904dc
drop
Final flow: unchanged
Megaflow: recirc_id=0x1cd,ct_state=+new-est-rel-inv+trk,eth,ip,in_port=4,vlan_tci=0x0000/0x1000,dl_src=00:00:00:00:00:00/01:00:00:00:00:00,dl_dst=02:d4:1d:8c:d9:9e,nw_src=10.0.0.10,nw_dst=20.0.0.64/26,nw_ttl=0,nw_frag=no
Datapath actions: ct_clear
用的上的命令：
# ovs-ofctl -O OpenFlow13 dump-meters br-int
OFPST_METER_CONFIG reply (OF1.3) (xid=0x2):
ovn-nbctl list Logical_Router_Port
# ovn-sbctl list Logical_Flow
理論參考：
https://www.jianshu.com/p/40868a1428fc
https://docs.openstack.org/networking-ovn/latest/
https://www.ibm.com/developerworks/cn/cloud/library/1605-ovn-introduction/
https://www.sdnlab.com/19802.html
http://patchwork.ozlabs.org/patch/720449/
https://www.sdnlab.com/19216.html
https://www.sdnlab.com/19842.html
https://www.ibm.com/developerworks/cn/cloud/library/1603-ovn-ovs-openvswitch/index.html
https://www.cnblogs.com/silvermagic/p/7666089.html
https://www.cnblogs.com/silvermagic/p/7666072.html
https://www.cnblogs.com/silvermagic/p/7666111.html
https://www.cnblogs.com/silvermagic/p/7666117.html
https://www.cnblogs.com/silvermagic/p/7666124.html
https://www.itread01.com/content/1541340929.html
https://blog.csdn.net/ptmozhu/article/details/78644825?utm_source=blogxgwz3
OVN理論部分
OVN的架構和分析
OVN/CMS Plugin 是Neutron的一個插件，作為OVN 和 CMS 之間的接口 。它將CMS中的數據（存儲在Neutron DB）翻譯成一種“中間格式”。這種中間格式就是邏輯網絡配置數據，這樣CMS中的網絡配置數據就能夠被OVN的Northbound DB 所理解。
Northbound DB 里面的幾乎所有的內容都是由 CMS 產生的，里面存的就是上面OVN/CMS Plugin翻譯之后的邏輯網絡的相關數據。比如 logical switch，logical router，logical port和ACL。OVN-northd 類似於一個集中的控制器，監聽Northbound DB 數據庫的內容變化，它把 Northbound DB 里面的邏輯網絡的相關數據翻譯成 Southbound DB 可理解的格式（logical datapath flows），並傳遞給 Southbound DB 進行存儲，進而被所有的chassis 讀取和應用
Southbound DB 處在 OVN 架構的核心，它是 OVN 中最重要的部分，它跟 OVN 的其他組件都有交互。 里面存的數據和 Northbound DB 語義完全不一樣，主要包含 3 類數據：
一、物理網絡數據，比如 hypervisor的 IP 地址，hypervisor的 tunnel 封裝格式；
二、邏輯網絡數據，比如報文如何在邏輯網絡中轉發；是OVN-northd 從Northbound DB 翻譯過來的
三、物理網絡和邏輯網絡的綁定關系，比如邏輯端口關聯到哪個 hypervisor上面。這類數據存儲在binding表中，字段有uuid,chassis, logical_datapath, logical_port, mac, parent_port, tag, tunnel_key。
ovn-controller 是 OVN 里面的 agent，類似於 Neutron 里面的 ovs-agent，它也是運行在每個 hypervisor和軟件網關之上。
它有下面2種功能：
（1）把物理網絡的信息寫到 Southbound DB 里面（這類信息就包括 Southbound DB中的第一類數據）；
（2）把 Southbound DB 里面存的一些數據轉化成 Openflow flow 配到本地的 OVS table 里面，來實現報文的轉發。
第2個功能的具體實現機制就是：
ovn-controller連接到到本地的ovsdb-server ，監控、讀取、管理OpenvSwitch的配置信息；
ovn-controller作為ovs-vswitchd 的Openflow 控制器來控制流量的轉發。另外，從架構圖中就可看出ovn-controller是一種分布式SDN控制器。
ovs-vswitchd 和 ovsdb-server 是 OVS 的兩個進程：
- ovs-vswitchd ：核心模塊，實現交換功能，和Linux內核模塊一起，實現基於流的交換；
- ovsdb-server ：是一個數據庫。其保存了整個OVS的配置信息，包括接口，流表和VLAN等；ovs-vswitchd從其查詢配置信息；
OVN Northbound DB
Northbound DB 是 OVN 和 CMS 之間的接口，Northbound DB 里面的幾乎所有的內容都是由 CMS 產生的，ovn-northd 監聽這個數據庫的內容變化，然后翻譯，保存到 Southbound DB 里面。
Northbound DB 里面主要有如下幾張表：
Logical_Switch：每一行代表一個邏輯交換機，邏輯交換機有兩種，一種是 overlay logical switches，對應於 neutron network，每創建一個 neutron network，networking-ovn 會在這張表里增加一行；另一種是 bridged logical switch，連接物理網絡和邏輯網絡，被 VTEP gateway 使用。Logical_Switch 里面保存了它包含的 logical port（指向 Logical_Port table）和應用在它上面的 ACL（指向 ACL table）。
Logical_Port：每一行代表一個邏輯端口，每創建一個 neutron port，networking-ovn 會在這張表里增加一行，每行保存的信息有端口的類型，比如 patch port，localnet port，端口的 IP 和 MAC 地址，端口的狀態 UP/Down。
ACL：每一行代表一個應用到邏輯交換機上的 ACL 規則，如果邏輯交換機上面的所有端口都沒有配置 security group，那么這個邏輯交換機上不應用 ACL。每條 ACL 規則包含匹配的內容，方向，還有動作。
Logical_Router：每一行代表一個邏輯路由器，每創建一個 neutron router，networking-ovn 會在這張表里增加一行，每行保存了它包含的邏輯的路由器端口。
Logical_Router_Port：每一行代表一個邏輯路由器端口，每創建一個 router interface，networking-ovn 會在這張表里加一行，它主要保存了路由器端口的 IP 和 MAC。
OVN Southbound DB
Southbound DB 里面有如下幾張表：
Chassis：每一行表示一個 HV 或者 VTEP 網關，由 ovn-controller/ovn-controller-vtep 填寫，包含 chassis 的名字和 chassis 支持的封裝的配置（指向表 Encap），如果 chassis 是 VTEP 網關，VTEP 網關上和 OVN 關聯的邏輯交換機也保存在這張表里。
Encap：保存着 tunnel 的類型和 tunnel endpoint IP 地址。
Logical_Flow：每一行表示一個邏輯的流表，這張表是 ovn-northd 根據 Nourthbound DB 里面二三層拓撲信息和 ACL 信息轉換而來的，ovn-controller 把這個表里面的流表轉換成 OVS 流表，配到 HV 上的 OVS table。流表主要包含匹配的規則，匹配的方向，優先級，table ID 和執行的動作。
Multicast_Group：每一行代表一個組播組，組播報文和廣播報文的轉發由這張表決定，它保存了組播組所屬的 datapath，組播組包含的端口，還有代表 logical egress port 的 tunnel_key。
Datapath_Binding：每一行代表一個 datapath 和物理網絡的綁定關系，每個 logical switch 和 logical router 對應一行。它主要保存了 OVN 給 datapath 分配的代表 logical datapath identifier 的 tunnel_key。
Port_Binding：這張表主要用來確定 logical port 處在哪個 chassis 上面。每一行包含的內容主要有 logical port 的 MAC 和 IP 地址，端口類型，端口屬於哪個 datapath binding，代表 logical input/output port identifier 的 tunnel_key, 以及端口處在哪個 chassis。端口所處的 chassis 由 ovn-controller/ovn-controller 設置，其余的值由 ovn-northd 設置。
表 Chassis 和表 Encap 包含的是物理網絡的數據，表 Logical_Flow 和表 Multicast_Group 包含的是邏輯網絡的數據，表 Datapath_Binding 和表 Port_Binding 包含的是邏輯網絡和物理網絡綁定關系的數據。
OVN security group 對比 Neutron security group
OVN tunnel
OVN 支持的 tunnel 類型有三種，分別是 Geneve，STT 和 VXLAN。HV 與 HV 之間的流量，只能用 Geneve 和 STT 兩種，HV 和 VTEP 網關之間的流量除了用Geneve和STT外，還能用 VXLAN，這是為了兼容硬件 VTEP 網關，因為大部分硬件VTEP網關只支持 VXLAN。
雖然 VXLAN 是數據中心常用的 tunnel 技術，但是 VXLAN header 是固定的，只能傳遞一個 VNID（VXLAN network identifier），如果想在 tunnel 里面傳遞更多的信息，VXLAN 實現不了。所以 OVN 選擇了 Geneve 和 STT，Geneve 的頭部有個 option 字段，支持 TLV 格式，用戶可以根據自己的需要進行擴展，而 STT 的頭部可以傳遞 64-bit 的數據，比 VXLAN 的 24-bit 大很多。
OVN tunnel 封裝時使用了三種數據，
Logical datapath identifier（邏輯的數據通道標識符）：datapath 是 OVS 里面的概念，報文需要送到 datapath 進行處理，一個 datapath 對應一個 OVN 里面的邏輯交換機或者邏輯路由器，類似於 tunnel ID。這個標識符有 24-bit，由 ovn-northd 分配的，全局唯一，保存在 Southbound DB 里面的表 Datapath_Binding 的列 tunnel_key 里。
Logical input port identifier（邏輯的入端口標識符）：進入 logical datapath 的端口標識符，15-bit 長，由 ovn-northd 分配的，在每個 datapath 里面唯一。它可用范圍是 1-32767，0 預留給內部使用。保存在 Southbound DB 里面的表 Port_Binding 的列 tunnel_key 里。
Logical output port identifier（邏輯的出端口標識符）：出 logical datapath 的端口標識符，16-bit 長，范圍 0-32767 和 logical input port identifier 含義一樣，范圍 32768-65535 給組播組使用。對於每個 logical port，input port identifier 和 output port identifier 相同。
如果 tunnel 類型是 Geneve，Geneve header 里面的 VNI 字段填 logical datapath identifier，Option 字段填 logical input port identifier 和 logical output port identifier，TLV 的 class 為 0xffff，type 為 0，value 為 1-bit 0 + 15-bit logical input port identifier + 16-bit logical output port identifier。
如果 tunnel 類型是 STT，上面三個值填在 Context ID 字段，格式為 9-bit 0 + 15-bit logical input port identifier + 16-bit logical output port identifier + 24-bit logical datapath identifier。
OVS 的 tunnel 封裝是由 Openflow 流表來做的，所以 ovn-controller 需要把這三個標識符寫到本地 HV 的 Openflow flow table 里面，對於每個進入 br-int 的報文，都會有這三個屬性，logical datapath identifier 和 logical input port identifier 在入口方向被賦值，分別存在 openflow metadata 字段和 Nicira 擴展寄存器 reg6 里面。報文經過 OVS 的 pipeline 處理后，如果需要從指定端口發出去，只需要把 Logical output port identifier 寫在 Nicira 擴展寄存器 reg7 里面。
OVN tunnel 里面所攜帶的 logical input port identifier 和 logical output port identifier 可以提高流表的查找效率，OVS 流表可以通過這兩個值來處理報文，不需要解析報文的字段。
從上一章節可以看到，OVN 里面的 tunnel 類型是由 HV 上面的 ovn-controller 來設置的，並不是由 CMS 指定的，並且 OVN 里面的 tunnel ID 又由 OVN 自己分配的，所以用 neutron 創建 network 時指定 tunnel 類型和 tunnel ID（比如 vnid）是無用的，OVN 不做處理。
OVN VTEP 網關
小結：
OVN存在的意義（目標）
- 可用於生產環境
- 簡潔的設計
- 支持1000台以上的物理機環境（也支持相當數量的虛擬機/容器環境）
- 基於已有的OpenStack OVS 插件 來提升性能和穩定性
- 成為OpenStack+OVS集成場景下的首選方案
已經實現從OVS 平滑升級到 OVN
OVN 對於運行平台沒有額外的要求，只要能夠運行 OVS，就可以運行 OVN，可以和 Linux，Docker，DPDK 還有 Hyper-V 兼容，所以從 OVS 升級到 OVN 是非常簡單快捷的。原有的網絡、路由等數據不會丟失，也不需要對這些數據導入導出來進行數據遷移
另外 OVN 可以和很多 CMS（Cloud Management System）集成到一起，尤其是 OpenStack Neutron，這些 CMS 只需要添加一個 plugin 來配置 OVN 即可。
OVN對neutron的改變（以Ocata版本中的OVN和OVS 2.9版本來看OVN帶來的變化）
OVN 里面數據的讀寫都是通過 OVSDB 協議來做的，取代了 neutron 里面的消息隊列機制，neutron 變成了一個 API server 來處理用戶的 REST 請求，其他的功能都交給 OVN 來做。
使得Neutron組件數量減少
OVN原生的ML2 driver替換掉 OVS ML2 driver 和 Neutron的OVS agent；
OVN原生支持L3和DHCP功能，這樣就不再需要Neutron 的L3 agent、 DHCP agent 和DVR。
從 OVN 的架構可以看出，OVN 里面數據的讀寫都是通過 OVSDB來做的，取代了 Neutron 的消息隊列機制，所以有了 OVN 之后，Neutron 里面所有的 agent 都不需要了，Neutron 變成了一個 API server 來處理用戶的 REST 請求，其他的功能都交給 OVN 來做，只需要在 Neutron 里面加一個 plugin 來調用配置 OVN。
Neutron 里面的子項目 networking-ovn 就是實現 OVN 的 plugin。Plugin 使用 OVSDB 協議來把用戶的配置寫在 Northbound DB 里，ovn-northd 監聽到 Northbound DB 配置發生改變，然后把配置翻譯到 Southbound DB 里面。 ovn-controller 監控到 Southbound DB 數據的發生變化之后，進而更新本地的流表。
OVN 里面報文的處理都是通過 OVS OpenFlow 流表來實現的，而在 Neutron 里面二層報文處理是通過 OVS OpenFlow 流表來實現，三層報文處理是通過 Linux TCP/IP 協議棧來實現。
OVN L3 對比 Neutron L3
Neutron 的三層功能主要有路由，SNAT 和 Floating IP（也叫 DNAT），它是通 Linux kernel 的namespace 來實現的，每個路由器對應一個 namespace，利用 Linux TCP/IP 協議棧來做路由轉發。
OVN 支持原生的三層功能，不需要借助 Linux TCP/IP stack，用OpenFlow 流表來實現路由查找，ARP 查找，TTL 和 MAC 地址的更改。OVN 的路由也是分布式的，路由器在每個計算節點上都有實例，有了 OVN 之后，不需要 Neutron L3 agent 了 和DVR了。
OVN和其它通用SDN控制器（比如OpenDayLight）的主要區別
OVN專注於實現雲計算管理平台場景下的SDN控制器
OVN專注於實現二層和三層網絡功能。除了在傳輸層實現了基於L4的ACL 外，基本上不在L4 ~ L7層實現某些功能。
OVN的實現了哪些功能？擁有哪些特性？
Logical switches：邏輯交換機，用來做二層轉發。
L2/L3/L4 ACLs：二到四層的 ACL，可以根據報文的 MAC 地址，IP 地址，端口號來做訪問控制。
Logical routers：邏輯路由器，分布式的，用來做三層轉發。
Multiple tunnel overlays：支持多種隧道封裝技術，有 Geneve，STT 和 VXLAN。
TOR switch or software logical switch gateways：支持使用硬件 TOR switch 或者軟件邏輯 switch 當作網關來連接物理網絡和虛擬網絡。
ovs ovn 學習資料
0、A Primer on OVN
http://blog.spinhirne.com/2016/09/a-primer-on-ovn.html
1、Open Virtual Networking With Docker
http://docs.openvswitch.org/en/latest/howto/docker/
2、Multi-Host Docker network
https://wiredcraft.com/blog/multi-host-docker-network/
3、ovn-namespace
https://github.com/shettyg/ovn-namespace
4、OVN簡介PPT
http://openvswitch.org/support/slides/OVN_Barcelona.pdf
5、What is Open Virtual Network (OVN)? How It Works （包含了各種關於網絡虛擬化的介紹的連接）
https://www.sdxcentral.com/sdn/network-virtualization/definitions/what-is-open-virtual-network-ovn-how-it-works/
6、Open vSwitch 相關論文
http://openvswitch.org/support/papers/
７、OVN, Bringing Native Virtual Networking to OVS
https://networkheresy.com/category/open-vswitch/
8、基於Open vSwitch的OpenFlow實踐
http://www.chenshake.com/based-on-openflow-practices-open-vswitch/
9、ovs源碼分析
http://blog.csdn.net/column/details/openvswitch.html
10、ovs orbit
https://ovsorbit.org/
11、introduction to ovn
http://galsagie.github.io/2015/04/20/ovn-1/
12、Russell Bryant的博客
https://blog.russellbryant.net/category/ovs/
13、ovn architecture
http://openvswitch.org/support/dist-docs/ovn-architecture.7.html
14、OVN Logical Flows and ovn-trace
https://blog.russellbryant.net/2016/11/11/ovn-logical-flows-and-ovn-trace/
15、Justin Pettit的個人主頁（其中包含了ovs, ovn相關的各種論文，博客和視頻）
http://yuba.stanford.edu/~jpettit/
16、ovs 2.5.0源碼分析
http://blog.csdn.net/one_clouder/article/category/6359278/1
17、netwoking-ovn - OpenStack Neutron integration with OVN
https://docs.openstack.org/networking-ovn/latest/
18、OVN路由功能詳解
https://www.ibm.com/developerworks/cn/cloud/library/1605-ovn-introduction/index.html
19、OVS博客
http://www.cnblogs.com/popsuper1982/p/5848879.html
20、OVSDB RFC
https://datatracker.ietf.org/doc/rfc7047/
21、openstack底層技術-openflow在ovs中的應用
http://www.isjian.com/openstack/openstack-base-openflow-in-openvswitch/
命令行
yum install -y openvswitch.x86_64
yum install -y openvswitch-devel.x86_64
yum install -y openvswitch-ovn-central.x86_64
yum install -y openvswitch-ovn-common.x86_64
yum install -y openvswitch-ovn-docker.x86_64
yum install -y openvswitch-ovn-host.x86_64
yum install -y openvswitch-ovn-vtep.x86_64
yum install -y openvswitch-test.noarch
yum install -y python2-openvswitch.noarch
export Centralip=10.33.46.4
ovn-nbctl set-connection ptcp:6641:$Centralip
ovn-sbctl set-connection ptcp:6642:$Centralip
ovs-vsctl set open . external-ids:ovn-remote=tcp:$Centralip:6642
ovs-vsctl set open . external-ids:ovn-encap-type=geneve
ovs-vsctl set open . external-ids:ovn-encap-ip=$Centralip
ovs-vsctl set open . external-ids:ovn-remote=tcp:$Centralip:6642
ovs-vsctl set open . external-ids:ovn-encap-type=geneve
export Nodeip=10.33.46.4
ovs-vsctl set open . external-ids:ovn-encap-ip=$Nodeip
ovn-nbctl ls-add inside
ovn-nbctl ls-add dmz
ovn-nbctl lr-add tenant1
ovn-nbctl lrp-add tenant1 tenant1-dmz 02:d4:1d:8c:d9:9f 20.0.0.1/24
ovn-nbctl lsp-add dmz dmz-tenant1
ovn-nbctl lsp-set-type dmz-tenant1 router
ovn-nbctl lsp-set-addresses dmz-tenant1 02:d4:1d:8c:d9:9f
ovn-nbctl lsp-set-options dmz-tenant1 router-port=tenant1-dmz
ovn-nbctl lrp-add tenant1 tenant1-inside 02:d4:1d:8c:d9:9e 10.0.0.1/24
ovn-nbctl lsp-add inside inside-tenant1
ovn-nbctl lsp-set-type inside-tenant1 router
ovn-nbctl lsp-set-addresses inside-tenant1 02:d4:1d:8c:d9:9e
ovn-nbctl lsp-set-options inside-tenant1 router-port=tenant1-inside
ovn-nbctl show
ovn-nbctl lsp-add dmz dmz-vm1
ovn-nbctl lsp-set-addresses dmz-vm1 "02:d4:1d:8c:d9:9d 20.0.0.10"
ovn-nbctl lsp-set-port-security dmz-vm1 "02:d4:1d:8c:d9:9d 20.0.0.10"
ovn-nbctl lsp-add dmz dmz-vm2
ovn-nbctl lsp-set-addresses dmz-vm2 "02:d4:1d:8c:d9:9c 20.0.0.20"
ovn-nbctl lsp-set-port-security dmz-vm2 "02:d4:1d:8c:d9:9c 20.0.0.20"
ovn-nbctl lsp-add inside inside-vm3
ovn-nbctl lsp-set-addresses inside-vm3 "02:d4:1d:8c:d9:9b 10.0.0.10"
ovn-nbctl lsp-set-port-security inside-vm3 "02:d4:1d:8c:d9:9b 10.0.0.10"
ovn-nbctl lsp-add inside inside-vm4
ovn-nbctl lsp-set-addresses inside-vm4 "02:d4:1d:8c:d9:9a 10.0.0.20"
ovn-nbctl lsp-set-port-security inside-vm4 "02:d4:1d:8c:d9:9a 10.0.0.20"
ovn-nbctl show
ovn-nbctl create DHCP_Options cidr=20.0.0.0/24 options="\"server_id\"=\"20.0.0.1\" \"server_mac\"=\"02:d4:1d:8c:d9:9f\" \"lease_time\"=\"36000\" \"router\"=\"20.0.0.1\""
ovn-nbctl create DHCP_Options cidr=10.0.0.0/24 options="\"server_id\"=\"10.0.0.1\" \"server_mac\"=\"02:d4:1d:8c:d9:9e\" \"lease_time\"=\"360000\" \"router\"=\"10.0.0.1\""
ovn-nbctl dhcp-options-list
ovn-nbctl lsp-set-dhcpv4-options dmz-vm1 a5c06e37-496f-4b9c-ad79-178bd266d128
ovn-nbctl lsp-set-dhcpv4-options dmz-vm2 a5c06e37-496f-4b9c-ad79-178bd266d128
ovn-nbctl lsp-set-dhcpv4-options inside-vm3 38975d0a-658c-4064-a203-361a708045b1
ovn-nbctl lsp-set-dhcpv4-options inside-vm4 38975d0a-658c-4064-a203-361a708045b1
ovn-nbctl dhcp-options-list
ovn-nbctl dhcp-options-show 38975d0a-658c-4064-a203-361a708045b1
ovn-nbctl dhcp-options-get-options 38975d0a-658c-4064-a203-361a708045b1
ovn-nbctl lsp-get-dhcpv4-options dmz-vm1
ovn-nbctl lsp-get-dhcpv4-options dmz-vm2
ovn-nbctl lsp-get-dhcpv4-options inside-vm3
ovn-nbctl lsp-get-dhcpv4-options inside-vm4
ovs-vsctl show
ovs-vsctl del-port br-int vm1
ip netns add vm1
ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
ip link set vm1 address 02:d4:1d:8c:d9:9d
ip link set vm1 netns vm1
ovs-vsctl set Interface vm1 external_ids:iface-id=dmz-vm1
ip netns exec vm1 dhclient vm1
ip netns exec vm1 ip addr show vm1
ip netns add vm2
ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
ip link set vm2 address 02:d4:1d:8c:d9:9c
ip link set vm2 netns vm2
ovs-vsctl set Interface vm2 external_ids:iface-id=dmz-vm2
ip netns exec vm2 killall dhclient
ip netns exec vm2 dhclient vm2
ip netns exec vm2 ip addr show vm2
ip netns add vm3
ovs-vsctl add-port br-int vm3 -- set interface vm3 type=internal
ip link set vm3 address 02:d4:1d:8c:d9:9b
ip link set vm3 netns vm3
ovs-vsctl set Interface vm3 external_ids:iface-id=inside-vm3
ip netns exec vm3 killall dhclient
ip netns exec vm3 dhclient vm3
ip netns exec vm3 ip addr show vm3
ip netns add vm4
ovs-vsctl add-port br-int vm4 -- set interface vm4 type=internal
ip link set vm4 address 02:d4:1d:8c:d9:9a
ip link set vm4 netns vm4
ovs-vsctl set Interface vm4 external_ids:iface-id=inside-vm4
ip netns exec vm4 killall dhclient
ip netns exec vm4 dhclient vm4
ip netns exec vm4 ip addr show vm4
ip netns exec vm1 ping -c 2 20.0.0.1
ip netns exec vm1 ping -c 2 20.0.0.20
ip netns exec vm1 ping -c 2 10.0.0.10
ovn-sbctl show
ovn-nbctl lr-add edge1
ovn-nbctl ls-add transit
ovn-nbctl lrp-add edge1 edge1-transit 02:d4:1d:8c:d9:ae 192.168.0.1/24
ovn-nbctl lsp-add transit transit-edge1
ovn-nbctl lsp-set-type transit-edge1 router
ovn-nbctl lsp-set-addresses transit-edge1 02:d4:1d:8c:d9:ae
ovn-nbctl lsp-set-options transit-edge1 router-port=edge1-transit
ovn-nbctl lrp-add tenant1 tenant1-transit 02:d4:1d:8c:d9:af 192.168.0.2/24
ovn-nbctl lsp-add transit transit-tenant1
ovn-nbctl lsp-set-type transit-tenant1 router
ovn-nbctl lsp-set-addresses transit-tenant1 02:d4:1d:8c:d9:af
ovn-nbctl lsp-set-options transit-tenant1 router-port=tenant1-transit
ovn-nbctl lr-route-list edge1
ip netns exec vm1 ping -c 2 192.168.0.1
ovn-nbctl lr-route-add tenant1 "0.0.0.0/0" 192.168.0.1
ip netns exec vm1 ping -c 2 192.168.0.1
ovn-nbctl lr-route-add edge1 "10.0.0.0/24" 192.168.0.2
ovn-nbctl lr-route-add edge1 "20.0.0.0/24" 192.168.0.2
ip netns exec vm1 ping -c 2 192.168.0.1
ovn-nbctl ls-add outside
ovn-nbctl lrp-add edge1 edge1-outside 02:d4:1d:8c:d9:be 192.168.200.16/24
ovn-nbctl lsp-add outside outside-edge1
ovn-nbctl lsp-set-type outside-edge1 router
ovn-nbctl lsp-set-addresses outside-edge1 02:d4:1d:8c:d9:be
ovn-nbctl lsp-set-options outside-edge1 router-port=edge1-outside
# Central節點
# 創建外網邏輯交換機，並配置網關到叫交換機的連接
ovn-nbctl ls-add outside
ovn-nbctl lrp-add edge1 edge1-outside 02:d4:1d:8c:d9:be 192.168.200.16/24
ovn-nbctl lsp-add outside outside-edge1
ovn-nbctl lsp-set-type outside-edge1 router
ovn-nbctl lsp-set-addresses outside-edge1 02:d4:1d:8c:d9:be
ovn-nbctl lsp-set-options outside-edge1 router-port=edge1-outside
# 為外網網卡ens4創建網橋
ovs-vsctl add-br br-ex
# 為外網網卡ens4創建網橋到網絡的映射
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=dataNet:br-ex
# 在邏輯交換機outside上添加本地網絡端口，並且本地網絡的名字為dataNet
ovn-nbctl lsp-add outside outside-localnet
ovn-nbctl lsp-set-addresses outside-localnet unknown
ovn-nbctl lsp-set-type outside-localnet localnet
ovn-nbctl lsp-set-options outside-localnet network_name=dataNet
# 關聯外網網卡ens4到網橋上
ovs-vsctl add-port br-ex ens3
# 測試連通性（需要注意vm2的ip地址是不是沒了，dhclient好像有些問題）
ip netns exec vm2 ping -c 2 192.168.200.16
# 設置網橋地址
ip addr add 192.168.200.17/24 dev br-ex
ip link set br-ex up
ip route del 192.168.200.0/24 dev ens3
#設置SNAT
# Central節點
# 設置網關chassis
ovn-nbctl lrp-set-gateway-chassis edge1-outside ddeaaec0-eb6e-4ae2-a4cd-2d97e6696e6d
# 配置SNAT規則
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=20.0.0.0/24 external_ip=192.168.200.16 -- add logical_router edge1 nat @nat
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=10.0.0.0/24 external_ip=192.168.200.16 -- add logical_router edge1 nat @nat
# 測試連通性
ip netns exec vm2 ping -c 2 192.168.200.17
# Node節點
ip netns exec vm4 ping -c 2 192.168.200.1
命令：
ovn-sbctl list SB_Globalovn-nbctl list NB_Global
ovn-sbctl list Port_Binding
ovn-nbctl find Logical_Switch_Port name="f6c17ce0-083b-4def-a6d8-9ebb7a69da04"ovn-sbctl list Connection
ovn-sbctl list SSL
ovn-sbctl list DHCP_Options
ovn-sbctl list DHCPv6_Options
ovn-sbctl lflow-list (邏輯流表項154個)
ovn-sbctl list Logical_Flow (邏輯流154個)
https://blog.csdn.net/zhengmx100/article/details/75426710
https://blog.csdn.net/zhengmx100/article/details/75426014
ovn-nbctl list NB_Global
ovn-nbctl list Logical_Switch_Port
ovn-nbctl list ACL
ovn-nbctl list Address_Set
ovn-nbctl list Connection 配置數據庫連接到ovsdb客戶端
ovn-nbctl list DHCP_Options
對IPv6的支持
DHCPv6 stateful
ipv6 neutron應用（一）
一、neutron支持ipv6，有2個重要的屬性
1、ipv6_ra_mode
2、ipv6_address_mode
這2個屬性都可以設置下面三個值
1、slaac
2、dhcpv6-stateful
3、dhcpv6-stateless
ipv6_address_mode用於客戶端如何獲取ipv6地址
ipv6_ra_mode用於在子網內路由器的通告，用icmpv6報文來實現，type為134的報文
slaac，無狀態地址自動配置，客戶端通過路由器返回的RA消息配置本地地址
dhcpv6-stateful，ip地址和dns等信息從dhcp服務器獲取
dhcpv6-stateless，ip地址從路由器RA消息獲取，dns等信息從dhcp服務器獲取
我們目前支持的是dhcpv6-stateful+dhcpv6-stateful
實例通過 DHCPv6服務器接收一個 IPv6 地址，使用 DHCPv6 服務器獲得其它可選信息，該DHCPv6服務來自openstack
二、neutron應用
1、創建ipv6網絡，子網
neutron subnet-create --name subnet_v6 --enable-dhcp --ip-version 6 --ipv6-ra-mode dhcpv6-stateful --ipv6-address-mode dhcpv6-stateful --dns-nameserver 2001:4860:4860::8844 ipv6_test 2001:1001::/64
2、網絡綁定路由器，開通該網絡虛機
neutron router-interface-add ipv6_test subnet=2e3f9776-3545-4271-8a96-2e6e285d9500
nova boot wx_ipv6_test1 --flavor 2 --image 78544ead-fc8b-4b0e-9ff4-f21cfae42a55 --nic net-id=9c7a3663-a779-444a-9f97-07b60a592a03
OpenStack 中的5種分配IPv6地址的方式
No options specified（Default），
SLAAC: Address discovered from an OpenStack router
SLAAC: Address discovered from an external router
DHCPv6 stateless : Address discovered from OpenStack Router and additional information from OpenStack DHCP
DHCPv6 stateful : Address discovered from OpenStack DHCP
接下來對上述五種方式進行說明：
手工配置，即需要管理員對虛擬機逐台手工配置IPv6地址。
SLAAC (StateLess Autoconfiguration)其實就是利用路由宣告消息（RA）來確定前綴和長度，利用EUI-64算法計算出接口ID。
OpenStack 中又將SLAAC細分為2種，第一種就是利用OpenStack router 路由宣告消息（RA）報文來生成IPv6地址、
第二種SLAAC:利用外部 router的路由宣告消息（RA）報文來生成IPv6地址
無狀態DHCPv6,利用OpenStack router 路由宣告消息（RA）報文來生成IPv6地址 ,從OpenStack DHCPv6服務器獲取其它信息，比如DNS服務器地址、NTP服務器地址、WINS服務器地址、TFTP服務器地址、IP電話服務器地址、證書服務器地址等。
有狀態DHCPv6，從從OpenStack DHCPv6服務器獲取IPv6地址及其它信息，比如DNS服務器地址、NTP服務器地址、WINS服務器地址、TFTP服務器地址、IP電話服務器地址、證書服務器地址等。
手動配置測試ipv6-dhcp
ovn-nbctl lsp-add 98e1793b-b6a7-4d0b-a9c8-336c467795bd ljx-vm1
ovn-nbctl lsp-set-addresses ljx-vm1 "02:d4:1d:8c:d9:10 2003::2"
ovn-nbctl create DHCP_Options cidr=2003::/64 options="\"server_id\"=\"02:d4:1d:8c:99:99\" \"ia_addr\"=\"2003::1\" \"dns_server\"=\"2003::1\" \"domain_search\"=\"openstacklocal\" "
ovn-nbctl set DHCP_Options deb8260c-241e-46f3-a275-24bf1eb8a590 options="\"server_id\"=\"02:d4:1d:8c:99:99\" \"ia_addr\"=\"2003::1\" \"dns_server\"=\"2003::1\" \"domain_search\"=\"openstacklocal\" "
ovn-nbctl list DHCP_OPTIONS
ovn-nbctl lsp-set-dhcpv6-options ljx-vm1 deb8260c-241e-46f3-a275-24bf1eb8a590
ovn-nbctl lsp-get-dhcpv6-options ljx-vm1
ip netns add ljx
ovs-vsctl add-port br-int ljx-vm1 -- set interface ljx-vm1 type=internal
ovs-vsctl set Interface ljx-vm1 external_ids:iface-id=ljx-vm1
ip link set ljx-vm1 netns ljx
ovs-vsctl set Interface ljx-vm1 external_ids:iface-id=ljx-vm1
ip netns exec ljx dhclient ljx-vm1
ip netns exec ljx ip addr show ljx-vm1
ovn-nbctl find Logical_Switch_Port name="ljx-vm1"
https://blog.csdn.net/zhengmx100/article/details/78854478