速記符	匹配項
ip	dl_type=0x800
ipv6	dl_type=0x86dd
icmp	dl_type=0x0800,nw_proto=1
icmp6	dl_type=0x86dd,nw_proto=58
tcp	dl_type=0x0800,nw_proto=6
tcp6	dl_type=0x86dd,nw_proto=6
udp	dl_type=0x0800,nw_proto=17
udp6	dl_type=0x86dd,nw_proto=17
sctp	dl_type=0x0800,nw_proto=132
sctp6	dl_type=0x86dd,nw_proto=132
arp	dl_type=0x0806
rarp	dl_type=0x8035
mpls	dl_type=0x8847
mplsm	dl_type=0x8848

NXM字段	報文字段
NXM_OF_ETH_SRC	源MAC
NXM_OF_ETH_DST	目的MAC
NXM_OF_ETH_TYPE	以太網類型
NXM_OF_VLAN_TCI	vid
NXM_OF_IP_PROTO	IP協議號
NXM_OF_IP_TOS	IP ToS值
NXM_NX_IP_ECN	IP ToS ECN
NXM_OF_IP_SRC	源IP
NXM_OF_IP_DST	目的IP
NXM_OF_TCP_SRC	TCP源端口
NXM_OF_TCP_DST	TCP目的端口
NXM_OF_UDP_SRC	UDP源端口
NXM_OF_UDP_DST	UDP目的端口
NXM_OF_SCTP_SRC	SCTP源端口
NXM_OF_SCTP_DST	SCTP目的端口

2.設計考慮

2.1.整體數據結構

ovs datapath classifier涉及的數據結構主要有如下。

網橋數據結構

struct datapath {

struct rcu_head rcu;

/* 該結構使用雙鏈表組織 */

struct list_head list_node;

/* 從屬該datapath的流表項 */

struct flow_table table;

/* 從屬該datapath的vport信息 */

struct hlist_head *ports;

…

}

流表數據結構

struct flow_table {

/* 用key組織的流表實例 */

struct table_instance *ti;

/* 用unique flow identifier(ufid)組織的流表實例 */

struct table_instance *ufid_ti;

/* 掩碼信息緩存表 */

struct mask_cache_entry *mask_cache;

/* 掩碼信息列表 */

struct mask_array *mask_array;

/* 記錄當前節拍數 */

unsigned long last_rehash;

/* ti流表實例中存儲的流表項數目 */

unsigned int count;

/* ufid_ti流表實例中存儲的流表項數目 */

unsigned int ufid_count;

}

流表實例數據結構

struct table_instance {

/* 哈希桶,用於組織各個流表項的具體信息. */

struct flex_array *buckets;

/* 哈希桶大小 */

unsigned int n_buckets;

struct rcu_head rcu;

int node_ver;

u32 hash_seed;

bool keep_flows;

}

掩碼信息列表數據結構

struct mask_array {

struct rcu_head rcu;

/* 當前已存儲的掩碼信息數目 */

int count;

/* 一共可以存儲掩碼信息的總數目 */

int max;

/* 具體的掩碼信息 */

struct sw_flow_mask *masks[];

}

struct sw_flow_mask {

/* 引用計數 */

int ref_count;

struct rcu_head rcu;

/* 匹配關鍵字的有效范圍,下文會具體解釋 */

struct sw_flow_key_range range;

/* 匹配關鍵字信息 */

struct sw_flow_key key;

}

掩碼信息緩存表數據結構

struct mask_cache_entry {

/* hash值 */

u32 skb_hash;

/* 對應哪一個掩碼信息,索引值 */

u32 mask_index;

}

哈希桶數據結構

struct flex_array {

union {

struct {

/* 實際等於sizeof(struct hlist_head) */

int element_size;

/* 一共有多少個struct hlist_head指針. */

int total_nr_elements;

/* 每個part(頁)可以存儲多少個struct hlist_head指針 */

int elems_per_part;

u32 reciprocal_value reciprocal_elems;

/* 具體存儲struct hlist_head指針的緩沖區 */

struct flex_array_part *parts[];

};

/* 填充字段.代表了整個結構的大小=頁大小(4096字節) */

char padding[FLEX_ARRAY_BASE_SIZE];

}

struct flex_array_part {

/* 緩沖區.一個頁大小,即:4096字節 */

char elements[FLEX_ARRAY_PART_SIZE];

};

上述這些數據結構是在源碼中的ovs_flow_tbl_init函數里面進行初始化操作的。初始化后以及上述這些數據結構之間的關系如下圖所示。

2.2.關鍵信息范圍

從上面2.1節可知，掩碼信息(struct sw_flow_mask結構)中記錄了一個范圍(struct sw_flow_key_range結構)，該范圍用於標識需要匹配的關鍵信息最小偏移和最大偏移。為什么需要這樣做？個人感覺和ovs-dpdk datapath classifier中描述的miniflow類似，即：匹配過程中並非使用報文的所有字段去匹配，而是使用部分字段，例如使用報文的五元組信息(源IP、目的IP、協議號、源端口、目的端口)。那么使用sw_flow_key_range結構來標識這五元組信息中最小偏移和最大偏移。實際源碼中，關鍵信息是使用struct sw_flow_key結構來描述的。由於該結構字段較多，這里不詳細給出。以報文五元組信息為例，這里給出的五元組信息所在sw_flow_key結構的位置和實際源碼對應的位置是不相同的，這里只是給出計算最小偏移和最大偏移的概念，如下圖所示，關鍵信息的有效范圍為：
最小偏移=M，最大偏移=N

2.3.更新過程

在源碼中，對應更新過程的入口函數是：ovs_flow_cmd_new。這個入口函數是處於內核模塊中，在接收到報文時，通過下面2.4節所述的查找過程，查找失敗時，會將報文的相關信息upcall到用戶空間，在用戶空間通過查找”慢路徑”將對應的actions和mask信息下發到內核空間，在內核空間，處理的入口函數正是ovs_flow_cmd_new。下面將對這個函數的處理過程作詳細描述。描述之前，先掌握一些相關的數據結構，如下所示：

struct sw_flow {

struct rcu_head rcu;

/* 哈希桶鏈表 */

struct {

struct hlist_node node[2];

u32 hash;

} flow_table, ufid_table;

int stats_last_writer;

/* 關鍵信息.經過掩碼運算后的 */

struct sw_flow_key key;

/* 流id信息 */

struct sw_flow_id id;

/* 指向的掩碼信息 */

struct sw_flow_mask *mask;

/* 指向的流動作信息 */

struct sw_flow_actions *sf_acts;

struct flow_stats *stats[];

}

struct sw_flow_id {

u32 ufid_len;

union {

/* ufid為128比特,共16字節 */

u32 ufid[4];

struct sw_flow_key *unmasked_key;

}

struct sw_flow_actions {

struct rcu_head rcu;

size_t orig_len;

u32 actions_len;

struct nlattr actions[];

}

struct sw_flow_match {

struct sw_flow_key *key;

struct sw_flow_key_range range;

struct sw_flow_mask *mask;

}

具體流程如下所示：
1)初始化struct sw_flow *new_flow。分配空間給new_flow，並設置部分的成員值，如下：
new_flow->sf_acts = null;
new_flow->mask = null;
new_flow->id.ufid_len=0;
new_flow->id.unmasked_key=null;
2)構建struct sw_flow_match match信息。根據接收到的關鍵信息和掩碼信息，存儲在match.key和match.mask中。

3)計算new_flow->key = match.key & match.mask。
4)將ufid信息提取出來存儲在new_flow->id.ufid_len和new_flow->id.ufid中。如果未提供任何的UFID信息(此時的new_flow->id.ufid_len為0)，則將match.key信息拷貝一份到new_flow->id.unmasked_key中存儲。
5)將actions信息存儲在變量struct sw_flow_actions acts中。
6)根據接收到的dp_ifindex獲取struct datapath dp信息。
7)如果new_flow->id.ufid_len不為0，則使用new_flow->id.ufid信息去dp->table.ufid_ti指向的哈希桶中找到對應的struct sw_flow信息。假設找不到。
8)而如果new_flow->id.ufid_len為0，則需要在dp->table.ti，同時配合在2)中構建出來的match.key和dp->table.mask_array配合找出對應的struct sw_flow信息。這個詳細過程可以參考下面的2.4節。這里也假設找不到。
9)經過上面7)和8)步都沒找到對應的struct sw_flow信息。首先設置new_flow->sf_acts=acts(在5)步中獲取)。然后使用match.mask去dp->table.mask_array中查找是否已存在該match.mask信息。如果不存在則接着分配一個struct sw_flow_mask *mask。設置mask->key=match.mask->key, mask->range=match.mask->range, mask->ref_count=1。並且在dp->table.mask_array.masks[i]中存儲該mask的地址。添加完畢之后，如下圖所示。變化的地方用綠色標注出來了。同時new_flow->mask=mask。

10)根據new_flow->key和new_flow->mask.range計算出new_flow->flow_table.hash值。
11)在dp->table.ti->buckets指向的哈希桶中插入new_flow。如下圖所示(假設插入的位置為part[0]對應的第0個位置)。dp->table.count。
12)如果new_flow->id.ufid_len不為0，則使用new_flow->id.ufid信息計算出new_flow->ufid_table.hash。然后根據這個hash值在dp->table.ufid_ti->buckets中找到合適的哈希桶存儲對應的new_flow信息。這里假設找到的位置為parts[0]的第0個位置。如下圖所示。插入new_flow信息之后，dp->table.ufid_count。

到此為止，更新過程完畢。

2.4.查找過程

在源碼中，對應查找過程的入口函數是：ovs_vport_receive。具體過程如下：
1)從接收到的報文信息中提取出關鍵信息。存儲到struct sw_flow_key key。
2)從接收到的報文信息中獲取struct vport信息，再從vport信息獲取struct datapath信息，假設為dp。
3)如果接收到的報文中未帶有skb_hash值信息，則執行：
遍歷每一個掩碼，即:dp->table.mask_array->masks[i]。這里簡單記為mask。將1)提取出來的關鍵新key與這個掩碼mask做邏輯與運算，得出掩碼過的信息，記為masked_key。通過masked_key和mask.range信息計算出hash值在dp->table.ti->buckets中找到對應的哈希桶，並遍歷該桶上的所有struct sw_flow信息，記為flow。執行比較：如果flow->mask指向的是mask，flow->flow_table.hash值和計算出的hash值相等，且flow->key和masked_key信息在mask.range指定的范圍內完全相同，則認為成功匹配，返回flow信息。否則匹配失敗，返回NULL。
4)如果接收到的報文中帶有skb_hash值信息，則執行：
4)-1：根據舊的skb_hash值和key->recirc_id值重新計算出新的skb_hash值。
4)-2：將skb_hash值分成4段，每段1字節，記為hash。遍歷每一段的hash值，執行：獲取dp->table.mask_cache[hash]表項，記為entry，如果entry.skb_hash值和skb_hash值相等，則使用entry.mask_index指向的掩碼信息去dp->table.ti->buckets中找到對應的struct sw_flow信息(過程和上述第3)步相同)。如果找到對應的flow，則返回flow，過程中會更新entry.mask_index值來實際指向具體的掩碼信息索引。否則，匹配失敗，將entry.skb_hash值置0，返回NULL。而如果每一個entry.skb_hash值和skb_hash值不相等，則遍歷完每一段hash值之后，從中選擇最佳候選entry，最佳候選的條件是：其skb_hash值最小者，假設記為ce。最后使用ce.mask_index指向的掩碼信息去dp->table.ti->buckets中找到對應的struct sw_flow信息(過程和上述第3)步相同)。如果找到對應的flow，則返回flow，同時將ce->skb_hash值更新為skb_hash值。當然在查找的過程中(上述第3)步)，也會更新ce->mask_index來指向實際的掩碼信息索引。
5) 如果在上述第3)和第4)步中都匹配失敗了，則需要將報文的信息upcall到用戶空間，由用戶空間負責查找對應的流動作信息，返回給內核。這個過程具體請參考上述2.3節。
到此為止，查找過程結束。

2.5.執行過程

在2.4節查找過程中，如果查找成功，則需要執行對應的流動作。入口函數是：ovs_execute_actions。
而如果查找失敗，upcall到用戶空間，找到對應的流動作之后，調用執行，到內核空間，入口函數對應的是：ovs_packet_cmd_execute。該函數最終會調用到ovs_execute_actions。

2.6.掩碼信息比較

在上述2.3和2.4節所述的更新過程和查找過程中都會遇見用新構造的掩碼信息與datapath結構的table.mask_array作比較，以檢測掩碼信息是否需要新增。而比較的方法如下：
假設新構造的掩碼信息為mask，與table.mask_array->masksi比較：
1)mask.range.start和exist_mask.range.start相等。
2)mask.range.end和exist_mask.range.end相等。
3)mask和exist_mask在range范圍內完全相同。
同時符合上面三個條件才認為兩個掩碼完全相同。

2.7.流信息比較

在上述2.4節所述的查找過程中，對接收到的報文進行匹配表項時，需要對流信息作比較。假設接收到的報文提取出來的關鍵信息為key，匹配的掩碼信息為mask，通過mask & key計算出掩碼后的報文關鍵信息，記為masked_key，通過masked_key和mask.range計算出hash值。根據這個hash值找到對應的哈希桶，遍歷這個哈希桶中存儲的每一個流信息，記為flow。現在需要比較flow信息進而找到匹配的流表項。比較如下：
1)flow->mask指向的是mask。
2)flow->flow_table.hash和計算出的hash值相等。
3)flow->key和masked_key在mask.range范圍內完全相同。
同時符合上面三個條件才認為找到匹配的流表項。

2.8.mask_cache表項

在ovs_dp_process_packet函數中查找匹配的流表項時，如果報文的skb中已經帶有skb_hash值，則將這個skb_hash值(32比特)分為4段，每段8比特，每段的哈希值暫記為hash，用這個hash值去datapath結構中的table.mask_cache緩存表中查找對應的掩碼信息索引。每個表項(struct mask_cache_entry結構)存儲了skb_hash和mask_index信息。初始化的時候，這個緩存表中所有表項都置0，因此，用報文的skb_hash值，分4段去查找，都無法找到合適的表項。這時需要從中選出最佳候選的表項，而最佳候選的表項為其skb_hash值最小。因此，初始化的時候，最終會選擇table.mask_cache[0]為最佳候選表項。
接着根據上述2.4節所述的查找過程，找到合適的流表項信息。如果匹配成功，則最佳候選表項table.mask_cache[0].mask_index記錄了掩碼信息索引(index)，即：table.mask_array->masks[index]。同時table.mask_cache[0].skb_hash賦值為skb_hash。假設index為0，則對應如下圖所示。
后續如果收到的報文帶的skb_hash值與table.mask_cache[0].skb_hash值相等時，則首先使用table.mask_cache[0].mask_index索引的掩碼信息去找匹配的流表信息。當然，如果匹配成功，table.mask_cache[0].mask_index可能並非為0(之前存儲的值)，有可能更新為其他值。而如果匹配失敗，則table.mask_cache[0].skb_hash置0。

2.9.mask_array擴充

當datapath結構中的table.mask_array->count >= table.mask_array->max時，則需要擴充mask_array空間。已當前table.mask_array->max * 2的大小進行擴充。擴充前后的效果如下圖所示。橙色線指向的掩碼信息在擴充之后，會釋放掉old的掩碼信息空間。擴充總是按照當前max數值的2倍大小進行擴充，例如：16 -> 32 -> 64 -> 128 -> …。從源碼中暫時未看到這個擴充的最大值。

2.10.table_instance擴充

table_instance擴充有兩個條件觸發：
1)當datapath結構中的table.count > table.ti->n_buckets時，觸發擴充。
2)datapath結構中的table.last_rehash記錄了上次執行擴充或初始化table時系統的jiffies值。如果超過10分鍾，則需要重新擴充，只是這次的擴充並非增大空間，而是以相同的大小重新分配空間。覺得這樣做的意義是重新生成table_instance結構中的hash_seed值，重新安排哈希桶的鏈表長度，分散存儲，減少匹配的比較次數。
我們主要以第一種條件為例，描述擴充的過程。這種擴充是在原有的哈希桶數目(n_buckets)基礎之上，以2倍的大小進行擴充。擴充前后，table_instance結構的變化如下圖所示。

根據上圖可知，擴充之后，table_instance中的hash_seed更新為新的隨機數了。這樣，在從舊的table_instance將哈希桶中對應的各個流信息拷貝到新的table_instance時，需要重新計算哈希桶的位置，重新安排了。這樣做的好處時：可以重新分散每個哈希桶中流信息鏈表的長度，減少在匹配時流的比較次數。如下圖所示，之前Flow_A和Flow_B都位於第0個哈希桶，擴充之后，Flow_A處於了第0個哈希桶，而Flow_B則處於第1025個哈希桶了。這樣在查找Flow_B的時候，比較的次數就減少了一次。在大規模查找的過程中，這種改變可以大大提高查找的效率。

2.11.示例

假設ovs用戶空間的”慢路徑”存儲的流表信息如下所示：

table=0, src_ip=11.11.11.0/24, dst_ip=192.0.0.0/8, actions=output:1

table=0, src_ip=2.2.2.0/24, dst_ip=2.0.0.0/8, actions=output:2

table=0, dst_ip=8.0.0.0/8, actions=drop

初始化的時候，ovs內核空間的”快路徑”沒有存儲任何的流路徑信息。如下圖所示：

收到第一個報文：src_ip=11.11.11.25, dst_ip=192.1.1.1
收到第一個報文，匹配結果Miss，upcall到用戶空間，查表將結果發送回內核空間的datapath。根據上述2.3節所述的更新過程，會新增相應的掩碼信息(mask_A)和流信息(flow_A)。如下圖所示。

收到第二個報文：src_ip=11.11.11.63, dst_ip=192.168.7.8
根據上述2.4節所述的查找過程，匹配成功。

收到第三個報文：src_ip=2.2.2.4, dst_ip=2.7.7.7
根據上述2.4節所述的查找過程，匹配失敗。因為掩碼信息和mask_A完全相同，因此無需新增掩碼信息，只需要將mask_A的ref_count引用計數加1即可。但是需要新增相應的流信息(flow_B)，如下圖所示。

收到第四個報文：dst_ip=8.12.34.56
根據上述2.4節所述的查找過程，匹配失敗。需要新增相應的掩碼信息(mask_B)和流信息(flow_C)。如下圖所示

As part of my work in OpenDaylight, we are looking at creating a router using Open vSwitch... Why? Well OpenStack requires some limited L3 capabilities and we think that we can handle those in a distributed router.

Test Topology

My test topology looks like this:

We have a host in an external network 172.16.1.0/24, one host in an internal network 10.10.10.0/24 and two hosts in another internal network 10.10.20.0/24.

As such, The hosts in the 10.x.x.x range should be able to speak to each other, but should not be able to speak to external hosts.

The host 10.10.10.2 has a floating IP of 172.16.1.10 and should be reachable on this address from the external 172.16.1.0/24network. To do this, we'll use DNAT for traffic from 172.16.1.2 -> 172.16.1.10 and SNAT for traffic back from10.10.10.2 -> 172.16.1.2

If you'd like to recreate this topology you can checkout the OpenDaylight OVSDB project source on GitHub and:

 
                       vagrant up mininet
vagrant ssh mininet
cd /vagrant/resources/mininet
sudo mn --custom topo.py --topo l3 
                      

The Pipeline

Our router is implemented using the following pipeline:

Table 0 - Classifier

In this table we work out what traffic is interesting for us before pushing it further along the pipeline

Table 100 - ACL

While we don't use this table today, the idea would be to filter traffic in this table (or series of tables) and to then resubmit to the classifier once we have scrubbed them.

Table 105 - ARP Responder

In this table we use some OVS-Jitsu to take an incoming ARP Request and turn it in to an ARP reply

Table 5 - L3 Rewrite

In this table, we make any L3 modifications we need to before a packet is routed

Table 10 - L3 Routing

The L3 routing table is where the routing magic happens. Here we modify the Source MAC address, Decrement the TTL and push to the L2 tables for forwarding

Table 15 - L3 Forwarding

In the L3 forwarding table we resolve a destination IP address to the correct MAC address for L2 forwarding.

Table 20 - L2 Rewrites

While we aren't using this table in this example, typically we would push/pop any L2 encapsulations here like a VLAN tag or a VXLAN/GRE Tunnel ID.

Table 25 - L2 Forwarding

In this table we do our L2 lookup and forward out the correct port. We also handle L2 BUM traffic here using OpenFlow Groups - no VLANs!

The Flows

To program the flows, paste the following in to mininet:

## Set Bridge to use OpenFlow 1.3
sh ovs-vsctl set Bridge s1 "protocols=OpenFlow13"

## Create Groups
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=1,type=all,bucket=output:1
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=2,type=all,bucket=output:2,4
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=3,type=all,bucket=output:3

## Table 0 - Classifier
# Send ARP to ARP Responder
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=1000, dl_type=0x0806, actions=goto_table=105"
# Send L3 traffic to L3 Rewrite Table
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:01, action=goto_table=5"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:02, action=goto_table=5"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:03, action=goto_table=5"
# Send L3 to L2 Rewrite Table
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=0, action=goto_table=20"

## Table 5 - L3 Rewrites
# Exclude connected subnets
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=65535, dl_type=0x0800, nw_dst=10.10.10.0/24 actions=goto_table=10"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=65535, dl_type=0x0800, nw_dst=10.10.20.0/24 actions=goto_table=10"
# DNAT
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=100, dl_type=0x0800,  nw_dst=172.16.1.10 actions=mod_nw_dst=10.10.10.2, goto_table=10"
# SNAT
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=100, dl_type=0x0800,  nw_src=10.10.10.2, actions=mod_nw_src=172.16.1.10,  goto_table=10"
# If no rewrite needed, continue to table 10
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=0, actions=goto_table=10"

## Table 10 - IPv4 Routing
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=10.10.10.0/24, actions=mod_dl_src=00:00:5E:00:02:01, dec_ttl, goto_table=15"
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=10.10.20.0/24, actions=mod_dl_src=00:00:5E:00:02:02, dec_ttl, goto_table=15"
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=172.16.1.0/24, actions=mod_dl_src=00:00:5E:00:02:03, dec_ttl, goto_table=15"
# Explicit drop if cannot route
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, priority=0, actions=output:0"

## Table 15 - L3 Forwarding
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.10.2, actions=mod_dl_dst:00:00:00:00:00:01, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.20.2, actions=mod_dl_dst:00:00:00:00:00:02, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.20.4, actions=mod_dl_dst:00:00:00:00:00:04, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=172.16.1.2, actions=mod_dl_dst:00:00:00:00:00:03, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, priority=0, actions=goto_table=20"

## Table 20 - L2 Rewrite
# Go to next table
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=20, priority=0, actions=goto_table=25"

## Table 25 - L2 Forwarding
# Use groups for BUM traffic
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=1, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=1"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=2, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=3, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=3"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=4, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:01,actions=output=1"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:02,actions=output=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:03,actions=output=3"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:04,actions=output=4"

## Table 105 - ARP Responder
# Respond to ARP for Router Addresses
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, dl_type=0x0806, nw_dst=10.10.10.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[], mod_dl_src:00:00:5E:00:02:01, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000201->NXM_NX_ARP_SHA[], load:0x0a0a0a01->NXM_OF_ARP_SPA[], in_port"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105,  dl_type=0x0806, nw_dst=10.10.20.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],  mod_dl_src:00:00:5E:00:02:02, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000202->NXM_NX_ARP_SHA[], load:0xa0a1401->NXM_OF_ARP_SPA[], in_port"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105,  dl_type=0x0806, nw_dst=172.16.1.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],  mod_dl_src:00:00:5E:00:02:03, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000203->NXM_NX_ARP_SHA[], load:0xac100101->NXM_OF_ARP_SPA[], in_port"
# Proxy ARP for all floating IPs go below
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, dl_type=0x0806, nw_dst=172.16.1.10, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[], mod_dl_src:00:00:5E:00:02:03, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000203->NXM_NX_ARP_SHA[], load:0xac10010a->NXM_OF_ARP_SPA[], in_port"
# if we made it here, the arp packet is to be handled as any other regular L2 packet
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, priority=0, action=resubmit(,20)"

If you'd like to paste these in without comments, you can use this Gist

Testing

We can run a pingall to test this out:

mininet> pingall
*** Ping: testing ping reachability
h1 -> h2 h3 h4
h2 -> h1 X h4
h3 -> X X X
h4 -> h1 h2 X
*** Results: 41% dropped (7/12 received)

hosts 1,2 and 4 can speak to each other. Everything initiated by h3 is dropped (as expected) but h1 can speak to h3 (thanks to NAT). We can test our DNAT and Proxy ARP for Floating IP's using this command:

mininet> h3 ping 172.16.1.10
PING 172.16.1.10 (172.16.1.10) 56(84) bytes of data.
64 bytes from 172.16.1.10: icmp_seq=1 ttl=63 time=1.30 ms
64 bytes from 172.16.1.10: icmp_seq=2 ttl=63 time=0.043 ms
64 bytes from 172.16.1.10: icmp_seq=3 ttl=63 time=0.045 ms
64 bytes from 172.16.1.10: icmp_seq=4 ttl=63 time=0.050 ms
^C
--- 172.16.1.10 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 2999ms
rtt min/avg/max/mdev = 0.043/0.360/1.305/0.545 ms

Cool! It works!

Conclusion

So far we've implemented the nuts and bolts of routing, but we are missing one crucial piece - ICMP handling. Without this useful things like Path MTU Discovery won't work and neither will diagnostics tools like Ping and Traceroute. I think we can do this using a Open vSwitch but first we'll need to add some new NXM fields to ICMP Data and ICMP Checksum and to also make the existing OXM's writable through set_field. I'm going to start talking to the OVS community about this to see if it's possible so watch this space!

@dave_tucker

Helpful Links and Further Reading

ovs-ofctl Man Page Neutron ARP Responder Write Up Address Resolution Protocol OpenFlow 1.3.1 Spec

基於Open vSwitch搭建虛擬路由器

Test Topology

My test topology looks like this:

We have a host in an external network 172.16.1.0/24, one host in an internal network 10.10.10.0/24 and two hosts in another internal network 10.10.20.0/24.

As such, The hosts in the 10.x.x.x range should be able to speak to each other, but should not be able to speak to external hosts.

If you'd like to recreate this topology you can checkout the OpenDaylight OVSDB project source on GitHub and:

 
                           vagrant up mininet
vagrant ssh mininet
cd /vagrant/resources/mininet
sudo mn --custom topo.py --topo l3 
                          

The Pipeline

Our router is implemented using the following pipeline:

Table 0 - Classifier

In this table we work out what traffic is interesting for us before pushing it further along the pipeline

Table 100 - ACL

While we don't use this table today, the idea would be to filter traffic in this table (or series of tables) and to then resubmit to the classifier once we have scrubbed them.

Table 105 - ARP Responder

In this table we use some OVS-Jitsu to take an incoming ARP Request and turn it in to an ARP reply

Table 5 - L3 Rewrite

In this table, we make any L3 modifications we need to before a packet is routed

Table 10 - L3 Routing

The L3 routing table is where the routing magic happens. Here we modify the Source MAC address, Decrement the TTL and push to the L2 tables for forwarding

Table 15 - L3 Forwarding

In the L3 forwarding table we resolve a destination IP address to the correct MAC address for L2 forwarding.

Table 20 - L2 Rewrites

While we aren't using this table in this example, typically we would push/pop any L2 encapsulations here like a VLAN tag or a VXLAN/GRE Tunnel ID.

Table 25 - L2 Forwarding

In this table we do our L2 lookup and forward out the correct port. We also handle L2 BUM traffic here using OpenFlow Groups - no VLANs!

The Flows

To program the flows, paste the following in to mininet:

## Set Bridge to use OpenFlow 1.3
sh ovs-vsctl set Bridge s1 "protocols=OpenFlow13"

## Create Groups
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=1,type=all,bucket=output:1
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=2,type=all,bucket=output:2,4
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=3,type=all,bucket=output:3

## Table 0 - Classifier
# Send ARP to ARP Responder
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=1000, dl_type=0x0806, actions=goto_table=105"
# Send L3 traffic to L3 Rewrite Table
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:01, action=goto_table=5"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:02, action=goto_table=5"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:03, action=goto_table=5"
# Send L3 to L2 Rewrite Table
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=0, action=goto_table=20"

## Table 5 - L3 Rewrites
# Exclude connected subnets
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=65535, dl_type=0x0800, nw_dst=10.10.10.0/24 actions=goto_table=10"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=65535, dl_type=0x0800, nw_dst=10.10.20.0/24 actions=goto_table=10"
# DNAT
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=100, dl_type=0x0800,  nw_dst=172.16.1.10 actions=mod_nw_dst=10.10.10.2, goto_table=10"
# SNAT
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=100, dl_type=0x0800,  nw_src=10.10.10.2, actions=mod_nw_src=172.16.1.10,  goto_table=10"
# If no rewrite needed, continue to table 10
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=0, actions=goto_table=10"

## Table 10 - IPv4 Routing
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=10.10.10.0/24, actions=mod_dl_src=00:00:5E:00:02:01, dec_ttl, goto_table=15"
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=10.10.20.0/24, actions=mod_dl_src=00:00:5E:00:02:02, dec_ttl, goto_table=15"
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=172.16.1.0/24, actions=mod_dl_src=00:00:5E:00:02:03, dec_ttl, goto_table=15"
# Explicit drop if cannot route
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, priority=0, actions=output:0"

## Table 15 - L3 Forwarding
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.10.2, actions=mod_dl_dst:00:00:00:00:00:01, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.20.2, actions=mod_dl_dst:00:00:00:00:00:02, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.20.4, actions=mod_dl_dst:00:00:00:00:00:04, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=172.16.1.2, actions=mod_dl_dst:00:00:00:00:00:03, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, priority=0, actions=goto_table=20"

## Table 20 - L2 Rewrite
# Go to next table
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=20, priority=0, actions=goto_table=25"

## Table 25 - L2 Forwarding
# Use groups for BUM traffic
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=1, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=1"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=2, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=3, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=3"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=4, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:01,actions=output=1"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:02,actions=output=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:03,actions=output=3"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:04,actions=output=4"

## Table 105 - ARP Responder
# Respond to ARP for Router Addresses
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, dl_type=0x0806, nw_dst=10.10.10.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[], mod_dl_src:00:00:5E:00:02:01, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000201->NXM_NX_ARP_SHA[], load:0x0a0a0a01->NXM_OF_ARP_SPA[], in_port"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105,  dl_type=0x0806, nw_dst=10.10.20.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],  mod_dl_src:00:00:5E:00:02:02, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000202->NXM_NX_ARP_SHA[], load:0xa0a1401->NXM_OF_ARP_SPA[], in_port"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105,  dl_type=0x0806, nw_dst=172.16.1.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],  mod_dl_src:00:00:5E:00:02:03, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000203->NXM_NX_ARP_SHA[], load:0xac100101->NXM_OF_ARP_SPA[], in_port"
# Proxy ARP for all floating IPs go below
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, dl_type=0x0806, nw_dst=172.16.1.10, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[], mod_dl_src:00:00:5E:00:02:03, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000203->NXM_NX_ARP_SHA[], load:0xac10010a->NXM_OF_ARP_SPA[], in_port"
# if we made it here, the arp packet is to be handled as any other regular L2 packet
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, priority=0, action=resubmit(,20)"

If you'd like to paste these in without comments, you can use this Gist

Testing

We can run a pingall to test this out:

mininet> pingall
*** Ping: testing ping reachability
h1 -> h2 h3 h4
h2 -> h1 X h4
h3 -> X X X
h4 -> h1 h2 X
*** Results: 41% dropped (7/12 received)

mininet> h3 ping 172.16.1.10
PING 172.16.1.10 (172.16.1.10) 56(84) bytes of data.
64 bytes from 172.16.1.10: icmp_seq=1 ttl=63 time=1.30 ms
64 bytes from 172.16.1.10: icmp_seq=2 ttl=63 time=0.043 ms
64 bytes from 172.16.1.10: icmp_seq=3 ttl=63 time=0.045 ms
64 bytes from 172.16.1.10: icmp_seq=4 ttl=63 time=0.050 ms
^C
--- 172.16.1.10 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 2999ms
rtt min/avg/max/mdev = 0.043/0.360/1.305/0.545 ms

Cool! It works!

Conclusion

@dave_tucker

Helpful Links and Further Reading

ovs-ofctl Man Page Neutron ARP Responder Write Up Address Resolution Protocol OpenFlow 1.3.1 Spec

端口	說明
p0	IP 地址: 192.168.1.100/24 網絡名稱空間: ns0 網絡接口 MAC地址: 66:4e:cc:ae:4d:20 OpenFlow Port Number: 100
p1	IP 地址: 192.168.1.101/24 網絡名稱空間: ns1 網絡接口 MAC地址: 46:54:8a:95:dd:f8 OpenFlow Port Number: 101
p2	IP 地址: 192.168.1.102/24, 網絡名稱空間: ns2 網絡接口 MAC地址: 86:3b:c8:d0:44:10 OpenFlow Port Number: 102

OVS——大雜燴

OVS架構解析--dpdk datapath數據通路

1. 架構說明

1.1. 整體架構

1.2. 核心組件及其關聯關系

1.3. 內部模塊及其關聯關系

2. vswitchd和ovsdb通信

2.1. 建立連接

2.2. 接收數據

2.3. json數據解析

2.4. 更新數據改變標記

3. dpdk加速處理

3.1. ovs dpdk加速配置

3.2. dpdk初始化

4. 數據結構抽象關系

4.1. 分層抽象

4.2. 主要數據結構關系

5. vswitchd啟動

6. 附錄-數據結構查詢

6.1. ofproto_class

6.2. dpif_class

6.3. netdev_class

6.4. bridge

6.5. ofproto

6.6. port

6.7. iface

6.8. ofport

6.9. ofport_dpif

6.10. ofproto_port

6.11. dpif_port

6.12. ofproto_dpif

6.13. dpif_backer

6.14. udpif

6.15. dpif

6.16. dpif_netdev

6.17. dp_netdev

6.18. dp_netdev_port

6.19. netdev

6.20. netdev_dpdk

6.21. dp_netdev_rxq

6.22. netdev_rxq

6.23. netdev_rxq_dpdk

6.24. dp_netdev_pmd_thread

6.25. rxq_poll

6.26. tx_port

OVS 架構

OVS 源碼結構

數據轉發流程

總結

創建一個OVS 交換機

創建的端口信息

使用 ovs-ofctl 創建並測試 OpenFlow 命令

查看 datapath 的信息

屏蔽數據包

修改數據包

重定向數據包

修改數據包的 VLAN Tag

其他 OpenFlow 常用的操作

OVS架構解析--dpdk datapath數據通路

https://blog.csdn.net/chen98765432101/article/details/79835435

1. 架構說明

1.1. 整體架構

1.2. 核心組件及其關聯關系

1.3. 內部模塊及其關聯關系

2. vswitchd和ovsdb通信

2.1. 建立連接

2.2. 接收數據

2.3. json數據解析

2.4. 更新數據改變標記

3. dpdk加速處理

3.1. ovs dpdk加速配置

3.2. dpdk初始化

4. 數據結構抽象關系

4.1. 分層抽象

4.2. 主要數據結構關系

5. vswitchd啟動

6. 附錄-數據結構查詢

6.1. ofproto_class

6.2. dpif_class

6.3. netdev_class