Openvswitch原理與代碼分析(4)：網絡包的處理過程

 

在上一節提到，Openvswitch的內核模塊openvswitch.ko會在網卡上注冊一個函數netdev_frame_hook，每當有網絡包到達網卡的時候，這個函數就會被調用。

 

1. static struct sk_buff *netdev_frame_hook(struct sk_buff *skb)
2. {
3.    if (unlikely(skb->pkt_type == PACKET_LOOPBACK))
4.       return skb;
5.  
6.    port_receive(skb);
7.    return NULL;
8. }

 

調用port_receive即是調用netdev_port_receive

#define port_receive(skb) netdev_port_receive(skb, NULL)

 

1. void netdev_port_receive(struct sk_buff *skb, struct ip_tunnel_info *tun_info)
2. {
3.    struct vport *vport;
4.  
5.    vport = ovs_netdev_get_vport(skb->dev);
6. ……
7.    skb_push(skb, ETH_HLEN);
8.    ovs_skb_postpush_rcsum(skb, skb->data, ETH_HLEN);
9.    ovs_vport_receive(vport, skb, tun_info);
10.    return;
11. error:
12.    kfree_skb(skb);
13. }

 

在函數int ovs_vport_receive(struct vport *vport, struct sk_buff *skb, const struct ip_tunnel_info *tun_info)實現如下

1. int ovs_vport_receive(struct vport *vport, struct sk_buff *skb,
2.             const struct ip_tunnel_info *tun_info)
3. {
4.    struct sw_flow_key key;
5.    ......
6.    /* Extract flow from 'skb' into 'key'. */
7.    error = ovs_flow_key_extract(tun_info, skb, &key);
8.    if (unlikely(error)) {
9.       kfree_skb(skb);
10.       return error;
11.    }
12.    ovs_dp_process_packet(skb, &key);
13.    return 0;
14. }

 

在這個函數里面，首先聲明了變量struct sw_flow_key key;

如果我們看這個key的定義

1. struct sw_flow_key {
2.    u8 tun_opts[255];
3.    u8 tun_opts_len;
4.    struct ip_tunnel_key tun_key; /* Encapsulating tunnel key. */
5.    struct {
6.       u32 priority; /* Packet QoS priority. */
7.       u32 skb_mark; /* SKB mark. */
8.       u16 in_port; /* Input switch port (or DP_MAX_PORTS). */
9.    } __packed phy; /* Safe when right after 'tun_key'. */
10.    u32 ovs_flow_hash; /* Datapath computed hash value. */
11.    u32 recirc_id; /* Recirculation ID. */
12.    struct {
13.       u8 src[ETH_ALEN]; /* Ethernet source address. */
14.       u8 dst[ETH_ALEN]; /* Ethernet destination address. */
15.       __be16 tci; /* 0 if no VLAN, VLAN_TAG_PRESENT set otherwise. */
16.       __be16 type; /* Ethernet frame type. */
17.    } eth;
18.    union {
19.       struct {
20.          __be32 top_lse; /* top label stack entry */
21.       } mpls;
22.       struct {
23.          u8 proto; /* IP protocol or lower 8 bits of ARP opcode. */
24.          u8 tos; /* IP ToS. */
25.          u8 ttl; /* IP TTL/hop limit. */
26.          u8 frag; /* One of OVS_FRAG_TYPE_*. */
27.       } ip;
28.    };
29.    struct {
30.       __be16 src; /* TCP/UDP/SCTP source port. */
31.       __be16 dst; /* TCP/UDP/SCTP destination port. */
32.       __be16 flags; /* TCP flags. */
33.    } tp;
34.    union {
35.       struct {
36.          struct {
37.             __be32 src; /* IP source address. */
38.             __be32 dst; /* IP destination address. */
39.          } addr;
40.          struct {
41.             u8 sha[ETH_ALEN]; /* ARP source hardware address. */
42.             u8 tha[ETH_ALEN]; /* ARP target hardware address. */
43.          } arp;
44.       } ipv4;
45.       struct {
46.          struct {
47.             struct in6_addr src; /* IPv6 source address. */
48.             struct in6_addr dst; /* IPv6 destination address. */
49.          } addr;
50.          __be32 label; /* IPv6 flow label. */
51.          struct {
52.             struct in6_addr target; /* ND target address. */
53.             u8 sll[ETH_ALEN]; /* ND source link layer address. */
54.             u8 tll[ETH_ALEN]; /* ND target link layer address. */
55.          } nd;
56.       } ipv6;
57.    };
58.    struct {
59.       /* Connection tracking fields. */
60.       u16 zone;
61.       u32 mark;
62.       u8 state;
63.       struct ovs_key_ct_labels labels;
64.    } ct;
65.  
66. } __aligned(BITS_PER_LONG/8); /* Ensure that we can do comparisons as longs. */

 

可見這個key里面是一個大雜燴，數據包里面的幾乎任何部分都可以作為key來查找flow表

• tunnel可以作為key
• 在物理層，in_port即包進入的網口的ID
• 在MAC層，源和目的MAC地址
• 在IP層，源和目的IP地址
• 在傳輸層，源和目的端口號
• IPV6

所以，要在內核態匹配流表，首先需要調用ovs_flow_key_extract，從包的正文中提取key的值。

接下來就是要調用ovs_dp_process_packet了。

1. void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
2. {
3.    const struct vport *p = OVS_CB(skb)->input_vport;
4.    struct datapath *dp = p->dp;
5.    struct sw_flow *flow;
6.    struct sw_flow_actions *sf_acts;
7.    struct dp_stats_percpu *stats;
8.    u64 *stats_counter;
9.    u32 n_mask_hit;
10.  
11.    stats = this_cpu_ptr(dp->stats_percpu);
12.  
13.    /* Look up flow. */
14.    flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
15.                 &n_mask_hit);
16.    if (unlikely(!flow)) {
17.       struct dp_upcall_info upcall;
18.       int error;
19.  
20.       memset(&upcall, 0, sizeof(upcall));
21.       upcall.cmd = OVS_PACKET_CMD_MISS;
22.       upcall.portid = ovs_vport_find_upcall_portid(p, skb);
23.       upcall.mru = OVS_CB(skb)->mru;
24.       error = ovs_dp_upcall(dp, skb, key, &upcall);
25.       if (unlikely(error))
26.          kfree_skb(skb);
27.       else
28.          consume_skb(skb);
29.       stats_counter = &stats->n_missed;
30.       goto out;
31.    }
32.  
33.    ovs_flow_stats_update(flow, key->tp.flags, skb);
34.    sf_acts = rcu_dereference(flow->sf_acts);
35.    ovs_execute_actions(dp, skb, sf_acts, key);
36.  
37.    stats_counter = &stats->n_hit;
38.  
39. out:
40.    /* Update datapath statistics. */
41.    u64_stats_update_begin(&stats->syncp);
42.    (*stats_counter)++;
43.    stats->n_mask_hit += n_mask_hit;
44.    u64_stats_update_end(&stats->syncp);
45. }

 

這個函數首先在內核里面的流表中查找符合key的flow，也即ovs_flow_tbl_lookup_stats，如果找到了，很好說明用戶態的流表已經放入內核，則走fast path就可了。於是直接調用ovs_execute_actions，執行這個key對應的action。

如果不能找到，則只好調用ovs_dp_upcall，讓用戶態去查找流表。會調用static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, const struct sw_flow_key *key, const struct dp_upcall_info *upcall_info)

它會調用err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);通過netlink將消息發送給用戶態。在用戶態，有線程監聽消息，一旦有消息，則觸發udpif_upcall_handler。

 

Slow Path & Fast Path

Slow Path:

當Datapath找不到flow rule對packet進行處理時

Vswitchd使用flow rule對packet進行處理。

 

Fast Path:

將slow path的flow rule放在內核態，對packet進行處理

 

Unknown Packet Processing

Datapath使用flow rule對packet進行處理，如果沒有，則有vswitchd使用flow rule進行處理

 

 

1. 從Device接收Packet交給事先注冊的event handler進行處理
2. 接收Packet后識別是否是unknown packet，是則交由upcall處理
3. vswitchd對unknown packet找到flow rule進行處理
4. 將Flow rule發送給datapath