在节点2上配置：
ip link add default_g1 type veth peer name default_g
ip link set default_g1 up ip link set default_g up #其中default_g1在evpn-vrf，default_g在default ip link set default_g1 master evpn-vrf ip addr add 5.5.5.253/24 dev default_g1 ip addr add 5.5.5.254/24 dev default_g ip route add default via 5.5.5.254 dev default_g1 table 100 nft add table nat nft add chain nat prerouting { type nat hook prerouting priority 0 \; } nft add chain nat postrouting { type nat hook postrouting priority 100 \; } nft add rule nat postrouting oifname default_g1 counter masquerade nft add rule nat postrouting oifname enp1s0 counter masquerade
节点2上bgp配置
evpn2.novalocal# show running-config
Building configuration...
Current configuration:
! frr version 7.3-MyOwnFRRVersion frr defaults traditional hostname evpn2.novalocal log file /var/log/frr/bgpd.log ! vrf evpn-vrf vni 100 exit-vrf ! router bgp 9999 bgp router-id 10.10.18.212 bgp bestpath as-path multipath-relax neighbor fabric peer-group neighbor fabric remote-as external neighbor 10.10.18.209 peer-group fabric neighbor 10.10.18.209 update-source 10.10.18.212 ! address-family l2vpn evpn neighbor fabric activate advertise-all-vni exit-address-family ! router bgp 9999 vrf evpn-vrf ! address-family ipv4 unicast network 0.0.0.0/0 network 9.9.9.0/24 exit-address-family ! address-family l2vpn evpn advertise ipv4 unicast exit-address-family ! line vty ! end
[root@evpn2 ~]# ip netns exec host2 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether 6e:7f:fc:df:5d:bb brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 9.9.9.1/24 scope global eth0 valid_lft forever preferred_lft forever [root@evpn2 ~]# ip netns exec host2 ping 10.10.18.212 PING 10.10.18.212 (10.10.18.212) 56(84) bytes of data. 64 bytes from 10.10.18.212: icmp_seq=1 ttl=64 time=0.094 ms 64 bytes from 10.10.18.212: icmp_seq=2 ttl=64 time=0.068 ms ^C --- 10.10.18.212 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 9ms rtt min/avg/max/mdev = 0.068/0.081/0.094/0.013 ms [root@evpn2 ~]# ip netns exec host2 ping 8.8.8.8 PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. 64 bytes from 8.8.8.8: icmp_seq=4 ttl=42 time=1920 ms 64 bytes from 8.8.8.8: icmp_seq=6 ttl=42 time=14.1 ms 64 bytes from 8.8.8.8: icmp_seq=7 ttl=42 time=14.1 ms 64 bytes from 8.8.8.8: icmp_seq=9 ttl=42 time=14.2 ms ^C --- 8.8.8.8 ping statistics --- 9 packets transmitted, 4 received, 55.5556% packet loss, time 256ms rtt min/avg/max/mdev = 14.068/490.570/1919.957/825.256 ms, pipe 2
[root@evpn2 ~]# ip route show vrf evpn-vrf
default via 5.5.5.254 dev default_g1 2.2.2.0/24 dev br30 proto kernel scope link src 2.2.2.254 3.3.3.2 via 10.10.18.209 dev br100 proto bgp metric 20 onlink 5.5.5.0/24 dev default_g1 proto kernel scope link src 5.5.5.253 9.9.9.0/24 dev br20 proto kernel scope link src 9.9.9.254
节点1
[root@evpn2 ~]# ip netns exec host2 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0@if10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether b6:7a:bc:9e:4e:95 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 3.3.3.2/24 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::b47a:bcff:fe9e:4e95/64 scope link valid_lft forever preferred_lft forever [root@evpn2 ~]# ip netns exec host2 ping 10.10.18.212 PING 10.10.18.212 (10.10.18.212) 56(84) bytes of data. 64 bytes from 10.10.18.212: icmp_seq=2 ttl=63 time=0.393 ms 64 bytes from 10.10.18.212: icmp_seq=3 ttl=63 time=0.370 ms ^C --- 10.10.18.212 ping statistics --- 3 packets transmitted, 2 received, 33.3333% packet loss, time 49ms rtt min/avg/max/mdev = 0.370/0.381/0.393/0.022 ms [root@evpn2 ~]# ip netns exec host2 ping 8.8.8.8 PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. 64 bytes from 8.8.8.8: icmp_seq=1 ttl=41 time=14.8 ms 64 bytes from 8.8.8.8: icmp_seq=3 ttl=41 time=14.9 ms 64 bytes from 8.8.8.8: icmp_seq=4 ttl=41 time=14.4 ms ^C --- 8.8.8.8 ping statistics --- 4 packets transmitted, 3 received, 25% packet loss, time 52ms rtt min/avg/max/mdev = 14.410/14.701/14.857/0.228 ms [root@evpn2 ~]# ip route show vrf evpn-vrf default via 10.10.18.212 dev br100 proto bgp metric 20 onlink 2.2.2.0/24 dev br10 proto kernel scope link src 2.2.2.254 3.3.3.0/24 dev br20 proto kernel scope link src 3.3.3.254 5.5.5.0/24 dev default_g1 proto kernel scope link src 5.5.5.253 9.9.9.0/24 via 10.10.18.212 dev br100 proto bgp metric 20 onlink
取消 network 0.0.0.0/0
no network 0.0.0.0/0
[root@evpn2 ~]# vtysh
Hello, this is FRRouting (version 7.3-MyOwnFRRVersion). Copyright 1996-2005 Kunihiro Ishiguro, et al. evpn2.novalocal# conf t evpn2.novalocal(config)# router bgp 9999 vrf evpn-vrf evpn2.novalocal(config-router)# address-family ipv4 unicast evpn2.novalocal(config-router-af)# no network 0.0.0.0/0 evpn2.novalocal(config-router-af)# exit-address-family evpn2.novalocal(config-router)# exit evpn2.novalocal(config)# exit evpn2.novalocal# wr mem Note: this version of vtysh never writes vtysh.conf Building Configuration... Configuration saved to /etc/frr/zebra.conf Configuration saved to /etc/frr/ospfd.conf Configuration saved to /etc/frr/bgpd.conf Configuration saved to /etc/frr/pimd.conf Configuration saved to /etc/frr/fabricd.conf Configuration saved to /etc/frr/staticd.conf
节点2
[root@evpn2 ~]# ip netns exec host2 ping 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.
64 bytes from 8.8.8.8: icmp_seq=4 ttl=42 time=1920 ms
64 bytes from 8.8.8.8: icmp_seq=5 ttl=42 time=880 ms
64 bytes from 8.8.8.8: icmp_seq=6 ttl=42 time=14.1 ms
64 bytes from 8.8.8.8: icmp_seq=7 ttl=42 time=14.3 ms
64 bytes from 8.8.8.8: icmp_seq=9 ttl=42 time=14.2 ms
64 bytes from 8.8.8.8: icmp_seq=10 ttl=42 time=14.3 ms
64 bytes from 8.8.8.8: icmp_seq=11 ttl=42 time=14.3 ms
64 bytes from 8.8.8.8: icmp_seq=12 ttl=42 time=14.3 ms
^C
--- 8.8.8.8 ping statistics ---
17 packets transmitted, 8 received, 52.9412% packet loss, time 432ms
rtt min/avg/max/mdev = 14.112/360.703/1920.068/653.987 ms, pipe 2
[root@evpn2 ~]# ip route show vrf evpn-vrf default via 5.5.5.254 dev default_g1 2.2.2.0/24 dev br30 proto kernel scope link src 2.2.2.254 3.3.3.2 via 10.10.18.209 dev br100 proto bgp metric 20 onlink 5.5.5.0/24 dev default_g1 proto kernel scope link src 5.5.5.253 9.9.9.0/24 dev br20 proto kernel scope link src 9.9.9.254
节点1
[root@evpn2 ~]# ip netns exec host2 ping 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. ^C --- 8.8.8.8 ping statistics --- 10 packets transmitted, 0 received, 100% packet loss, time 387ms [root@evpn2 ~]# ip route show vrf evpn-vrf 2.2.2.0/24 dev br10 proto kernel scope link src 2.2.2.254 3.3.3.0/24 dev br20 proto kernel scope link src 3.3.3.254 5.5.5.0/24 dev default_g1 proto kernel scope link src 5.5.5.253 9.9.9.0/24 via 10.10.18.212 dev br100 proto bgp metric 20 onlink
增加其他网段
一开始节点1是无法访问10.10.16.0/24网段
[root@evpn1 ~]# ip netns exec host2 ping 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. ^C --- 8.8.8.8 ping statistics --- 1953 packets transmitted, 0 received, 100% packet loss, time 2138ms
在节点2上添加network 10.10.16.0/24网段通告
evpn2.novalocal(config)# router bgp 9999 vrf evpn-vrf
evpn2.novalocal(config-router)# address-family ipv4 unicast evpn2.novalocal(config-router-af)# network 10.10.16.0/24 evpn2.novalocal(config-router-af)# exit-address-family evpn2.novalocal(config-router)# exit evpn2.novalocal(config)# exit evpn2.novalocal# wr mem Note: this version of vtysh never writes vtysh.conf Building Configuration... Configuration saved to /etc/frr/zebra.conf Configuration saved to /etc/frr/ospfd.conf Configuration saved to /etc/frr/bgpd.conf Configuration saved to /etc/frr/pimd.conf Configuration saved to /etc/frr/fabricd.conf Configuration saved to /etc/frr/staticd.conf
节点1可以访问10.10.16.0/24网段
[root@evpn1 ~]# ip netns exec host2 ping 10.10.16.81 PING 10.10.16.81 (10.10.16.81) 56(84) bytes of data. 64 bytes from 10.10.16.81: icmp_seq=14 ttl=60 time=899 ms 64 bytes from 10.10.16.81: icmp_seq=15 ttl=60 time=0.529 ms 64 bytes from 10.10.16.81: icmp_seq=16 ttl=60 time=0.501 ms 64 bytes from 10.10.16.81: icmp_seq=17 ttl=60 time=0.540 ms 64 bytes from 10.10.16.81: icmp_seq=18 ttl=60 time=0.549 ms 64 bytes from 10.10.16.81: icmp_seq=19 ttl=60 time=0.461 ms 64 bytes from 10.10.16.81: icmp_seq=20 ttl=60 time=0.477 ms 64 bytes from 10.10.16.81: icmp_seq=21 ttl=60 time=0.445 ms 64 bytes from 10.10.16.81: icmp_seq=22 ttl=60 time=0.471 ms ^C
取消节点2上network 10.10.16.0/24，在节点1上添加network 10.10.16.0/24
evpn1(config)# router bgp 8888 vrf evpn-vrf
evpn1(config-router)# address-family l2vpn evpn evpn1(config-router-af)# exit evpn1(config-router)# router bgp 8888 vrf evpn-vrf evpn1(config-router)# address-family ipv4 unicast evpn1(config-router-af)# network 10.10.16.0/24 evpn1(config-router-af)# exit-address-family evpn1(config-router)# exit
[root@evpn1 ~]# ip netns exec host2 ping 10.10.16.81
PING 10.10.16.81 (10.10.16.81) 56(84) bytes of data. ^C --- 10.10.16.81 ping statistics --- 311 packets transmitted, 0 received, 100% packet loss, time 700ms
原因是节点一上没有做snat,添加nat
[root@evpn1 ~]# ip route list vrf evpn-vrf
default via 5.5.5.254 dev default_g1 2.2.2.0/24 dev br10 proto kernel scope link src 2.2.2.254 3.3.3.0/24 dev br20 proto kernel scope link src 3.3.3.254 5.5.5.0/24 dev default_g1 proto kernel scope link src 5.5.5.253 9.9.9.0/24 via 10.10.18.212 dev br100 proto bgp metric 20 onlink [root@evpn1 ~]# iptables -t nat -A POSTROUTING -o default_g1 -j MASQUERADE [root@evpn1 ~]# iptables -t nat -A POSTROUTING -o enp1s0 -j MASQUERADE [root@evpn1 ~]# ip netns exec host2 ping 8.8.8.8 PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. 64 bytes from 8.8.8.8: icmp_seq=1 ttl=42 time=14.4 ms 64 bytes from 8.8.8.8: icmp_seq=2 ttl=42 time=14.1 ms 64 bytes from 8.8.8.8: icmp_seq=3 ttl=42 time=14.2 ms 64 bytes from 8.8.8.8: icmp_seq=4 ttl=42 time=14.1 ms ^C --- 8.8.8.8 ping statistics --- 4 packets transmitted, 4 received, 0% packet loss, time 7ms rtt min/avg/max/mdev = 14.054/14.171/14.350/0.146 ms
[root@evpn1 ~]# ip netns exec host2 ping 10.10.16.254
PING 10.10.16.254 (10.10.16.254) 56(84) bytes of data. 64 bytes from 10.10.16.254: icmp_seq=4 ttl=252 time=1920 ms 64 bytes from 10.10.16.254: icmp_seq=5 ttl=252 time=880 ms 64 bytes from 10.10.16.254: icmp_seq=6 ttl=252 time=0.908 ms 64 bytes from 10.10.16.254: icmp_seq=7 ttl=252 time=0.832 ms 64 bytes from 10.10.16.254: icmp_seq=8 ttl=252 time=0.910 ms 64 bytes from 10.10.16.254: icmp_seq=9 ttl=252 time=0.951 ms 64 bytes from 10.10.16.254: icmp_seq=10 ttl=252 time=0.978 ms ^C --- 10.10.16.254 ping statistics --- 10 packets transmitted, 7 received, 30% packet loss, time 318ms rtt min/avg/max/mdev = 0.832/400.666/1920.024/690.473 ms, pipe 2
节点2上
[root@evpn2 ~]# bridge fdb show 01:00:5e:00:00:01 dev dev enp1s0 self permanent 33:33:00:00:00:01 dev dev enp1s0 self permanent 33:33:00:00:00:02 dev dev enp1s0 self permanent d2:74:ee:ec:29:70 dev dev veth2 vlan 1 master br20 permanent d2:74:ee:ec:29:70 dev dev veth2 master br20 permanent 33:33:00:00:00:01 dev dev veth2 self permanent 33:33:00:00:00:02 dev dev veth2 self permanent 01:00:5e:00:00:01 dev dev veth2 self permanent 33:33:ff:ec:29:70 dev dev veth2 self permanent 33:33:ff:00:00:00 dev dev veth2 self permanent 01:00:5e:00:00:01 dev dev br20 self permanent 33:33:00:00:00:02 dev dev br20 self permanent 33:33:00:00:00:01 dev dev br20 self permanent f6:ef:f3:68:0a:b8 dev dev veth3 vlan 1 master br30 permanent f6:ef:f3:68:0a:b8 dev dev veth3 master br30 permanent 33:33:00:00:00:01 dev dev veth3 self permanent 33:33:00:00:00:02 dev dev veth3 self permanent 01:00:5e:00:00:01 dev dev veth3 self permanent 33:33:ff:68:0a:b8 dev dev veth3 self permanent 33:33:ff:00:00:00 dev dev veth3 self permanent 01:00:5e:00:00:01 dev dev br30 self permanent 33:33:00:00:00:02 dev dev br30 self permanent 33:33:00:00:00:01 dev dev br30 self permanent 33:33:ff:02:03:10 dev dev br30 self permanent 33:33:ff:00:00:00 dev dev br30 self permanent 00:00:01:02:03:10 dev dev br30 vlan 1 master br30 permanent 00:00:01:02:03:10 dev dev br30 master br30 permanent 02:20:03:9f:74:90 dev dev vxlan10 vlan 1 master br30 permanent 02:20:03:9f:74:90 dev dev vxlan10 master br30 permanent 00:00:00:00:00:00 dev dev vxlan10 dst 10.10.18.209 self permanent 01:00:5e:00:00:01 dev dev br100 self permanent 33:33:00:00:00:02 dev dev br100 self permanent 33:33:00:00:00:01 dev dev br100 self permanent 33:33:ff:02:03:05 dev dev br100 self permanent 33:33:ff:00:00:00 dev dev br100 self permanent 00:00:01:02:03:05 dev dev br100 vlan 1 master br100 permanent 00:00:01:02:03:05 dev dev br100 master br100 permanent e6:83:12:4c:78:fc dev dev vxlan100 vlan 1 master br100 permanent e6:83:12:4c:78:fc dev dev vxlan100 master br100 permanent 33:33:00:00:00:01 dev dev evpn-vrf self permanent 01:00:5e:00:00:0d dev dev evpn-vrf self permanent 01:00:5e:00:00:01 dev dev evpn-vrf self permanent 33:33:00:00:00:01 dev dev default_g self permanent 33:33:00:00:00:02 dev dev default_g self permanent 01:00:5e:00:00:01 dev dev default_g self permanent 33:33:ff:da:b8:9e dev dev default_g self permanent 33:33:ff:00:00:00 dev dev default_g self permanent 01:00:5e:00:00:01 dev dev default_g1 self permanent 33:33:00:00:00:02 dev dev default_g1 self permanent 33:33:00:00:00:01 dev dev default_g1 self permanent 33:33:ff:6e:c7:28 dev dev default_g1 self permanent 33:33:ff:00:00:00 dev dev default_g1 self permanent