VXLAN MP-BGP EVPN eBGP Multi-AS Configuration for Nexus 9000 (NX-OS) Switches

Spine-1# sh run Spine-2# sh run

<some output omitted below> <some output omitted below>

nv overlay evpn nv overlay evpn

feature bgp feature bgp
feature pim feature pim
feature lldp feature lldp
feature nv overlay feature nv overlay

ip pim rp-address 111.1.1.1 group-list 224.0.0.0/4 ip pim rp-address 111.1.1.1 group-list 224.0.0.0/4
ip pim ssm range 232.0.0.0/8 ip pim ssm range 232.0.0.0/8
ip pim anycast-rp 111.1.1.1 123.1.1.1 ip pim anycast-rp 111.1.1.1 123.1.1.1
ip pim anycast-rp 111.1.1.1 222.2.2.2 ip pim anycast-rp 111.1.1.1 222.2.2.2

route-map UNCHANGED permit 10 route-map UNCHANGED permit 10

set ip next-hop unchanged set ip next-hop unchanged

interface Ethernet1/3 interface Ethernet1/3

description connected-to-Leaf2-Ethernet1/3 description connected-to-Leaf1-Ethernet1/3
no switchport no switchport
mtu 9216 mtu 9216
no ip redirects no ip redirects
ip address 192.168.3.1/30 ip address 172.16.3.1/30
no ipv6 redirects no ipv6 redirects
ip pim sparse-mode ip pim sparse-mode
no shutdown no shutdown

interface Ethernet1/4 interface Ethernet1/4

description connected-to-Leaf2-Ethernet1/4 description connected-to-Leaf1-Ethernet1/4
no switchport no switchport
mtu 9216 mtu 9216
no ip redirects no ip redirects
ip address 192.168.4.1/30 ip address 172.16.4.1/30
no ipv6 redirects no ipv6 redirects
ip pim sparse-mode ip pim sparse-mode
no shutdown no shutdown

interface Ethernet1/5 interface Ethernet1/5

description connected-to-Leaf1-Ethernet1/5 description connected-to-Leaf2-Ethernet1/5
no switchport no switchport
mtu 9216 mtu 9216
no ip redirects no ip redirects
ip address 192.168.5.1/30 ip address 172.16.5.1/30
no ipv6 redirects no ipv6 redirects
ip pim sparse-mode ip pim sparse-mode
no shutdown no shutdown

interface Ethernet1/6 interface Ethernet1/6

description connected-to-Leaf1-Ethernet1/6 description connected-to-Leaf2-Ethernet1/6
no switchport no switchport
mtu 9216 mtu 9216
no ip redirects no ip redirects
ip address 192.168.6.1/30 ip address 172.16.6.1/30
no ipv6 redirects no ipv6 redirects
ip pim sparse-mode ip pim sparse-mode
no shutdown no shutdown

interface loopback0 interface loopback0

description Routing loopback interface description Routing loopback interface
ip address 10.2.0.1/32 ip address 10.2.0.3/32
ip pim sparse-mode ip pim sparse-mode

interface loopback100 interface loopback100

description RP interface description RP interface
ip address 222.2.2.2/32 ip address 123.1.1.1/32
ip pim sparse-mode ip pim sparse-mode

interface loopback101 interface loopback101

description Anycast-RP interface description Anycast-RP interface
ip address 111.1.1.1/32 ip address 111.1.1.1/32
ip pim sparse-mode ip pim sparse-mode

router bgp 65536 router bgp 65536

router-id 10.2.0.1 router-id 10.2.0.3
address-family ipv4 unicast address-family ipv4 unicast
network 10.2.0.1/32 network 10.2.0.3/32
network 111.1.1.1/32 network 111.1.1.1/32
network 222.2.2.2/32 network 123.1.1.1/32
address-family l2vpn evpn address-family l2vpn evpn
nexthop route-map UNCHANGED nexthop route-map UNCHANGED
retain route-target all retain route-target all
template peer LEAF-2-PEER template peer LEAF-2-PEER
remote-as 65552 remote-as 65552
address-family ipv4 unicast address-family ipv4 unicast
template peer LEAF-1-PEER template peer LEAF-1-PEER
remote-as 65551 remote-as 65551
address-family ipv4 unicast address-family ipv4 unicast
neighbor 10.2.0.2 neighbor 10.2.0.2
remote-as 65552 remote-as 65552
update-source loopback0 update-source loopback0
ebgp-multihop 5 ebgp-multihop 5
address-family l2vpn evpn address-family l2vpn evpn
disable-peer-as-check disable-peer-as-check
send-community send-community
send-community extended send-community extended
route-map UNCHANGED out route-map UNCHANGED out
rewrite-evpn-rt-asn rewrite-evpn-rt-asn
neighbor 10.2.0.4 neighbor 10.2.0.4
remote-as 65551 remote-as 65551
update-source loopback0 update-source loopback0
ebgp-multihop 5 ebgp-multihop 5
address-family l2vpn evpn address-family l2vpn evpn
disable-peer-as-check disable-peer-as-check
send-community send-community
send-community extended send-community extended
route-map UNCHANGED out route-map UNCHANGED out
rewrite-evpn-rt-asn rewrite-evpn-rt-asn
neighbor 192.168.3.2 neighbor 172.16.3.2
inherit peer LEAF-2-PEER inherit peer LEAF-1-PEER
neighbor 192.168.4.2 neighbor 172.16.4.2
inherit peer LEAF-2-PEER inherit peer LEAF-1-PEER
neighbor 192.168.5.2 neighbor 172.16.5.2
inherit peer LEAF-1-PEER inherit peer LEAF-2-PEER
neighbor 192.168.6.2 neighbor 172.16.6.2
inherit peer LEAF-1-PEER inherit peer LEAF-2-PEER

Spine-1 (N9K-C9372PX; 9.3(9)) Spine-2 (N9K-C9372PX; 9.3(7))

BGP ASN
65536
Eth1/3-4 Eth1/3-4
Eth1/5-6 Eth1/5-6

Eth1/3-4 Eth1/3-4
Eth1/5-6 Eth1/5-6

Server-1 Server-2
BGP ASN BGP ASN
10.10.10.10 Eth1/7 Eth1/7 10.10.10.20
65551 65552
20.20.20.10 20.20.20.20

Leaf-1 (N9K-C9372PX; 9.3(10)) Leaf-2 (N9K-C9372PX; 9.3(7))

Leaf-1# sh run Leaf-2# sh run

Server-1# ping 10.10.10.20 source 10.10.10.10 Server-2# ping 10.10.10.10 source 10.10.10.20
Type escape sequence to abort. <some output omitted below> <some output omitted below> Type escape sequence to abort.
Sending 5, 100-byte ICMP Echos to 10.10.10.20, timeout is 2 seconds: Sending 5, 100-byte ICMP Echos to 10.10.10.10, timeout is 2 seconds:
Packet sent with a source address of 10.10.10.10 nv overlay evpn nv overlay evpn Packet sent with a source address of 10.10.10.20
!!!!! feature bgp feature bgp !!!!!
Success rate is 100 percent (5/5), round-trip min/avg/max = 3/3/4 ms feature pim feature pim Success rate is 100 percent (5/5), round-trip min/avg/max = 1/4/10 ms
feature fabric forwarding feature fabric forwarding
Server-1# ping 10.10.10.20 source 20.20.20.10 feature interface-vlan feature interface-vlan Server-2# ping 20.20.20.10 source 10.10.10.20
Type escape sequence to abort. feature vn-segment-vlan-based feature vn-segment-vlan-based Type escape sequence to abort.
Sending 5, 100-byte ICMP Echos to 10.10.10.20, timeout is 2 seconds: feature lldp feature lldp Sending 5, 100-byte ICMP Echos to 20.20.20.10, timeout is 2 seconds:
Packet sent with a source address of 20.20.20.10 feature nv overlay feature nv overlay Packet sent with a source address of 10.10.10.20
!!!!! !!!!!
Success rate is 100 percent (5/5), round-trip min/avg/max = 3/4/6 ms fabric forwarding anycast-gateway-mac 2020.0000.00aa fabric forwarding anycast-gateway-mac 2020.0000.00aa Success rate is 100 percent (5/5), round-trip min/avg/max = 1/2/10 ms
ip pim rp-address 111.1.1.1 group-list 224.0.0.0/4 ip pim rp-address 111.1.1.1 group-list 224.0.0.0/4
Server-1# ping 20.20.20.20 source 10.10.10.10 ip pim ssm range 232.0.0.0/8 ip pim ssm range 232.0.0.0/8 Server-2# ping 10.10.10.10 source 20.20.20.20
Type escape sequence to abort. vlan 10,20,30 vlan 10,20,30 Type escape sequence to abort.
Sending 5, 100-byte ICMP Echos to 20.20.20.20, timeout is 2 seconds: vlan 10 vlan 10 Sending 5, 100-byte ICMP Echos to 10.10.10.10, timeout is 2 seconds:
Packet sent with a source address of 10.10.10.10 vn-segment 10010 vn-segment 10010 Packet sent with a source address of 20.20.20.20
!!!!! vlan 20 vlan 20 !!!!!
Success rate is 100 percent (5/5), round-trip min/avg/max = 3/3/4 ms vn-segment 20020 vn-segment 20020 Success rate is 100 percent (5/5), round-trip min/avg/max = 1/4/10 ms
vlan 30 vlan 30
Server-1# ping 20.20.20.20 source 20.20.20.10 vn-segment 30030 vn-segment 30030 Server-2# ping 20.20.20.10 source 20.20.20.20
Type escape sequence to abort. Type escape sequence to abort.
Sending 5, 100-byte ICMP Echos to 20.20.20.20, timeout is 2 seconds: vrf context TENANT-1 vrf context TENANT-1 Sending 5, 100-byte ICMP Echos to 20.20.20.10, timeout is 2 seconds:
Packet sent with a source address of 20.20.20.10 vni 30030 vni 30030 Packet sent with a source address of 20.20.20.20
!!!!! rd auto rd auto !!!!!
Success rate is 100 percent (5/5), round-trip min/avg/max = 3/3/4 ms address-family ipv4 unicast address-family ipv4 unicast Success rate is 100 percent (5/5), round-trip min/avg/max = 1/6/10 ms
route-target import 65552:30030 route-target import 65551:30030
route-target import 65552:30030 evpn route-target import 65551:30030 evpn
route-target export 65551:30030 route-target export 65552:30030
route-target export 65551:30030 evpn route-target export 65552:30030 evpn

interface Vlan10 interface Vlan10

no shutdown no shutdown
vrf member TENANT-1 vrf member TENANT-1
no ip redirects no ip redirects
ip address 10.10.10.254/24 ip address 10.10.10.254/24
no ipv6 redirects no ipv6 redirects
fabric forwarding mode anycast-gateway fabric forwarding mode anycast-gateway

interface Vlan20 interface Vlan20

no shutdown no shutdown
vrf member TENANT-1 vrf member TENANT-1
no ip redirects no ip redirects
ip address 20.20.20.254/24 ip address 20.20.20.254/24
no ipv6 redirects no ipv6 redirects
fabric forwarding mode anycast-gateway fabric forwarding mode anycast-gateway

interface Vlan30 interface Vlan30

no shutdown no shutdown
vrf member TENANT-1 vrf member TENANT-1
no ip redirects no ip redirects
ip forward ip forward
no ipv6 redirects no ipv6 redirects

interface nve1 interface nve1

no shutdown no shutdown
host-reachability protocol bgp host-reachability protocol bgp
source-interface loopback1 source-interface loopback1
member vni 10010 member vni 10010
mcast-group 225.5.5.5 mcast-group 225.5.5.5
member vni 20020 member vni 20020
mcast-group 227.7.7.7 mcast-group 227.7.7.7
member vni 30030 associate-vrf member vni 30030 associate-vrf

interface Ethernet1/3 interface Ethernet1/3

description connected-to-Spine2-Ethernet1/3 description connected-to-Spine1-Ethernet1/3
no switchport no switchport
mtu 9216 mtu 9216
no ip redirects no ip redirects
ip address 172.16.3.2/30 ip address 192.168.3.2/30
no ipv6 redirects no ipv6 redirects
ip pim sparse-mode ip pim sparse-mode
no shutdown no shutdown

interface Ethernet1/4 interface Ethernet1/4

description connected-to-Spine2-Ethernet1/4 description connected-to-Spine1-Ethernet1/4
no switchport no switchport
mtu 9216 mtu 9216
no ip redirects no ip redirects
ip address 172.16.4.2/30 ip address 192.168.4.2/30
no ipv6 redirects no ipv6 redirects
ip pim sparse-mode ip pim sparse-mode
no shutdown no shutdown

interface Ethernet1/5 interface Ethernet1/5

description connected-to-Spine1-Ethernet1/5 description connected-to-Spine2-Ethernet1/5
no switchport no switchport
mtu 9216 mtu 9216
no ip redirects no ip redirects
ip address 192.168.5.2/30 ip address 172.16.5.2/30
no ipv6 redirects no ipv6 redirects
ip pim sparse-mode ip pim sparse-mode
no shutdown no shutdown

interface Ethernet1/6 interface Ethernet1/6

description connected-to-Spine1-Ethernet1/6 description connected-to-Spine2-Ethernet1/6
no switchport no switchport
mtu 9216 mtu 9216
no ip redirects no ip redirects
ip address 192.168.6.2/30 ip address 172.16.6.2/30
no ipv6 redirects no ipv6 redirects
ip pim sparse-mode ip pim sparse-mode
no shutdown no shutdown

interface Ethernet1/7 interface Ethernet1/7

switchport mode trunk switchport mode trunk

interface loopback0 interface loopback0

description Routing loopback interface description Routing loopback interface
ip address 10.2.0.4/32 ip address 10.2.0.2/32
ip pim sparse-mode ip pim sparse-mode

interface loopback1 interface loopback1

description VTEP loopback interface description VTEP loopback interface
ip address 10.3.0.2/32 ip address 10.3.0.1/32
ip pim sparse-mode ip pim sparse-mode

router bgp 65551 router bgp 65552

router-id 10.2.0.4 router-id 10.2.0.2
address-family ipv4 unicast address-family ipv4 unicast
network 10.2.0.4/32 network 10.2.0.2/32
network 10.3.0.2/32 network 10.3.0.1/32
template peer SPINE-PEER template peer SPINE-PEER
remote-as 65536 remote-as 65536
address-family ipv4 unicast address-family ipv4 unicast
neighbor 10.2.0.1 neighbor 10.2.0.1
remote-as 65536 remote-as 65536
update-source loopback0 update-source loopback0
disable-connected-check disable-connected-check
ebgp-multihop 5 ebgp-multihop 5
address-family l2vpn evpn address-family l2vpn evpn
disable-peer-as-check disable-peer-as-check
send-community send-community
send-community extended send-community extended
rewrite-evpn-rt-asn rewrite-evpn-rt-asn
neighbor 10.2.0.3 neighbor 10.2.0.3
remote-as 65536 remote-as 65536
update-source loopback0 update-source loopback0
disable-connected-check disable-connected-check
ebgp-multihop 5 ebgp-multihop 5
address-family l2vpn evpn address-family l2vpn evpn
disable-peer-as-check disable-peer-as-check
send-community send-community
send-community extended send-community extended
rewrite-evpn-rt-asn rewrite-evpn-rt-asn
neighbor 172.16.3.1 neighbor 172.16.5.1
inherit peer SPINE-PEER inherit peer SPINE-PEER
neighbor 172.16.4.1 neighbor 172.16.6.1
inherit peer SPINE-PEER inherit peer SPINE-PEER
neighbor 192.168.5.1 neighbor 192.168.3.1
inherit peer SPINE-PEER inherit peer SPINE-PEER
neighbor 192.168.6.1 neighbor 192.168.4.1
inherit peer SPINE-PEER inherit peer SPINE-PEER
vrf TENANT-1 vrf TENANT-1
address-family ipv4 unicast address-family ipv4 unicast
evpn evpn
vni 10010 l2 vni 10010 l2
rd auto rd auto
route-target import 65552:10010 route-target import 65551:10010
route-target export 65551:10010 route-target export 65552:10010
vni 20020 l2 vni 20020 l2
rd auto rd auto
route-target import 65552:20020 route-target import 65551:20020
route-target export 65551:20020 route-target export 65552:20020

Created by Matthew Rich, Systems Architect, Cisco Systems, Inc.