From 1f12a8db7bb30bf7367c121e5daf1b6cdcdc6957 Mon Sep 17 00:00:00 2001 From: Longxiang Lyu <35479537+lolyu@users.noreply.github.com> Date: Sat, 1 Jun 2024 01:05:17 +0800 Subject: [PATCH] [subnet decap] Add subnet decap rule based on overlay ECMP (#3153) * [subnet decap] Add subnet decap rule based on overlay ECMP vxlan tunnel route What I did Support dynamic decap rule generation based on the Vxlan tunnel route of Overlay ECMP. This depends on: #3117 Why I did it To enable SONiC with the capability to decap IPinIP packets with dest IP in the Overlay ECMP Vxlan tunnel route prefix. --- orchagent/vnetorch.cpp | 50 ++++++- orchagent/vnetorch.h | 4 + tests/test_vnet.py | 324 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 376 insertions(+), 2 deletions(-) diff --git a/orchagent/vnetorch.cpp b/orchagent/vnetorch.cpp index b976c728a7..5c482d726d 100644 --- a/orchagent/vnetorch.cpp +++ b/orchagent/vnetorch.cpp @@ -21,6 +21,7 @@ #include "neighorch.h" #include "crmorch.h" #include "routeorch.h" +#include "tunneldecaporch.h" #include "flowcounterrouteorch.h" extern sai_virtual_router_api_t* sai_virtual_router_api; @@ -43,6 +44,7 @@ extern RouteOrch *gRouteOrch; extern MacAddress gVxlanMacAddress; extern BfdOrch *gBfdOrch; extern SwitchOrch *gSwitchOrch; +extern TunnelDecapOrch *gTunneldecapOrch; /* * VRF Modeling and VNetVrf class definitions */ @@ -334,7 +336,7 @@ VNetVrfObject::~VNetVrfObject() set vr_ent = getVRids(); for (auto it : vr_ent) { - if (it != gVirtualRouterId) + if (it != gVirtualRouterId) { sai_status_t status = sai_virtual_router_api->remove_virtual_router(it); if (status != SAI_STATUS_SUCCESS) @@ -717,7 +719,8 @@ static bool update_route(sai_object_id_t vr_id, sai_ip_prefix_t& ip_pfx, sai_obj } VNetRouteOrch::VNetRouteOrch(DBConnector *db, vector &tableNames, VNetOrch *vnetOrch) - : Orch2(db, tableNames, request_), vnet_orch_(vnetOrch), bfd_session_producer_(db, APP_BFD_SESSION_TABLE_NAME) + : Orch2(db, tableNames, request_), vnet_orch_(vnetOrch), bfd_session_producer_(db, APP_BFD_SESSION_TABLE_NAME), + app_tunnel_decap_term_producer_(db, APP_TUNNEL_DECAP_TERM_TABLE_NAME) { SWSS_LOG_ENTER(); @@ -1432,6 +1435,39 @@ bool VNetRouteOrch::updateTunnelRoute(const string& vnet, IpPrefix& ipPrefix, return true; } +inline void VNetRouteOrch::createSubnetDecapTerm(const IpPrefix &ipPrefix) +{ + const SubnetDecapConfig &config = gTunneldecapOrch->getSubnetDecapConfig(); + if (!config.enable || subnet_decap_terms_created_.find(ipPrefix) != subnet_decap_terms_created_.end()) + { + return; + } + SWSS_LOG_NOTICE("Add subnet decap term for %s", ipPrefix.to_string().c_str()); + static const vector data = { + {"term_type", "MP2MP"}, + {"subnet_type", "vip"} + }; + string tunnel_name = ipPrefix.isV4() ? config.tunnel : config.tunnel_v6; + string key = tunnel_name + ":" + ipPrefix.to_string(); + app_tunnel_decap_term_producer_.set(key, data); + subnet_decap_terms_created_.insert(ipPrefix); +} + +inline void VNetRouteOrch::removeSubnetDecapTerm(const IpPrefix &ipPrefix) +{ + const SubnetDecapConfig &config = gTunneldecapOrch->getSubnetDecapConfig(); + auto it = subnet_decap_terms_created_.find(ipPrefix); + if (it == subnet_decap_terms_created_.end()) + { + return; + } + SWSS_LOG_NOTICE("Remove subnet decap term for %s", ipPrefix.to_string().c_str()); + string tunnel_name = ipPrefix.isV4() ? config.tunnel : config.tunnel_v6; + string key = tunnel_name + ":" + ipPrefix.to_string(); + app_tunnel_decap_term_producer_.del(key); + subnet_decap_terms_created_.erase(it); +} + template<> bool VNetRouteOrch::doRouteTask(const string& vnet, IpPrefix& ipPrefix, nextHop& nh, string& op) @@ -2088,6 +2124,14 @@ void VNetRouteOrch::postRouteState(const string& vnet, IpPrefix& ipPrefix, NextH removeRouteAdvertisement(prefix_to_use); } } + if (route_state == "active") + { + createSubnetDecapTerm(prefix_to_use); + } + else if (route_state == "inactive") + { + removeSubnetDecapTerm(prefix_to_use); + } } void VNetRouteOrch::removeRouteState(const string& vnet, IpPrefix& ipPrefix) @@ -2101,11 +2145,13 @@ void VNetRouteOrch::removeRouteState(const string& vnet, IpPrefix& ipPrefix) if(adv_prefix_refcount_[adv_pfx] == 1) { removeRouteAdvertisement(adv_pfx); + removeSubnetDecapTerm(adv_pfx); } } else { removeRouteAdvertisement(ipPrefix); + removeSubnetDecapTerm(ipPrefix); } } diff --git a/orchagent/vnetorch.h b/orchagent/vnetorch.h index 0cffa115fd..e2ba25d0a5 100644 --- a/orchagent/vnetorch.h +++ b/orchagent/vnetorch.h @@ -463,6 +463,8 @@ class VNetRouteOrch : public Orch2, public Subject, public Observer void updateVnetTunnel(const BfdUpdate&); void updateVnetTunnelCustomMonitor(const MonitorUpdate& update); bool updateTunnelRoute(const string& vnet, IpPrefix& ipPrefix, NextHopGroupKey& nexthops, string& op); + void createSubnetDecapTerm(const IpPrefix &ipPrefix); + void removeSubnetDecapTerm(const IpPrefix &ipPrefix); template bool doRouteTask(const string& vnet, IpPrefix& ipPrefix, NextHopGroupKey& nexthops, string& op, string& profile, @@ -485,7 +487,9 @@ class VNetRouteOrch : public Orch2, public Subject, public Observer std::map nexthop_info_; std::map prefix_to_adv_prefix_; std::map adv_prefix_refcount_; + std::set subnet_decap_terms_created_; ProducerStateTable bfd_session_producer_; + ProducerStateTable app_tunnel_decap_term_producer_; unique_ptr monitor_session_producer_; shared_ptr state_db_; shared_ptr app_db_; diff --git a/tests/test_vnet.py b/tests/test_vnet.py index c28d7cf320..be08a52c69 100644 --- a/tests/test_vnet.py +++ b/tests/test_vnet.py @@ -1,4 +1,5 @@ import time +import ipaddress import json import random import time @@ -541,6 +542,62 @@ def check_syslog(dvs, marker, err_log): assert num.strip() == "0" +def create_fvs(**kwargs): + return swsscommon.FieldValuePairs(list(kwargs.items())) + + +def create_subnet_decap_tunnel(dvs, tunnel_name, **kwargs): + """Create tunnel and verify all needed entries in state DB exists.""" + appdb = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) + statedb = swsscommon.DBConnector(swsscommon.STATE_DB, dvs.redis_sock, 0) + fvs = create_fvs(**kwargs) + # create tunnel entry in DB + ps = swsscommon.ProducerStateTable(appdb, "TUNNEL_DECAP_TABLE") + ps.set(tunnel_name, fvs) + + # wait till config will be applied + time.sleep(1) + + # validate the tunnel entry in state db + tunnel_state_table = swsscommon.Table(statedb, "TUNNEL_DECAP_TABLE") + + tunnels = tunnel_state_table.getKeys() + for tunnel in tunnels: + status, fvs = tunnel_state_table.get(tunnel) + assert status == True + + for field, value in fvs: + if field == "tunnel_type": + assert value == "IPINIP" + elif field == "dscp_mode": + assert value == kwargs["dscp_mode"] + elif field == "ecn_mode": + assert value == kwargs["ecn_mode"] + elif field == "ttl_mode": + assert value == kwargs["ttl_mode"] + elif field == "encap_ecn_mode": + assert value == kwargs["encap_ecn_mode"] + else: + assert False, "Field %s is not tested" % field + + +def delete_subnet_decap_tunnel(dvs, tunnel_name): + """Delete tunnel and checks that state DB is cleared.""" + appdb = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) + statedb = swsscommon.DBConnector(swsscommon.STATE_DB, dvs.redis_sock, 0) + tunnel_app_table = swsscommon.Table(appdb, "TUNNEL_DECAP_TABLE") + tunnel_state_table = swsscommon.Table(statedb, "TUNNEL_DECAP_TABLE") + + ps = swsscommon.ProducerStateTable(appdb, "TUNNEL_DECAP_TABLE") + ps._del(tunnel_name) + + # wait till config will be applied + time.sleep(1) + + assert len(tunnel_app_table.getKeys()) == 0 + assert len(tunnel_state_table.getKeys()) == 0 + + loopback_id = 0 def_vr_id = 0 switch_mac = None @@ -577,11 +634,27 @@ class VnetVxlanVrfTunnel(object): ASIC_BFD_SESSION = "ASIC_STATE:SAI_OBJECT_TYPE_BFD_SESSION" APP_VNET_MONITOR = "VNET_MONITOR_TABLE" + ecn_modes_map = { + "standard" : "SAI_TUNNEL_DECAP_ECN_MODE_STANDARD", + "copy_from_outer": "SAI_TUNNEL_DECAP_ECN_MODE_COPY_FROM_OUTER" + } + + dscp_modes_map = { + "pipe" : "SAI_TUNNEL_DSCP_MODE_PIPE_MODEL", + "uniform" : "SAI_TUNNEL_DSCP_MODE_UNIFORM_MODEL" + } + + ttl_modes_map = { + "pipe" : "SAI_TUNNEL_TTL_MODE_PIPE_MODEL", + "uniform" : "SAI_TUNNEL_TTL_MODE_UNIFORM_MODEL" + } + def __init__(self): self.tunnel_map_ids = set() self.tunnel_map_entry_ids = set() self.tunnel_ids = set() self.tunnel_term_ids = set() + self.ipinip_tunnel_term_ids = {} self.tunnel_map_map = {} self.tunnel = {} self.vnet_vr_ids = set() @@ -611,6 +684,61 @@ def fetch_exist_entries(self, dvs): if switch_mac is None: switch_mac = get_switch_mac(dvs) + def check_ipinip_tunnel(self, dvs, tunnel_name, dscp_mode, ecn_mode, ttl_mode): + asic_db = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0) + + tunnel_id = get_created_entry(asic_db, self.ASIC_TUNNEL_TABLE, self.tunnel_ids) + tunnel_attrs = { + 'SAI_TUNNEL_ATTR_TYPE': 'SAI_TUNNEL_TYPE_IPINIP', + 'SAI_TUNNEL_ATTR_ENCAP_DSCP_MODE': self.dscp_modes_map[dscp_mode], + 'SAI_TUNNEL_ATTR_ENCAP_ECN_MODE': self.ecn_modes_map[ecn_mode], + 'SAI_TUNNEL_ATTR_ENCAP_TTL_MODE': self.ttl_modes_map[ttl_mode] + } + check_object(asic_db, self.ASIC_TUNNEL_TABLE, tunnel_id, tunnel_attrs) + + self.tunnel_ids.add(tunnel_id) + self.tunnel[tunnel_name] = tunnel_id + + def check_del_ipinip_tunnel(self, dvs, tunnel_name): + asic_db = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0) + + tunnel_id = get_deleted_entries(asic_db, self.ASIC_TUNNEL_TABLE, self.tunnel_ids, 1)[0] + check_deleted_object(asic_db, self.ASIC_TUNNEL_TABLE, tunnel_id) + self.tunnel_ids.remove(tunnel_id) + assert tunnel_id == self.tunnel[tunnel_name] + self.tunnel.pop(tunnel_name) + + def check_ipinip_tunnel_decap_term(self, dvs, tunnel_name, dst_ip, src_ip): + asic_db = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0) + + dst_ip = ipaddress.ip_network(dst_ip) + src_ip = ipaddress.ip_network(src_ip) + tunnel_term_id = get_created_entry(asic_db, self.ASIC_TUNNEL_TERM_ENTRY, self.tunnel_term_ids) + tunnel_term_attrs = { + 'SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_TYPE': 'SAI_TUNNEL_TERM_TABLE_ENTRY_TYPE_MP2MP', + 'SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_TUNNEL_TYPE': 'SAI_TUNNEL_TYPE_IPINIP', + 'SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_DST_IP': str(dst_ip.network_address), + 'SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_DST_IP_MASK': str(dst_ip.netmask), + 'SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_SRC_IP': str(src_ip.network_address), + 'SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_SRC_IP_MASK': str(src_ip.netmask), + 'SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_ACTION_TUNNEL_ID': self.tunnel[tunnel_name] + } + check_object(asic_db, self.ASIC_TUNNEL_TERM_ENTRY, tunnel_term_id, tunnel_term_attrs) + + self.tunnel_term_ids.add(tunnel_term_id) + self.ipinip_tunnel_term_ids[(tunnel_name, src_ip, dst_ip)] = tunnel_term_id + + def check_del_ipinip_tunnel_decap_term(self, dvs, tunnel_name, dst_ip, src_ip): + asic_db = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0) + + dst_ip = ipaddress.ip_network(dst_ip) + src_ip = ipaddress.ip_network(src_ip) + tunnel_term_id = get_deleted_entries(asic_db, self.ASIC_TUNNEL_TERM_ENTRY, self.tunnel_term_ids, 1)[0] + check_deleted_object(asic_db, self.ASIC_TUNNEL_TERM_ENTRY, tunnel_term_id) + self.tunnel_term_ids.remove(tunnel_term_id) + assert self.ipinip_tunnel_term_ids[(tunnel_name, src_ip, dst_ip)] == tunnel_term_id + self.ipinip_tunnel_term_ids.pop((tunnel_name, src_ip, dst_ip)) + def check_vxlan_tunnel(self, dvs, tunnel_name, src_ip): asic_db = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0) global loopback_id, def_vr_id @@ -1099,6 +1227,30 @@ def check_custom_monitor_deleted(self, dvs, prefix, endpoint): class TestVnetOrch(object): + CFG_SUBNET_DECAP_TABLE_NAME = "SUBNET_DECAP" + + @pytest.fixture + def setup_subnet_decap(self, dvs): + + def _apply_subnet_decap_config(subnet_decap_config): + """Apply subnet decap config to CONFIG_DB.""" + subnet_decap_tbl = swsscommon.Table(configdb, self.CFG_SUBNET_DECAP_TABLE_NAME) + fvs = create_fvs(**subnet_decap_config) + subnet_decap_tbl.set("AZURE", fvs) + + def _cleanup_subnet_decap_config(): + """Cleanup subnet decap config in CONFIG_DB.""" + subnet_decap_tbl = swsscommon.Table(configdb, self.CFG_SUBNET_DECAP_TABLE_NAME) + for key in subnet_decap_tbl.getKeys(): + subnet_decap_tbl._del(key) + + configdb = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0) + _cleanup_subnet_decap_config() + + yield _apply_subnet_decap_config + + _cleanup_subnet_decap_config() + def get_vnet_obj(self): return VnetVxlanVrfTunnel() @@ -3524,6 +3676,178 @@ def test_vnet_orch_25(self, dvs, testlog): vnet_obj.check_del_vnet_entry(dvs, 'Vnet25') delete_vxlan_tunnel(dvs, tunnel_name) + ''' + Test 26 - Test for vnet tunnel routes with ECMP nexthop group with subnet decap enable + ''' + def test_vnet_orch_26(self, dvs, setup_subnet_decap): + # apply subnet decap config + subnet_decap_config = { + "status": "enable", + "src_ip": "10.10.10.0/24", + "src_ip_v6": "20c1:ba8::/64" + } + setup_subnet_decap(subnet_decap_config) + + vnet_obj = self.get_vnet_obj() + vnet_obj.fetch_exist_entries(dvs) + + # Add the subnet decap tunnel + create_subnet_decap_tunnel(dvs, "IPINIP_SUBNET", tunnel_type="IPINIP", + dscp_mode="uniform", ecn_mode="standard", ttl_mode="pipe") + vnet_obj.check_ipinip_tunnel(dvs, "IPINIP_SUBNET", "uniform", "standard", "pipe") + + vnet_obj.fetch_exist_entries(dvs) + tunnel_name = 'tunnel_26' + create_vxlan_tunnel(dvs, tunnel_name, '26.26.26.26') + create_vnet_entry(dvs, 'Vnet26', tunnel_name, '10026', "", advertise_prefix=True) + + vnet_obj.check_vnet_entry(dvs, 'Vnet26') + vnet_obj.check_vxlan_tunnel_entry(dvs, tunnel_name, 'Vnet26', '10026') + vnet_obj.check_vxlan_tunnel(dvs, tunnel_name, '26.26.26.26') + + vnet_obj.fetch_exist_entries(dvs) + create_vnet_routes(dvs, "100.100.1.1/32", 'Vnet26', '26.0.0.1,26.0.0.2,26.0.0.3', ep_monitor='26.1.0.1,26.1.0.2,26.1.0.3', profile="test_profile") + + with pytest.raises(AssertionError): + vnet_obj.check_ipinip_tunnel_decap_term(dvs, "IPINIP_SUBNET", "100.100.1.1/32", "10.10.10.0/24") + + # default bfd status is down, route should not be programmed in this status + vnet_obj.check_del_vnet_routes(dvs, 'Vnet26', ["100.100.1.1/32"]) + check_state_db_routes(dvs, 'Vnet26', "100.100.1.1/32", []) + check_remove_routes_advertisement(dvs, "100.100.1.1/32") + + # Route should be properly configured when all bfd session states go up + update_bfd_session_state(dvs, '26.1.0.1', 'Up') + + time.sleep(2) + # subnet decap term should be created as one bfd session state go up + vnet_obj.check_ipinip_tunnel_decap_term(dvs, "IPINIP_SUBNET", "100.100.1.1/32", "10.10.10.0/24") + + update_bfd_session_state(dvs, '26.1.0.2', 'Up') + update_bfd_session_state(dvs, '26.1.0.3', 'Up') + time.sleep(2) + vnet_obj.check_vnet_ecmp_routes(dvs, 'Vnet26', ['26.0.0.1', '26.0.0.2', '26.0.0.3'], tunnel_name) + check_state_db_routes(dvs, 'Vnet26', "100.100.1.1/32", ['26.0.0.1', '26.0.0.2', '26.0.0.3']) + check_routes_advertisement(dvs, "100.100.1.1/32", "test_profile") + + # Set all endpoint to down state + update_bfd_session_state(dvs, '26.1.0.1', 'Down') + update_bfd_session_state(dvs, '26.1.0.2', 'Down') + update_bfd_session_state(dvs, '26.1.0.3', 'Down') + time.sleep(2) + + # subnet decap term should be removed as all bfd session states go down + vnet_obj.check_del_ipinip_tunnel_decap_term(dvs, "IPINIP_SUBNET", "100.100.1.1/32", "10.10.10.0/24") + + # Confirm the tunnel route is updated in ASIC + vnet_obj.check_del_vnet_routes(dvs, 'Vnet26', ["100.100.1.1/32"]) + check_state_db_routes(dvs, 'Vnet26', "100.100.1.1/32", []) + check_remove_routes_advertisement(dvs, "100.100.1.1/32") + + # Remove tunnel route + delete_vnet_routes(dvs, "100.100.1.1/32", 'Vnet26') + vnet_obj.check_del_vnet_routes(dvs, 'Vnet26', ["100.100.1.1/32"]) + check_remove_state_db_routes(dvs, 'Vnet26', "100.100.1.1/32") + check_remove_routes_advertisement(dvs, "100.100.1.1/32") + + # Confirm the BFD sessions are removed + check_del_bfd_session(dvs, ['12.1.0.1', '12.1.0.2', '12.1.0.3']) + + delete_vnet_entry(dvs, 'Vnet26') + vnet_obj.check_del_vnet_entry(dvs, 'Vnet26') + delete_vxlan_tunnel(dvs, tunnel_name) + + # Remove the subnet decap tunnel + vnet_obj.fetch_exist_entries(dvs) + delete_subnet_decap_tunnel(dvs, "IPINIP_SUBNET") + vnet_obj.check_del_ipinip_tunnel(dvs, "IPINIP_SUBNET") + + ''' + Test 27 - Test for IPv6 vnet tunnel routes with ECMP nexthop group with subnet decap enable + ''' + def test_vnet_orch_27(self, dvs, setup_subnet_decap): + subnet_decap_config = { + "status": "enable", + "src_ip": "10.10.10.0/24", + "src_ip_v6": "20c1:ba8::/64" + } + setup_subnet_decap(subnet_decap_config) + + vnet_obj = self.get_vnet_obj() + vnet_obj.fetch_exist_entries(dvs) + + # Add the subnet decap tunnel + create_subnet_decap_tunnel(dvs, "IPINIP_SUBNET_V6", tunnel_type="IPINIP", + dscp_mode="uniform", ecn_mode="standard", ttl_mode="pipe") + vnet_obj.check_ipinip_tunnel(dvs, "IPINIP_SUBNET_V6", "uniform", "standard", "pipe") + + vnet_obj.fetch_exist_entries(dvs) + tunnel_name = 'tunnel_27' + vnet_name = 'Vnet26' + create_vxlan_tunnel(dvs, tunnel_name, 'fd:10::32') + create_vnet_entry(dvs, vnet_name, tunnel_name, '10010', "", advertise_prefix=True) + + vnet_obj.check_vnet_entry(dvs, vnet_name) + vnet_obj.check_vxlan_tunnel_entry(dvs, tunnel_name, vnet_name, '10010') + vnet_obj.check_vxlan_tunnel(dvs, tunnel_name, 'fd:10::32') + + vnet_obj.fetch_exist_entries(dvs) + create_vnet_routes(dvs, "fd:10:10::1/128", vnet_name, 'fd:10:1::1,fd:10:1::2,fd:10:1::3', ep_monitor='fd:10:2::1,fd:10:2::2,fd:10:2::3', profile="test_profile") + + with pytest.raises(AssertionError): + vnet_obj.check_ipinip_tunnel_decap_term(dvs, "IPINIP_SUBNET_V6", "100.100.1.1/32", "10.10.10.0/24") + + # default bfd status is down, route should not be programmed in this status + vnet_obj.check_del_vnet_routes(dvs, vnet_name, ["fd:10:10::1/128"]) + check_state_db_routes(dvs, vnet_name, "fd:10:10::1/128", []) + check_remove_routes_advertisement(dvs, "fd:10:10::1/128") + + # Route should be properly configured when all bfd session states go up + update_bfd_session_state(dvs, 'fd:10:2::2', 'Up') + + time.sleep(2) + # subnet decap term should be created as one bfd session state go up + vnet_obj.check_ipinip_tunnel_decap_term(dvs, "IPINIP_SUBNET_V6", "fd:10:10::1/128", "20c1:ba8::/64") + + update_bfd_session_state(dvs, 'fd:10:2::3', 'Up') + update_bfd_session_state(dvs, 'fd:10:2::1', 'Up') + time.sleep(2) + vnet_obj.check_vnet_ecmp_routes(dvs, vnet_name, ['fd:10:1::1', 'fd:10:1::2', 'fd:10:1::3'], tunnel_name) + check_state_db_routes(dvs, vnet_name, "fd:10:10::1/128", ['fd:10:1::1', 'fd:10:1::2', 'fd:10:1::3']) + check_routes_advertisement(dvs, "fd:10:10::1/128", "test_profile") + + # Set all endpoint to down state + update_bfd_session_state(dvs, 'fd:10:2::1', 'Down') + update_bfd_session_state(dvs, 'fd:10:2::2', 'Down') + update_bfd_session_state(dvs, 'fd:10:2::3', 'Down') + time.sleep(2) + + # subnet decap term should be removed as all bfd session states go down + vnet_obj.check_del_ipinip_tunnel_decap_term(dvs, "IPINIP_SUBNET_V6", "fd:10:10::1/128", "20c1:ba8::/64") + + # Confirm the tunnel route is updated in ASIC + vnet_obj.check_del_vnet_routes(dvs, vnet_name, ["fd:10:10::1/128"]) + check_state_db_routes(dvs, vnet_name, "fd:10:10::1/128", []) + check_remove_routes_advertisement(dvs, "fd:10:10::1/128") + + # Remove tunnel route + delete_vnet_routes(dvs, "fd:10:10::1/128", vnet_name) + vnet_obj.check_del_vnet_routes(dvs, vnet_name, ["fd:10:10::1/128"]) + check_remove_state_db_routes(dvs, vnet_name, "fd:10:10::1/128") + check_remove_routes_advertisement(dvs, "fd:10:10::1/128") + + # Confirm the BFD sessions are removed + check_del_bfd_session(dvs, ['fd:10:2::1', 'fd:10:2::2', 'fd:10:2::3']) + + delete_vnet_entry(dvs, vnet_name) + vnet_obj.check_del_vnet_entry(dvs, vnet_name) + delete_vxlan_tunnel(dvs, tunnel_name) + + # Remove the subnet decap tunnel + vnet_obj.fetch_exist_entries(dvs) + delete_subnet_decap_tunnel(dvs, "IPINIP_SUBNET_V6") + vnet_obj.check_del_ipinip_tunnel(dvs, "IPINIP_SUBNET_V6") + # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying def test_nonflaky_dummy():