[Reclaiming buffer] Test cases for reclaiming buffer on both traditio…

…nal and dynamic model (sonic-net#4561)

### Description of PR

Summary: Test cases for reclaiming buffer on both traditional and dynamic model

### Approach
#### What is the motivation for this PR?
Add regression test cases of reclaiming buffer for both traditional and dynamic model

#### How did you do it?
1. Adjust dynamic buffer test cases to verify reclaiming buffer in port admin down test
2. Add deployment test to verify correct profiles in `APPL_DB`, and `ASIC_DB` have been applied on each port.
    This is for both traditional and dynamic buffer models.
3. Adjust the script which transmits a switch from traditional model to dynamic model.
4. Shut down BGP neighbors before test and start up them during teardown as we need to run the test in topologies other than `ptf32`.

tests/common/helpers/enable-dynamic-buffer.py

@@ -7,8 +7,36 @@
 from swsscommon.swsscommon import ConfigDBConnector
 
 lossless_profile_name_pattern = 'pg_lossless_([1-9][0-9]*000)_([1-9][0-9]*m)_profile'
+zero_profile_name_pattern = '.*zero_profile'
+zero_pool_name_pattern = '.*zero_pool'
+zero_profiles_to_normal_profiles = {
+    '[BUFFER_PROFILE|ingress_lossy_pg_zero_profile]': '[BUFFER_PROFILE|ingress_lossy_profile]',
+    '[BUFFER_PROFILE|ingress_lossless_zero_profile]': '[BUFFER_PROFILE|ingress_lossless_profile]',
+    '[BUFFER_PROFILE|ingress_lossy_zero_profile]': '[BUFFER_PROFILE|ingress_lossy_profile]',
+    '[BUFFER_PROFILE|egress_lossless_zero_profile]': '[BUFFER_PROFILE|egress_lossless_profile]',
+    '[BUFFER_PROFILE|egress_lossy_zero_profile]': '[BUFFER_PROFILE|egress_lossy_profile]',
+    'ingress_lossy_pg_zero_profile': 'ingress_lossy_profile',
+    'ingress_lossless_zero_profile': 'ingress_lossless_profile',
+    'ingress_lossy_zero_profile': 'ingress_lossy_profile',
+    'egress_lossless_zero_profile': 'egress_lossless_profile',
+    'egress_lossy_zero_profile': 'egress_lossy_profile'
+    }
 logger = Logger()
 
+def _replace_buffer_profile_lists(config_db, table):
+    ingress_profile_lists = config_db.get_table(table)
+    for key, profile_list in ingress_profile_lists.items():
+        if re.search(zero_profile_name_pattern, profile_list['profile_list']):
+            zero_profiles = profile_list['profile_list'].split(',')
+            normal_profiles = ''
+            for profile in zero_profiles:
+                normal_profile = zero_profiles_to_normal_profiles.get(profile)
+                if normal_profile:
+                    normal_profiles += normal_profile + ','
+            profile_list['profile_list'] = normal_profiles[:-1]
+            config_db.set_entry(table, key, profile_list)
+
+
 def stop_traditional_buffer_model(config_db):
     """
     Stop the traditional buffer model
@@ -27,24 +55,62 @@ def stop_traditional_buffer_model(config_db):
     # Remove lossless PGs from BUFFER_PG table
     # A PG whose profile matches pg_lossless_<speed>_<cable-length>_profile is treated as a lossless PG
     pgs = config_db.get_table('BUFFER_PG')
-    lossless_pgs = {};
+    lossless_pgs = {}
+    zero_pgs = []
     for key, pg in pgs.items():
         if re.search(lossless_profile_name_pattern, pg['profile']):
             config_db.set_entry('BUFFER_PG', key, None)
             pg['profile'] = 'NULL'
             lossless_pgs[key] = pg
+            # We can not apply profile as NULL for now. The traditional buffer manager can not handle them
+
+        if re.search(zero_profile_name_pattern, pg['profile']):
+            normal_profile = zero_profiles_to_normal_profiles.get(pg['profile'])
+            if normal_profile:
+                pg['profile'] = normal_profile
+                config_db.set_entry('BUFFER_PG', key, pg)
+                zero_pgs.append(key)
+
+    logger.log_notice("Lossless PGs have been removed from BUFFER_PG and will be applied after dynamic buffer manager starts {}".format(lossless_pgs))
+    logger.log_notice("Zero PGs have been replaced by normal profile {}".format(zero_pgs))
 
-    logger.log_notice("Lossless PGs have been removed from BUFFER_PG: {}".format(lossless_pgs.keys()))
+    queues = config_db.get_table('BUFFER_QUEUE')
+    zero_queues = []
+    for key, queue in queues.items():
+        if re.search(zero_profile_name_pattern, queue['profile']):
+            normal_profile = zero_profiles_to_normal_profiles.get(queue['profile'])
+            if normal_profile:
+                queue['profile'] = normal_profile
+                config_db.set_entry('BUFFER_QUEUE', key, queue)
+                zero_queues.append(key)
+
+    logger.log_notice("Queues referencing zero profiles have been removed from BUFFER_QUEUE and will be replaced by normal profile: {}".format(zero_queues))
+
+    _replace_buffer_profile_lists(config_db, 'BUFFER_PORT_INGRESS_PROFILE_LIST')
+    _replace_buffer_profile_lists(config_db, 'BUFFER_PORT_EGRESS_PROFILE_LIST')
 
     # Remove dynamically generated profiles
     profiles = config_db.get_table('BUFFER_PROFILE')
     dynamic_profile = []
+    zero_profile = []
     for key, profile in profiles.items():
         if re.search(lossless_profile_name_pattern, key):
             config_db.set_entry('BUFFER_PROFILE', key, None)
             dynamic_profile.append(key)
+        elif re.search(zero_profile_name_pattern, key):
+            config_db.set_entry('BUFFER_PROFILE', key, None)
+            zero_profile.append(key)
 
-    logger.log_notice("Dynamically generated profiles have been removed from BUFFER_PROFILE: {}".format(dynamic_profile))
+    logger.log_notice("Dynamically generated profiles and zero profiles have been removed from BUFFER_PROFILE: {} {}".format(dynamic_profile, zero_profile))
+
+    pools = config_db.get_table('BUFFER_POOL')
+    zero_pool = []
+    for key, pool in pools.items():
+        if re.search(zero_pool_name_pattern, key):
+            config_db.set_entry('BUFFER_POOL', key, None)
+            zero_pool.append(key)
+
+    logger.log_notice("Zero pools have been removed from BUFFER_TABLE: {}".format(zero_pool))
 
     # Stop the buffermgrd
     # We don't stop the buffermgrd at the beginning
@@ -85,29 +151,36 @@ def start_dynamic_buffer_model(config_db, lossless_pgs, metadata):
     # By default, all pools except egress_lossless_pool are dynamic size pools
     dynamic_size_pools = ['ingress_lossless_pool', 'ingress_lossy_pool', 'egress_lossy_pool']
     pools = config_db.get_table('BUFFER_POOL')
+    shared_headroom_pool = False;
     for key, pool in pools.items():
         if key in dynamic_size_pools:
             config_db.set_entry('BUFFER_POOL', key, None)
             if 'size' in pool.keys():
                 pool.pop('size')
+            if 'xoff' in pool.keys():
+                pool.pop('xoff')
+                shared_headroom_pool = True
             config_db.set_entry('BUFFER_POOL', key, pool)
 
     logger.log_notice("Sizes have been removed from {}".format(dynamic_size_pools))
 
     # Create lossless PGs
     if lossless_pgs:
         for key, pg in lossless_pgs.items():
-            pg['profile'] = 'NULL'
             config_db.set_entry('BUFFER_PG', key, pg)
         logger.log_notice("Lossless PGs have been created for {}".format(lossless_pgs.keys()))
     else:
+        # The lossless_pgs can be None if this script is called immediately after reloading minigraph
+        # because the lossless PGs hasn't been inserted into CONFIG_DB by traditional buffer manager
         ports = config_db.get_keys('PORT')
         for port in ports:
             config_db.set_entry('BUFFER_PG', '{}|3-4'.format(port), {'profile': 'NULL'})
-        logger.log_notice("No lossless PG in CONFIG_DB, lossless PGs have been created for all ports {}".format(ports)) 
+        logger.log_notice("No lossless PG in CONFIG_DB, lossless PGs have been created for all ports {}".format(ports))
 
     # Add necessary tables to run dynamic model
     default_lossless_param = {'default_dynamic_th': '0'}
+    if shared_headroom_pool:
+        default_lossless_param['over_subscribe_ratio'] = '2'
     config_db.set_entry('DEFAULT_LOSSLESS_BUFFER_PARAMETER', 'AZURE', default_lossless_param)
 
     logger.log_notice("DEFAULT_LOSSLESS_BUFFER_PARAMETER|AZURE has been created")
@@ -145,6 +218,10 @@ def start_dynamic_buffer_model(config_db, lossless_pgs, metadata):
     print("Don't enable dynamic buffer calculation for non-default SKUs")
     exit(0)
 
+if 'dynamic' == metadata.get('buffer_model'):
+    print("The current model is already dynamic model")
+    exit(0)
+
 lossless_pgs = stop_traditional_buffer_model(config_db)
 
 start_dynamic_buffer_model(config_db, lossless_pgs, metadata)

tests/qos/files/dynamic_buffer_param.json

@@ -25,13 +25,26 @@
 		"dynamic_th": "2"
 	    }
 	},
-	"lossy_pg": {
-	    "8": "37888",
-	    "default": "19456"
+	"extra_overhead": {
+	    "400000": "95232",
+	    "default": "58368"
 	},
 	"shared-headroom-pool": {
 	    "size": "1024000",
 	    "private_pg_headroom": "10240"
+	},
+	"admin-down": {
+	    "BUFFER_PG_TABLE": {
+		"0": "[BUFFER_PROFILE_TABLE:ingress_lossy_pg_zero_profile]"
+	    },
+	    "BUFFER_QUEUE_TABLE": {
+		"0-2": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]",
+		"3-4": "[BUFFER_PROFILE_TABLE:egress_lossless_zero_profile]",
+		"5-6": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]",
+		"7-15": "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]"
+	    },
+	    "BUFFER_PORT_INGRESS_PROFILE_LIST_TABLE": ["[BUFFER_PROFILE_TABLE:ingress_lossless_zero_profile]"],
+	    "BUFFER_PORT_EGRESS_PROFILE_LIST_TABLE": ["[BUFFER_PROFILE_TABLE:egress_lossless_zero_profile]", "[BUFFER_PROFILE_TABLE:egress_lossy_zero_profile]"]
 	}
     }
 }