Update for the procedures for insertion/hot swap of Switch Fabric Module(SFM) by using "config chassis modules shutdown/startup" commands

config/chassis_modules.py

@@ -1,8 +1,9 @@
 #!/usr/sbin/env python
 
 import click
-
+import time
 import utilities_common.cli as clicommon
+from .fabric_module_set_admin_status import fabric_module_set_admin_status
 
 #
 # 'chassis_modules' group ('config chassis_modules ...')
@@ -17,6 +18,28 @@ def modules():
     """Configure chassis modules"""
     pass
 
+def get_config_module_state(db, chassis_module_name):
+    config_db = db.cfgdb
+    fvs = config_db.get_entry('CHASSIS_MODULE', chassis_module_name)
+    if not fvs:
+        return None
+    else:
+        return fvs['admin_status']
+
+TIMEOUT_SECS = 10
+
+# Name: get_config_module_state_timeout
+# return: True: timeout, False: not timeout
+def get_config_module_state_timeout(ctx, db, chassis_module_name, state):
+    counter = 0
+    while  get_config_module_state(db, chassis_module_name) != state:
+        time.sleep(1)
+        counter += 1
+        if counter >= TIMEOUT_SECS:
+            ctx.fail("get_config_module_state {} timeout".format(chassis_module_name))
+            return True
+    return False
+
 #
 # 'shutdown' subcommand ('config chassis_modules shutdown ...')
 #
@@ -33,8 +56,14 @@ def shutdown_chassis_module(db, chassis_module_name):
        not chassis_module_name.startswith("FABRIC-CARD"):
         ctx.fail("'module_name' has to begin with 'SUPERVISOR', 'LINE-CARD' or 'FABRIC-CARD'")
 
+    #To avoid duplicate operation
+    if get_config_module_state(db, chassis_module_name) == 'down':
+        click.echo("Duplicate operation for " + chassis_module_name)
+        return
     fvs = {'admin_status': 'down'}
     config_db.set_entry('CHASSIS_MODULE', chassis_module_name, fvs)
+    if not get_config_module_state_timeout(ctx, db, chassis_module_name, 'down'):
+        fabric_module_set_admin_status(chassis_module_name, 'down')
 
 #
 # 'startup' subcommand ('config chassis_modules startup ...')
@@ -45,5 +74,13 @@ def shutdown_chassis_module(db, chassis_module_name):
 def startup_chassis_module(db, chassis_module_name):
     """Chassis-module startup of module"""
     config_db = db.cfgdb
+    ctx = click.get_current_context()
+
+    #To avoid duplicate operation
+    if get_config_module_state(db, chassis_module_name) is None:
+        click.echo("Duplicate operation for " + chassis_module_name)
+        return
 
     config_db.set_entry('CHASSIS_MODULE', chassis_module_name, None)
+    if not get_config_module_state_timeout(ctx, db, chassis_module_name, None):
+        fabric_module_set_admin_status(chassis_module_name, 'up')

config/fabric_module_set_admin_status.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+#
+import re
+import subprocess
+import time
+from swsscommon.swsscommon import SonicV2Connector
+from sonic_py_common.logger import Logger
+import utilities_common.cli as clicommon
+
+# Name: fabric_module_set_admin_status.py, version: 1.0
+# Syntax: fabric_module_set_admin_status <module_name> <up/down>
+def fabric_module_set_admin_status(module, state):
+    logger = Logger("fabric_module_set_admin_status.py")
+    logger.set_min_log_priority_info()
+
+    if not module.startswith("FABRIC-CARD"):
+        logger.log_warning("Failed to set {} state. Admin state can only be set on Fabric module.".format(module))
+        return
+
+    if (state != "up" and state != "down"):
+        logger.log_warning("Failed to set {}. Admin state can only be set to up or down.".format(state))
+        return
+
+    num = int(re.search(r"(\d+)$", module).group())
+    chassisdb = SonicV2Connector(host="127.0.0.1")
+    chassisdb.connect("CHASSIS_STATE_DB")
+
+    if state == "down":
+        asics_keys_list = chassisdb.keys("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE*")
+        asic_list = []
+        for service in asics_keys_list:
+            name = chassisdb.get("CHASSIS_STATE_DB",service,"name")
+            if name == module:
+                asic_id = int(re.search(r"(\d+)$", service).group())
+                asic_list.append(asic_id)
+        if len(asic_list) == 0:
+            logger.log_warning("Failed to get {}'s asic list.".format(module))
+            return
+        logger.log_info("Shutting down chassis module {}".format(module))
+        for asic in asic_list:
+            logger.log_info("Stopping swss@{} and syncd@{} ...".format(asic, asic))
+            clicommon.run_command('sudo systemctl stop swss@{}.service'.format(asic))
+            # wait for service is down
+            time.sleep(5)
+            chassisdb.delete("CHASSIS_STATE_DB","CHASSIS_FABRIC_ASIC_TABLE|asic" + str(asic))
+            logger.log_info("Start swss@{} and syncd@{} ...".format(asic, asic))
+            clicommon.run_command('sudo systemctl start swss@{}.service'.format(asic))
+    else:
+        # wait SFM HW init done.
+        time.sleep(5)
+        asics_keys_list = chassisdb.keys("CHASSIS_STATE_DB", "CHASSIS_FABRIC_ASIC_TABLE*")
+        asic_list = []
+        for service in asics_keys_list:
+            name = chassisdb.get("CHASSIS_STATE_DB",service,"name")
+            if name == module:
+                asic_id = int(re.search(r"(\d+)$", service).group())
+                asic_list.append(asic_id)
+        if len(asic_list) == 0:
+            return
+        for asic in asic_list:
+            logger.log_info("Start swss@{} and syncd@{} ...".format(asic, asic))
+            clicommon.run_command('sudo systemctl start swss@{}.service'.format(asic))