diff --git a/tests/common/plugins/sanity_check/constants.py b/tests/common/plugins/sanity_check/constants.py index fa0e4abfaa..ddd581726a 100644 --- a/tests/common/plugins/sanity_check/constants.py +++ b/tests/common/plugins/sanity_check/constants.py @@ -12,8 +12,8 @@ # Recover related definitions RECOVER_METHODS = { - "config_reload": {"cmd": "config reload -y", "reboot": False, "adaptive": False, 'recover_wait': 60}, - "load_minigraph": {"cmd": "config load_minigraph -y", "reboot": False, "adaptive": False, 'recover_wait': 60}, + "config_reload": {"cmd": "bash -c 'config reload -y &>/dev/null'", "reboot": False, "adaptive": False, 'recover_wait': 60}, + "load_minigraph": {"cmd": "bash -c 'config load_minigraph -y &>/dev/null'", "reboot": False, "adaptive": False, 'recover_wait': 60}, "reboot": {"cmd": "reboot", "reboot": True, "adaptive": False, 'recover_wait': 120}, "warm_reboot": {"cmd": "warm-reboot", "reboot": True, "adaptive": False, 'recover_wait': 120}, "fast_reboot": {"cmd": "fast_reboot", "reboot": True, "adaptive": False, 'recover_wait': 120}, diff --git a/tests/common/plugins/sanity_check/recover.py b/tests/common/plugins/sanity_check/recover.py index 0abacd5818..a0c2633cbe 100644 --- a/tests/common/plugins/sanity_check/recover.py +++ b/tests/common/plugins/sanity_check/recover.py @@ -30,13 +30,21 @@ def reboot_dut(dut, localhost, cmd, wait_time): def __recover_interfaces(dut, fanouthosts, result, wait_time): + action = None for port in result['down_ports']: logging.info("Restoring port {}".format(port)) + + pn = str(port).lower() + if 'portchannel' in pn or 'vlan' in pn: + action = 'config_reload' + continue + fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port) if fanout and fanout_port: fanout.no_shutdown(fanout_port) dut.no_shutdown(port) wait(wait_time, msg="Wait {} seconds for interface(s) to restore.".format(wait_time)) + return action def __recover_services(dut, result): @@ -57,16 +65,18 @@ def adaptive_recover(dut, localhost, fanouthosts, check_results, wait_time): if result['failed']: logging.info("Restoring {}".format(result)) if result['check_item'] == 'interfaces': - __recover_interfaces(dut, fanouthosts, result, wait_time) - elif result['check_item'] in ['services', 'processes']: - action = __recover_services(dut, result) - # Only allow outstanding_action be overridden when it is - # None. In case the outstanding_action has already been - # been set to 'reboot'. - if not outstanding_action: - outstanding_action = action + action = __recover_interfaces(dut, fanouthosts, result, wait_time) + elif result['check_item'] == 'services': + action = __recover_services(dut, result) + elif result['check_item'] == 'processes': + action = 'config_reload' else: - outstanding_action = 'reboot' + action = 'reboot' + + # Any action can override no action or 'config_reload'. + # 'reboot' is last resort and cannot be overridden. + if action and (not outstanding_action or outstanding_action == 'config_reload'): + outstanding_action = action if outstanding_action: method = constants.RECOVER_METHODS[outstanding_action]