[GNOI E2E Testing] Add E2E Tests for GNOI KillProcess API (sonic-net#…

…12478)

### Type of change

- [x] Test case(new/improvement)


### Approach
#### What is the motivation for this PR?
GNOI KillProcess API introduced in this PR: sonic-net/sonic-gnmi#213 requires thorough E2E tests of supported services, and possible failure scenarios
#### How did you do it?
Add E2E tests

#### How did you verify/test it?
Run E2E test on physical DUT

### Documentation
Confirmed passing on latest master: 
---------------------------------------------------------------------------------- live log teardown -----------------------------------------------------------------------------------
22:57:39 gu_utils.rollback                        L0247 INFO   | Commands: config rollback test_setup_checkpoint
22:58:06 conftest.core_dump_and_config_check      L2165 INFO   | Dumping Disk and Memory Space informataion after test on str3-s6100-acs-6
22:58:09 conftest.core_dump_and_config_check      L2169 INFO   | Collecting core dumps after test on str3-s6100-acs-6
22:58:10 conftest.core_dump_and_config_check      L2186 INFO   | Collecting running config after test on str3-s6100-acs-6
22:58:13 conftest.core_dump_and_config_check      L2317 INFO   | Core dump and config check passed for test_gnoi_killprocess.py


=================================================================================== warnings summary ===================================================================================
../../../../usr/local/lib/python3.8/dist-packages/paramiko/transport.py:236
  /usr/local/lib/python3.8/dist-packages/paramiko/transport.py:236: CryptographyDeprecationWarning: Blowfish has been deprecated
    "class": algorithms.Blowfish,

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
------------------------------------------------------------ generated xml file: /var/src/sonic-mgmt-int/tests/logs/tr.xml -------------------------------------------------------------
-------------------------------------------------------------------------------- live log sessionfinish --------------------------------------------------------------------------------
22:58:13 __init__.pytest_terminal_summary         L0067 INFO   | Can not get Allure report URL. Please check logs
====================================================================== 16 passed, 1 warning in 375.00s (0:06:14) =======================================================================

tests/gnmi/helper.py

@@ -251,3 +251,21 @@ def gnoi_reboot(duthost, method, delay, message):
         return -1, output['stderr']
     else:
         return 0, output['stdout']
+
+
+def gnoi_request(duthost, localhost, rpc, request_json_data):
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
+    ip = duthost.mgmt_ip
+    port = env.gnmi_port
+    cmd = "docker exec %s gnoi_client -target %s:%s " % (env.gnmi_container, ip, port)
+    cmd += "-cert /etc/sonic/telemetry/gnmiclient.crt "
+    cmd += "-key /etc/sonic/telemetry/gnmiclient.key "
+    cmd += "-ca /etc/sonic/telemetry/gnmiCA.pem "
+    cmd += "-logtostderr -rpc {} ".format(rpc)
+    cmd += f'-jsonin \'{request_json_data}\''
+    output = duthost.shell(cmd, module_ignore_errors=True)
+    if output['stderr']:
+        logger.error(output['stderr'])
+        return -1, output['stderr']
+    else:
+        return 0, output['stdout']

tests/gnmi/test_gnoi_killprocess.py

@@ -0,0 +1,79 @@
+import pytest
+from .helper import gnoi_request
+from tests.common.helpers.assertions import pytest_assert
+from tests.common.helpers.dut_utils import is_container_running
+
+
+# This test ensures functionality of KillProcess API to kill and restart a process when a valid process name is passed
+# When an invalid process name is passed, this test ensures that the expected error is returned
+@pytest.mark.parametrize("process,is_valid, expected_msg", [
+    ("gnmi", False, "Dbus does not support gnmi service management"),
+    ("nonexistent", False, "Dbus does not support nonexistent service management"),
+    ("", False, "Dbus stop_service called with no service specified"),
+    ("snmp", True, ""),
+    ("dhcp_relay", True, ""),
+    ("radv", True, ""),
+    ("restapi", True, ""),
+    ("lldp", True, ""),
+    ("sshd", True, ""),
+    ("swss", True, ""),
+    ("pmon", True, ""),
+    ("rsyslog", True, ""),
+    ("telemetry", True, "")
+])
+def test_gnoi_killprocess_then_restart(duthosts, rand_one_dut_hostname, localhost, process, is_valid, expected_msg):
+    duthost = duthosts[rand_one_dut_hostname]
+
+    if process and process != "nonexistent":
+        pytest_assert(duthost.is_host_service_running(process),
+                      "{} should be running before KillProcess test attempts to kill this process".format(process))
+
+    request_kill_json_data = '{{"name": "{}", "signal": 1}}'.format(process)
+    ret, msg = gnoi_request(duthost, localhost, "KillProcess", request_kill_json_data)
+    if is_valid:
+        pytest_assert(ret == 0, "KillProcess API unexpectedly reported failure")
+        pytest_assert(not is_container_running(duthost, process),
+                      "{} found running after KillProcess reported success".format(process))
+
+        request_restart_json_data = '{{"name": "{}", "restart": true, "signal": 1}}'.format(process)
+        ret, msg = gnoi_request(duthost, localhost, "KillProcess", request_restart_json_data)
+        pytest_assert(ret == 0,
+                      "KillProcess API unexpectedly reported failure when attempting to restart {}".format(process))
+        pytest_assert(duthost.is_host_service_running(process),
+                      "{} not running after KillProcess reported successful restart".format(process))
+    else:
+        pytest_assert(ret != 0, "KillProcess API unexpectedly succeeded with invalid request parameters")
+        pytest_assert(expected_msg in msg, "Unexpected error message in response to invalid gNOI request")
+
+    pytest_assert(duthost.critical_services_fully_started, "System unhealthy after gNOI API request")
+
+
+# This test performs additional verification of the restart request under KillProcess API
+# This test focuses on edge conditions of restart value in the request, so we only test against one service: snmp
+@pytest.mark.parametrize("request_restart_value, is_valid", [
+    ("invalid", False),
+    ("", False)
+])
+def test_gnoi_killprocess_restart(duthosts, rand_one_dut_hostname, localhost, request_restart_value, is_valid):
+    duthost = duthosts[rand_one_dut_hostname]
+    request_json_data = f'{{"name": "snmp", "restart": {request_restart_value}, "signal": 1}}'
+    ret, msg = gnoi_request(duthost, localhost, "KillProcess", request_json_data)
+    if is_valid:
+        pytest_assert(ret == 0, "KillProcess API unexpectedly reported failure")
+        pytest_assert(is_container_running(duthost, "snmp"),
+                      "snmp not running after KillProcess API reported successful restart")
+    else:
+        pytest_assert(ret != 0, "KillProcess API unexpectedly succeeded with invalid request parameters")
+        pytest_assert("panic" in msg, "Unexpected error message in response to invalid gNOI request")
+    pytest_assert(duthost.critical_services_fully_started, "System unhealthy after gNOI API request")
+
+
+def test_invalid_signal(duthosts, rand_one_dut_hostname, localhost):
+    duthost = duthosts[rand_one_dut_hostname]
+    request_json_data = '{"name": "snmp", "restart": true, "signal": 2}'
+    ret, msg = gnoi_request(duthost, localhost, "KillProcess", request_json_data)
+
+    pytest_assert(ret != 0, "KillProcess API unexpectedly succeeded with invalid request parameters")
+    pytest_assert("KillProcess only supports SIGNAL_TERM (option 1)" in msg,
+                  "Unexpected error message in response to invalid gNOI request")
+    pytest_assert(duthost.critical_services_fully_started, "System unhealthy after gNOI API request")