From a3df644b96473f09e78128997ba5703b3f81d74c Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 6 Oct 2023 09:50:21 +0100 Subject: [PATCH 1/8] Fixed send_command function as per suggestion from @alabou --- nmostesting/IS12Utils.py | 84 +++++++++++++++++--------------- nmostesting/suites/IS1201Test.py | 3 +- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 922a3dd1..c9219904 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -151,6 +151,7 @@ def __init__(self, url, spec_path, spec_branch): self.ROOT_BLOCK_OID = 1 self.ncp_websocket = None self.command_handle = 0 + self.expect_notifications = False self.notifications = [] def load_is12_schemas(self, spec_path): @@ -230,47 +231,50 @@ def send_command(self, test, command_json): self.ncp_websocket.send(json.dumps(command_json)) - # Wait for server to respond + results = [] start_time = time.time() while time.time() < start_time + WS_MESSAGE_TIMEOUT: - if self.ncp_websocket.is_messages_received(): + if not self.ncp_websocket.is_messages_received(): + time.sleep(0.2) + continue + + # find the response to our request + for message in self.ncp_websocket.get_messages(): + parsed_message = json.loads(message) + + if self.message_type_to_schema_name(parsed_message.get("messageType")): + self.validate_is12_schema( + test, + parsed_message, + self.message_type_to_schema_name(parsed_message["messageType"]), + context=self.message_type_to_schema_name(parsed_message["messageType"]) + ": ") + else: + raise NMOSTestException(test.FAIL("Unrecognised message type: " + parsed_message.get("messageType"), + "https://specs.amwa.tv/is-12/branches/{}" + "/docs/Protocol_messaging.html#command-message-type" + .format(self.spec_branch))) + + if parsed_message["messageType"] == MessageTypes.CommandResponse: + responses = parsed_message["responses"] + for response in responses: + if response["handle"] == command_handle: + if response["result"]["status"] != NcMethodStatus.OK: + raise NMOSTestException(test.FAIL(response["result"])) + results.append(response) + if parsed_message["messageType"] == MessageTypes.SubscriptionResponse: + results.append(parsed_message["subscriptions"]) + if parsed_message["messageType"] == MessageTypes.Notification: + self.notifications += parsed_message["notifications"] + if parsed_message["messageType"] == MessageTypes.Error: + raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}" + "/docs/Protocol_messaging.html#error-messages" + .format(self.spec_branch))) + + if not self.expect_notifications and len(results) != 0: + break + if self.expect_notifications and len(results) != 0 and len(self.notifications) != 0: break - time.sleep(0.2) - - messages = self.ncp_websocket.get_messages() - results = [] - # find the response to our request - for message in messages: - parsed_message = json.loads(message) - - if self.message_type_to_schema_name(parsed_message["messageType"]): - self.validate_is12_schema( - test, - parsed_message, - self.message_type_to_schema_name(parsed_message["messageType"]), - context=self.message_type_to_schema_name(parsed_message["messageType"]) + ": ") - else: - raise NMOSTestException(test.FAIL("Unrecognised message type: " + parsed_message["messageType"], - "https://specs.amwa.tv/is-12/branches/{}" - "/docs/Protocol_messaging.html#command-message-type" - .format(self.spec_branch))) - - if parsed_message["messageType"] == MessageTypes.CommandResponse: - responses = parsed_message["responses"] - for response in responses: - if response["handle"] == command_handle: - if response["result"]["status"] != NcMethodStatus.OK: - raise NMOSTestException(test.FAIL(response["result"])) - results.append(response) - if parsed_message["messageType"] == MessageTypes.SubscriptionResponse: - results.append(parsed_message["subscriptions"]) - if parsed_message["messageType"] == MessageTypes.Notification: - self.notifications += parsed_message["notifications"] - if parsed_message["messageType"] == MessageTypes.Error: - raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}" - "/docs/Protocol_messaging.html#error-messages" - .format(self.spec_branch))) if len(results) == 0: raise NMOSTestException(test.FAIL("No Message Response received.", "https://specs.amwa.tv/is-12/branches/{}" @@ -285,9 +289,13 @@ def send_command(self, test, command_json): def get_notifications(self): return self.notifications - def reset_notifications(self): + def start_logging_notifications(self): + self.expect_notifications = True self.notifications = [] + def stop_logging_notifications(self): + self.expect_notifications = False + def create_command_JSON(self, oid, method_id, arguments): """for sending over websocket""" self.command_handle += 1 diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index f8ab015c..3120a777 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -1385,8 +1385,9 @@ def test_35(self, test): context = "oid: " + str(oid) + ", " for label in [new_user_label, old_user_label]: - self.is12_utils.reset_notifications() + self.is12_utils.start_logging_notifications() self.is12_utils.set_property(test, oid, NcObjectProperties.USER_LABEL.value, label) + self.is12_utils.stop_logging_notifications() if len(self.is12_utils.get_notifications()) == 0: error = True From 891584623034397f2aa2044502804595af311c2e Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Thu, 3 Aug 2023 21:25:26 +0100 Subject: [PATCH 2/8] Bump Node 12.x to Node 16.x (cherry picked from commit 4caeca489a5c428141cdfdc0a668b851383176ae) --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e7992318..68175cbb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,7 +18,7 @@ jobs: - name: Install Node.js uses: actions/setup-node@v1 with: - node-version: '12.x' + node-version: '16.x' - name: Setup for python lint run: pip install flake8 - name: Lint python From 10e231b0afe556473545ed84db8055422d77ddce Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Mon, 14 Aug 2023 15:23:57 +0100 Subject: [PATCH 3/8] python-zeroconf 0.75.0 strict=False for _nmos-registration._tcp (cherry picked from commit bc7e9971908157bd76c328b2d932d069ca43631d) --- nmostesting/suites/IS0401Test.py | 20 ++++++++++---------- requirements.txt | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nmostesting/suites/IS0401Test.py b/nmostesting/suites/IS0401Test.py index 1fde8b8c..461b719f 100644 --- a/nmostesting/suites/IS0401Test.py +++ b/nmostesting/suites/IS0401Test.py @@ -32,10 +32,6 @@ from ..IS04Utils import IS04Utils from ..TestHelper import get_default_ip, is_ip_address, load_resolved_schema, check_content_type -# monkey patch zeroconf to allow us to advertise "_nmos-registration._tcp" -from zeroconf import service_type_name -service_type_name.__kwdefaults__['strict'] = False - NODE_API_KEY = "node" RECEIVER_CAPS_KEY = "receiver-caps" CAPS_REGISTER_KEY = "caps-register" @@ -94,6 +90,10 @@ def tear_down_tests(self): if self.dns_server: self.dns_server.reset() + def _strict_service_name(self, info): + # avoid zeroconf._exceptions.BadTypeInNameException: Service name (nmos-registration) must be <= 15 bytes + return len(info.type[1:info.type.find('.')]) <= 15 + def _mdns_info(self, port, service_type, txt={}, api_ver=None, api_proto=None, api_auth=None, ip=None): """Get an mDNS ServiceInfo object in order to create an advertisement""" if api_ver is None: @@ -192,9 +192,9 @@ def do_registry_basics_prereqs(self): if CONFIG.DNS_SD_MODE == "multicast": # Advertise the primary registry and invalid ones at pri 0, and allow the Node to do a basic registration if self.is04_utils.compare_api_version(self.apis[NODE_API_KEY]["version"], "v1.0") != 0: - self.zc.register_service(registry_mdns[0]) - self.zc.register_service(registry_mdns[1]) - self.zc.register_service(registry_mdns[2]) + self.zc.register_service(registry_mdns[0], strict=self._strict_service_name(registry_mdns[0])) + self.zc.register_service(registry_mdns[1], strict=self._strict_service_name(registry_mdns[1])) + self.zc.register_service(registry_mdns[2], strict=self._strict_service_name(registry_mdns[2])) # Wait for n seconds after advertising the service for the first POST from a Node start_time = time.time() @@ -226,7 +226,7 @@ def do_registry_basics_prereqs(self): if CONFIG.DNS_SD_MODE == "multicast": for info in registry_mdns[3:]: - self.zc.register_service(info) + self.zc.register_service(info, strict=self._strict_service_name(info)) # Kill registries one by one to collect data around failover self.invalid_registry.disable() @@ -1461,7 +1461,7 @@ def test_21(self, test): if CONFIG.DNS_SD_MODE == "multicast": # Advertise a registry at pri 0 and allow the Node to do a basic registration - self.zc.register_service(registry_info) + self.zc.register_service(registry_info, strict=self._strict_service_name(registry_info)) # Wait for n seconds after advertising the service for the first POST and then DELETE from a Node self.primary_registry.wait_for_registration(CONFIG.DNS_SD_ADVERT_TIMEOUT) @@ -2166,7 +2166,7 @@ def collect_mdns_announcements(self): if CONFIG.DNS_SD_MODE == "multicast": # Advertise a registry at pri 0 and allow the Node to do a basic registration - self.zc.register_service(registry_info) + self.zc.register_service(registry_info, strict=self._strict_service_name(registry_info)) # Wait for n seconds after advertising the service for the first POST from a Node self.primary_registry.wait_for_registration(CONFIG.DNS_SD_ADVERT_TIMEOUT) diff --git a/requirements.txt b/requirements.txt index 5a51fa50..93bb689e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ flask>=2.0.0 wtforms jsonschema -zeroconf>=0.32.0 +zeroconf>=0.75.0 requests netifaces gitpython From 4ee4077f4b1b84d21210c700cf240c01411807b8 Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Fri, 18 Aug 2023 14:00:25 +0100 Subject: [PATCH 4/8] Python 3.6 (and 3.7!) is End-Of-Life, and so is Node.js 14.x Libraries like python-zeroconf are starting to depend on more recent Python features (cherry picked from commit b10ff7157f3e0e97e736e5a820c57a6a4cc0383f) --- Dockerfile | 10 +++++----- docs/1.1. Installation - Local.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 08fcfef8..8c000875 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:bionic +FROM ubuntu:focal WORKDIR /home/nmos-testing ADD . . @@ -7,9 +7,9 @@ ADD .git .git RUN apt-get update \ && export DEBIAN_FRONTEND=noninteractive \ && apt-get install -y wget \ - && wget https://deb.nodesource.com/setup_14.x \ - && chmod 755 setup_14.x \ - && /home/nmos-testing/setup_14.x \ + && wget https://deb.nodesource.com/setup_16.x \ + && chmod 755 setup_16.x \ + && /home/nmos-testing/setup_16.x \ && apt-get install -y --no-install-recommends \ gcc openssl libssl-dev wget ca-certificates avahi-daemon avahi-utils libnss-mdns libavahi-compat-libdnssd-dev \ python3 python3-pip python3-dev nodejs \ @@ -26,7 +26,7 @@ RUN apt-get update \ && rm v3.0.7.tar.gz \ && npm config set unsafe-perm true \ && npm install -g AMWA-TV/sdpoker#v0.3.0 \ - && rm /home/nmos-testing/setup_14.x \ + && rm /home/nmos-testing/setup_16.x \ && apt-get remove -y wget \ && apt-get clean -y --no-install-recommends \ && apt-get autoclean -y --no-install-recommends \ diff --git a/docs/1.1. Installation - Local.md b/docs/1.1. Installation - Local.md index d7249421..57fd68fa 100644 --- a/docs/1.1. Installation - Local.md +++ b/docs/1.1. Installation - Local.md @@ -4,7 +4,7 @@ Please ensure that the following dependencies are installed on your system first. -- Python 3.6 or higher, including the 'pip' package manager +- Python 3.8 or higher, including the 'pip' package manager - Git - [testssl.sh](https://testssl.sh) (required for BCP-003-01 testing, see our [README](../testssl/README.md) for instructions) - [OpenSSL](https://www.openssl.org/) (required for BCP-003-01 OCSP testing) From 027a3ac9759381a6a9d965f8c24db08dd8f201cc Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Thu, 5 Oct 2023 09:42:50 +0100 Subject: [PATCH 5/8] Pass IP address of API under test to testssl.sh So that same one is used by Python tests and testssl.sh tests Co-authored-by: Simon Lo (cherry picked from commit 3f10a8d478b9148c10f12ddef7af485002e4c9c2) --- nmostesting/suites/BCP00301Test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nmostesting/suites/BCP00301Test.py b/nmostesting/suites/BCP00301Test.py index 08fa1da9..de6cf2c1 100644 --- a/nmostesting/suites/BCP00301Test.py +++ b/nmostesting/suites/BCP00301Test.py @@ -56,7 +56,9 @@ def perform_test_ssl(self, test, args=None): "--openssl-timeout", str(CONFIG.HTTP_TIMEOUT), "--add-ca", - CONFIG.CERT_TRUST_ROOT_CA + CONFIG.CERT_TRUST_ROOT_CA, + "--ip", + self.apis[SECURE_API_KEY]["ip"] ] + args + ["{}:{}".format(self.apis[SECURE_API_KEY]["hostname"], self.apis[SECURE_API_KEY]["port"])] ) From 3db3689d165bfb4d726a22fd3a45f9ba3c1f8128 Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:15:19 +0100 Subject: [PATCH 6/8] Add MOCK_SERVICES_WARM_UP_DELAY (cherry picked from commit 796a11507c979ea6343717ce0a7fc3cdfe0332ea) --- nmostesting/Config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nmostesting/Config.py b/nmostesting/Config.py index 07242a05..72c399aa 100644 --- a/nmostesting/Config.py +++ b/nmostesting/Config.py @@ -19,6 +19,9 @@ # Please consult the documentation for instructions on how to adjust these values for common testing setups including # unicast DNS-SD and HTTPS testing. +# Number of seconds to wait after starting the mock DNS server, authorization server, etc. before running tests. +# This gives the API or client under test a chance to use these services before any test case is run. +MOCK_SERVICES_WARM_UP_DELAY = 0 # Enable or disable DNS-SD advertisements. Browsing is always permitted. # The IS-04 Node tests create a mock registry on the network unless the `ENABLE_DNS_SD` parameter is set to `False`. From e6e73a62b2ad81cce89bd3681b473f86ce275c2b Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:22:42 +0100 Subject: [PATCH 7/8] Delay running tests until after MOCK_SERVICES_WARM_UP_DELAY Co-authored-by: Simon Lo (cherry picked from commit 43a1f764284eb4da960e9f5d75463ae91ead52d9) --- nmostesting/NMOSTesting.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index 470e2eeb..f49ce567 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -1189,6 +1189,13 @@ def main(args): print(" * Testing tool running on 'http://{}:{}'. Version '{}'" .format(get_default_ip(), core_app.config['PORT'], TOOL_VERSION)) + # Give an API or client that is already running a chance to use the mock services + # before running any test cases + if CONFIG.MOCK_SERVICES_WARM_UP_DELAY: + print(" * Warm up your engines, testing begins in {} seconds..." + .format(CONFIG.MOCK_SERVICES_WARM_UP_DELAY)) + time.sleep(CONFIG.MOCK_SERVICES_WARM_UP_DELAY) + exit_code = 0 if "suite" not in vars(CMD_ARGS): # Interactive testing mode. Await user input. From 986aa29e5099dd0cc599ea3a9832ed4c861e63c9 Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Thu, 5 Oct 2023 12:43:06 +0100 Subject: [PATCH 8/8] Tweak message Co-authored-by: jonathan-r-thorpe <64410119+jonathan-r-thorpe@users.noreply.github.com> (cherry picked from commit 215f736469e5260b7629592e5adf8769c50555ca) --- nmostesting/NMOSTesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index f49ce567..df51a4d6 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -1192,7 +1192,7 @@ def main(args): # Give an API or client that is already running a chance to use the mock services # before running any test cases if CONFIG.MOCK_SERVICES_WARM_UP_DELAY: - print(" * Warm up your engines, testing begins in {} seconds..." + print(" * Waiting for {} seconds to allow discovery of mock services" .format(CONFIG.MOCK_SERVICES_WARM_UP_DELAY)) time.sleep(CONFIG.MOCK_SERVICES_WARM_UP_DELAY)