From 50e7ac19960d990887ddd10039931931ad0e12b3 Mon Sep 17 00:00:00 2001 From: Kaiwalya Joshi Date: Tue, 15 Jan 2019 14:46:11 -0800 Subject: [PATCH 1/6] Replace item.get_marker with item.get_closest_marker as it was removed upstream as mentioned here: https://github.com/pytest-dev/pytest/issues/4606#issuecomment-451761580 Quoted here incase that link breaks in the future: "We have removed item.get_marker, users should use item.get_closest_marker now (it's a long history, this was not an arbitrary change). Indeed I suspect pytest-cov might need to be updated." item.get_marker was removed in this PR https://github.com/pytest-dev/pytest/pull/4564 --- testing/sdk_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/sdk_utils.py b/testing/sdk_utils.py index 403bb53ad01..d4ab07728b5 100644 --- a/testing/sdk_utils.py +++ b/testing/sdk_utils.py @@ -111,7 +111,7 @@ def your_test_here(): ... In order for this annotation to take effect, this function must be called by a pytest_runtest_setup() hook. """ - min_version_mark = item.get_marker("dcos_min_version") + min_version_mark = item.get_closest_marker("dcos_min_version") if min_version_mark: min_version = min_version_mark.args[0] message = "Feature only supported in DC/OS {} and up".format(min_version) From c6567e98e99951f8cac1eda76fcc4da5bb1ece85 Mon Sep 17 00:00:00 2001 From: Kaiwalya Joshi Date: Tue, 15 Jan 2019 16:38:23 -0800 Subject: [PATCH 2/6] Determine security mode of the cluster by querying the cluster instead of a local environment-variable. --- testing/sdk_dcos.py | 28 ++++++++++++++++++++++++++++ testing/sdk_utils.py | 4 +++- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 testing/sdk_dcos.py diff --git a/testing/sdk_dcos.py b/testing/sdk_dcos.py new file mode 100644 index 00000000000..c922d7ef2ba --- /dev/null +++ b/testing/sdk_dcos.py @@ -0,0 +1,28 @@ +'''Utilities relating to getting information about DC/OS itself + +************************************************************************ +FOR THE TIME BEING WHATEVER MODIFICATIONS ARE APPLIED TO THIS FILE +SHOULD ALSO BE APPLIED TO sdk_dcos IN ANY OTHER PARTNER REPOS +************************************************************************ +''' +from enum import Enum + +import sdk_cmd + + +class DCOS_SECURITY(Enum): + disabled = 1 + permissive = 2 + strict = 3 + + +def get_metadata(): + return sdk_cmd.cluster_request('GET', + 'dcos-metadata/bootstrap-config.json', + retry=False) + + +def get_security_mode() -> DCOS_SECURITY: + r = get_metadata().json() + mode = r['security'] + return DCOS_SECURITY[mode] diff --git a/testing/sdk_utils.py b/testing/sdk_utils.py index d4ab07728b5..d2397c346e5 100644 --- a/testing/sdk_utils.py +++ b/testing/sdk_utils.py @@ -14,7 +14,9 @@ import string import sdk_cmd +import sdk_dcos +from sdk_dcos import DCOS_SECURITY from distutils.version import LooseVersion log = logging.getLogger(__name__) @@ -129,7 +131,7 @@ def is_open_dcos(): def is_strict_mode(): """Determine if the tests are being run on a strict mode cluster.""" - return os.environ.get("SECURITY", "") == "strict" + return sdk_dcos.get_security_mode() == DCOS_SECURITY.strict """Annotation which may be used to mark test suites or test cases as EE-only. From c026cf1efd5c8b8d278628c3582fb40c75595ab4 Mon Sep 17 00:00:00 2001 From: Kaiwalya Joshi Date: Tue, 15 Jan 2019 16:39:36 -0800 Subject: [PATCH 3/6] Remove underscores from services names, this breaks for scenarios such as 'nonessential_tasks' where underscores are not allowed in task names. --- frameworks/helloworld/tests/scale/test_scale.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frameworks/helloworld/tests/scale/test_scale.py b/frameworks/helloworld/tests/scale/test_scale.py index daa246a5c57..b5cfa4a43d6 100644 --- a/frameworks/helloworld/tests/scale/test_scale.py +++ b/frameworks/helloworld/tests/scale/test_scale.py @@ -20,7 +20,8 @@ def test_scaling_load(service_count, """ # TODO: parallelize account creation and installation if time is an issue in scale tests for index in range(service_count): - service_name = "{}-{}-{}".format(config.PACKAGE_NAME, scenario, index) + #Note service-names *cannot* have underscores in them. + service_name = ("{}-{}-{}".format(config.PACKAGE_NAME, scenario, index)).replace("_", "-") security_info = _create_service_account(service_name) _install_service(service_name, scenario, From 620e8dbab317156897cd4aeac35fb02696c9d429 Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Thu, 21 Feb 2019 21:46:34 -0500 Subject: [PATCH 4/6] Update frameworks/helloworld/tests/scale/test_scale.py Co-Authored-By: kaiwalyajoshi --- frameworks/helloworld/tests/scale/test_scale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frameworks/helloworld/tests/scale/test_scale.py b/frameworks/helloworld/tests/scale/test_scale.py index b5cfa4a43d6..f9922be12c7 100644 --- a/frameworks/helloworld/tests/scale/test_scale.py +++ b/frameworks/helloworld/tests/scale/test_scale.py @@ -20,7 +20,7 @@ def test_scaling_load(service_count, """ # TODO: parallelize account creation and installation if time is an issue in scale tests for index in range(service_count): - #Note service-names *cannot* have underscores in them. + # Note service-names *cannot* have underscores in them. service_name = ("{}-{}-{}".format(config.PACKAGE_NAME, scenario, index)).replace("_", "-") security_info = _create_service_account(service_name) _install_service(service_name, From 87c08f825529b261840296e1e473a24e78f5a4e7 Mon Sep 17 00:00:00 2001 From: Kaiwalya Joshi Date: Fri, 22 Feb 2019 11:36:58 -0800 Subject: [PATCH 5/6] Address comments from code-review --- testing/sdk_dcos.py | 28 ---------------------------- testing/sdk_utils.py | 22 +++++++++++++++++++--- 2 files changed, 19 insertions(+), 31 deletions(-) delete mode 100644 testing/sdk_dcos.py diff --git a/testing/sdk_dcos.py b/testing/sdk_dcos.py deleted file mode 100644 index c922d7ef2ba..00000000000 --- a/testing/sdk_dcos.py +++ /dev/null @@ -1,28 +0,0 @@ -'''Utilities relating to getting information about DC/OS itself - -************************************************************************ -FOR THE TIME BEING WHATEVER MODIFICATIONS ARE APPLIED TO THIS FILE -SHOULD ALSO BE APPLIED TO sdk_dcos IN ANY OTHER PARTNER REPOS -************************************************************************ -''' -from enum import Enum - -import sdk_cmd - - -class DCOS_SECURITY(Enum): - disabled = 1 - permissive = 2 - strict = 3 - - -def get_metadata(): - return sdk_cmd.cluster_request('GET', - 'dcos-metadata/bootstrap-config.json', - retry=False) - - -def get_security_mode() -> DCOS_SECURITY: - r = get_metadata().json() - mode = r['security'] - return DCOS_SECURITY[mode] diff --git a/testing/sdk_utils.py b/testing/sdk_utils.py index d2397c346e5..496493437c7 100644 --- a/testing/sdk_utils.py +++ b/testing/sdk_utils.py @@ -14,10 +14,9 @@ import string import sdk_cmd -import sdk_dcos -from sdk_dcos import DCOS_SECURITY from distutils.version import LooseVersion +from enum import Enum log = logging.getLogger(__name__) @@ -26,6 +25,11 @@ # Service/task names ### +class DCOS_SECURITY(Enum): + disabled = 1 + permissive = 2 + strict = 3 + def get_package_name(default: str) -> str: return os.environ.get("INTEGRATION_TEST__PACKAGE_NAME") or default @@ -131,7 +135,19 @@ def is_open_dcos(): def is_strict_mode(): """Determine if the tests are being run on a strict mode cluster.""" - return sdk_dcos.get_security_mode() == DCOS_SECURITY.strict + return get_security_mode() == DCOS_SECURITY.strict + + +def get_security_mode() -> DCOS_SECURITY: + r = get_metadata().json() + mode = r['security'] + return DCOS_SECURITY[mode] + + +def get_metadata(): + return sdk_cmd.cluster_request('GET', + 'dcos-metadata/bootstrap-config.json', + retry=False) """Annotation which may be used to mark test suites or test cases as EE-only. From f8e13e464c4b1b975777528ce4e2a311895d25a6 Mon Sep 17 00:00:00 2001 From: Kaiwalya Joshi Date: Fri, 22 Feb 2019 12:13:08 -0800 Subject: [PATCH 6/6] Add support for soack configuration and custom service names, this can be used to differentiate the various hello-worlds. ie --service-name='orchestration/hello-world' --- frameworks/helloworld/tests/conftest.py | 7 ++++ .../helloworld/tests/scale/test_scale.py | 32 ++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/frameworks/helloworld/tests/conftest.py b/frameworks/helloworld/tests/conftest.py index 5e249b96c9d..a05c7906d97 100644 --- a/frameworks/helloworld/tests/conftest.py +++ b/frameworks/helloworld/tests/conftest.py @@ -14,6 +14,8 @@ def pytest_addoption(parser): help='Number of hello world services to deploy with a given scenario') parser.addoption("--scenario", action='store', default='', help="hello world service yml to use") + parser.addoption("--service-name", action='store', default='hello-world', + help="custom service name to be used instead of default 'hello-world'") @pytest.fixture @@ -24,3 +26,8 @@ def service_count(request) -> int: @pytest.fixture def scenario(request) -> str: return str(request.config.getoption('--scenario')) + + +@pytest.fixture +def service_name(request) -> str: + return str(request.config.getoption('--service-name')) diff --git a/frameworks/helloworld/tests/scale/test_scale.py b/frameworks/helloworld/tests/scale/test_scale.py index f9922be12c7..75cca845288 100644 --- a/frameworks/helloworld/tests/scale/test_scale.py +++ b/frameworks/helloworld/tests/scale/test_scale.py @@ -8,22 +8,46 @@ log = logging.getLogger(__name__) +@pytest.mark.soak +def test_soak_load(service_name, + scenario) -> None: + """Launch a soak test scenario. This does not verify the results + of the test, but does ensure the instances were created. + + Args: + service_name: name of the service to install as + scenario: yaml scenario to run helloworld with (normal, crashloop) are added for this case + """ + # Note service-names *cannot* have underscores in them. + soak_service_name = ("{}-{}-soak".format(service_name, scenario)).replace("_", "-") + # service-names can have '/'s in them but service account names cannot, sanitize here. + service_account_name = soak_service_name.replace("/", "__") + security_info = _create_service_account(service_account_name) + _install_service(soak_service_name, + scenario, + security_info) + + @pytest.mark.scale -def test_scaling_load(service_count, +def test_scaling_load(service_name, + service_count, scenario) -> None: """Launch a load test scenario. This does not verify the results of the test, but does ensure the instances were created. Args: + service_name: name of the service to install as service_count: number of helloworld services to install scenario: yaml scenario to run helloworld with (normal, crashloop) are added for this case """ # TODO: parallelize account creation and installation if time is an issue in scale tests for index in range(service_count): # Note service-names *cannot* have underscores in them. - service_name = ("{}-{}-{}".format(config.PACKAGE_NAME, scenario, index)).replace("_", "-") - security_info = _create_service_account(service_name) - _install_service(service_name, + scale_service_name = ("{}-{}-{}".format(service_name, scenario, index)).replace("_", "-") + # service-names can have '/'s in them but service account names cannot, sanitize here. + service_account_name = scale_service_name.replace("/", "__") + security_info = _create_service_account(service_account_name) + _install_service(scale_service_name, scenario, security_info)