Automation testing

.github/workflows/web-health-checks.yaml

@@ -0,0 +1,48 @@
+name: Health Checks
+
+on:
+  schedule:
+    - cron: '*/5 * * * *'
+
+jobs:
+  health_check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      run: |
+        pip install pytest requests
+
+    - name: Determine Tags
+      run: |
+        if [[ "${GITHUB_REF##*/}" == "main" ]]; then
+          echo "TAGS=main" >> $GITHUB_ENV
+        elif [[ "${GITHUB_REF##*/}" == release/* ]]; then
+          VERSION=$(grep 'version_base =' src/sparsezoo/version.py | sed -E "s/version_base = \"([0-9]+\.[0-9]+\.[0-9]+)\"/\1/")
+          echo "TAGS=release-$VERSION" >> $GITHUB_ENV
+        else
+          echo "TAGS=dev" >> $GITHUB_ENV
+        fi
+      shell: bash
+
+    - name: Run Health Checks
+      run: |
+        testmo automation:run:submit \
+          --instance "$TESTMO_URL" \
+          --project-id 7 \
+          --name "SparseZoo Health Checks" \
+          --source "web" \
+          --results results/*.xml \
+          --tags TAGS \
+          -- pytest tests/automation/health/ --junitxml=results/test-results.xml
+      env:
+          TESTMO_URL: ${{ secrets.TESTMO_URL }}
+          TESTMO_TOKEN: ${{ secrets.TESTMO_TOKEN }}

tests/automation/README.md

@@ -0,0 +1,2 @@
+# Automation Tests
+

tests/automation/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/automation/core/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/automation/health/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/automation/health/conftest.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/automation/health/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/automation/health/tests/test_api_health.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import requests
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+API_URL = "https://api.neuralmagic.com/v2/graphql"
+HEADERS = {
+    "Content-Type": "application/json",
+}
+
+
+def test_graphql_api_health():
+    try:
+        payload = {
+            "operationName": "GetModelsList",
+            "variables": {},
+            "query": """query GetModelsList {
+                models(isPubliclyVisible: true) {
+                    ...ModelFields
+                    __typename
+                }
+            }
+            fragment ModelFields on Model {
+              architecture
+              baseModelId
+              displayDescription
+              displayName
+              domain
+              domainDisplayName
+              baseModelId
+              downloadCount
+              framework
+              modelId
+              modelOnnxSizeCompressedBytes
+              modelOnnxSizeCompressionRatio
+              repo
+              repoName
+              repoDisplayName
+              sourceDataset
+              sparseTag
+              subArchitecture
+              stub
+              tags {
+                displayName
+                __typename
+              }
+              task
+              taskDisplayName
+              trainingDataset
+              modelCardDownloadUrl
+              __typename
+            }""",
+        }
+
+        response = requests.post(API_URL, headers=HEADERS, json=payload, timeout=30)
+        response.raise_for_status()
+
+        data = response.json()
+        assert (
+            "data" in data and "models" in data["data"]
+        ), "Expected 'models' field in the response"
+        assert (
+            len(data["data"]["models"]) > 100
+        ), f"Expected more than 100 models but got {len(data['data']['models'])}"
+
+    except requests.RequestException as e:
+        LOGGER.error(f"Error when accessing GraphQL API: {e}")
+        raise AssertionError(f"Failed to access GraphQL API: {e}") from e
+    except AssertionError as e:
+        LOGGER.error(f"Data verification error: {e}")
+        raise e

tests/automation/health/tests/test_website_health.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import requests
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_sparsezoo_website_health():
+    try:
+        response = requests.get("https://sparsezoo.neuralmagic.com", timeout=30)
+        response.raise_for_status()
+    except requests.RequestException as e:
+        LOGGER.error(f"Error when accessing SparseZoo website: {e}")
+        raise AssertionError(f"Failed to access SparseZoo website: {e}") from e

tests/automation/models/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.