Merge pull request #634 from linode/dev

v5.51.0

.github/labels.yml

@@ -2,6 +2,9 @@
 - name: new-feature
   description: for new features in the changelog.
   color: 225fee
+- name: project
+  description: for new projects in the changelog.
+  color: 46BAF0
 - name: improvement
   description: for improvements in existing functionality in the changelog.
   color: 22ee47

.github/release.yml

@@ -3,7 +3,10 @@ changelog:
     labels:
       - ignore-for-release
   categories:
-    - title: ⚠️  Breaking Change
+    - title: 📋 New Project
+      labels:
+        - project
+    - title: ⚠️ Breaking Change
       labels:
         - breaking-change
     - title: 🐛 Bug Fixes
@@ -18,7 +21,7 @@ changelog:
     - title: 🧪 Testing Improvements
       labels:
         - testing
-    - title: ⚙️  Repo/CI Improvements
+    - title: ⚙️ Repo/CI Improvements
       labels:
         - repo-ci-improvement
     - title: 📖 Documentation

.github/workflows/e2e-suite-pr.yml → .github/workflows/e2e-suite-windows.yml

@@ -15,99 +15,6 @@ on:
 name: PR E2E Tests
 
 jobs:
-  integration-fork-ubuntu:
-    runs-on: ubuntu-latest
-    if:
-      github.event_name == 'workflow_dispatch' && inputs.sha != ''
-
-    steps:
-      - uses: actions-ecosystem/action-regex-match@v2
-        id: validate-tests
-        with:
-          text: ${{ inputs.test_path }}
-          regex: '[^a-z0-9-:.\/_]'  # Tests validation
-          flags: gi
-
-      # Check out merge commit
-      - name: Checkout PR
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.sha }}
-
-      - name: Get the hash value of the latest commit from the PR branch
-        uses: octokit/graphql-action@v2.x
-        id: commit-hash
-        if: ${{ inputs.pull_request_number != '' }}
-        with:
-          query: |
-            query PRHeadCommitHash($owner: String!, $repo: String!, $pr_num: Int!) {
-              repository(owner:$owner, name:$repo) {
-                pullRequest(number: $pr_num) {
-                  headRef {
-                    target {
-                      ... on Commit {
-                        oid
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          owner: ${{ github.event.repository.owner.login }}
-          repo: ${{ github.event.repository.name }}
-          pr_num: ${{ fromJSON(inputs.pull_request_number) }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Update system packages
-        run: sudo apt-get update -y
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-
-      - name: Install Python deps
-        run: pip install .[dev,obj]
-
-      - name: Install the CLI
-        run: make install
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - run: make INTEGRATION_TEST_PATH="${{ inputs.test_path }}" testint
-        if: ${{ steps.validate-tests.outputs.match == '' }}
-        env:
-          LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }}
-
-      - uses: actions/github-script@v6
-        id: update-check-run
-        if: ${{ inputs.pull_request_number != '' && fromJson(steps.commit-hash.outputs.data).repository.pullRequest.headRef.target.oid == inputs.sha }}
-        env:
-          number: ${{ inputs.pull_request_number }}
-          job: ${{ github.job }}
-          conclusion: ${{ job.status }}
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const { data: pull } = await github.rest.pulls.get({
-              ...context.repo,
-              pull_number: process.env.number
-            });
-            const ref = pull.head.sha;
-            const { data: checks } = await github.rest.checks.listForRef({
-              ...context.repo,
-              ref
-            });
-            const check = checks.check_runs.filter(c => c.name === process.env.job);
-            const { data: result } = await github.rest.checks.update({
-              ...context.repo,
-              check_run_id: check[0].id,
-              status: 'completed',
-              conclusion: process.env.conclusion
-            });
-            return result;
-
   integration-fork-windows:
     runs-on: windows-latest
     if:

.github/workflows/e2e-suite.yml

@@ -1,23 +1,82 @@
 name: Integration Tests
+
 on:
-  workflow_dispatch: null
+  workflow_dispatch:
+    inputs:
+      use_minimal_test_account:
+        description: 'Use minimal test account'
+        required: false
+        default: 'false'
+      test_path:
+        description: "The path from 'test/integration' to the target to be tested, e.g. 'cli'"
+        required: false
+      sha:
+        description: 'The hash value of the commit.'
+        required: false
+        default: ''
+      pull_request_number:
+        description: 'The number of the PR. Ensure sha value is provided'
+        required: false
   push:
     branches:
       - main
       - dev
+
 jobs:
   integration-tests:
-    name: Run integration tests
+    name: Run integration tests on Ubuntu
     runs-on: ubuntu-latest
-    env:
-      EXIT_STATUS: 0
+    if: github.event_name == 'workflow_dispatch' && inputs.sha != '' || github.event_name == 'push' || github.event_name == 'pull_request'
     steps:
-      - name: Clone Repository
-        uses: actions/checkout@v3
+      - name: Validate Test Path
+        uses: actions-ecosystem/action-regex-match@v2
+        id: validate-tests
+        if: ${{ inputs.test_path != '' }}
+        with:
+          text: ${{ inputs.test_path }}
+          regex: '[^a-z0-9-:.\/_]'  # Tests validation
+          flags: gi
+
+      - name: Checkout Repository with SHA
+        if: ${{ inputs.sha != '' }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+          ref: ${{ inputs.sha }}
+
+      - name: Checkout Repository without SHA
+        if: ${{ inputs.sha == '' }}
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: 'recursive'
 
+      - name: Get the hash value of the latest commit from the PR branch
+        uses: octokit/graphql-action@v2.x
+        id: commit-hash
+        if: ${{ inputs.pull_request_number != '' }}
+        with:
+          query: |
+            query PRHeadCommitHash($owner: String!, $repo: String!, $pr_num: Int!) {
+              repository(owner:$owner, name:$repo) {
+                pullRequest(number: $pr_num) {
+                  headRef {
+                    target {
+                      ... on Commit {
+                        oid
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          owner: ${{ github.event.repository.owner.login }}
+          repo: ${{ github.event.repository.name }}
+          pr_num: ${{ fromJSON(inputs.pull_request_number) }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Update system packages
         run: sudo apt-get update -y
 
@@ -26,39 +85,85 @@ jobs:
         with:
           python-version: '3.x'
 
-      - name: Install Python deps
-        run: pip install wheel boto3
-
-      - name: Update cert
-        run: pip install certifi -U
+      - name: Install Python dependencies and update cert
+        run: |
+          pip install wheel boto3 && \
+          pip install certifi -U && \
+          pip install .[obj,dev]
 
-      - name: Install deps
-        run: pip install .[obj,dev]
+      - name: Download kubectl and calicoctl for LKE clusters
+        run: |
+          curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
+          curl -LO "https://github.com/projectcalico/calico/releases/download/v3.25.0/calicoctl-linux-amd64"
+          chmod +x calicoctl-linux-amd64 kubectl
+          mv calicoctl-linux-amd64 /usr/local/bin/calicoctl
+          mv kubectl /usr/local/bin/kubectl
 
       - name: Install Package
         run: make install
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set LINODE_CLI_TOKEN
+        run: |
+          echo "LINODE_CLI_TOKEN=${{ secrets[inputs.use_minimal_test_account == 'true' && 'MINIMAL_LINODE_TOKEN' || 'LINODE_TOKEN'] }}" >> $GITHUB_ENV
+
       - name: Run the integration test suite
         run: |
           timestamp=$(date +'%Y%m%d%H%M')
           report_filename="${timestamp}_cli_test_report.xml"
           make testint TEST_ARGS="--junitxml=${report_filename}"
+        if: ${{ steps.validate-tests.outputs.match == '' || inputs.test_path == '' }}
+        env:
+          LINODE_CLI_TOKEN: ${{ env.LINODE_CLI_TOKEN }}
+
+      - name: Apply Calico Rules to LKE
+        if: always()
+        run: |
+          cd scripts && ./lke_calico_rules_e2e.sh
         env:
-          LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }}
+          LINODE_TOKEN: ${{ env.LINODE_CLI_TOKEN }}
 
       - name: Upload test results
         if: always()
         run: |
           filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$')
-          python tod_scripts/add_to_xml_test_report.py \
+          python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/add_gha_info_to_xml.py \
           --branch_name "${GITHUB_REF#refs/*/}" \
           --gha_run_id "$GITHUB_RUN_ID" \
           --gha_run_number "$GITHUB_RUN_NUMBER" \
           --xmlfile "${filename}"
           sync
-          python tod_scripts/test_report_upload_script.py "${filename}"
+          python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/xml_to_obj.py "${filename}"
         env:
           LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }}
-          LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }}
+          LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }}
+
+      - name: Update PR Check Run
+        uses: actions/github-script@v6
+        id: update-check-run
+        if: ${{ inputs.pull_request_number != '' && fromJson(steps.commit-hash.outputs.data).repository.pullRequest.headRef.target.oid == inputs.sha }}
+        env:
+          number: ${{ inputs.pull_request_number }}
+          job: ${{ github.job }}
+          conclusion: ${{ job.status }}
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { data: pull } = await github.rest.pulls.get({
+              ...context.repo,
+              pull_number: process.env.number
+            });
+            const ref = pull.head.sha;
+            const { data: checks } = await github.rest.checks.listForRef({
+              ...context.repo,
+              ref
+            });
+            const check = checks.check_runs.filter(c => c.name === process.env.job);
+            const { data: result } = await github.rest.checks.update({
+              ...context.repo,
+              check_run_id: check[0].id,
+              status: 'completed',
+              conclusion: process.env.conclusion
+            });
+            return result;

.github/workflows/nightly-smoke-tests.yml

@@ -32,4 +32,4 @@ jobs:
         run: |
           make smoketest
         env:
-          LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN_2 }}
+          LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }}

.gitignore

@@ -13,3 +13,4 @@ test/.env
 .tmp*
 MANIFEST
 venv
+openapi*.yaml

.gitmodules

@@ -4,6 +4,6 @@
 [submodule "test/test_helper/bats-support"]
 	path = test/test_helper/bats-support
 	url = https://github.com/ztombol/bats-support
-[submodule "tod_scripts"]
-	path = tod_scripts
-	url = https://github.com/linode/TOD-test-report-uploader.git
+[submodule "e2e_scripts"]
+	path = e2e_scripts
+	url = https://github.com/linode/dx-e2e-test-scripts