support for scheduled runs

Altinity · Oct 1, 2024 · 2039ec0 · 2039ec0
1 parent 5978424
commit 2039ec0
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 163 deletions.
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
@@ -24,6 +24,7 @@ on: # yamllint disable-line rule:truthy
   push:
     branches:
       - 'releases/24.3**'
+  workflow_dispatch:
 
 jobs:
   RunConfig:
@@ -37,19 +38,11 @@ jobs:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get version
           filter: tree:0
-      # - name: Labels check
-      #   run: |
-      #     cd "$GITHUB_WORKSPACE/tests/ci"
-      #     python3 run_check.py
       - name: Python unit tests
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           echo "Testing the main ci directory"
           python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
@@ -83,15 +76,6 @@ jobs:
       test_name: Compatibility check (amd64)
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
-  # CompatibilityCheckAarch64:
-  #   needs: [RunConfig, BuilderDebAarch64]
-  #   if: ${{ !failure() && !cancelled() }}
-  #   uses: ./.github/workflows/reusable_test.yml
-  #   secrets: inherit
-  #   with:
-  #     test_name: Compatibility check (aarch64)
-  #     runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04
-  #     data: ${{ needs.RunConfig.outputs.data }}
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
 #########################################################################################
@@ -106,17 +90,6 @@ jobs:
       data: ${{ needs.RunConfig.outputs.data }}
       # always rebuild on release branches to be able to publish from any commit
       force: true
-  # BuilderDebAarch64:
-  #   needs: [RunConfig, BuildDockers]
-  #   if: ${{ !failure() && !cancelled() }}
-  #   uses: ./.github/workflows/reusable_build.yml
-  #   secrets: inherit
-  #   with:
-  #     build_name: package_aarch64
-  #     checkout_depth: 0
-  #     data: ${{ needs.RunConfig.outputs.data }}
-  #     # always rebuild on release branches to be able to publish from any commit
-  #     force: true
   BuilderDebAsan:
     needs: [RunConfig, BuildDockers]
     if: ${{ !failure() && !cancelled() }}
@@ -141,14 +114,6 @@ jobs:
     with:
       build_name: package_tsan
       data: ${{ needs.RunConfig.outputs.data }}
-  BuilderDebMsan:
-    needs: [RunConfig, BuildDockers]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    secrets: inherit
-    with:
-      build_name: package_msan
-      data: ${{ needs.RunConfig.outputs.data }}
   BuilderDebDebug:
     needs: [RunConfig, BuildDockers]
     if: ${{ !failure() && !cancelled() }}
@@ -157,29 +122,6 @@ jobs:
     with:
       build_name: package_debug
       data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinDarwin:
-    needs: [RunConfig, BuildDockers]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    secrets: inherit
-    with:
-      build_name: binary_darwin
-      checkout_depth: 0
-      data: ${{ needs.RunConfig.outputs.data }}
-      # always rebuild on release branches to be able to publish from any commit
-      force: true
-  # BuilderBinDarwinAarch64:
-  #   needs: [RunConfig, BuildDockers]
-  #   if: ${{ !failure() && !cancelled() }}
-  #   uses: ./.github/workflows/reusable_build.yml
-  #   secrets: inherit
-  #   with:
-  #     build_name: binary_darwin_aarch64
-  #     checkout_depth: 0
-  #     data: ${{ needs.RunConfig.outputs.data }}
-  #     # always rebuild on release branches to be able to publish from any commit
-  #     force: true
-
 ############################################################################################
 ##################################### Docker images  #######################################
 ############################################################################################
@@ -211,38 +153,20 @@ jobs:
     needs:
       - RunConfig
       - BuilderDebRelease
-      # - BuilderDebAarch64
       - BuilderDebAsan
       - BuilderDebTsan
       - BuilderDebUBsan
-      - BuilderDebMsan
       - BuilderDebDebug
     uses: ./.github/workflows/reusable_test.yml
     secrets: inherit
     with:
       test_name: ClickHouse build check
       runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
-  BuilderSpecialReport:
-    # run report check for failed builds to indicate the CI error
-    if: ${{ !cancelled() }}
-    needs:
-      - RunConfig
-      - BuilderBinDarwin
-      # - BuilderBinDarwinAarch64
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: ClickHouse special build check
-      runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04
-      data: ${{ needs.RunConfig.outputs.data }}
   MarkReleaseReady:
     if: ${{ !failure() && !cancelled() }}
     needs:
-      - BuilderBinDarwin
-      # - BuilderBinDarwinAarch64
       - BuilderDebRelease
-      # - BuilderDebAarch64
     runs-on: [self-hosted, altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04]
     steps:
       - name: Debug
@@ -281,17 +205,6 @@ jobs:
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 install_check.py "$CHECK_NAME"
-  # InstallPackagesTestAarch64:
-  #   needs: [RunConfig, BuilderDebAarch64]
-  #   if: ${{ !failure() && !cancelled() }}
-  #   uses: ./.github/workflows/reusable_test.yml
-  #   secrets: inherit
-  #   with:
-  #     test_name: Install packages (arm64)
-  #     runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04
-  #     data: ${{ needs.RunConfig.outputs.data }}
-  #     run_command: |
-  #       python3 install_check.py "$CHECK_NAME"
 ##############################################################################################
 ########################### FUNCTIONAl STATELESS TESTS #######################################
 ##############################################################################################
@@ -304,15 +217,6 @@ jobs:
       test_name: Stateless tests (release)
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
-  # FunctionalStatelessTestAarch64:
-  #   needs: [RunConfig, BuilderDebAarch64]
-  #   if: ${{ !failure() && !cancelled() }}
-  #   uses: ./.github/workflows/reusable_test.yml
-  #   secrets: inherit
-  #   with:
-  #     test_name: Stateless tests (aarch64)
-  #     runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04
-  #     data: ${{ needs.RunConfig.outputs.data }}
   FunctionalStatelessTestAsan:
     needs: [RunConfig, BuilderDebAsan]
     if: ${{ !failure() && !cancelled() }}
@@ -331,15 +235,6 @@ jobs:
       test_name: Stateless tests (tsan)
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: Stateless tests (msan)
-      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
-      data: ${{ needs.RunConfig.outputs.data }}
   FunctionalStatelessTestUBsan:
     needs: [RunConfig, BuilderDebUBsan]
     if: ${{ !failure() && !cancelled() }}
@@ -370,15 +265,6 @@ jobs:
       test_name: Stateful tests (release)
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
-  # FunctionalStatefulTestAarch64:
-  #   needs: [RunConfig, BuilderDebAarch64]
-  #   if: ${{ !failure() && !cancelled() }}
-  #   uses: ./.github/workflows/reusable_test.yml
-  #   secrets: inherit
-  #   with:
-  #     test_name: Stateful tests (aarch64)
-  #     runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04
-  #     data: ${{ needs.RunConfig.outputs.data }}
   FunctionalStatefulTestAsan:
     needs: [RunConfig, BuilderDebAsan]
     if: ${{ !failure() && !cancelled() }}
@@ -397,15 +283,6 @@ jobs:
       test_name: Stateful tests (tsan)
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: Stateful tests (msan)
-      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
-      data: ${{ needs.RunConfig.outputs.data }}
   FunctionalStatefulTestUBsan:
     needs: [RunConfig, BuilderDebUBsan]
     if: ${{ !failure() && !cancelled() }}
@@ -445,15 +322,6 @@ jobs:
       test_name: Stress test (tsan)
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
-  StressTestMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: Stress test (msan)
-      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
-      data: ${{ needs.RunConfig.outputs.data }}
   StressTestUBsan:
     needs: [RunConfig, BuilderDebUBsan]
     if: ${{ !failure() && !cancelled() }}
@@ -521,19 +389,9 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression
-      commit: 0c78d66f01dd08f324c57ae2efec5cd6bd7d19e3
+      commit: 91b2de501b77cd90ae9345abb7b56e5fa8c1a921
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-  # RegressionTestsAarch64:
-  #   needs: [BuilderDebAarch64]
-  #   if: ${{ !failure() && !cancelled() }}
-  #   uses: ./.github/workflows/regression.yml
-  #   secrets: inherit
-  #   with:
-  #     runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression
-  #     commit: c5e1513a2214ee33696c29717935e0a94989ac2a
-  #     arch: aarch64
-  #     build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
   SignRelease:
     needs: [RunConfig, BuilderDebRelease]
     if: ${{ !failure() && !cancelled() }}
@@ -543,49 +401,32 @@ jobs:
       test_name: Sign release
       runner_type: altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
-  # SignAarch64:
-  #   needs: [RunConfig, BuilderDebAarch64]
-  #   if: ${{ !failure() && !cancelled() }}
-  #   uses: ./.github/workflows/reusable_sign.yml
-  #   secrets: inherit
-  #   with:
-  #     test_name: Sign aarch64
-  #     runner_type: altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
-  #     data: ${{ needs.RunConfig.outputs.data }}
   FinishCheck:
     if: ${{ !failure() && !cancelled() }}
     needs:
       - DockerServerImage
       - DockerKeeperImage
       - BuilderReport
-      - BuilderSpecialReport
       - MarkReleaseReady
       - FunctionalStatelessTestDebug
       - FunctionalStatelessTestRelease
-      # - FunctionalStatelessTestAarch64
       - FunctionalStatelessTestAsan
       - FunctionalStatelessTestTsan
-      - FunctionalStatelessTestMsan
       - FunctionalStatelessTestUBsan
       - FunctionalStatefulTestDebug
       - FunctionalStatefulTestRelease
-      # - FunctionalStatefulTestAarch64
       - FunctionalStatefulTestAsan
       - FunctionalStatefulTestTsan
-      - FunctionalStatefulTestMsan
       - FunctionalStatefulTestUBsan
       - StressTestDebug
       - StressTestAsan
       - StressTestTsan
-      - StressTestMsan
       - StressTestUBsan
       - IntegrationTestsAsan
       - IntegrationTestsTsan
       - IntegrationTestsRelease
       - CompatibilityCheckX86
-      # - CompatibilityCheckAarch64
       - RegressionTestsRelease
-      # - RegressionTestsAarch64
       - SignRelease
     runs-on: [self-hosted, altinity-on-demand, altinity-type-cax11, altinity-image-arm-system-ubuntu-22.04, altinity-setup-regression]
     steps:

diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
@@ -12,6 +12,7 @@
 from cache_utils import CargoCache
 
 from env_helper import (
+    GITHUB_RUN_ID,
     REPO_COPY,
     S3_ACCESS_KEY_ID,
     S3_BUILDS_BUCKET,
@@ -128,6 +129,9 @@ def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str,
     # It should be fixed in performance-comparison image eventually
     # For performance tests we always set PRs prefix
     performance_pr = "PRs/0"
+    if pr_info.event_type == "dispatch":
+        # for dispatch maintenance run we use major version and time
+        return f"maintenance/{pr_info.base_ref}/{GITHUB_RUN_ID}", performance_pr
     if "release" in pr_info.labels or "release-lts" in pr_info.labels:
         # for release pull requests we use branch names prefixes, not pr numbers
         return pr_info.head_ref, performance_pr

diff --git a/tests/ci/ci.py b/tests/ci/ci.py
@@ -1876,8 +1876,10 @@ def main() -> int:
             else {}
         )
 
-        if not args.skip_jobs and pr_info.has_changes_in_documentation_only():
-            _update_config_for_docs_only(jobs_data)
+        if pr_info.event_type != "dispatch":
+            # Avoid calling pr_info.has_changes_in_documentation_only() during workflow_dispatch event
+            if not args.skip_jobs and pr_info.has_changes_in_documentation_only():
+                _update_config_for_docs_only(jobs_data)
 
         if not args.skip_jobs:
             ci_cache = CiCache(s3, jobs_data["digests"])