ReleaseBranchCI #1590
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # yamllint disable rule:comments-indentation | |
| name: ReleaseBranchCI | |
| env: | |
| # Force the stdout and stderr streams to be unbuffered | |
| PYTHONUNBUFFERED: 1 | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| on: # yamllint disable-line rule:truthy | |
| pull_request: | |
| types: | |
| - synchronize | |
| - reopened | |
| - opened | |
| branches: | |
| # Anything/24.3 (e.g customizations/24.3.x) | |
| - '**/24.3*' | |
| release: | |
| types: | |
| - published | |
| - prereleased | |
| push: | |
| branches: | |
| - 'releases/24.3**' | |
| workflow_dispatch: | |
| jobs: | |
| RunConfig: | |
| runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04] | |
| outputs: | |
| data: ${{ steps.runconfig.outputs.CI_DATA }} | |
| steps: | |
| - name: Check out repository code | |
| uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 | |
| with: | |
| clear-repository: true # to ensure correct digests | |
| fetch-depth: 0 # to get version | |
| filter: tree:0 | |
| - name: Python unit tests | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| echo "Testing the main ci directory" | |
| python3 -m unittest discover -s . -p 'test_*.py' | |
| - name: PrepareRunConfig | |
| id: runconfig | |
| run: | | |
| echo "::group::configure CI run" | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json | |
| echo "::endgroup::" | |
| echo "::group::CI run configure results" | |
| python3 -m json.tool ${{ runner.temp }}/ci_run_data.json | |
| echo "::endgroup::" | |
| { | |
| echo 'CI_DATA<<EOF' | |
| cat ${{ runner.temp }}/ci_run_data.json | |
| echo 'EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Re-create GH statuses for skipped jobs if any | |
| run: | | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses | |
| BuildDockers: | |
| needs: [RunConfig] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_docker.yml | |
| secrets: inherit | |
| with: | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| CompatibilityCheckX86: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Compatibility check (amd64) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ######################################################################################### | |
| #################################### ORDINARY BUILDS #################################### | |
| ######################################################################################### | |
| BuilderDebRelease: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| secrets: inherit | |
| with: | |
| build_name: package_release | |
| checkout_depth: 0 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| # always rebuild on release branches to be able to publish from any commit | |
| force: true | |
| BuilderDebAsan: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| secrets: inherit | |
| with: | |
| build_name: package_asan | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderDebUBsan: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| secrets: inherit | |
| with: | |
| build_name: package_ubsan | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderDebTsan: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| secrets: inherit | |
| with: | |
| build_name: package_tsan | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderDebDebug: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| secrets: inherit | |
| with: | |
| build_name: package_debug | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################ | |
| ##################################### Docker images ####################################### | |
| ############################################################################################ | |
| DockerServerImage: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Docker server image | |
| runner_type: altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| DockerKeeperImage: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Docker keeper image | |
| runner_type: altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################ | |
| ##################################### BUILD REPORTER ####################################### | |
| ############################################################################################ | |
| BuilderReport: | |
| # run report check for failed builds to indicate the CI error | |
| if: ${{ !cancelled() }} | |
| needs: | |
| - RunConfig | |
| - BuilderDebRelease | |
| - BuilderDebAsan | |
| - BuilderDebTsan | |
| - BuilderDebUBsan | |
| - BuilderDebDebug | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: ClickHouse build check | |
| runner_type: altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| MarkReleaseReady: | |
| if: ${{ !failure() && !cancelled() }} | |
| needs: | |
| - BuilderDebRelease | |
| runs-on: [self-hosted, altinity-on-demand, altinity-setup-reporter, altinity-type-cax11, altinity-in-hel1, altinity-image-arm-system-ubuntu-22.04] | |
| steps: | |
| - name: Debug | |
| run: | | |
| echo need with different filters | |
| cat << 'EOF' | |
| ${{ toJSON(needs) }} | |
| ${{ toJSON(needs.*.result) }} | |
| no failures ${{ !contains(needs.*.result, 'failure') }} | |
| no skips ${{ !contains(needs.*.result, 'skipped') }} | |
| no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| EOF | |
| - name: Not ready | |
| # fail the job to be able restart it | |
| if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }} | |
| run: exit 1 | |
| - name: Check out repository code | |
| if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 | |
| - name: Mark Commit Release Ready | |
| if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 mark_release_ready.py | |
| ############################################################################################ | |
| #################################### INSTALL PACKAGES ###################################### | |
| ############################################################################################ | |
| InstallPackagesTestRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Install packages (amd64) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| run_command: | | |
| python3 install_check.py "$CHECK_NAME" | |
| ############################################################################################## | |
| ########################### FUNCTIONAl STATELESS TESTS ####################################### | |
| ############################################################################################## | |
| FunctionalStatelessTestRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateless tests (release) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateless tests (asan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestTsan: | |
| needs: [RunConfig, BuilderDebTsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateless tests (tsan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestUBsan: | |
| needs: [RunConfig, BuilderDebUBsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateless tests (ubsan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestDebug: | |
| needs: [RunConfig, BuilderDebDebug] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateless tests (debug) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################## | |
| ############################ FUNCTIONAl STATEFUL TESTS ####################################### | |
| ############################################################################################## | |
| FunctionalStatefulTestRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateful tests (release) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateful tests (asan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestTsan: | |
| needs: [RunConfig, BuilderDebTsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateful tests (tsan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestUBsan: | |
| needs: [RunConfig, BuilderDebUBsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateful tests (ubsan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestDebug: | |
| needs: [RunConfig, BuilderDebDebug] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stateful tests (debug) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################## | |
| ######################################### STRESS TESTS ####################################### | |
| ############################################################################################## | |
| StressTestAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stress test (asan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| StressTestTsan: | |
| needs: [RunConfig, BuilderDebTsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stress test (tsan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| StressTestUBsan: | |
| needs: [RunConfig, BuilderDebUBsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stress test (ubsan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| StressTestDebug: | |
| needs: [RunConfig, BuilderDebDebug] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Stress test (debug) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################# | |
| ############################# INTEGRATION TESTS ############################################# | |
| ############################################################################################# | |
| IntegrationTestsAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Integration tests (asan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| IntegrationTestsAnalyzerAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Integration tests (asan, old analyzer) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| IntegrationTestsTsan: | |
| needs: [RunConfig, BuilderDebTsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Integration tests (tsan) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| IntegrationTestsRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| secrets: inherit | |
| with: | |
| test_name: Integration tests (release) | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################# | |
| ##################################### REGRESSION TESTS ###################################### | |
| ############################################################################################# | |
| RegressionTestsRelease: | |
| needs: [BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/regression.yml | |
| secrets: inherit | |
| with: | |
| runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression | |
| commit: 91b2de501b77cd90ae9345abb7b56e5fa8c1a921 | |
| arch: release | |
| build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| SignRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_sign.yml | |
| secrets: inherit | |
| with: | |
| test_name: Sign release | |
| runner_type: altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FinishCheck: | |
| if: ${{ !failure() && !cancelled() }} | |
| needs: | |
| - DockerServerImage | |
| - DockerKeeperImage | |
| - BuilderReport | |
| - MarkReleaseReady | |
| - FunctionalStatelessTestDebug | |
| - FunctionalStatelessTestRelease | |
| - FunctionalStatelessTestAsan | |
| - FunctionalStatelessTestTsan | |
| - FunctionalStatelessTestUBsan | |
| - FunctionalStatefulTestDebug | |
| - FunctionalStatefulTestRelease | |
| - FunctionalStatefulTestAsan | |
| - FunctionalStatefulTestTsan | |
| - FunctionalStatefulTestUBsan | |
| - StressTestDebug | |
| - StressTestAsan | |
| - StressTestTsan | |
| - StressTestUBsan | |
| - IntegrationTestsAsan | |
| - IntegrationTestsTsan | |
| - IntegrationTestsRelease | |
| - CompatibilityCheckX86 | |
| - RegressionTestsRelease | |
| - SignRelease | |
| runs-on: [self-hosted, altinity-on-demand, altinity-type-cax11, altinity-image-arm-system-ubuntu-22.04, altinity-setup-regression] | |
| steps: | |
| - name: Check out repository code | |
| uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 | |
| with: | |
| clear-repository: true | |
| - name: Finish label | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 finish_check.py |