Automatic performance testing #232
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "QA" | |
| # Runs on each push and tests flowR for the default configuration | |
| # Depending on the targets etc. this may perform many more checks! | |
| 'on': | |
| push: | |
| paths-ignore: | |
| - '**/*.md' | |
| - 'LICENSE' | |
| - '.gitignore' | |
| - 'scripts/**' | |
| pull_request: | |
| types: [ opened, synchronize ] | |
| branches: [ main ] | |
| workflow_dispatch: | |
| inputs: | |
| force-full: | |
| description: "Force all steps" | |
| required: true | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| deployments: write | |
| contents: write | |
| jobs: | |
| debug: | |
| name: "Debug" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: "Debug state" | |
| run: | | |
| echo "Event: ${{ github.event_name }}" | |
| echo "Ref: ${{ github.ref }}" | |
| echo "Base Ref: ${{ github.base_ref }}" | |
| lint: | |
| needs: [ debug ] | |
| name: "Linting" | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.base.ref != 'main' }} | |
| uses: Code-Inspect/flowr/.github/workflows/run.yaml@main | |
| with: | |
| r-version: '' | |
| action: lint | |
| test: | |
| needs: [ debug ] | |
| name: "Test Suite (coverage)" | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.base.ref != 'main' }} | |
| uses: Code-Inspect/flowr/.github/workflows/run.yaml@main | |
| with: | |
| action: test-full | |
| coverage: true | |
| performance-test: | |
| needs: [ test ] | |
| name: "Run Performance Tests (only on main)" | |
| # TODO: if everything works: && (inputs.force-full || (github.event_name == 'push' && github.ref == 'refs/heads/main')) | |
| if: ${{ always() && !failure() }} | |
| uses: Code-Inspect/flowr/.github/workflows/run.yaml@161-setup-an-automatic-and-reproducible-performance-benchmark-with-the-ci | |
| with: | |
| action: performance-test | |
| coverage: false | |
| benchmark: true | |
| timeout-minutes: 25 | |
| performance-test-upload: | |
| name: "Upload Performance Test Results" | |
| runs-on: ubuntu-latest | |
| needs: [ performance-test ] | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: | |
| - name: 'artificial' | |
| tool: 'customSmallerIsBetter' | |
| steps: | |
| - name: "Checkout Repository" | |
| uses: actions/checkout@v4 | |
| - name: "⬇️ Get benchmark Artifacts" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: benchmark-results | |
| path: benchmark/ | |
| - name: Name and Email for Git (config) | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| - name: "Merge and Produce Performance Results" | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: '"${{ matrix.name }}" Benchmark Suite' | |
| tool: '${{ matrix.tool }}' | |
| output-file-path: benchmark/${{ matrix.name }}/${{ matrix.name }}-graph.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| fail-on-alert: false | |
| summary-always: true | |
| comment-on-alert: true | |
| comment-always: true | |
| skip-fetch-gh-pages: true | |
| benchmark-data-dir-path: wiki/stats/benchmark/${{ matrix.name }}/ | |
| auto-push: false | |
| - name: "⬆️ Push benchmark results" | |
| if: github.event_name != 'pull_request' | |
| # we manipulate the name of the last commit message | |
| run: | | |
| git add -f "wiki/stats/benchmark/${{ matrix.name }}/" | |
| git commit --amend -m "[skip ci] doc: new benchmark data for ${{ matrix.name }}" | |
| - name: Push changes | |
| uses: ad-m/github-push-action@master | |
| with: | |
| branch: demo # todo: change | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| force: true | |
| full-test: | |
| needs: [ lint ] | |
| name: "Run Full Test Suite (only on PR to main)" | |
| if: ${{ always() && !failure() && (inputs.force-full || (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main')) }} | |
| uses: Code-Inspect/flowr/.github/workflows/full-test.yaml@main | |
| deploy-doc: | |
| needs: [ test, full-test, performance-test-upload ] | |
| name: "Deploy Documentation (only on main)" | |
| if: ${{ always() && !failure() && (inputs.force-full || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }} | |
| uses: Code-Inspect/flowr/.github/workflows/deploy-doc.yaml@main |