diff --git a/.github/workflows/ProverBenchFromHalo2.yml b/.github/workflows/ProverBenchFromHalo2.yml index 548c323f1f..4f5d2691de 100644 --- a/.github/workflows/ProverBenchFromHalo2.yml +++ b/.github/workflows/ProverBenchFromHalo2.yml @@ -1,2 +1,45 @@ -# TODO: We plan to bring this Github action back when merge to main -# name: Prover Bench on halo2 PR +name: Prover Bench on halo2 PR +on: + workflow_dispatch: + inputs: + event-type: + required: true + type: string + ghuser: + required: true + type: string + revision: + required: true + type: string + halo2pr: + required: true + type: string +jobs: + Exec-ProverBench-on-halo2-PR: + runs-on: pse-runner + env: + GH_USER: ${{ github.event.inputs.ghuser }} + HALO2PR: ${{ github.event.inputs.halo2pr }} + steps: + - run: echo "halo2 PR ${{ env.HALO2PR }} submitted by ${{ env.GH_USER }}" + - run: echo "Running prover benchmarking due to ${{ github.event.inputs.event-type }}." + - uses: actions/checkout@v2 + with: + ref: 'halo2benchmarking' + - run: .github/proverCiScripts/wakeUpProver.sh + shell: bash + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/rsSysstat.sh + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/prepareProver.sh "${{ github.event.inputs.halo2pr }}[halo2]" "${{ github.workspace }}" + - run: .github/proverCiScripts/deployToProver.sh "${{ github.event.inputs.halo2pr }}[halo2]" "${{ github.workspace }}" + shell: bash + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/injectHalo2Revision.sh "${{ github.event.inputs.halo2pr }}[halo2]" "${{ github.event.inputs.revision }}" + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/execBench.sh "${{ github.event.inputs.halo2pr }}[halo2]" 19 + shell: bash + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/getSysstat.sh "${{ github.event.inputs.halo2pr }}[halo2]" + - run: .github/proverCiScripts/shutdownProver.sh + shell: bash diff --git a/.github/workflows/gh-actions-prover-benches.yml b/.github/workflows/gh-actions-prover-benches.yml index 0130e3b9cb..7e0199ac33 100644 --- a/.github/workflows/gh-actions-prover-benches.yml +++ b/.github/workflows/gh-actions-prover-benches.yml @@ -1,2 +1,36 @@ -# TODO: We plan to bring the following Github action back when merge to main -# name: GH Actions Prover Benches on PR +name: GH Actions Prover Benches on PR +on: + pull_request: + types: [ labeled , ready_for_review] +jobs: + Prover_benchmarking_Automation: + if: contains(github.event.label.name, 'benchmarks') + runs-on: pse-runner + env: + PR_NUMBER: ${{ github.event.number }} + + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - run: .github/proverCiScripts/wakeUpProver.sh + shell: bash + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/rsSysstat.sh "${{ env.PR_NUMBER }}" + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/prepareProver.sh "${{ env.PR_NUMBER }}" "${{ github.workspace }}" + - run: .github/proverCiScripts/deployToProver.sh "${{ env.PR_NUMBER }}" "${{ github.workspace }}" + shell: bash + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/execBench.sh "${{ env.PR_NUMBER }}" 19 '"${{ github.event.label.name }}"' + shell: bash + - run: | + ssh prover "bash -s" -- < .github/proverCiScripts/getSysstat.sh "${{ env.PR_NUMBER }}" + - name: Calculate Benchmark Result + if: success() || failure() + run: .github/proverCiScripts/processResults.sh "${{ env.PR_NUMBER }}" '"${{ github.event.label.name }}"' "19" + shell: bash + - name: PowerOff prover + if: always() + run: .github/proverCiScripts/shutdownProver.sh + shell: bash + diff --git a/.github/workflows/github-ops.yml b/.github/workflows/github-ops.yml index 3949a39f87..b55df33546 100644 --- a/.github/workflows/github-ops.yml +++ b/.github/workflows/github-ops.yml @@ -1,2 +1,27 @@ -# TODO: We plan to bring the following Github action back when merge to main -# name: github-ops +name: github-ops + +on: + workflow_dispatch: + workflow_run: + workflows: ["*"] + types: + - requested + - completed + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + ping: + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: Ping runner workflow + env: + # only needs github actions (write) `workflow_dispatch` + GH_OPS_PAT: ${{ secrets.PAT_OPS_TRIGGER }} + # change `zkevm-chain.yml` to the desired workflow + run: | + curl -H "authorization: token $GH_OPS_PAT" 'https://github.com/gitapi/repos/privacy-scaling-explorations/github-ops/actions/workflows/zkevm-circuits.yml/dispatches' -d '{"ref": "master"}' + diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index af2a4159c4..e2183badad 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,2 +1,109 @@ -# TODO: We plan to bring the following Github action back when merge to main -# name: Integration Tests +name: Integration Tests + +on: + merge_group: + schedule: + - cron: '50 1 * * SUN' + pull_request: + types: [synchronize, labeled] + push: + branches: + - main + workflow_dispatch: + inputs: + provertype: + description: 'invoke real vs mock prover (with or without root)' + required: true + default: 'sub_mock_prover' + type: choice + options: + - sub_real_prover + - sub_mock_prover + - root_mock_prover + - root_real_prover + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + set-outputs: + if: github.event.pull_request.draft == false + + runs-on: ubuntu-latest + outputs: + instancetype: ${{ steps.set-outputs.outputs.instancetype }} + provertype: ${{ steps.set-outputs.outputs.provertype }} + steps: + - id: set-outputs + name: Select instance and prover type + run: | + if [ "${{ github.event.inputs.provertype }}" = "root_real_prover" ] || [ "${{ github.event_name }}" = "schedule" ]; then + echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT" + echo "provertype=root_real_prover" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event.inputs.provertype }}" = "root_mock_prover" ]; then + echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT" + echo "provertype=root_mock_prover" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event.inputs.provertype }}" = "sub_real_prover" ]; then + echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT" + echo "provertype=sub_real_prover" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event.inputs.provertype }}" = "sub_mock_prover" ] || [ -z ${{ github.event.inputs.provertype }} ]; then + echo "instancetype=c5.9xlarge" >> "$GITHUB_OUTPUT" + echo "provertype=sub_mock_prover" >> "$GITHUB_OUTPUT" + else + exit 1 + fi + + integration-tests: + if: github.event.pull_request.draft == false + + name: Integration Tests + runs-on: ["${{github.run_id}}", self-hosted, "${{needs.set-outputs.outputs.instancetype}}"] + needs: set-outputs + + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + override: false + - name: Setup golang + uses: actions/setup-go@v3 + with: + go-version: ~1.19 + # Go cache for building geth-utils + - name: Go cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Cargo cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + # Run an initial build in a separate step to split the build time from execution time + - name: Build bins + run: cargo build --bin gen_blockchain_data + - name: Build tests + run: for testname in rpc circuit_input_builder circuits; do cargo test --profile release --test $testname --features $testname --no-run; done + - run: ./run.sh --steps "setup" + working-directory: integration-tests + - run: ./run.sh --steps "gendata" + working-directory: integration-tests + - run: ./run.sh --steps "tests" --tests "rpc" + working-directory: integration-tests + - run: ./run.sh --steps "tests" --tests "circuit_input_builder" + working-directory: integration-tests + - run: RUST_TEST_THREADS=1 ./run.sh --steps "tests" --tests "circuits::${{ needs.set-outputs.outputs.provertype }}" + working-directory: integration-tests + - run: ./run.sh --steps "cleanup" + working-directory: integration-tests diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000000..27719aa160 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,11 @@ +name: "Pull Request Labeler" +on: + - pull_request_target + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v3 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/test-features.yml b/.github/workflows/test-features.yml index 1a538c6233..eb43f61c97 100644 --- a/.github/workflows/test-features.yml +++ b/.github/workflows/test-features.yml @@ -1,2 +1,82 @@ -# TODO: We plan to bring this Github action back when merge to main -# name: Test feature flags +name: Test feature flags + +on: + merge_group: + pull_request: + types: [synchronize, opened, reopened, ready_for_review] + push: + branches: + - main + +jobs: + skip_check: + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@v5 + with: + cancel_others: 'true' + concurrent_skipping: 'same_content_newer' + paths_ignore: '["**/README.md"]' + + test_features: + needs: [skip_check] + if: | + github.event.pull_request.draft == false && + (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true') + timeout-minutes: 30 + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + crate: + - zkevm-circuits + feature: + - default + - test + - test-circuits + - warn-unimplemented + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + override: false + + # Go cache for building geth-utils + - name: Go cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ github.workflow }}-${{ matrix.crate }}-${{ matrix.feature }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + + - name: Cargo cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ github.workflow }}-${{ matrix.crate }}-${{ matrix.feature }}-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: '${{ matrix.crate }}: ${{ matrix.feature }}' + run: | + GIT_ROOT=$(pwd) + cd /tmp + cargo new foobar + cd foobar + cp "${GIT_ROOT}/rust-toolchain" . || true + cargo add --path "${GIT_ROOT}/${{ matrix.crate }}" --features '${{ matrix.feature }}' + cd ../ + rm -rf foobar + + test_features_complete: + needs: [test_features] + runs-on: ubuntu-latest + steps: + - run: echo dummy