retain all ci

privacy-scaling-explorations · Jun 30, 2023 · bde3eb2 · bde3eb2
1 parent 4f4b32a
commit bde3eb2
Show file tree

Hide file tree

Showing 6 changed files with 310 additions and 10 deletions.
diff --git a/.github/workflows/ProverBenchFromHalo2.yml b/.github/workflows/ProverBenchFromHalo2.yml
@@ -1,2 +1,45 @@
-# TODO: We plan to bring this Github action back when merge to main
-# name: Prover Bench on halo2 PR
+name: Prover Bench on halo2 PR
+on:
+  workflow_dispatch:
+    inputs:
+      event-type:
+        required: true
+        type: string
+      ghuser:
+        required: true
+        type: string
+      revision:
+        required: true
+        type: string
+      halo2pr:
+        required: true
+        type: string
+jobs:
+  Exec-ProverBench-on-halo2-PR:
+    runs-on: pse-runner
+    env:
+      GH_USER: ${{ github.event.inputs.ghuser }}
+      HALO2PR: ${{ github.event.inputs.halo2pr }}
+    steps:
+      - run: echo "halo2 PR ${{ env.HALO2PR }} submitted by ${{ env.GH_USER }}"
+      - run: echo "Running prover benchmarking due to ${{ github.event.inputs.event-type }}."        
+      - uses: actions/checkout@v2
+        with:
+          ref: 'halo2benchmarking'
+      - run: .github/proverCiScripts/wakeUpProver.sh
+        shell: bash
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/rsSysstat.sh
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/prepareProver.sh "${{ github.event.inputs.halo2pr }}[halo2]" "${{ github.workspace }}"
+      - run: .github/proverCiScripts/deployToProver.sh "${{ github.event.inputs.halo2pr }}[halo2]" "${{ github.workspace }}"
+        shell: bash
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/injectHalo2Revision.sh "${{ github.event.inputs.halo2pr }}[halo2]" "${{ github.event.inputs.revision }}"
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/execBench.sh "${{ github.event.inputs.halo2pr }}[halo2]" 19
+        shell: bash
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/getSysstat.sh "${{ github.event.inputs.halo2pr }}[halo2]"
+      - run: .github/proverCiScripts/shutdownProver.sh
+        shell: bash
diff --git a/.github/workflows/gh-actions-prover-benches.yml b/.github/workflows/gh-actions-prover-benches.yml
@@ -1,2 +1,36 @@
-# TODO: We plan to bring the following Github action back when merge to main
-# name: GH Actions Prover Benches on PR
+name: GH Actions Prover Benches on PR
+on:
+  pull_request:
+    types: [ labeled ,  ready_for_review]
+jobs:
+  Prover_benchmarking_Automation:
+    if: contains(github.event.label.name, 'benchmarks')
+    runs-on: pse-runner
+    env:
+      PR_NUMBER: ${{ github.event.number }}
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - run: .github/proverCiScripts/wakeUpProver.sh
+        shell: bash
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/rsSysstat.sh "${{ env.PR_NUMBER }}"
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/prepareProver.sh "${{ env.PR_NUMBER }}" "${{ github.workspace }}"
+      - run: .github/proverCiScripts/deployToProver.sh "${{ env.PR_NUMBER }}" "${{ github.workspace }}"
+        shell: bash
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/execBench.sh "${{ env.PR_NUMBER }}" 19 '"${{ github.event.label.name }}"'
+        shell: bash
+      - run: |
+          ssh prover "bash -s" -- < .github/proverCiScripts/getSysstat.sh "${{ env.PR_NUMBER }}"
+      - name: Calculate Benchmark Result
+        if: success() || failure()
+        run: .github/proverCiScripts/processResults.sh "${{ env.PR_NUMBER }}" '"${{ github.event.label.name }}"' "19"
+        shell: bash
+      - name: PowerOff prover
+        if: always()
+        run: .github/proverCiScripts/shutdownProver.sh
+        shell: bash
+
diff --git a/.github/workflows/github-ops.yml b/.github/workflows/github-ops.yml
@@ -1,2 +1,27 @@
-# TODO: We plan to bring the following Github action back when merge to main
-# name: github-ops
+name: github-ops
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["*"]
+    types:
+      - requested
+      - completed
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  ping:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+    steps:
+      - name: Ping runner workflow
+        env:
+          # only needs github actions (write) `workflow_dispatch`
+          GH_OPS_PAT: ${{ secrets.PAT_OPS_TRIGGER }}
+          # change `zkevm-chain.yml` to the desired workflow
+        run: |
+          curl -H "authorization: token $GH_OPS_PAT" 'https://github.com/gitapi/repos/privacy-scaling-explorations/github-ops/actions/workflows/zkevm-circuits.yml/dispatches' -d '{"ref": "master"}'
+
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -1,2 +1,109 @@
-# TODO: We plan to bring the following Github action back when merge to main
-# name: Integration Tests
+name: Integration Tests
+
+on:
+  merge_group:
+  schedule:
+    - cron: '50 1 * * SUN'
+  pull_request:
+    types: [synchronize, labeled]
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      provertype:
+        description: 'invoke real vs mock prover (with or without root)'
+        required: true
+        default: 'sub_mock_prover'
+        type: choice
+        options:
+        - sub_real_prover
+        - sub_mock_prover
+        - root_mock_prover
+        - root_real_prover
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  set-outputs:
+    if: github.event.pull_request.draft == false
+
+    runs-on: ubuntu-latest
+    outputs:
+      instancetype: ${{ steps.set-outputs.outputs.instancetype }}
+      provertype: ${{ steps.set-outputs.outputs.provertype }}
+    steps:
+      - id: set-outputs
+        name: Select instance and prover type
+        run: |
+          if [ "${{ github.event.inputs.provertype }}" = "root_real_prover" ] || [ "${{ github.event_name }}" = "schedule" ]; then
+            echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT"
+            echo "provertype=root_real_prover" >> "$GITHUB_OUTPUT"
+          elif [ "${{ github.event.inputs.provertype }}" = "root_mock_prover" ]; then
+            echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT"
+            echo "provertype=root_mock_prover" >> "$GITHUB_OUTPUT"
+          elif [ "${{ github.event.inputs.provertype }}" = "sub_real_prover" ]; then
+            echo "instancetype=r6i.32xlarge" >> "$GITHUB_OUTPUT"
+            echo "provertype=sub_real_prover" >> "$GITHUB_OUTPUT"
+          elif [ "${{ github.event.inputs.provertype }}" = "sub_mock_prover" ] || [ -z ${{ github.event.inputs.provertype }} ]; then
+            echo "instancetype=c5.9xlarge" >> "$GITHUB_OUTPUT"
+            echo "provertype=sub_mock_prover" >> "$GITHUB_OUTPUT"
+          else
+            exit 1
+          fi
+
+  integration-tests:
+    if: github.event.pull_request.draft == false
+
+    name: Integration Tests
+    runs-on: ["${{github.run_id}}", self-hosted, "${{needs.set-outputs.outputs.instancetype}}"]
+    needs: set-outputs
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: false
+      - name: Setup golang
+        uses: actions/setup-go@v3
+        with:
+          go-version: ~1.19
+      # Go cache for building geth-utils
+      - name: Go cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Cargo cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      # Run an initial build in a separate step to split the build time from execution time
+      - name: Build bins
+        run: cargo build --bin gen_blockchain_data
+      - name: Build tests
+        run: for testname in rpc circuit_input_builder circuits; do cargo test --profile release --test $testname --features $testname --no-run; done
+      - run: ./run.sh --steps "setup"
+        working-directory: integration-tests
+      - run: ./run.sh --steps "gendata"
+        working-directory: integration-tests
+      - run: ./run.sh --steps "tests" --tests "rpc"
+        working-directory: integration-tests
+      - run: ./run.sh --steps "tests" --tests "circuit_input_builder"
+        working-directory: integration-tests
+      - run: RUST_TEST_THREADS=1 ./run.sh --steps "tests" --tests "circuits::${{ needs.set-outputs.outputs.provertype }}"
+        working-directory: integration-tests
+      - run: ./run.sh --steps "cleanup"
+        working-directory: integration-tests
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
@@ -0,0 +1,11 @@
+name: "Pull Request Labeler"
+on:
+  - pull_request_target
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/labeler@v3
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/.github/workflows/test-features.yml b/.github/workflows/test-features.yml
@@ -1,2 +1,82 @@
-# TODO: We plan to bring this Github action back when merge to main
-# name: Test feature flags
+name: Test feature flags
+
+on:
+  merge_group:
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+  push:
+    branches:
+      - main
+
+jobs:
+  skip_check:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@v5
+        with:
+          cancel_others: 'true'
+          concurrent_skipping: 'same_content_newer'
+          paths_ignore: '["**/README.md"]'
+
+  test_features:
+    needs: [skip_check]
+    if: |
+      github.event.pull_request.draft == false &&
+      (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        crate:
+          - zkevm-circuits
+        feature:
+          - default
+          - test
+          - test-circuits
+          - warn-unimplemented
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: false
+
+      # Go cache for building geth-utils
+      - name: Go cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ github.workflow }}-${{ matrix.crate }}-${{ matrix.feature }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+
+      - name: Cargo cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ github.workflow }}-${{ matrix.crate }}-${{ matrix.feature }}-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: '${{ matrix.crate }}: ${{ matrix.feature }}'
+        run: |
+          GIT_ROOT=$(pwd)
+          cd /tmp
+          cargo new foobar
+          cd foobar
+          cp "${GIT_ROOT}/rust-toolchain" . || true
+          cargo add --path "${GIT_ROOT}/${{ matrix.crate }}" --features '${{ matrix.feature }}'
+          cd ../
+          rm -rf foobar
+
+  test_features_complete:
+    needs: [test_features]
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo dummy