Move benchmark config generation to build_e2e_test_artifacts (#13291)

When working on #13273, I noticed that benchmark configs and modules are uploaded in 3 different jobs. These artifacts can be uploaded to different GCS dirs when some of them are re-run due to failures (because re-run jobs will create [GCS_DIR with new run attempts](https://github.com/openxla/iree/blob/main/.github/workflows/benchmark_compilation.yml#L44-L50)). As a result, users might not be able to download all benchmark artifacts from a single GCS dir URL, which can be confusing. This PR changes the workflow to generate all benchmark modules and configs in `build_e2e_test_artifacts` to avoid such issue. All files are uploaded to `${GCS_URL}/e2e-test-artifacts`, a single path to download all benchmark artifacts. Besides the reason above, I think during the CMake build we should generate the benchmark configs under `${IREE_BUILD_DIR}/e2e_test_artifacts` instead of calling `export_benchmark_config.py` separately. There are some questions about how to pass the benchmark presets/filters through CMake configuration, so I decided to defer that as the next step.
iree-org · May 1, 2023 · 251e59b · 251e59b
1 parent eca1e07
commit 251e59b
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 66 deletions.
diff --git a/.github/workflows/benchmark_compilation.yml b/.github/workflows/benchmark_compilation.yml
@@ -68,54 +68,43 @@ jobs:
     steps:
       - name: "Checking out repository"
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: "Exporting configs"
-        id: "export"
-        env:
-          COMPILATION_CONFIG: "compilation-config.json"
-        run: |
-          ./build_tools/benchmarks/export_benchmark_config.py \
-            compilation \
-            --output="${COMPILATION_CONFIG}"
-          echo "compilation-config=${COMPILATION_CONFIG}" >> "${GITHUB_OUTPUT}"
       - name: "Downloading assets"
         id: "download-assets"
         env:
-          COMPILATION_CONFIG: ${{ steps.export.outputs.compilation-config }}
+          BENCHMARK_CONFIG: ${{ env.E2E_TEST_ARTIFACTS_DIR }}/compilation-benchmark-config.json
+          BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ env.E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR }}/compilation-benchmark-config.json
         run: |
+          mkdir -p "${E2E_TEST_ARTIFACTS_DIR}"
+          gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}"
           gcloud storage cp \
             "${E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT}" \
             "${E2E_TEST_ARTIFACTS_BUILD_LOG}"
-          mkdir -p "${E2E_TEST_ARTIFACTS_DIR}"
           jq -r \
             --arg GCS_ARTIFACT_DIR "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}" \
             '.module_dir_paths | map("\($GCS_ARTIFACT_DIR)/\(.)") | join("\n")' \
-            "${COMPILATION_CONFIG}" | \
+            "${BENCHMARK_CONFIG}" | \
             gcloud storage cp -r --read-paths-from-stdin \
               "${E2E_TEST_ARTIFACTS_DIR}"
+          echo "benchmark-config=${BENCHMARK_CONFIG}" >> "${GITHUB_OUTPUT}"
       - name: "Collecting compilation statistics"
         id: collect
         env:
-          COMPILATION_CONFIG: ${{ steps.export.outputs.compilation-config }}
-          GENERATION_CONFIG: generation-config.json
+          BENCHMARK_CONFIG: ${{ steps.download-assets.outputs.benchmark-config }}
           COMPILE_STATS_RESULTS: benchmark-results/compile-stats-results.json
         run: |
           mkdir -p benchmark-results
           ./build_tools/benchmarks/collect_compilation_statistics.py alpha \
             --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR}" \
             --build_log="${E2E_TEST_ARTIFACTS_BUILD_LOG}" \
-            --compilation_benchmark_config="${COMPILATION_CONFIG}" \
+            --compilation_benchmark_config="${BENCHMARK_CONFIG}" \
             --output="${COMPILE_STATS_RESULTS}"
           echo "compile-stats-results=${COMPILE_STATS_RESULTS}" >> "${GITHUB_OUTPUT}"
       - name: "Uploading benchmark results"
         id: upload
         env:
-          COMPILATION_CONFIG: ${{ steps.export.outputs.compilation-config }}
           COMPILE_STATS_RESULTS: ${{ steps.collect.outputs.compile-stats-results }}
           COMPILE_STATS_RESULTS_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.collect.outputs.compile-stats-results }}
         run: |
-          # Upload files with two commands since they go into different GCS
-          # directories.
-          gcloud storage cp "${COMPILATION_CONFIG}" "${GCS_DIR}"
           gcloud storage cp \
             "${COMPILE_STATS_RESULTS}" \
             "${COMPILE_STATS_RESULTS_GCS_ARTIFACT}"

diff --git a/.github/workflows/benchmark_execution.yml b/.github/workflows/benchmark_execution.yml
@@ -28,9 +28,6 @@ on:
       benchmark-tools-gcs-artifact-dir:
         required: true
         type: string
-      benchmark-presets:
-        required: true
-        type: string
     outputs:
       benchmark-results-dir:
         description: |
@@ -50,61 +47,43 @@ env:
   # last ran in. It therefore can't be passed in via inputs because the env
   # context isn't available there.
   GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}
+  BENCHMARK_CONFIG: ${{ inputs.e2e-test-artifacts-dir }}/execution-benchmark-config.json
+  BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ inputs.e2e-test-artifacts-gcs-artifact-dir }}/execution-benchmark-config.json
 
 jobs:
-  export_benchmark_config:
+  generate_matrix:
     runs-on:
       - self-hosted  # must come first
       - runner-group=${{ inputs.runner-group }}
       - environment=${{ inputs.runner-env }}
       - cpu
       - os-family=Linux
     outputs:
-      benchmark-matrix: ${{ steps.export.outputs.benchmark-matrix }}
-      benchmark-config: ${{ steps.export.outputs.benchmark-config }}
-      benchmark-config-gcs-artifact: ${{ steps.upload.outputs.benchmark-config-gcs-artifact }}
+      benchmark-matrix: ${{ steps.generate.outputs.benchmark-matrix }}
     steps:
       - name: "Checking out repository"
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
       - name: "Checking out runtime submodules"
         run: ./build_tools/scripts/git/update_runtime_submodules.sh
-      - name: "Exporting benchmark run config"
-        id: export
-        env:
-          BENCHMARK_CONFIG: "benchmark-config.json"
-          BENCHMARK_PRESETS: ${{ inputs.benchmark-presets }}
+      - name: "Generating benchmark matrix"
+        id: generate
         run: |
-          ./build_tools/benchmarks/export_benchmark_config.py \
-            execution \
-            --benchmark_presets="${BENCHMARK_PRESETS}" \
-            --output="${BENCHMARK_CONFIG}"
-          echo "benchmark-config=${BENCHMARK_CONFIG}" >> "${GITHUB_OUTPUT}"
+          gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}"
           echo benchmark-matrix=$(jq \
               'to_entries | map({"device_name": .key, "host_environment": .value.host_environment})' \
               "${BENCHMARK_CONFIG}") \
             >> "${GITHUB_OUTPUT}"
-      - name: "Uploading benchmark config"
-        id: upload
-        if: steps.export.outputs.benchmark-matrix != '[]'
-        env:
-          BENCHMARK_CONFIG: ${{ steps.export.outputs.benchmark-config }}
-          BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.export.outputs.benchmark-config }}
-        run: |
-          gcloud storage cp \
-            "${BENCHMARK_CONFIG}" \
-            "${BENCHMARK_CONFIG_GCS_ARTIFACT}"
-          echo "benchmark-config-gcs-artifact=${BENCHMARK_CONFIG_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
 
   run_benchmarks:
-    needs: [export_benchmark_config]
-    if: needs.export_benchmark_config.outputs.benchmark-matrix != '[]'
+    needs: [generate_matrix]
+    if: needs.generate_matrix.outputs.benchmark-matrix != '[]'
     strategy:
-      # Matrix is dynamically generated by the job export_benchmark_config. So
-      # we only runs the benchmarks specified in inputs.benchmark-presets.
+      # Matrix is dynamically generated by the job generate_matrix. So we only
+      # runs the benchmarks specified in ${BENCHMARK_CONFIG}.
       # All tasks in matrix are seen as a single job in Github CI and the job
       # can output a single set of values.
       matrix:
-        benchmark: ${{ fromJSON(needs.export_benchmark_config.outputs.benchmark-matrix) }}
+        benchmark: ${{ fromJSON(needs.generate_matrix.outputs.benchmark-matrix) }}
     runs-on:
       - self-hosted  # must come first
       - runner-group=${{ inputs.runner-group }}
@@ -138,9 +117,6 @@ jobs:
           echo "benchmark-tools-archive=${BENCHMARK_TOOLS_ARCHIVE}" >> "${GITHUB_OUTPUT}"
       - name: "Downloading benchmark assets"
         id: download-assets
-        env:
-          BENCHMARK_CONFIG: ${{ needs.export_benchmark_config.outputs.benchmark-config }}
-          BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ needs.export_benchmark_config.outputs.benchmark-config-gcs-artifact }}
         run: |
           gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}"
           mkdir -p "${E2E_TEST_ARTIFACTS_DIR}"

diff --git a/.github/workflows/benchmark_long.yml b/.github/workflows/benchmark_long.yml
@@ -55,6 +55,7 @@ jobs:
       build-dir: ${{ needs.build_all.outputs.build-dir }}
       build-dir-archive: ${{ needs.build_all.outputs.build-dir-archive }}
       build-dir-gcs-artifact: ${{ needs.build_all.outputs.build-dir-gcs-artifact }}
+      benchmark-presets: cuda-long
 
   compilation_benchmarks:
     needs: [setup, build_e2e_test_artifacts]
@@ -80,7 +81,6 @@ jobs:
       e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
       e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
       benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }}
-      benchmark-presets: cuda-long
 
   summary:
     # Even if you have an explicit if condition, you still need to override

diff --git a/.github/workflows/build_e2e_test_artifacts.yml b/.github/workflows/build_e2e_test_artifacts.yml
@@ -28,6 +28,9 @@ on:
       build-dir-gcs-artifact:
         required: true
         type: string
+      benchmark-presets:
+        required: true
+        type: string
     outputs:
       e2e-test-artifacts-dir:
         description: |
@@ -92,6 +95,21 @@ jobs:
             "${BUILD_E2E_TEST_ARTIFACTS_DIR}"
           echo "e2e-test-artifacts-dir=${BUILD_E2E_TEST_ARTIFACTS_DIR}/e2e_test_artifacts" >> "${GITHUB_OUTPUT}"
           echo "e2e-test-artifacts-build-log=${BUILD_E2E_TEST_ARTIFACTS_DIR}/.ninja_log" >> "${GITHUB_OUTPUT}"
+      - name: "Exporting benchmark configs"
+        id: export
+        if: inputs.benchmark-presets != ''
+        env:
+          COMPILATION_BENCHMARK_CONFIG: ${{ steps.build.outputs.e2e-test-artifacts-dir }}/compilation-benchmark-config.json
+          EXECUTION_BENCHMARK_CONFIG: ${{ steps.build.outputs.e2e-test-artifacts-dir }}/execution-benchmark-config.json
+          BENCHMARK_PRESETS: ${{ inputs.benchmark-presets }}
+        run: |
+          ./build_tools/benchmarks/export_benchmark_config.py \
+            compilation \
+            --output="${COMPILATION_BENCHMARK_CONFIG}"
+          ./build_tools/benchmarks/export_benchmark_config.py \
+            execution \
+            --benchmark_presets="${BENCHMARK_PRESETS}" \
+            --output="${EXECUTION_BENCHMARK_CONFIG}"
       - name: "Uploading e2e test artifacts"
         id: upload
         env:
@@ -105,7 +123,7 @@ jobs:
           # Not archiving the directory to allow fetching each file as needed
           # separately.
           find "${E2E_TEST_ARTIFACTS_DIR}" -maxdepth 1 \
-            -name "iree_*" -o -name "model_*.mlir" | \
+            -name "iree_*" -o -name "model_*.mlir" -o -name "*.json" | \
             gcloud storage cp --read-paths-from-stdin -r \
               "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}"
           gcloud storage cp "${E2E_TEST_ARTIFACTS_BUILD_LOG}" \

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -720,6 +720,7 @@ jobs:
       build-dir: ${{ needs.build_all.outputs.build-dir }}
       build-dir-archive: ${{ needs.build_all.outputs.build-dir-archive }}
       build-dir-gcs-artifact: ${{ needs.build_all.outputs.build-dir-gcs-artifact }}
+      benchmark-presets: ${{ needs.setup.outputs.benchmark-presets }}
 
   compilation_benchmarks:
     needs: [setup, build_e2e_test_artifacts]
@@ -745,7 +746,6 @@ jobs:
       e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
       e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
       benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }}
-      benchmark-presets: ${{ needs.setup.outputs.benchmark-presets }}
 
   process_benchmark_results:
     needs: [setup, compilation_benchmarks, execution_benchmarks]

diff --git a/docs/developers/developing_iree/benchmark_suites.md b/docs/developers/developing_iree/benchmark_suites.md
@@ -158,18 +158,14 @@ export E2E_TEST_ARTIFACTS_DIR="e2e_test_artifacts"
 # Download all artifacts
 mkdir "${E2E_TEST_ARTIFACTS_DIR?}"
 gcloud storage cp -r "${GCS_URL?}/e2e-test-artifacts" "${E2E_TEST_ARTIFACTS_DIR?}"
-
-# Download benchmark configs
-gcloud storage cp "${GCS_URL?}/benchmark-config.json" "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json"
-gcloud storage cp "${GCS_URL?}/compilation-config.json" "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json"
 ```
 
 Run the helper tool to dump benchmark commands from benchmark configs:
 
 ```sh
 build_tools/benchmarks/benchmark_helper.py dump-cmds \
-  --execution_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json" \
-  --compilation_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json" \
+  --execution_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/execution-benchmark-config.json" \
+  --compilation_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/compilation-benchmark-config.json" \
   --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR?}" \
   --benchmark_id="<benchmark_id>"
 ```
@@ -239,10 +235,14 @@ Execution and compilation benchmark configs can be downloaded at:
 
 ```sh
 # Execution benchmark config:
-gcloud storage cp "${GCS_URL?}/benchmark-config.json" "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json"
+gcloud storage cp \
+  "${GCS_URL?}/e2e-test-artifacts/execution-benchmark-config.json" \
+  "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json"
 
 # Compilation benchmark config:
-gcloud storage cp "${GCS_URL?}/compilation-config.json" "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json"
+gcloud storage cp \
+  "${GCS_URL?}/e2e-test-artifacts/compilation-benchmark-config.json" \
+  "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json"
 ```
 
 Benchmark raw results and traces can be downloaded at: