Refactor

CMakeLists.txt

@@ -662,26 +662,30 @@ endif()
 # dependencies to them.
 #-------------------------------------------------------------------------------
 
-if(IREE_BUILD_LEGACY_BENCHMARKS OR IREE_BUILD_E2E_TEST_ARTIFACTS)
+# TODO(#11263): Remove together with IREE_BUILD_LEGACY_BENCHMARKS.
+if(IREE_BUILD_LEGACY_BENCHMARKS)
   # Add top-level custom targets to drive generating benchmark suites.
 
   # iree-benchmark-import-models imports benchmark models from their source
   # formats, such as .tflite flatbuffers, to IREE-compatible .mlir files.
   add_custom_target(iree-benchmark-import-models)
 
-  # iree-benchmark-suites* fully prepares benchmark models for benchmarking:
+  # iree-benchmark-suites fully prepares benchmark models for benchmarking:
   #   * importing from source formats to IREE-compatible .mlir files
   #   * compiling from .mlir files to benchmark-ready .vmfb files
   #   * generating flagfiles for executing the benchmark .vmfb files
-  # Build defualt benchmark suites.
   add_custom_target(iree-benchmark-suites)
-  # Build long-running benchmark suites with larger models.
-  add_custom_target(iree-benchmark-suites-long)
 endif()
 
-# TODO(#11263): This conditional block should be merged with the block above
-# once we remove IREE_BUILD_LEGACY_BENCHMARKS.
 if(IREE_BUILD_E2E_TEST_ARTIFACTS)
+  # iree-benchmark-suites* fully prepares benchmark models for benchmarking:
+  #   * importing from source formats to IREE-compatible .mlir files
+  #   * compiling from .mlir files to benchmark-ready .vmfb files
+  # Build defualt benchmark suites.
+  add_custom_target(iree-benchmark-suites)
+  # Build long-running benchmark suites.
+  add_custom_target(iree-benchmark-suites-long)
+
   # iree-e2e-compile-stats-suites* compiles the benchmark models with specific
   # flags to collect the compilation statistics.
   # Build for defualt benchmark suites.

build_tools/benchmarks/export_benchmark_config.py

@@ -39,7 +39,7 @@
 import json
 import textwrap
 
-from benchmark_suites.iree import benchmark_collections
+from benchmark_suites.iree import benchmark_tags, benchmark_collections
 from e2e_test_artifacts import iree_artifacts
 from e2e_test_framework import serialization
 from e2e_test_framework.definitions import common_definitions, iree_definitions
@@ -50,10 +50,12 @@
     "x86_64":
         lambda config: config.target_device_spec.architecture.architecture ==
         "x86_64",
-    "cuda": (lambda config: "cuda" in config.tags and "long-running" not in
-             config.tags),
+    "cuda":
+        lambda config: (benchmark_tags.CUDA_TAG in config.tags and
+                        benchmark_tags.LONG_RUNNING_TAG not in config.tags),
     "cuda-long":
-        lambda config: "cuda" in config.tags and "long-running" in config.tags,
+        lambda config: (benchmark_tags.CUDA_TAG in config.tags and
+                        benchmark_tags.LONG_RUNNING_TAG in config.tags),
     "vulkan-nvidia":
         lambda config: "vulkan-nvidia" in config.tags,
     "android-cpu":
@@ -148,7 +150,7 @@ def _export_compilation_handler(_args: argparse.Namespace):
   all_gen_configs, _ = benchmark_collections.generate_benchmarks()
   compile_stats_gen_configs = [
       config for config in all_gen_configs
-      if benchmark_collections.COMPILE_STATS_TAG in config.compile_config.tags
+      if benchmark_tags.COMPILE_STATS_TAG in config.compile_config.tags
   ]
 
   distinct_module_dir_paths = _get_distinct_module_dir_paths(

build_tools/benchmarks/run_benchmarks_on_android.py

@@ -286,7 +286,6 @@ def __run_capture(self, android_case_dir: pathlib.PurePosixPath,
     stdout_redirect = None if self.verbose else subprocess.DEVNULL
     execute_cmd(capture_cmd, verbose=self.verbose, stdout=stdout_redirect)
 
-
   # TODO(#13187): These logics are inherited from the legacy benchmark suites,
   # which only work for a few specific phones. We should define the topology
   # in their device specs.

build_tools/python/benchmark_suites/iree/benchmark_collections.py

@@ -9,15 +9,13 @@
 
 from e2e_test_artifacts import iree_artifacts
 from e2e_test_framework.definitions import iree_definitions
-from benchmark_suites.iree import (riscv_benchmarks, x86_64_benchmarks,
-                                   adreno_benchmarks, armv8_a_benchmarks,
-                                   cuda_benchmarks, mali_benchmarks,
-                                   vulkan_nvidia_benchmarks, vmvx_benchmarks)
+from benchmark_suites.iree import (benchmark_tags, riscv_benchmarks,
+                                   x86_64_benchmarks, adreno_benchmarks,
+                                   armv8_a_benchmarks, cuda_benchmarks,
+                                   mali_benchmarks, vulkan_nvidia_benchmarks,
+                                   vmvx_benchmarks)
 
 COMPILE_STATS_ID_SUFFIX = "-compile-stats"
-# Tag that indicates this compile config is generated for collecting compilation
-# statistics.
-COMPILE_STATS_TAG = "compile-stats"
 
 
 def generate_benchmarks(
@@ -49,7 +47,7 @@ def generate_benchmarks(
     scheduling_stats_path = f"{iree_definitions.MODULE_DIR_VARIABLE}/{iree_artifacts.SCHEDULING_STATS_FILENAME}"
     compile_stats_config = iree_definitions.CompileConfig.build(
         id=compile_config.id + COMPILE_STATS_ID_SUFFIX,
-        tags=compile_config.tags + [COMPILE_STATS_TAG],
+        tags=compile_config.tags + [benchmark_tags.COMPILE_STATS_TAG],
         compile_targets=compile_config.compile_targets,
         extra_flags=compile_config.extra_flags + [
             # Enable zip polyglot to provide component sizes.

build_tools/python/benchmark_suites/iree/cuda_benchmarks.py

@@ -6,7 +6,7 @@
 """Defines IREE CUDA benchmarks."""
 
 from typing import List, Tuple, Sequence
-from benchmark_suites.iree import module_execution_configs
+from benchmark_suites.iree import benchmark_tags, module_execution_configs
 from e2e_test_framework import unique_ids
 from e2e_test_framework.definitions import common_definitions, iree_definitions
 from e2e_test_framework.device_specs import device_collections
@@ -71,24 +71,25 @@ def generate(
   ) -> Tuple[List[iree_definitions.ModuleGenerationConfig],
              List[iree_definitions.E2EModelRunConfig]]:
     """Generates IREE compile and run configs."""
-    # The `cuda` tag is required to put them into the CUDA benchmark preset.
-    gen_configs, run_configs = self._generate_configs(model_groups.CUDA_MODELS,
-                                                      self.SM_80_COMPILE_CONFIG,
-                                                      tags=["cuda"])
+    # The CUDA tag is required to put them into the CUDA benchmark preset.
+    gen_configs, run_configs = self._generate_configs(
+        model_groups.CUDA_MODELS,
+        self.SM_80_COMPILE_CONFIG,
+        tags=[benchmark_tags.CUDA_TAG])
     ubench_gen_configs, ubench_run_configs = self._generate_configs(
         model_groups.MICRO_MATMUL,
         self.SM_80_UBENCH_MATMUL_COMPILE_CONFIG,
         execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG,
-        tags=["cuda"])
+        tags=[benchmark_tags.CUDA_TAG])
     ubench_splitk_gen_configs, ubench_splitk_run_configs = self._generate_configs(
         model_groups.MICRO_MATMUL_SPLITK,
         self.SM_80_UBENCH_MATMUL_SPLITK_COMPILE_CONFIG,
         execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG,
-        tags=["cuda"])
+        tags=[benchmark_tags.CUDA_TAG])
     long_running_gen_configs, long_running_module_configs = self._generate_configs(
         model_groups.CUDA_MODELS_LONG,
         self.SM_80_COMPILE_CONFIG,
-        tags=["cuda", "long-running"])
+        tags=[benchmark_tags.CUDA_TAG, benchmark_tags.LONG_RUNNING_TAG])
     return (gen_configs + ubench_gen_configs + ubench_splitk_gen_configs +
             long_running_gen_configs, run_configs + ubench_run_configs +
             ubench_splitk_run_configs + long_running_module_configs)

build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py

@@ -10,7 +10,7 @@
 from typing import Dict, List, Sequence
 import pathlib
 
-from benchmark_suites.iree import benchmark_collections
+from benchmark_suites.iree import benchmark_tags
 from e2e_test_artifacts import iree_artifacts
 from e2e_test_artifacts.cmake_generator import model_rule_generator
 from e2e_test_framework.definitions import iree_definitions
@@ -21,7 +21,7 @@
 BENCHMARK_SUITES_CMAKE_TARGET = "iree-benchmark-suites"
 # Compilation statistics suites for default benchmarks.
 E2E_COMPILE_STATS_SUITES = "iree-e2e-compile-stats-suites"
-# Long-running benchmark suites with larger models.
+# Long-running benchmark suites.
 LONG_BENCHMARK_SUITES_CMAKE_TARGET = "iree-benchmark-suites-long"
 # Compilation statistics suites for long-running benchmarks.
 LONG_E2E_COMPILE_STATS_SUITES = "iree-e2e-compile-stats-suites-long"
@@ -177,9 +177,9 @@ def generate_rules(
         module_generation_config=gen_config,
         output_file_path=module_dir_path / iree_artifacts.MODULE_FILENAME)
 
-    has_compile_stats_tag = (benchmark_collections.COMPILE_STATS_TAG
+    has_compile_stats_tag = (benchmark_tags.COMPILE_STATS_TAG
                              in gen_config.compile_config.tags)
-    if "long-running" in gen_config.tags:
+    if benchmark_tags.LONG_RUNNING_TAG in gen_config.tags:
       suite_target = (LONG_E2E_COMPILE_STATS_SUITES if has_compile_stats_tag
                       else LONG_BENCHMARK_SUITES_CMAKE_TARGET)
     else: