Split benchamrk suite build target into default and long-running

CMakeLists.txt

@@ -662,7 +662,8 @@ endif()
 # dependencies to them.
 #-------------------------------------------------------------------------------
 
-if(IREE_BUILD_LEGACY_BENCHMARKS OR IREE_BUILD_E2E_TEST_ARTIFACTS)
+# TODO(#11263): Remove together with IREE_BUILD_LEGACY_BENCHMARKS.
+if(IREE_BUILD_LEGACY_BENCHMARKS)
   # Add top-level custom targets to drive generating benchmark suites.
 
   # iree-benchmark-import-models imports benchmark models from their source
@@ -676,20 +677,35 @@ if(IREE_BUILD_LEGACY_BENCHMARKS OR IREE_BUILD_E2E_TEST_ARTIFACTS)
   add_custom_target(iree-benchmark-suites)
 endif()
 
-# TODO(#11263): This conditional block should be merged with the block above
-# once we remove IREE_BUILD_LEGACY_BENCHMARKS.
 if(IREE_BUILD_E2E_TEST_ARTIFACTS)
-  # iree-e2e-compile-stats-suites compiles the benchmark models with specific
+  # iree-benchmark-import-models imports benchmark models from their source
+  # formats, such as .tflite flatbuffers, to IREE-compatible .mlir files.
+  add_custom_target(iree-benchmark-import-models)
+
+  # iree-benchmark-suites* fully prepares benchmark models for benchmarking:
+  #   * importing from source formats to IREE-compatible .mlir files
+  #   * compiling from .mlir files to benchmark-ready .vmfb files
+  # Build default benchmark suites.
+  add_custom_target(iree-benchmark-suites)
+  # Build long-running benchmark suites.
+  add_custom_target(iree-benchmark-suites-long)
+
+  # iree-e2e-compile-stats-suites* compiles the benchmark models with specific
   # flags to collect the compilation statistics.
+  # Build for default benchmark suites.
   add_custom_target(iree-e2e-compile-stats-suites)
+  # Build for long-running benchmark suites.
+  add_custom_target(iree-e2e-compile-stats-suites-long)
 
   # iree-e2e-test-artifacts builds all e2e test artifacts, including benchmarks
   # and model tests.
   add_custom_target(iree-e2e-test-artifacts)
   add_dependencies(iree-e2e-test-artifacts
     iree-benchmark-import-models
     iree-benchmark-suites
+    iree-benchmark-suites-long
     iree-e2e-compile-stats-suites
+    iree-e2e-compile-stats-suites-long
   )
 endif()
 

build_tools/benchmarks/export_benchmark_config.py

@@ -39,7 +39,7 @@
 import json
 import textwrap
 
-from benchmark_suites.iree import benchmark_collections
+from benchmark_suites.iree import benchmark_collections, benchmark_tags
 from e2e_test_artifacts import iree_artifacts
 from e2e_test_framework import serialization
 from e2e_test_framework.definitions import common_definitions, iree_definitions
@@ -50,12 +50,14 @@
     "x86_64":
         lambda config: config.target_device_spec.architecture.architecture ==
         "x86_64",
-    "cuda": (lambda config: "cuda" in config.tags and "long-running" not in
-             config.tags),
+    "cuda":
+        lambda config: (benchmark_tags.CUDA in config.tags and benchmark_tags.
+                        LONG_RUNNING not in config.tags),
     "cuda-long":
-        lambda config: "cuda" in config.tags and "long-running" in config.tags,
+        lambda config: (benchmark_tags.CUDA in config.tags and benchmark_tags.
+                        LONG_RUNNING in config.tags),
     "vulkan-nvidia":
-        lambda config: "vulkan-nvidia" in config.tags,
+        lambda config: benchmark_tags.VULKAN_NVIDIA in config.tags,
     "android-cpu":
         lambda config:
         (config.target_device_spec.architecture.type == common_definitions.
@@ -148,7 +150,7 @@ def _export_compilation_handler(_args: argparse.Namespace):
   all_gen_configs, _ = benchmark_collections.generate_benchmarks()
   compile_stats_gen_configs = [
       config for config in all_gen_configs
-      if benchmark_collections.COMPILE_STATS_TAG in config.compile_config.tags
+      if benchmark_tags.COMPILE_STATS in config.compile_config.tags
   ]
 
   distinct_module_dir_paths = _get_distinct_module_dir_paths(

build_tools/python/benchmark_suites/iree/benchmark_collections.py

@@ -9,15 +9,13 @@
 
 from e2e_test_artifacts import iree_artifacts
 from e2e_test_framework.definitions import iree_definitions
-from benchmark_suites.iree import (riscv_benchmarks, x86_64_benchmarks,
-                                   adreno_benchmarks, armv8_a_benchmarks,
-                                   cuda_benchmarks, mali_benchmarks,
-                                   vulkan_nvidia_benchmarks, vmvx_benchmarks)
+from benchmark_suites.iree import (benchmark_tags, riscv_benchmarks,
+                                   x86_64_benchmarks, adreno_benchmarks,
+                                   armv8_a_benchmarks, cuda_benchmarks,
+                                   mali_benchmarks, vulkan_nvidia_benchmarks,
+                                   vmvx_benchmarks)
 
 COMPILE_STATS_ID_SUFFIX = "-compile-stats"
-# Tag that indicates this compile config is generated for collecting compilation
-# statistics.
-COMPILE_STATS_TAG = "compile-stats"
 
 
 def generate_benchmarks(
@@ -49,7 +47,7 @@ def generate_benchmarks(
     scheduling_stats_path = f"{iree_definitions.MODULE_DIR_VARIABLE}/{iree_artifacts.SCHEDULING_STATS_FILENAME}"
     compile_stats_config = iree_definitions.CompileConfig.build(
         id=compile_config.id + COMPILE_STATS_ID_SUFFIX,
-        tags=compile_config.tags + [COMPILE_STATS_TAG],
+        tags=compile_config.tags + [benchmark_tags.COMPILE_STATS],
         compile_targets=compile_config.compile_targets,
         extra_flags=compile_config.extra_flags + [
             # Enable zip polyglot to provide component sizes.
@@ -63,7 +61,8 @@ def generate_benchmarks(
     compile_stats_gen_configs.append(
         iree_definitions.ModuleGenerationConfig.build(
             imported_model=gen_config.imported_model,
-            compile_config=compile_stats_config))
+            compile_config=compile_stats_config,
+            tags=gen_config.tags))
   all_gen_configs += compile_stats_gen_configs
 
   return (all_gen_configs, all_run_configs)

build_tools/python/benchmark_suites/iree/benchmark_tags.py

@@ -0,0 +1,18 @@
+## Copyright 2023 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Tag that indicates this compile config is generated for collecting compilation
+# statistics.
+COMPILE_STATS = "compile-stats"
+
+# Tag for long-running benchmarks.
+LONG_RUNNING = "long-running"
+
+# Tag for CUDA benchamrks.
+CUDA = "cuda"
+
+# Tag for Vulkan NVIDIA benchamrks.
+VULKAN_NVIDIA = "vulkan-nvidia"

build_tools/python/benchmark_suites/iree/cuda_benchmarks.py

@@ -6,7 +6,7 @@
 """Defines IREE CUDA benchmarks."""
 
 from typing import List, Tuple, Sequence
-from benchmark_suites.iree import module_execution_configs
+from benchmark_suites.iree import benchmark_tags, module_execution_configs
 from e2e_test_framework import unique_ids
 from e2e_test_framework.definitions import common_definitions, iree_definitions
 from e2e_test_framework.device_specs import device_collections
@@ -46,14 +46,14 @@ def _generate_configs(
       compile_config: iree_definitions.CompileConfig,
       execution_config: iree_definitions.
       ModuleExecutionConfig = module_execution_configs.CUDA_CONFIG,
-      run_tags: Sequence[str] = [],
+      tags: Sequence[str] = (),
   ) -> Tuple[List[iree_definitions.ModuleGenerationConfig],
              List[iree_definitions.E2EModelRunConfig]]:
     gen_configs = [
         iree_definitions.ModuleGenerationConfig.build(
             compile_config=compile_config,
-            imported_model=iree_definitions.ImportedModel.from_model(model))
-        for model in models
+            imported_model=iree_definitions.ImportedModel.from_model(model),
+            tags=tags) for model in models
     ]
     sm80_devices = device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs(
         architecture=common_definitions.DeviceArchitecture.NVIDIA_AMPERE,
@@ -62,7 +62,7 @@ def _generate_configs(
         module_generation_configs=gen_configs,
         module_execution_configs=[execution_config],
         device_specs=sm80_devices,
-        tags=run_tags)
+        tags=tags)
 
     return (gen_configs, run_module_configs)
 
@@ -71,24 +71,25 @@ def generate(
   ) -> Tuple[List[iree_definitions.ModuleGenerationConfig],
              List[iree_definitions.E2EModelRunConfig]]:
     """Generates IREE compile and run configs."""
-    # The `cuda` tag is required to put them into the CUDA benchmark preset.
-    gen_configs, run_configs = self._generate_configs(model_groups.CUDA_MODELS,
-                                                      self.SM_80_COMPILE_CONFIG,
-                                                      run_tags=["cuda"])
+    # The CUDA tag is required to put them into the CUDA benchmark preset.
+    gen_configs, run_configs = self._generate_configs(
+        model_groups.CUDA_MODELS,
+        self.SM_80_COMPILE_CONFIG,
+        tags=[benchmark_tags.CUDA])
     ubench_gen_configs, ubench_run_configs = self._generate_configs(
         model_groups.MICRO_MATMUL,
         self.SM_80_UBENCH_MATMUL_COMPILE_CONFIG,
         execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG,
-        run_tags=["cuda"])
+        tags=[benchmark_tags.CUDA])
     ubench_splitk_gen_configs, ubench_splitk_run_configs = self._generate_configs(
         model_groups.MICRO_MATMUL_SPLITK,
         self.SM_80_UBENCH_MATMUL_SPLITK_COMPILE_CONFIG,
         execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG,
-        run_tags=["cuda"])
+        tags=[benchmark_tags.CUDA])
     long_running_gen_configs, long_running_module_configs = self._generate_configs(
         model_groups.CUDA_MODELS_LONG,
         self.SM_80_COMPILE_CONFIG,
-        run_tags=["cuda", "long-running"])
+        tags=[benchmark_tags.CUDA, benchmark_tags.LONG_RUNNING])
     return (gen_configs + ubench_gen_configs + ubench_splitk_gen_configs +
             long_running_gen_configs, run_configs + ubench_run_configs +
             ubench_splitk_run_configs + long_running_module_configs)

build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py

@@ -6,7 +6,7 @@
 """Defines IREE Vulkan NVIDIA benchmarks."""
 
 from typing import List, Tuple, Sequence
-from benchmark_suites.iree import module_execution_configs
+from benchmark_suites.iree import benchmark_tags, module_execution_configs
 from e2e_test_framework import unique_ids
 from e2e_test_framework.definitions import common_definitions, iree_definitions
 from e2e_test_framework.device_specs import device_collections
@@ -58,14 +58,14 @@ def _generate_configs(
       compile_config: iree_definitions.CompileConfig,
       execution_config: iree_definitions.
       ModuleExecutionConfig = module_execution_configs.VULKAN_CONFIG,
-      run_tags: Sequence[str] = [],
+      tags: Sequence[str] = [],
   ) -> Tuple[List[iree_definitions.ModuleGenerationConfig],
              List[iree_definitions.E2EModelRunConfig]]:
     gen_configs = [
         iree_definitions.ModuleGenerationConfig.build(
             compile_config=compile_config,
-            imported_model=iree_definitions.ImportedModel.from_model(model))
-        for model in models
+            imported_model=iree_definitions.ImportedModel.from_model(model),
+            tags=tags) for model in models
     ]
     # We use the same NVIDIA Ampere GPU for benchmarking code generated for
     # both Pascal and Ampere architectures. What we care is not exactly these
@@ -79,7 +79,7 @@ def _generate_configs(
         module_generation_configs=gen_configs,
         module_execution_configs=[execution_config],
         device_specs=ampere_devices,
-        tags=run_tags)
+        tags=tags)
 
     return (gen_configs, run_module_configs)
 
@@ -88,15 +88,15 @@ def generate(
   ) -> Tuple[List[iree_definitions.ModuleGenerationConfig],
              List[iree_definitions.E2EModelRunConfig]]:
     """Generates IREE compile and run configs."""
-    # The `vulkan-nvidia` tag is required to put them into the Vulkan NVIDIA
+    # The `vulkan-nvidia`` tag is required to put them into the Vulkan NVIDIA
     # benchmark preset.
     tensorcore_gen_configs, tensorcore_run_configs = self._generate_configs(
         model_groups.VULKAN_MODELS,
         self.TENSORCORE_COMPILE_CONFIG,
-        run_tags=["vulkan-nvidia"])
+        tags=[benchmark_tags.VULKAN_NVIDIA])
     simt_gen_configs, simt_run_configs = self._generate_configs(
         model_groups.VULKAN_MODELS,
         self.SIMT_COMPILE_CONFIG,
-        run_tags=["vulkan-nvidia"])
+        tags=[benchmark_tags.VULKAN_NVIDIA])
     return (tensorcore_gen_configs + simt_gen_configs,
             tensorcore_run_configs + simt_run_configs)

build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py

@@ -5,19 +5,26 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 """Generates CMake rules to build IREE artifacts."""
 
+import collections
 from dataclasses import dataclass
 from typing import Dict, List, Sequence
 import pathlib
 
-from benchmark_suites.iree import benchmark_collections
+from benchmark_suites.iree import benchmark_tags
 from e2e_test_artifacts import iree_artifacts
 from e2e_test_artifacts.cmake_generator import model_rule_generator
-from e2e_test_framework.definitions import common_definitions, iree_definitions
+from e2e_test_framework.definitions import iree_definitions
 import cmake_builder.rules
 
 BENCHMARK_IMPORT_MODELS_CMAKE_TARGET = "iree-benchmark-import-models"
+# Default benchmark suites.
 BENCHMARK_SUITES_CMAKE_TARGET = "iree-benchmark-suites"
+# Compilation statistics suites for default benchmarks.
 E2E_COMPILE_STATS_SUITES = "iree-e2e-compile-stats-suites"
+# Long-running benchmark suites.
+LONG_BENCHMARK_SUITES_CMAKE_TARGET = "iree-benchmark-suites-long"
+# Compilation statistics suites for long-running benchmarks.
+LONG_E2E_COMPILE_STATS_SUITES_CMAKE_TARGET = "iree-e2e-compile-stats-suites-long"
 
 
 @dataclass(frozen=True)
@@ -159,8 +166,7 @@ def generate_rules(
     model_import_rule_map[imported_model_id] = model_import_rule
     cmake_rules.extend(model_import_rule.cmake_rules)
 
-  module_target_names = []
-  compile_stats_module_target_names = []
+  suite_target_names = collections.defaultdict(list)
   for gen_config in module_generation_configs:
     model_import_rule = model_import_rule_map[
         gen_config.imported_model.composite_id]
@@ -170,10 +176,22 @@ def generate_rules(
         model_import_rule=model_import_rule,
         module_generation_config=gen_config,
         output_file_path=module_dir_path / iree_artifacts.MODULE_FILENAME)
-    if benchmark_collections.COMPILE_STATS_TAG in gen_config.compile_config.tags:
-      compile_stats_module_target_names.append(module_compile_rule.target_name)
+
+    is_compile_stats = (benchmark_tags.COMPILE_STATS
+                        in gen_config.compile_config.tags)
+    if benchmark_tags.LONG_RUNNING in gen_config.tags:
+      if is_compile_stats:
+        suite_target = LONG_E2E_COMPILE_STATS_SUITES_CMAKE_TARGET
+      else:
+        suite_target = LONG_BENCHMARK_SUITES_CMAKE_TARGET
     else:
-      module_target_names.append(module_compile_rule.target_name)
+      if is_compile_stats:
+        suite_target = E2E_COMPILE_STATS_SUITES
+      else:
+        suite_target = BENCHMARK_SUITES_CMAKE_TARGET
+
+    suite_target_names[suite_target].append(module_compile_rule.target_name)
+
     cmake_rules.extend(module_compile_rule.cmake_rules)
 
   if len(model_import_rule_map) > 0:
@@ -184,21 +202,14 @@ def generate_rules(
                 rule_builder.build_target_path(rule.target_name)
                 for rule in model_import_rule_map.values()
             ]))
-  if len(module_target_names) > 0:
+
+  for suite_target, module_target_names in suite_target_names.items():
     cmake_rules.append(
         cmake_builder.rules.build_add_dependencies(
-            target=BENCHMARK_SUITES_CMAKE_TARGET,
+            target=suite_target,
             deps=[
                 rule_builder.build_target_path(target_name)
                 for target_name in module_target_names
             ]))
-  if len(compile_stats_module_target_names) > 0:
-    cmake_rules.append(
-        cmake_builder.rules.build_add_dependencies(
-            target=E2E_COMPILE_STATS_SUITES,
-            deps=[
-                rule_builder.build_target_path(target_name)
-                for target_name in compile_stats_module_target_names
-            ]))
 
   return cmake_rules

build_tools/python/e2e_test_framework/definitions/iree_definitions.py

@@ -235,6 +235,7 @@ class ModuleGenerationConfig(object):
   """Describes a compile target to generate the module."""
   composite_id: str
   name: str
+  tags: List[str]
   imported_model: ImportedModel
   compile_config: CompileConfig
   # Full list of flags to compile with, derived from sub-components, with
@@ -267,7 +268,10 @@ def _replace_module_dir_placeholder(value: str) -> str:
                                  map_funcs=[_replace_module_dir_placeholder])
 
   @classmethod
-  def build(cls, imported_model: ImportedModel, compile_config: CompileConfig):
+  def build(cls,
+            imported_model: ImportedModel,
+            compile_config: CompileConfig,
+            tags: Sequence[str] = ()):
     composite_id = unique_ids.hash_composite_id(
         [imported_model.composite_id, compile_config.id])
     # Format: <imported_model_name> <compile_config_name>
@@ -276,6 +280,7 @@ def build(cls, imported_model: ImportedModel, compile_config: CompileConfig):
         compile_config, imported_model.import_config.dialect_type)
     return cls(composite_id=composite_id,
                name=name,
+               tags=list(tags),
                imported_model=imported_model,
                compile_config=compile_config,
                compile_flags=compile_flags)