From 6517313ae9666393fef33769c794f5b43681a884 Mon Sep 17 00:00:00 2001 From: Che-Yu Wu Date: Mon, 15 May 2023 18:05:53 +0000 Subject: [PATCH 1/2] Create long-running benchmark targets --- CMakeLists.txt | 12 ++- .../benchmark_suites/iree/cuda_benchmarks.py | 16 ++-- .../cmake_generator/iree_rule_generator.py | 38 +++++---- .../definitions/iree_definitions.py | 7 +- .../generated_e2e_test_iree_artifacts.cmake | 77 ++++++++++--------- 5 files changed, 86 insertions(+), 64 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b7cf9f97d0c..8e8ca3b8f28d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -669,19 +669,25 @@ if(IREE_BUILD_LEGACY_BENCHMARKS OR IREE_BUILD_E2E_TEST_ARTIFACTS) # formats, such as .tflite flatbuffers, to IREE-compatible .mlir files. add_custom_target(iree-benchmark-import-models) - # iree-benchmark-suites fully prepares benchmark models for benchmarking: + # iree-benchmark-suites* fully prepares benchmark models for benchmarking: # * importing from source formats to IREE-compatible .mlir files # * compiling from .mlir files to benchmark-ready .vmfb files # * generating flagfiles for executing the benchmark .vmfb files + # Build defualt benchmark suites. add_custom_target(iree-benchmark-suites) + # Build long-running benchmark suites with larger models. + add_custom_target(iree-benchmark-suites-long) endif() # TODO(#11263): This conditional block should be merged with the block above # once we remove IREE_BUILD_LEGACY_BENCHMARKS. if(IREE_BUILD_E2E_TEST_ARTIFACTS) - # iree-e2e-compile-stats-suites compiles the benchmark models with specific + # iree-e2e-compile-stats-suites* compiles the benchmark models with specific # flags to collect the compilation statistics. + # Build for defualt benchmark suites. add_custom_target(iree-e2e-compile-stats-suites) + # Build for long-running benchmark suites. + add_custom_target(iree-e2e-compile-stats-suites-long) # iree-e2e-test-artifacts builds all e2e test artifacts, including benchmarks # and model tests. @@ -689,7 +695,9 @@ if(IREE_BUILD_E2E_TEST_ARTIFACTS) add_dependencies(iree-e2e-test-artifacts iree-benchmark-import-models iree-benchmark-suites + iree-benchmark-suites-long iree-e2e-compile-stats-suites + iree-e2e-compile-stats-suites-long ) endif() diff --git a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py index 2d139875217d..12b56a2d5074 100644 --- a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py @@ -46,14 +46,14 @@ def _generate_configs( compile_config: iree_definitions.CompileConfig, execution_config: iree_definitions. ModuleExecutionConfig = module_execution_configs.CUDA_CONFIG, - run_tags: Sequence[str] = [], + tags: Sequence[str] = (), ) -> Tuple[List[iree_definitions.ModuleGenerationConfig], List[iree_definitions.E2EModelRunConfig]]: gen_configs = [ iree_definitions.ModuleGenerationConfig.build( compile_config=compile_config, - imported_model=iree_definitions.ImportedModel.from_model(model)) - for model in models + imported_model=iree_definitions.ImportedModel.from_model(model), + tags=tags) for model in models ] sm80_devices = device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( architecture=common_definitions.DeviceArchitecture.NVIDIA_AMPERE, @@ -62,7 +62,7 @@ def _generate_configs( module_generation_configs=gen_configs, module_execution_configs=[execution_config], device_specs=sm80_devices, - tags=run_tags) + tags=tags) return (gen_configs, run_module_configs) @@ -74,21 +74,21 @@ def generate( # The `cuda` tag is required to put them into the CUDA benchmark preset. gen_configs, run_configs = self._generate_configs(model_groups.CUDA_MODELS, self.SM_80_COMPILE_CONFIG, - run_tags=["cuda"]) + tags=["cuda"]) ubench_gen_configs, ubench_run_configs = self._generate_configs( model_groups.MICRO_MATMUL, self.SM_80_UBENCH_MATMUL_COMPILE_CONFIG, execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG, - run_tags=["cuda"]) + tags=["cuda"]) ubench_splitk_gen_configs, ubench_splitk_run_configs = self._generate_configs( model_groups.MICRO_MATMUL_SPLITK, self.SM_80_UBENCH_MATMUL_SPLITK_COMPILE_CONFIG, execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG, - run_tags=["cuda"]) + tags=["cuda"]) long_running_gen_configs, long_running_module_configs = self._generate_configs( model_groups.CUDA_MODELS_LONG, self.SM_80_COMPILE_CONFIG, - run_tags=["cuda", "long-running"]) + tags=["cuda", "long-running"]) return (gen_configs + ubench_gen_configs + ubench_splitk_gen_configs + long_running_gen_configs, run_configs + ubench_run_configs + ubench_splitk_run_configs + long_running_module_configs) diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py index eaeda0c926c9..326f320c0325 100644 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py +++ b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py @@ -6,18 +6,25 @@ """Generates CMake rules to build IREE artifacts.""" from dataclasses import dataclass +import collections from typing import Dict, List, Sequence import pathlib from benchmark_suites.iree import benchmark_collections from e2e_test_artifacts import iree_artifacts from e2e_test_artifacts.cmake_generator import model_rule_generator -from e2e_test_framework.definitions import common_definitions, iree_definitions +from e2e_test_framework.definitions import iree_definitions import cmake_builder.rules BENCHMARK_IMPORT_MODELS_CMAKE_TARGET = "iree-benchmark-import-models" +# Default benchmark suites. BENCHMARK_SUITES_CMAKE_TARGET = "iree-benchmark-suites" +# Compilation statistics suites for default benchmarks. E2E_COMPILE_STATS_SUITES = "iree-e2e-compile-stats-suites" +# Long-running benchmark suites with larger models. +LONG_BENCHMARK_SUITES_CMAKE_TARGET = "iree-benchmark-suites-long" +# Compilation statistics suites for long-running benchmarks. +LONG_E2E_COMPILE_STATS_SUITES = "iree-e2e-compile-stats-suites-long" @dataclass(frozen=True) @@ -159,8 +166,7 @@ def generate_rules( model_import_rule_map[imported_model_id] = model_import_rule cmake_rules.extend(model_import_rule.cmake_rules) - module_target_names = [] - compile_stats_module_target_names = [] + suite_target_names = collections.defaultdict(list) for gen_config in module_generation_configs: model_import_rule = model_import_rule_map[ gen_config.imported_model.composite_id] @@ -170,10 +176,17 @@ def generate_rules( model_import_rule=model_import_rule, module_generation_config=gen_config, output_file_path=module_dir_path / iree_artifacts.MODULE_FILENAME) - if benchmark_collections.COMPILE_STATS_TAG in gen_config.compile_config.tags: - compile_stats_module_target_names.append(module_compile_rule.target_name) + + has_compile_stats_tag = (benchmark_collections.COMPILE_STATS_TAG + in gen_config.compile_config.tags) + if "long-running" in gen_config.tags: + suite_target = (LONG_E2E_COMPILE_STATS_SUITES if has_compile_stats_tag + else LONG_BENCHMARK_SUITES_CMAKE_TARGET) else: - module_target_names.append(module_compile_rule.target_name) + suite_target = (E2E_COMPILE_STATS_SUITES if has_compile_stats_tag else + BENCHMARK_SUITES_CMAKE_TARGET) + suite_target_names[suite_target].append(module_compile_rule.target_name) + cmake_rules.extend(module_compile_rule.cmake_rules) if len(model_import_rule_map) > 0: @@ -184,21 +197,14 @@ def generate_rules( rule_builder.build_target_path(rule.target_name) for rule in model_import_rule_map.values() ])) - if len(module_target_names) > 0: + + for suite_target, module_target_names in suite_target_names.items(): cmake_rules.append( cmake_builder.rules.build_add_dependencies( - target=BENCHMARK_SUITES_CMAKE_TARGET, + target=suite_target, deps=[ rule_builder.build_target_path(target_name) for target_name in module_target_names ])) - if len(compile_stats_module_target_names) > 0: - cmake_rules.append( - cmake_builder.rules.build_add_dependencies( - target=E2E_COMPILE_STATS_SUITES, - deps=[ - rule_builder.build_target_path(target_name) - for target_name in compile_stats_module_target_names - ])) return cmake_rules diff --git a/build_tools/python/e2e_test_framework/definitions/iree_definitions.py b/build_tools/python/e2e_test_framework/definitions/iree_definitions.py index 2792be524107..a10ffa4029e3 100644 --- a/build_tools/python/e2e_test_framework/definitions/iree_definitions.py +++ b/build_tools/python/e2e_test_framework/definitions/iree_definitions.py @@ -235,6 +235,7 @@ class ModuleGenerationConfig(object): """Describes a compile target to generate the module.""" composite_id: str name: str + tags: List[str] imported_model: ImportedModel compile_config: CompileConfig # Full list of flags to compile with, derived from sub-components, with @@ -267,7 +268,10 @@ def _replace_module_dir_placeholder(value: str) -> str: map_funcs=[_replace_module_dir_placeholder]) @classmethod - def build(cls, imported_model: ImportedModel, compile_config: CompileConfig): + def build(cls, + imported_model: ImportedModel, + compile_config: CompileConfig, + tags: Sequence[str] = ()): composite_id = unique_ids.hash_composite_id( [imported_model.composite_id, compile_config.id]) # Format: @@ -276,6 +280,7 @@ def build(cls, imported_model: ImportedModel, compile_config: CompileConfig): compile_config, imported_model.import_config.dialect_type) return cls(composite_id=composite_id, name=name, + tags=list(tags), imported_model=imported_model, compile_config=compile_config, compile_flags=compile_flags) diff --git a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake index f61fa675c5cd..b57f60b79d10 100644 --- a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake +++ b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake @@ -6324,43 +6324,6 @@ add_dependencies(iree-benchmark-suites ${PACKAGE_NAME}_iree-module-2502249d103b48bf09a1d8d659fc143f1fb6e9479256e3e621061a2d06dcef2f ${PACKAGE_NAME}_iree-module-e6503d42c202eff85bcddb53e9869017943e756af09bfe2c05ffc27844ac77e9 ${PACKAGE_NAME}_iree-module-59025e9cf180a4a72de8f609ac9c4e4a3d855a603eedf62d9577e738f1ec12f4 - ${PACKAGE_NAME}_iree-module-58f67edb580a13aa98d3fd05ad31f207d62462c9de990825656b19d2fbacda3c - ${PACKAGE_NAME}_iree-module-1655ee92ee23748886eef280d7d6f96c8a227fae7d6d78abd3466ce8614f16ba - ${PACKAGE_NAME}_iree-module-496bdea78d06c9c32d8deae84b088b0eb34f6d1982ac790e2580a50dec4281ae - ${PACKAGE_NAME}_iree-module-5aa3c57b33d354d5ebfe51626e2dac78485a548aa9bf2fba7a3e7a248f8b909e - ${PACKAGE_NAME}_iree-module-a52ef9f5a8c064a4cbf52ea82b694d2c0bb9a0dd43d3576131335ab5453b62a4 - ${PACKAGE_NAME}_iree-module-a6f73626c6b41d410da11809d66d613293a0e3c298d491d6fb25e3729cf6685e - ${PACKAGE_NAME}_iree-module-369f2b1edbd94d2c19cf18be7ee8bb7aa937fe0669fbe6e5fb9a5c06185ebb42 - ${PACKAGE_NAME}_iree-module-52f3e54f0a7479eecdae97afb32b90e3952c8be195a861722fd8e8ece857ff40 - ${PACKAGE_NAME}_iree-module-30baf74b5974ab7269c57015554a0fb0447cbde5fb77ab44ed9cf9cd2bc0e351 - ${PACKAGE_NAME}_iree-module-09dd0099b9a0924cc1bcadfb355bccbeea9cb3c29fc48c5181fbb87d80af28a9 - ${PACKAGE_NAME}_iree-module-3385106a53075a52f4ae60523e09ad42cdad60475d97e84695a9c68584d9182a - ${PACKAGE_NAME}_iree-module-2021b32a685c21ba21823416591c14fd79075595c101665abe565ff92335c3ef - ${PACKAGE_NAME}_iree-module-57080e57e3ba061d70d35d8579824a8eab7ee05ef3e0eea02cc251bab87cfd10 - ${PACKAGE_NAME}_iree-module-88f03be31249d371d00c63d44d44732ad4c348bf2354bd2b3b042465d86f7183 - ${PACKAGE_NAME}_iree-module-b3f2a0c268a0a8752ccbb56f013455197c466635637b535778a494dab316cf9e - ${PACKAGE_NAME}_iree-module-bf4bda91fb17ec7f3402e7b301cb3ab0b7a9b0b00d8cee29fd4cfa46262a5ea1 - ${PACKAGE_NAME}_iree-module-9900b1a0c52b31fc38f1d82ff088102f7131ecbb3fc1b8274929481337d031c5 - ${PACKAGE_NAME}_iree-module-25f1a116fad68b01401dfbd8462c7b2729ef2d504df0954509cd488cd5da17a1 - ${PACKAGE_NAME}_iree-module-89643b9db9da0c7b822ba8e2ead4ec12fc3093dbde9b7a8570de1be3d46bd54b - ${PACKAGE_NAME}_iree-module-c16715c83936394998fe81f8d33f6e7901f17ff662dfd2941a3982f85c3a9e17 - ${PACKAGE_NAME}_iree-module-7ba873f0f155e857b45f991cbb4e8776f8b137d59ddc5f81a7a23ed2a48eecc9 - ${PACKAGE_NAME}_iree-module-374d3219ba3a0064fc5eccb0857f9ae7c37ad2c8183c930b59b33a6a6248d109 - ${PACKAGE_NAME}_iree-module-154de838dc7742304c9e27a9f315645a915493bc4f84160e29b15b7fc6dc475e - ${PACKAGE_NAME}_iree-module-21a83414dc1feb2a3b5fb6afadc36c1022a0ab747380291dbb309637d5f32eab - ${PACKAGE_NAME}_iree-module-7d93a3e9b342045ed2546960ee3d08f60be237fd8974fe35447a41234f322148 - ${PACKAGE_NAME}_iree-module-995595b7b80370d9c484413e9d06c2de928db32777101c91d3bac0d3797058ec - ${PACKAGE_NAME}_iree-module-45ed9ca2efb9bb645316e856a5c464b76e56902e930ff1d4e54fdef38043b33f - ${PACKAGE_NAME}_iree-module-8c26886533593b666597f8698e33c0e9c98b349bdf8ac01c3d122cc20b741def - ${PACKAGE_NAME}_iree-module-4df03cf0d71376b43f2e93ca35d252f052733fa8de905af146f25d4216b2b4d2 - ${PACKAGE_NAME}_iree-module-c947d0e7889a515a4e58bdbd1688f028ca88351dbd7ec501d9fb13a08c27503f - ${PACKAGE_NAME}_iree-module-9c8197e62299f79c373535d91c48acd1b3522264bd7a3d885683570bfc20164c - ${PACKAGE_NAME}_iree-module-222490941c581c67f5ce710e9ea141482cd8074294a43a5dc67a01a127037cd4 - ${PACKAGE_NAME}_iree-module-39556c12d84502be71243197b99b0f8c22949093c561c55c01ca906812a86288 - ${PACKAGE_NAME}_iree-module-9e1870e43977999573defd36f2a4540f70b69acf7a330f5133fd8170265cff23 - ${PACKAGE_NAME}_iree-module-99b28083e2bcb855cc6ead27a57d4efbd6a8c54205897ab226fe15bf321e239a - ${PACKAGE_NAME}_iree-module-0c02fdba447db54fce59698955c728295a98135ef702ce24061834bdea2d1d90 - ${PACKAGE_NAME}_iree-module-e9496836f964df336279914bb907889f071893aac3a0d069f6a318e9a74e4c23 ${PACKAGE_NAME}_iree-module-68f0eb37bb72d0d6605ecdf42691c64125960e122844b0beeae350871a445b1c ${PACKAGE_NAME}_iree-module-a7a1553d0739151f06bbc00a3ef8b67b0606463eab4b6607069aa94ea0bfd92f ${PACKAGE_NAME}_iree-module-e80d71ed8e86c0756226b2323e27e2c7c0fff8eddde59ba69e9222d36ee3eef6 @@ -6440,6 +6403,46 @@ add_dependencies(iree-benchmark-suites ${PACKAGE_NAME}_iree-module-bd0ea10065a27dea6875ceb70f769e7e0f67a08e857920ca0d0322593441e9ed ) +add_dependencies(iree-benchmark-suites-long + ${PACKAGE_NAME}_iree-module-58f67edb580a13aa98d3fd05ad31f207d62462c9de990825656b19d2fbacda3c + ${PACKAGE_NAME}_iree-module-1655ee92ee23748886eef280d7d6f96c8a227fae7d6d78abd3466ce8614f16ba + ${PACKAGE_NAME}_iree-module-496bdea78d06c9c32d8deae84b088b0eb34f6d1982ac790e2580a50dec4281ae + ${PACKAGE_NAME}_iree-module-5aa3c57b33d354d5ebfe51626e2dac78485a548aa9bf2fba7a3e7a248f8b909e + ${PACKAGE_NAME}_iree-module-a52ef9f5a8c064a4cbf52ea82b694d2c0bb9a0dd43d3576131335ab5453b62a4 + ${PACKAGE_NAME}_iree-module-a6f73626c6b41d410da11809d66d613293a0e3c298d491d6fb25e3729cf6685e + ${PACKAGE_NAME}_iree-module-369f2b1edbd94d2c19cf18be7ee8bb7aa937fe0669fbe6e5fb9a5c06185ebb42 + ${PACKAGE_NAME}_iree-module-52f3e54f0a7479eecdae97afb32b90e3952c8be195a861722fd8e8ece857ff40 + ${PACKAGE_NAME}_iree-module-30baf74b5974ab7269c57015554a0fb0447cbde5fb77ab44ed9cf9cd2bc0e351 + ${PACKAGE_NAME}_iree-module-09dd0099b9a0924cc1bcadfb355bccbeea9cb3c29fc48c5181fbb87d80af28a9 + ${PACKAGE_NAME}_iree-module-3385106a53075a52f4ae60523e09ad42cdad60475d97e84695a9c68584d9182a + ${PACKAGE_NAME}_iree-module-2021b32a685c21ba21823416591c14fd79075595c101665abe565ff92335c3ef + ${PACKAGE_NAME}_iree-module-57080e57e3ba061d70d35d8579824a8eab7ee05ef3e0eea02cc251bab87cfd10 + ${PACKAGE_NAME}_iree-module-88f03be31249d371d00c63d44d44732ad4c348bf2354bd2b3b042465d86f7183 + ${PACKAGE_NAME}_iree-module-b3f2a0c268a0a8752ccbb56f013455197c466635637b535778a494dab316cf9e + ${PACKAGE_NAME}_iree-module-bf4bda91fb17ec7f3402e7b301cb3ab0b7a9b0b00d8cee29fd4cfa46262a5ea1 + ${PACKAGE_NAME}_iree-module-9900b1a0c52b31fc38f1d82ff088102f7131ecbb3fc1b8274929481337d031c5 + ${PACKAGE_NAME}_iree-module-25f1a116fad68b01401dfbd8462c7b2729ef2d504df0954509cd488cd5da17a1 + ${PACKAGE_NAME}_iree-module-89643b9db9da0c7b822ba8e2ead4ec12fc3093dbde9b7a8570de1be3d46bd54b + ${PACKAGE_NAME}_iree-module-c16715c83936394998fe81f8d33f6e7901f17ff662dfd2941a3982f85c3a9e17 + ${PACKAGE_NAME}_iree-module-7ba873f0f155e857b45f991cbb4e8776f8b137d59ddc5f81a7a23ed2a48eecc9 + ${PACKAGE_NAME}_iree-module-374d3219ba3a0064fc5eccb0857f9ae7c37ad2c8183c930b59b33a6a6248d109 + ${PACKAGE_NAME}_iree-module-154de838dc7742304c9e27a9f315645a915493bc4f84160e29b15b7fc6dc475e + ${PACKAGE_NAME}_iree-module-21a83414dc1feb2a3b5fb6afadc36c1022a0ab747380291dbb309637d5f32eab + ${PACKAGE_NAME}_iree-module-7d93a3e9b342045ed2546960ee3d08f60be237fd8974fe35447a41234f322148 + ${PACKAGE_NAME}_iree-module-995595b7b80370d9c484413e9d06c2de928db32777101c91d3bac0d3797058ec + ${PACKAGE_NAME}_iree-module-45ed9ca2efb9bb645316e856a5c464b76e56902e930ff1d4e54fdef38043b33f + ${PACKAGE_NAME}_iree-module-8c26886533593b666597f8698e33c0e9c98b349bdf8ac01c3d122cc20b741def + ${PACKAGE_NAME}_iree-module-4df03cf0d71376b43f2e93ca35d252f052733fa8de905af146f25d4216b2b4d2 + ${PACKAGE_NAME}_iree-module-c947d0e7889a515a4e58bdbd1688f028ca88351dbd7ec501d9fb13a08c27503f + ${PACKAGE_NAME}_iree-module-9c8197e62299f79c373535d91c48acd1b3522264bd7a3d885683570bfc20164c + ${PACKAGE_NAME}_iree-module-222490941c581c67f5ce710e9ea141482cd8074294a43a5dc67a01a127037cd4 + ${PACKAGE_NAME}_iree-module-39556c12d84502be71243197b99b0f8c22949093c561c55c01ca906812a86288 + ${PACKAGE_NAME}_iree-module-9e1870e43977999573defd36f2a4540f70b69acf7a330f5133fd8170265cff23 + ${PACKAGE_NAME}_iree-module-99b28083e2bcb855cc6ead27a57d4efbd6a8c54205897ab226fe15bf321e239a + ${PACKAGE_NAME}_iree-module-0c02fdba447db54fce59698955c728295a98135ef702ce24061834bdea2d1d90 + ${PACKAGE_NAME}_iree-module-e9496836f964df336279914bb907889f071893aac3a0d069f6a318e9a74e4c23 +) + add_dependencies(iree-e2e-compile-stats-suites ${PACKAGE_NAME}_iree-module-28e38bd436b036babc0fabe98b6e7c68ca3a7088e73dffff2c538adfa7d6af4c ${PACKAGE_NAME}_iree-module-439f7c958ce1d3200ea96935174cabde8e8fe6917a007f5e238553e9c2aa7625 From 70d345f2e130a7f977fe8b41fe61e916252227dc Mon Sep 17 00:00:00 2001 From: Che-Yu Wu Date: Mon, 15 May 2023 20:02:15 +0000 Subject: [PATCH 2/2] Refactor --- CMakeLists.txt | 24 ++++-- .../benchmarks/export_benchmark_config.py | 14 ++-- .../iree/benchmark_collections.py | 17 ++-- .../benchmark_suites/iree/benchmark_tags.py | 18 +++++ .../benchmark_suites/iree/cuda_benchmarks.py | 17 ++-- .../iree/vulkan_nvidia_benchmarks.py | 16 ++-- .../cmake_generator/iree_rule_generator.py | 27 ++++--- .../generated_e2e_test_iree_artifacts.cmake | 77 ++++++++++--------- 8 files changed, 123 insertions(+), 87 deletions(-) create mode 100644 build_tools/python/benchmark_suites/iree/benchmark_tags.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e8ca3b8f28d..c349f92beead 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -662,29 +662,37 @@ endif() # dependencies to them. #------------------------------------------------------------------------------- -if(IREE_BUILD_LEGACY_BENCHMARKS OR IREE_BUILD_E2E_TEST_ARTIFACTS) +# TODO(#11263): Remove together with IREE_BUILD_LEGACY_BENCHMARKS. +if(IREE_BUILD_LEGACY_BENCHMARKS) # Add top-level custom targets to drive generating benchmark suites. # iree-benchmark-import-models imports benchmark models from their source # formats, such as .tflite flatbuffers, to IREE-compatible .mlir files. add_custom_target(iree-benchmark-import-models) - # iree-benchmark-suites* fully prepares benchmark models for benchmarking: + # iree-benchmark-suites fully prepares benchmark models for benchmarking: # * importing from source formats to IREE-compatible .mlir files # * compiling from .mlir files to benchmark-ready .vmfb files # * generating flagfiles for executing the benchmark .vmfb files - # Build defualt benchmark suites. add_custom_target(iree-benchmark-suites) - # Build long-running benchmark suites with larger models. - add_custom_target(iree-benchmark-suites-long) endif() -# TODO(#11263): This conditional block should be merged with the block above -# once we remove IREE_BUILD_LEGACY_BENCHMARKS. if(IREE_BUILD_E2E_TEST_ARTIFACTS) + # iree-benchmark-import-models imports benchmark models from their source + # formats, such as .tflite flatbuffers, to IREE-compatible .mlir files. + add_custom_target(iree-benchmark-import-models) + + # iree-benchmark-suites* fully prepares benchmark models for benchmarking: + # * importing from source formats to IREE-compatible .mlir files + # * compiling from .mlir files to benchmark-ready .vmfb files + # Build default benchmark suites. + add_custom_target(iree-benchmark-suites) + # Build long-running benchmark suites. + add_custom_target(iree-benchmark-suites-long) + # iree-e2e-compile-stats-suites* compiles the benchmark models with specific # flags to collect the compilation statistics. - # Build for defualt benchmark suites. + # Build for default benchmark suites. add_custom_target(iree-e2e-compile-stats-suites) # Build for long-running benchmark suites. add_custom_target(iree-e2e-compile-stats-suites-long) diff --git a/build_tools/benchmarks/export_benchmark_config.py b/build_tools/benchmarks/export_benchmark_config.py index d444e6a845c0..ca84ad752dc4 100755 --- a/build_tools/benchmarks/export_benchmark_config.py +++ b/build_tools/benchmarks/export_benchmark_config.py @@ -39,7 +39,7 @@ import json import textwrap -from benchmark_suites.iree import benchmark_collections +from benchmark_suites.iree import benchmark_collections, benchmark_tags from e2e_test_artifacts import iree_artifacts from e2e_test_framework import serialization from e2e_test_framework.definitions import common_definitions, iree_definitions @@ -50,12 +50,14 @@ "x86_64": lambda config: config.target_device_spec.architecture.architecture == "x86_64", - "cuda": (lambda config: "cuda" in config.tags and "long-running" not in - config.tags), + "cuda": + lambda config: (benchmark_tags.CUDA in config.tags and benchmark_tags. + LONG_RUNNING not in config.tags), "cuda-long": - lambda config: "cuda" in config.tags and "long-running" in config.tags, + lambda config: (benchmark_tags.CUDA in config.tags and benchmark_tags. + LONG_RUNNING in config.tags), "vulkan-nvidia": - lambda config: "vulkan-nvidia" in config.tags, + lambda config: benchmark_tags.VULKAN_NVIDIA in config.tags, "android-cpu": lambda config: (config.target_device_spec.architecture.type == common_definitions. @@ -148,7 +150,7 @@ def _export_compilation_handler(_args: argparse.Namespace): all_gen_configs, _ = benchmark_collections.generate_benchmarks() compile_stats_gen_configs = [ config for config in all_gen_configs - if benchmark_collections.COMPILE_STATS_TAG in config.compile_config.tags + if benchmark_tags.COMPILE_STATS in config.compile_config.tags ] distinct_module_dir_paths = _get_distinct_module_dir_paths( diff --git a/build_tools/python/benchmark_suites/iree/benchmark_collections.py b/build_tools/python/benchmark_suites/iree/benchmark_collections.py index df2cb6fad221..a2f3a14db20e 100644 --- a/build_tools/python/benchmark_suites/iree/benchmark_collections.py +++ b/build_tools/python/benchmark_suites/iree/benchmark_collections.py @@ -9,15 +9,13 @@ from e2e_test_artifacts import iree_artifacts from e2e_test_framework.definitions import iree_definitions -from benchmark_suites.iree import (riscv_benchmarks, x86_64_benchmarks, - adreno_benchmarks, armv8_a_benchmarks, - cuda_benchmarks, mali_benchmarks, - vulkan_nvidia_benchmarks, vmvx_benchmarks) +from benchmark_suites.iree import (benchmark_tags, riscv_benchmarks, + x86_64_benchmarks, adreno_benchmarks, + armv8_a_benchmarks, cuda_benchmarks, + mali_benchmarks, vulkan_nvidia_benchmarks, + vmvx_benchmarks) COMPILE_STATS_ID_SUFFIX = "-compile-stats" -# Tag that indicates this compile config is generated for collecting compilation -# statistics. -COMPILE_STATS_TAG = "compile-stats" def generate_benchmarks( @@ -49,7 +47,7 @@ def generate_benchmarks( scheduling_stats_path = f"{iree_definitions.MODULE_DIR_VARIABLE}/{iree_artifacts.SCHEDULING_STATS_FILENAME}" compile_stats_config = iree_definitions.CompileConfig.build( id=compile_config.id + COMPILE_STATS_ID_SUFFIX, - tags=compile_config.tags + [COMPILE_STATS_TAG], + tags=compile_config.tags + [benchmark_tags.COMPILE_STATS], compile_targets=compile_config.compile_targets, extra_flags=compile_config.extra_flags + [ # Enable zip polyglot to provide component sizes. @@ -63,7 +61,8 @@ def generate_benchmarks( compile_stats_gen_configs.append( iree_definitions.ModuleGenerationConfig.build( imported_model=gen_config.imported_model, - compile_config=compile_stats_config)) + compile_config=compile_stats_config, + tags=gen_config.tags)) all_gen_configs += compile_stats_gen_configs return (all_gen_configs, all_run_configs) diff --git a/build_tools/python/benchmark_suites/iree/benchmark_tags.py b/build_tools/python/benchmark_suites/iree/benchmark_tags.py new file mode 100644 index 000000000000..e3a324e123f5 --- /dev/null +++ b/build_tools/python/benchmark_suites/iree/benchmark_tags.py @@ -0,0 +1,18 @@ +## Copyright 2023 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Tag that indicates this compile config is generated for collecting compilation +# statistics. +COMPILE_STATS = "compile-stats" + +# Tag for long-running benchmarks. +LONG_RUNNING = "long-running" + +# Tag for CUDA benchamrks. +CUDA = "cuda" + +# Tag for Vulkan NVIDIA benchamrks. +VULKAN_NVIDIA = "vulkan-nvidia" diff --git a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py index 12b56a2d5074..9633b940c937 100644 --- a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py @@ -6,7 +6,7 @@ """Defines IREE CUDA benchmarks.""" from typing import List, Tuple, Sequence -from benchmark_suites.iree import module_execution_configs +from benchmark_suites.iree import benchmark_tags, module_execution_configs from e2e_test_framework import unique_ids from e2e_test_framework.definitions import common_definitions, iree_definitions from e2e_test_framework.device_specs import device_collections @@ -71,24 +71,25 @@ def generate( ) -> Tuple[List[iree_definitions.ModuleGenerationConfig], List[iree_definitions.E2EModelRunConfig]]: """Generates IREE compile and run configs.""" - # The `cuda` tag is required to put them into the CUDA benchmark preset. - gen_configs, run_configs = self._generate_configs(model_groups.CUDA_MODELS, - self.SM_80_COMPILE_CONFIG, - tags=["cuda"]) + # The CUDA tag is required to put them into the CUDA benchmark preset. + gen_configs, run_configs = self._generate_configs( + model_groups.CUDA_MODELS, + self.SM_80_COMPILE_CONFIG, + tags=[benchmark_tags.CUDA]) ubench_gen_configs, ubench_run_configs = self._generate_configs( model_groups.MICRO_MATMUL, self.SM_80_UBENCH_MATMUL_COMPILE_CONFIG, execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG, - tags=["cuda"]) + tags=[benchmark_tags.CUDA]) ubench_splitk_gen_configs, ubench_splitk_run_configs = self._generate_configs( model_groups.MICRO_MATMUL_SPLITK, self.SM_80_UBENCH_MATMUL_SPLITK_COMPILE_CONFIG, execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG, - tags=["cuda"]) + tags=[benchmark_tags.CUDA]) long_running_gen_configs, long_running_module_configs = self._generate_configs( model_groups.CUDA_MODELS_LONG, self.SM_80_COMPILE_CONFIG, - tags=["cuda", "long-running"]) + tags=[benchmark_tags.CUDA, benchmark_tags.LONG_RUNNING]) return (gen_configs + ubench_gen_configs + ubench_splitk_gen_configs + long_running_gen_configs, run_configs + ubench_run_configs + ubench_splitk_run_configs + long_running_module_configs) diff --git a/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py b/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py index fc9c764ef61c..1bf07942066d 100644 --- a/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py @@ -6,7 +6,7 @@ """Defines IREE Vulkan NVIDIA benchmarks.""" from typing import List, Tuple, Sequence -from benchmark_suites.iree import module_execution_configs +from benchmark_suites.iree import benchmark_tags, module_execution_configs from e2e_test_framework import unique_ids from e2e_test_framework.definitions import common_definitions, iree_definitions from e2e_test_framework.device_specs import device_collections @@ -58,14 +58,14 @@ def _generate_configs( compile_config: iree_definitions.CompileConfig, execution_config: iree_definitions. ModuleExecutionConfig = module_execution_configs.VULKAN_CONFIG, - run_tags: Sequence[str] = [], + tags: Sequence[str] = [], ) -> Tuple[List[iree_definitions.ModuleGenerationConfig], List[iree_definitions.E2EModelRunConfig]]: gen_configs = [ iree_definitions.ModuleGenerationConfig.build( compile_config=compile_config, - imported_model=iree_definitions.ImportedModel.from_model(model)) - for model in models + imported_model=iree_definitions.ImportedModel.from_model(model), + tags=tags) for model in models ] # We use the same NVIDIA Ampere GPU for benchmarking code generated for # both Pascal and Ampere architectures. What we care is not exactly these @@ -79,7 +79,7 @@ def _generate_configs( module_generation_configs=gen_configs, module_execution_configs=[execution_config], device_specs=ampere_devices, - tags=run_tags) + tags=tags) return (gen_configs, run_module_configs) @@ -88,15 +88,15 @@ def generate( ) -> Tuple[List[iree_definitions.ModuleGenerationConfig], List[iree_definitions.E2EModelRunConfig]]: """Generates IREE compile and run configs.""" - # The `vulkan-nvidia` tag is required to put them into the Vulkan NVIDIA + # The `vulkan-nvidia`` tag is required to put them into the Vulkan NVIDIA # benchmark preset. tensorcore_gen_configs, tensorcore_run_configs = self._generate_configs( model_groups.VULKAN_MODELS, self.TENSORCORE_COMPILE_CONFIG, - run_tags=["vulkan-nvidia"]) + tags=[benchmark_tags.VULKAN_NVIDIA]) simt_gen_configs, simt_run_configs = self._generate_configs( model_groups.VULKAN_MODELS, self.SIMT_COMPILE_CONFIG, - run_tags=["vulkan-nvidia"]) + tags=[benchmark_tags.VULKAN_NVIDIA]) return (tensorcore_gen_configs + simt_gen_configs, tensorcore_run_configs + simt_run_configs) diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py index 326f320c0325..b3b4a7f6a975 100644 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py +++ b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py @@ -5,12 +5,12 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception """Generates CMake rules to build IREE artifacts.""" -from dataclasses import dataclass import collections +from dataclasses import dataclass from typing import Dict, List, Sequence import pathlib -from benchmark_suites.iree import benchmark_collections +from benchmark_suites.iree import benchmark_tags from e2e_test_artifacts import iree_artifacts from e2e_test_artifacts.cmake_generator import model_rule_generator from e2e_test_framework.definitions import iree_definitions @@ -21,10 +21,10 @@ BENCHMARK_SUITES_CMAKE_TARGET = "iree-benchmark-suites" # Compilation statistics suites for default benchmarks. E2E_COMPILE_STATS_SUITES = "iree-e2e-compile-stats-suites" -# Long-running benchmark suites with larger models. +# Long-running benchmark suites. LONG_BENCHMARK_SUITES_CMAKE_TARGET = "iree-benchmark-suites-long" # Compilation statistics suites for long-running benchmarks. -LONG_E2E_COMPILE_STATS_SUITES = "iree-e2e-compile-stats-suites-long" +LONG_E2E_COMPILE_STATS_SUITES_CMAKE_TARGET = "iree-e2e-compile-stats-suites-long" @dataclass(frozen=True) @@ -177,14 +177,19 @@ def generate_rules( module_generation_config=gen_config, output_file_path=module_dir_path / iree_artifacts.MODULE_FILENAME) - has_compile_stats_tag = (benchmark_collections.COMPILE_STATS_TAG - in gen_config.compile_config.tags) - if "long-running" in gen_config.tags: - suite_target = (LONG_E2E_COMPILE_STATS_SUITES if has_compile_stats_tag - else LONG_BENCHMARK_SUITES_CMAKE_TARGET) + is_compile_stats = (benchmark_tags.COMPILE_STATS + in gen_config.compile_config.tags) + if benchmark_tags.LONG_RUNNING in gen_config.tags: + if is_compile_stats: + suite_target = LONG_E2E_COMPILE_STATS_SUITES_CMAKE_TARGET + else: + suite_target = LONG_BENCHMARK_SUITES_CMAKE_TARGET else: - suite_target = (E2E_COMPILE_STATS_SUITES if has_compile_stats_tag else - BENCHMARK_SUITES_CMAKE_TARGET) + if is_compile_stats: + suite_target = E2E_COMPILE_STATS_SUITES + else: + suite_target = BENCHMARK_SUITES_CMAKE_TARGET + suite_target_names[suite_target].append(module_compile_rule.target_name) cmake_rules.extend(module_compile_rule.cmake_rules) diff --git a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake index b57f60b79d10..46567b67f963 100644 --- a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake +++ b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake @@ -6489,43 +6489,6 @@ add_dependencies(iree-e2e-compile-stats-suites ${PACKAGE_NAME}_iree-module-f9df9c8ecc1b369ddc960f07ab7097a9c1dd8587441cbd9cf91e443e2639bf28 ${PACKAGE_NAME}_iree-module-371a68ee85e41abf2d2a2847fe6c9f85fdc51f76516d74e3576b9d72670c1a0b ${PACKAGE_NAME}_iree-module-1af884e597c283d319b14e8d55156eac66a86045fcb7e521c3f50aa8a6c04126 - ${PACKAGE_NAME}_iree-module-2fe6a4ac93d74732ebabbd87e84e6ec56d2aeb9f6a94f5b595f74f947498c579 - ${PACKAGE_NAME}_iree-module-32dfb79e574ef282dd352a156620d44d545dda0f49ea1ac27394028d2ced46a8 - ${PACKAGE_NAME}_iree-module-3421bd5d457a3ed62e7aa74559afb7199105c1f60b65090a247ee3e6d9aec69c - ${PACKAGE_NAME}_iree-module-5663bccd89aa0616691d5b804b337c8989a2ca94c1bf2673d41dbbab9e511f20 - ${PACKAGE_NAME}_iree-module-353a915204b8eb401d0caac543588920cfab76f826463394cc1e99546cab13bb - ${PACKAGE_NAME}_iree-module-c5f0532455c22436b0fce487797b957939acb0d1f235644187d63bca29a70654 - ${PACKAGE_NAME}_iree-module-1bac11dad2160652ffcfb201d96418f4c044fcd79771cf6a783f311998850a2e - ${PACKAGE_NAME}_iree-module-3edb70502ef7a26db736a64e24a677ff085fb800a8475e750dd2a9d119baf407 - ${PACKAGE_NAME}_iree-module-7f45c4f644d293bb984007aa1a91c919c5a75d01bf38f70ba7257c0b21c654f3 - ${PACKAGE_NAME}_iree-module-bb96ff5237b0ad603b1ee7c59601a1c957a2ed5c95cbfd94f7ec6d707aad3660 - ${PACKAGE_NAME}_iree-module-5b499a05fc60f78b5b11b1e9ba89ba797450e5b49b0613b3c23d996c6f767c64 - ${PACKAGE_NAME}_iree-module-40b0bd95d986d079ae0fc8cd57b473a150289a5f3ee6029b388ebc86edd4a60c - ${PACKAGE_NAME}_iree-module-03736c12afe2b27f77101146eddaa80c8911cdbc2120ab574225c08b088e14f3 - ${PACKAGE_NAME}_iree-module-b180d82d1a87fe29e414ecd55bf8a82ff92b80d7c644d31769729160b27aa467 - ${PACKAGE_NAME}_iree-module-6265e9506389bb29c551ad62f4ea05f54bd0e549c2d29bf425bac9d56aa55ef6 - ${PACKAGE_NAME}_iree-module-1ee530ad72c5711ee3abf11b27eed04e11d261fb6c0d23a6cb68c80c1a42580b - ${PACKAGE_NAME}_iree-module-38e99a06cd01399fe48cd9d860f65474e4fd7df1b3aca491766ff5f825315a79 - ${PACKAGE_NAME}_iree-module-db86dfb7ef5e59b31bf513df65fffd397f49e3fe3b5c03149140ccf3b6b2f821 - ${PACKAGE_NAME}_iree-module-89966a2ac764b543e721d822acdbe015526866e0a8d56a6782f240eeb3cddea1 - ${PACKAGE_NAME}_iree-module-6da0542e65f9e5eff1bc142506866aec435b931b36eb8e86139f79b4945863fd - ${PACKAGE_NAME}_iree-module-f55259e9c9778885d48b20cfb18242984fa9dde9840fa0f92c7de379c88c8230 - ${PACKAGE_NAME}_iree-module-d931d256d015f057a71fb67c6c7d7d4099910286d5bbcd6c1c71b43b2b3756e8 - ${PACKAGE_NAME}_iree-module-999f2edcdf9fbc84e0969923f8605e9069810a63849973a5b74488f83d14a2fe - ${PACKAGE_NAME}_iree-module-5176d0f5fa331fa047395186a866870fe4210f637472ef4bb0a3f1ba3a62a749 - ${PACKAGE_NAME}_iree-module-e67816e321544bb61b63ed8cd8b7faa8ff7d35cca15832b5fbc117f4693b3e78 - ${PACKAGE_NAME}_iree-module-d746ecf3d747f18306b6dea4cb6b9e9dbf987fe7fd4d0b27b39a57a213e75dd9 - ${PACKAGE_NAME}_iree-module-439dab16cd6df449fc83eb3e1603fa86ad811e749bcad2c3e3176976c56848e5 - ${PACKAGE_NAME}_iree-module-815e44d1ac31402e86d0fef72e79474a25dfad3a5c15b8cdd3642e101274342d - ${PACKAGE_NAME}_iree-module-fd0f6049f3cd4fee97c182e824f8e65eeabef47ef86a3797ae278ae1fc101705 - ${PACKAGE_NAME}_iree-module-48ad70710c3602b589431df74468589c74f2a5b54f56d8f86af86c70408ae36d - ${PACKAGE_NAME}_iree-module-07014af4305cf75cd094171519987b1ed1a6a6b70e489d2770301b623b7afe0c - ${PACKAGE_NAME}_iree-module-681f456f27dbb79e7d8d0266bf835866d9f29f87eafad7e867ac13c84602742f - ${PACKAGE_NAME}_iree-module-3b0ae1403ef444d812f0c7b37fda7311e2cc4ea407850ee7b91e6984b9c86100 - ${PACKAGE_NAME}_iree-module-ff26aa2d1b8f0d80a8ab10aab0a87ae89e36c141dc6c758da61c339d51e2db8d - ${PACKAGE_NAME}_iree-module-666922620019ce61f2ffa56e9e141fc3ef2736a04eab3d7c6d3b0fbeeaef4275 - ${PACKAGE_NAME}_iree-module-b36c4a1b93f09dee52b255982410f5b9e8e351010fe22eadb3f5afb6719f6873 - ${PACKAGE_NAME}_iree-module-50c0cb0ea6df2e4c1c1337a607252366fb8cd17ffe72cd8aeb76a9f9a8d6712f ${PACKAGE_NAME}_iree-module-16b5b80aaf1271b5ad782570340cc0c7c1c97e10b7e6c6cc6e5f3ede8393cb6c ${PACKAGE_NAME}_iree-module-65fa033050b916e8143d44b5081ee45db3b1946a5d77de223328a7fe92a1cc66 ${PACKAGE_NAME}_iree-module-16ef56b6869d10b17e983fec62e9f48e6bb87e9a348ab52a0b2faabca2b03578 @@ -6604,3 +6567,43 @@ add_dependencies(iree-e2e-compile-stats-suites ${PACKAGE_NAME}_iree-module-02b72f9538e4dfc9c789e63d722d5eab4333f3f55f8375503f433a790da119cc ${PACKAGE_NAME}_iree-module-e7bd41e564750501f39ac9690c18d1a2e77dc7999da710d0c0bf80751dda84a0 ) + +add_dependencies(iree-e2e-compile-stats-suites-long + ${PACKAGE_NAME}_iree-module-2fe6a4ac93d74732ebabbd87e84e6ec56d2aeb9f6a94f5b595f74f947498c579 + ${PACKAGE_NAME}_iree-module-32dfb79e574ef282dd352a156620d44d545dda0f49ea1ac27394028d2ced46a8 + ${PACKAGE_NAME}_iree-module-3421bd5d457a3ed62e7aa74559afb7199105c1f60b65090a247ee3e6d9aec69c + ${PACKAGE_NAME}_iree-module-5663bccd89aa0616691d5b804b337c8989a2ca94c1bf2673d41dbbab9e511f20 + ${PACKAGE_NAME}_iree-module-353a915204b8eb401d0caac543588920cfab76f826463394cc1e99546cab13bb + ${PACKAGE_NAME}_iree-module-c5f0532455c22436b0fce487797b957939acb0d1f235644187d63bca29a70654 + ${PACKAGE_NAME}_iree-module-1bac11dad2160652ffcfb201d96418f4c044fcd79771cf6a783f311998850a2e + ${PACKAGE_NAME}_iree-module-3edb70502ef7a26db736a64e24a677ff085fb800a8475e750dd2a9d119baf407 + ${PACKAGE_NAME}_iree-module-7f45c4f644d293bb984007aa1a91c919c5a75d01bf38f70ba7257c0b21c654f3 + ${PACKAGE_NAME}_iree-module-bb96ff5237b0ad603b1ee7c59601a1c957a2ed5c95cbfd94f7ec6d707aad3660 + ${PACKAGE_NAME}_iree-module-5b499a05fc60f78b5b11b1e9ba89ba797450e5b49b0613b3c23d996c6f767c64 + ${PACKAGE_NAME}_iree-module-40b0bd95d986d079ae0fc8cd57b473a150289a5f3ee6029b388ebc86edd4a60c + ${PACKAGE_NAME}_iree-module-03736c12afe2b27f77101146eddaa80c8911cdbc2120ab574225c08b088e14f3 + ${PACKAGE_NAME}_iree-module-b180d82d1a87fe29e414ecd55bf8a82ff92b80d7c644d31769729160b27aa467 + ${PACKAGE_NAME}_iree-module-6265e9506389bb29c551ad62f4ea05f54bd0e549c2d29bf425bac9d56aa55ef6 + ${PACKAGE_NAME}_iree-module-1ee530ad72c5711ee3abf11b27eed04e11d261fb6c0d23a6cb68c80c1a42580b + ${PACKAGE_NAME}_iree-module-38e99a06cd01399fe48cd9d860f65474e4fd7df1b3aca491766ff5f825315a79 + ${PACKAGE_NAME}_iree-module-db86dfb7ef5e59b31bf513df65fffd397f49e3fe3b5c03149140ccf3b6b2f821 + ${PACKAGE_NAME}_iree-module-89966a2ac764b543e721d822acdbe015526866e0a8d56a6782f240eeb3cddea1 + ${PACKAGE_NAME}_iree-module-6da0542e65f9e5eff1bc142506866aec435b931b36eb8e86139f79b4945863fd + ${PACKAGE_NAME}_iree-module-f55259e9c9778885d48b20cfb18242984fa9dde9840fa0f92c7de379c88c8230 + ${PACKAGE_NAME}_iree-module-d931d256d015f057a71fb67c6c7d7d4099910286d5bbcd6c1c71b43b2b3756e8 + ${PACKAGE_NAME}_iree-module-999f2edcdf9fbc84e0969923f8605e9069810a63849973a5b74488f83d14a2fe + ${PACKAGE_NAME}_iree-module-5176d0f5fa331fa047395186a866870fe4210f637472ef4bb0a3f1ba3a62a749 + ${PACKAGE_NAME}_iree-module-e67816e321544bb61b63ed8cd8b7faa8ff7d35cca15832b5fbc117f4693b3e78 + ${PACKAGE_NAME}_iree-module-d746ecf3d747f18306b6dea4cb6b9e9dbf987fe7fd4d0b27b39a57a213e75dd9 + ${PACKAGE_NAME}_iree-module-439dab16cd6df449fc83eb3e1603fa86ad811e749bcad2c3e3176976c56848e5 + ${PACKAGE_NAME}_iree-module-815e44d1ac31402e86d0fef72e79474a25dfad3a5c15b8cdd3642e101274342d + ${PACKAGE_NAME}_iree-module-fd0f6049f3cd4fee97c182e824f8e65eeabef47ef86a3797ae278ae1fc101705 + ${PACKAGE_NAME}_iree-module-48ad70710c3602b589431df74468589c74f2a5b54f56d8f86af86c70408ae36d + ${PACKAGE_NAME}_iree-module-07014af4305cf75cd094171519987b1ed1a6a6b70e489d2770301b623b7afe0c + ${PACKAGE_NAME}_iree-module-681f456f27dbb79e7d8d0266bf835866d9f29f87eafad7e867ac13c84602742f + ${PACKAGE_NAME}_iree-module-3b0ae1403ef444d812f0c7b37fda7311e2cc4ea407850ee7b91e6984b9c86100 + ${PACKAGE_NAME}_iree-module-ff26aa2d1b8f0d80a8ab10aab0a87ae89e36c141dc6c758da61c339d51e2db8d + ${PACKAGE_NAME}_iree-module-666922620019ce61f2ffa56e9e141fc3ef2736a04eab3d7c6d3b0fbeeaef4275 + ${PACKAGE_NAME}_iree-module-b36c4a1b93f09dee52b255982410f5b9e8e351010fe22eadb3f5afb6719f6873 + ${PACKAGE_NAME}_iree-module-50c0cb0ea6df2e4c1c1337a607252366fb8cd17ffe72cd8aeb76a9f9a8d6712f +)