-
Notifications
You must be signed in to change notification settings - Fork 581
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Split benchamrk suite build target into default and long-running #13129
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
## Copyright 2023 The IREE Authors | ||
# | ||
# Licensed under the Apache License v2.0 with LLVM Exceptions. | ||
# See https://llvm.org/LICENSE.txt for license information. | ||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
|
||
# Tag that indicates this compile config is generated for collecting compilation | ||
# statistics. | ||
COMPILE_STATS = "compile-stats" | ||
|
||
# Tag for long-running benchmarks. | ||
LONG_RUNNING = "long-running" | ||
|
||
# Tag for CUDA benchamrks. | ||
CUDA = "cuda" | ||
|
||
# Tag for Vulkan NVIDIA benchamrks. | ||
VULKAN_NVIDIA = "vulkan-nvidia" |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,7 +6,7 @@ | |
"""Defines IREE CUDA benchmarks.""" | ||
|
||
from typing import List, Tuple, Sequence | ||
from benchmark_suites.iree import module_execution_configs | ||
from benchmark_suites.iree import benchmark_tags, module_execution_configs | ||
from e2e_test_framework import unique_ids | ||
from e2e_test_framework.definitions import common_definitions, iree_definitions | ||
from e2e_test_framework.device_specs import device_collections | ||
|
@@ -46,14 +46,14 @@ def _generate_configs( | |
compile_config: iree_definitions.CompileConfig, | ||
execution_config: iree_definitions. | ||
ModuleExecutionConfig = module_execution_configs.CUDA_CONFIG, | ||
run_tags: Sequence[str] = [], | ||
tags: Sequence[str] = (), | ||
) -> Tuple[List[iree_definitions.ModuleGenerationConfig], | ||
List[iree_definitions.E2EModelRunConfig]]: | ||
gen_configs = [ | ||
iree_definitions.ModuleGenerationConfig.build( | ||
compile_config=compile_config, | ||
imported_model=iree_definitions.ImportedModel.from_model(model)) | ||
for model in models | ||
imported_model=iree_definitions.ImportedModel.from_model(model), | ||
tags=tags) for model in models | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmmm but that means each model only belongs to exactly one preset There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The presets can be a list, so each benchmark can still belong to multiple presets, if needed |
||
] | ||
sm80_devices = device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( | ||
architecture=common_definitions.DeviceArchitecture.NVIDIA_AMPERE, | ||
|
@@ -62,7 +62,7 @@ def _generate_configs( | |
module_generation_configs=gen_configs, | ||
module_execution_configs=[execution_config], | ||
device_specs=sm80_devices, | ||
tags=run_tags) | ||
tags=tags) | ||
|
||
return (gen_configs, run_module_configs) | ||
|
||
|
@@ -71,24 +71,25 @@ def generate( | |
) -> Tuple[List[iree_definitions.ModuleGenerationConfig], | ||
List[iree_definitions.E2EModelRunConfig]]: | ||
"""Generates IREE compile and run configs.""" | ||
# The `cuda` tag is required to put them into the CUDA benchmark preset. | ||
gen_configs, run_configs = self._generate_configs(model_groups.CUDA_MODELS, | ||
self.SM_80_COMPILE_CONFIG, | ||
run_tags=["cuda"]) | ||
# The CUDA tag is required to put them into the CUDA benchmark preset. | ||
gen_configs, run_configs = self._generate_configs( | ||
model_groups.CUDA_MODELS, | ||
self.SM_80_COMPILE_CONFIG, | ||
tags=[benchmark_tags.CUDA]) | ||
ubench_gen_configs, ubench_run_configs = self._generate_configs( | ||
model_groups.MICRO_MATMUL, | ||
self.SM_80_UBENCH_MATMUL_COMPILE_CONFIG, | ||
execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG, | ||
run_tags=["cuda"]) | ||
tags=[benchmark_tags.CUDA]) | ||
ubench_splitk_gen_configs, ubench_splitk_run_configs = self._generate_configs( | ||
model_groups.MICRO_MATMUL_SPLITK, | ||
self.SM_80_UBENCH_MATMUL_SPLITK_COMPILE_CONFIG, | ||
execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG, | ||
run_tags=["cuda"]) | ||
tags=[benchmark_tags.CUDA]) | ||
long_running_gen_configs, long_running_module_configs = self._generate_configs( | ||
model_groups.CUDA_MODELS_LONG, | ||
self.SM_80_COMPILE_CONFIG, | ||
run_tags=["cuda", "long-running"]) | ||
tags=[benchmark_tags.CUDA, benchmark_tags.LONG_RUNNING]) | ||
return (gen_configs + ubench_gen_configs + ubench_splitk_gen_configs + | ||
long_running_gen_configs, run_configs + ubench_run_configs + | ||
ubench_splitk_run_configs + long_running_module_configs) |
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -6,7 +6,7 @@ | |||||
"""Defines IREE Vulkan NVIDIA benchmarks.""" | ||||||
|
||||||
from typing import List, Tuple, Sequence | ||||||
from benchmark_suites.iree import module_execution_configs | ||||||
from benchmark_suites.iree import benchmark_tags, module_execution_configs | ||||||
from e2e_test_framework import unique_ids | ||||||
from e2e_test_framework.definitions import common_definitions, iree_definitions | ||||||
from e2e_test_framework.device_specs import device_collections | ||||||
|
@@ -58,14 +58,14 @@ def _generate_configs( | |||||
compile_config: iree_definitions.CompileConfig, | ||||||
execution_config: iree_definitions. | ||||||
ModuleExecutionConfig = module_execution_configs.VULKAN_CONFIG, | ||||||
run_tags: Sequence[str] = [], | ||||||
tags: Sequence[str] = [], | ||||||
) -> Tuple[List[iree_definitions.ModuleGenerationConfig], | ||||||
List[iree_definitions.E2EModelRunConfig]]: | ||||||
gen_configs = [ | ||||||
iree_definitions.ModuleGenerationConfig.build( | ||||||
compile_config=compile_config, | ||||||
imported_model=iree_definitions.ImportedModel.from_model(model)) | ||||||
for model in models | ||||||
imported_model=iree_definitions.ImportedModel.from_model(model), | ||||||
tags=tags) for model in models | ||||||
] | ||||||
# We use the same NVIDIA Ampere GPU for benchmarking code generated for | ||||||
# both Pascal and Ampere architectures. What we care is not exactly these | ||||||
|
@@ -79,7 +79,7 @@ def _generate_configs( | |||||
module_generation_configs=gen_configs, | ||||||
module_execution_configs=[execution_config], | ||||||
device_specs=ampere_devices, | ||||||
tags=run_tags) | ||||||
tags=tags) | ||||||
|
||||||
return (gen_configs, run_module_configs) | ||||||
|
||||||
|
@@ -88,15 +88,15 @@ def generate( | |||||
) -> Tuple[List[iree_definitions.ModuleGenerationConfig], | ||||||
List[iree_definitions.E2EModelRunConfig]]: | ||||||
"""Generates IREE compile and run configs.""" | ||||||
# The `vulkan-nvidia` tag is required to put them into the Vulkan NVIDIA | ||||||
# The `vulkan-nvidia`` tag is required to put them into the Vulkan NVIDIA | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Revert extra backtick? |
||||||
# benchmark preset. | ||||||
tensorcore_gen_configs, tensorcore_run_configs = self._generate_configs( | ||||||
model_groups.VULKAN_MODELS, | ||||||
self.TENSORCORE_COMPILE_CONFIG, | ||||||
run_tags=["vulkan-nvidia"]) | ||||||
tags=[benchmark_tags.VULKAN_NVIDIA]) | ||||||
simt_gen_configs, simt_run_configs = self._generate_configs( | ||||||
model_groups.VULKAN_MODELS, | ||||||
self.SIMT_COMPILE_CONFIG, | ||||||
run_tags=["vulkan-nvidia"]) | ||||||
tags=[benchmark_tags.VULKAN_NVIDIA]) | ||||||
return (tensorcore_gen_configs + simt_gen_configs, | ||||||
tensorcore_run_configs + simt_run_configs) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Especially when speaking about compilation of the models, is "long running" really the right name? That to me implies execution time. It seems like "large" would be simpler, shorter, and more accurate
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Talked to @mariecwhite, I think we can rename this to
large
. I can do in a follow-up change.