Skip to content

Commit

Permalink
Dump run and compile flags into benchmark JSON config (#12397)
Browse files Browse the repository at this point in the history
  • Loading branch information
Jerry Wu authored Feb 27, 2023
1 parent e6970d8 commit 0246bbf
Show file tree
Hide file tree
Showing 27 changed files with 383 additions and 284 deletions.
2 changes: 1 addition & 1 deletion build_tools/benchmarks/collect_compilation_statistics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def get_module_map_from_generation_config(
model_source=model.source_type.value,
target_arch=f"[{','.join(target_archs)}]",
compile_tags=tuple(compile_config.tags),
gen_config_id=gen_config.composite_id())
gen_config_id=gen_config.composite_id)
module_dir_path = iree_artifacts.get_module_dir_path(
module_generation_config=gen_config, root_path=e2e_test_artifacts_dir)
module_path = module_dir_path / iree_artifacts.MODULE_FILENAME
Expand Down
16 changes: 5 additions & 11 deletions build_tools/benchmarks/collect_compilation_statistics_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,16 +120,11 @@ def test_get_module_map_from_generation_config(self):
target_architecture=common_definitions.DeviceArchitecture.
RV64_GENERIC,
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_abi=iree_definitions.TargetABI.LINUX_GNU),
iree_definitions.CompileTarget(
target_architecture=common_definitions.DeviceArchitecture.
RV32_GENERIC,
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
])
gen_config_a = iree_definitions.ModuleGenerationConfig(
gen_config_a = iree_definitions.ModuleGenerationConfig.with_flag_generation(
imported_model=imported_model_a, compile_config=compile_config_a)
gen_config_b = iree_definitions.ModuleGenerationConfig(
gen_config_b = iree_definitions.ModuleGenerationConfig.with_flag_generation(
imported_model=imported_model_a, compile_config=compile_config_b)
serialized_gen_config = json.dumps(
serialization.serialize_and_pack([gen_config_a, gen_config_b]))
Expand All @@ -145,17 +140,16 @@ def test_get_module_map_from_generation_config(self):
model_source=model_a.source_type.value,
target_arch=f"[cpu-x86_64-cascadelake-linux-gnu]",
compile_tags=tuple(gen_config_a.compile_config.tags),
gen_config_id=gen_config_a.composite_id())
gen_config_id=gen_config_a.composite_id)
module_a_path = iree_artifacts.get_module_dir_path(
gen_config_a, root_dir) / iree_artifacts.MODULE_FILENAME
compile_info_b = common.benchmark_definition.CompilationInfo(
model_name=model_a.name,
model_tags=tuple(model_a.tags),
model_source=model_a.source_type.value,
target_arch=
f"[cpu-riscv_64-generic-linux-gnu,cpu-riscv_32-generic-linux-gnu]",
target_arch=f"[cpu-riscv_64-generic-linux-gnu]",
compile_tags=tuple(gen_config_a.compile_config.tags),
gen_config_id=gen_config_b.composite_id())
gen_config_id=gen_config_b.composite_id)
module_b_path = iree_artifacts.get_module_dir_path(
gen_config_b, root_dir) / iree_artifacts.MODULE_FILENAME
self.assertEqual(module_map, {
Expand Down
2 changes: 1 addition & 1 deletion build_tools/benchmarks/common/benchmark_driver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ def __get_benchmark_info_from_case(
compile_tags=compile_tags,
driver_info=benchmark_case.driver_info,
device_info=self.device_info,
run_config_id=benchmark_case.run_config.composite_id())
run_config_id=benchmark_case.run_config.composite_id)

def __get_available_drivers_and_loaders(
self) -> Tuple[Sequence[str], Sequence[str]]:
Expand Down
34 changes: 19 additions & 15 deletions build_tools/benchmarks/common/benchmark_suite_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,32 +169,36 @@ def test_load_from_run_configs(self):
architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
host_environment=common_definitions.HostEnvironment.LINUX_X86_64,
device_parameters=[])
run_config_a = iree_definitions.E2EModelRunConfig(
module_generation_config=iree_definitions.ModuleGenerationConfig(
compile_target = iree_definitions.CompileTarget(
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=iree_definitions.ModuleGenerationConfig.
with_flag_generation(
imported_model=iree_definitions.ImportedModel.from_model(
model_tflite),
compile_config=iree_definitions.CompileConfig(id="1",
tags=[],
compile_targets=[])),
compile_config=iree_definitions.CompileConfig(
id="1", tags=[], compile_targets=[compile_target])),
module_execution_config=exec_config_a,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
run_config_b = iree_definitions.E2EModelRunConfig(
module_generation_config=iree_definitions.ModuleGenerationConfig(
run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=iree_definitions.ModuleGenerationConfig.
with_flag_generation(
imported_model=iree_definitions.ImportedModel.from_model(
model_tflite),
compile_config=iree_definitions.CompileConfig(id="2",
tags=[],
compile_targets=[])),
compile_config=iree_definitions.CompileConfig(
id="2", tags=[], compile_targets=[compile_target])),
module_execution_config=exec_config_b,
target_device_spec=device_spec_b,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
run_config_c = iree_definitions.E2EModelRunConfig(
module_generation_config=iree_definitions.ModuleGenerationConfig(
run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=iree_definitions.ModuleGenerationConfig.
with_flag_generation(
imported_model=iree_definitions.ImportedModel.from_model(model_tf),
compile_config=iree_definitions.CompileConfig(id="3",
tags=[],
compile_targets=[])),
compile_config=iree_definitions.CompileConfig(
id="3", tags=[], compile_targets=[compile_target])),
module_execution_config=exec_config_a,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
Expand Down
48 changes: 29 additions & 19 deletions build_tools/benchmarks/export_benchmark_config_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,18 @@
source_url="",
entry_function="predict",
input_types=["1xf32"])
COMMON_GEN_CONFIG = iree_definitions.ModuleGenerationConfig(
COMMON_GEN_CONFIG = iree_definitions.ModuleGenerationConfig.with_flag_generation(
imported_model=iree_definitions.ImportedModel.from_model(COMMON_MODEL),
compile_config=iree_definitions.CompileConfig(id="1",
tags=[],
compile_targets=[]))
compile_config=iree_definitions.CompileConfig(
id="1",
tags=[],
compile_targets=[
iree_definitions.CompileTarget(
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_architecture=common_definitions.DeviceArchitecture.
RV64_GENERIC,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
]))
COMMON_EXEC_CONFIG = iree_definitions.ModuleExecutionConfig(
id="exec",
tags=[],
Expand All @@ -48,17 +55,17 @@ def test_filter_and_group_run_configs_set_all_filters(self):
device_name="dev_c",
architecture=common_definitions.DeviceArchitecture.CUDA_SM80,
host_environment=common_definitions.HostEnvironment.LINUX_X86_64)
matched_run_config_a = iree_definitions.E2EModelRunConfig(
matched_run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
unmatched_run_config_b = iree_definitions.E2EModelRunConfig(
unmatched_run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_b,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
matched_run_config_c = iree_definitions.E2EModelRunConfig(
matched_run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_c,
Expand Down Expand Up @@ -96,17 +103,17 @@ def test_filter_and_group_run_configs_include_all(self):
device_name="dev_a_gpu",
architecture=common_definitions.DeviceArchitecture.ADRENO_GENERIC,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
run_config_a = iree_definitions.E2EModelRunConfig(
run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
run_config_b = iree_definitions.E2EModelRunConfig(
run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_b,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
run_config_c = iree_definitions.E2EModelRunConfig(
run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_c,
Expand All @@ -133,12 +140,12 @@ def test_filter_and_group_run_configs_set_target_device_names(self):
device_name="dev_b",
architecture=common_definitions.DeviceArchitecture.VALHALL_MALI,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
run_config_a = iree_definitions.E2EModelRunConfig(
run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
run_config_b = iree_definitions.E2EModelRunConfig(
run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_b,
Expand Down Expand Up @@ -170,13 +177,16 @@ def test_filter_and_group_run_configs_set_preset_matchers(self):
source_url="",
entry_function="predict",
input_types=["1xf32"])
compile_config = iree_definitions.CompileConfig(id="1",
tags=[],
compile_targets=[])
small_gen_config = iree_definitions.ModuleGenerationConfig(
compile_target = iree_definitions.CompileTarget(
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
compile_config = iree_definitions.CompileConfig(
id="1", tags=[], compile_targets=[compile_target])
small_gen_config = iree_definitions.ModuleGenerationConfig.with_flag_generation(
imported_model=iree_definitions.ImportedModel.from_model(small_model),
compile_config=compile_config)
big_gen_config = iree_definitions.ModuleGenerationConfig(
big_gen_config = iree_definitions.ModuleGenerationConfig.with_flag_generation(
imported_model=iree_definitions.ImportedModel.from_model(big_model),
compile_config=compile_config)
device_spec_a = common_definitions.DeviceSpec(
Expand All @@ -189,12 +199,12 @@ def test_filter_and_group_run_configs_set_preset_matchers(self):
device_name="dev_b",
architecture=common_definitions.DeviceArchitecture.VALHALL_MALI,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
run_config_a = iree_definitions.E2EModelRunConfig(
run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=small_gen_config,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
run_config_b = iree_definitions.E2EModelRunConfig(
run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=big_gen_config,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_b,
Expand Down
7 changes: 1 addition & 6 deletions build_tools/benchmarks/run_benchmarks_on_linux.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,7 @@ def __build_tool_cmds(self, benchmark_case: BenchmarkCase,
run_config.module_generation_config,
root_path=self.config.root_benchmark_dir)
cmds += [f"--module={module_dir_path / iree_artifacts.MODULE_FILENAME}"]
cmds += run_module_utils.build_run_flags_for_model(
model=run_config.module_generation_config.imported_model.model,
model_input_data=run_config.input_data)
cmds += run_module_utils.build_run_flags_for_execution_config(
module_execution_config=run_config.module_execution_config,
gpu_id=self.gpu_id)
cmds += run_config.materialize_run_flags(gpu_id=self.gpu_id)

return cmds

Expand Down
6 changes: 3 additions & 3 deletions build_tools/python/benchmark_suites/iree/adreno_benchmarks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,19 @@ def generate(
tflite_models.MOBILENET_V3SMALL,
]
default_gen_configs = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.DEFAULT_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in default_models
]
fuse_padding_gen_configs = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.FUSE_PADDING_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in default_models
]
fuse_padding_repeated_kernel_gen_configs = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.FUSE_PADDING_REPEATED_KERNEL_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in [
Expand Down
6 changes: 3 additions & 3 deletions build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,18 +71,18 @@ def generate(
]

default_gen_confings = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.DEFAULT_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in self.NONQUANT_MODELS + self.QUANT_MODELS
]
experimental_gen_confings = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.MMT4D_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in self.NONQUANT_MODELS
] + [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.MMT4D_AND_DOTPROD_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in self.QUANT_MODELS
Expand Down
2 changes: 1 addition & 1 deletion build_tools/python/benchmark_suites/iree/benchmark_collections.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def generate_benchmarks(
"--iree-llvm-debug-symbols=false"
])
compile_stats_gen_configs.append(
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
imported_model=gen_config.imported_model,
compile_config=compile_stats_config))
all_gen_configs += compile_stats_gen_configs
Expand Down
2 changes: 1 addition & 1 deletion build_tools/python/benchmark_suites/iree/cuda_benchmarks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def generate(
"""Generates IREE compile and run configs."""

gen_configs = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.SM_80_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in model_groups.LARGE
Expand Down
6 changes: 3 additions & 3 deletions build_tools/python/benchmark_suites/iree/mali_benchmarks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,17 +114,17 @@ def _get_module_generation_configs(
extra_flags=compile_config.extra_flags +
["--iree-flow-demote-f32-to-f16"])
return [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=compile_config,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in fp32_models
] + [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=demote_compile_config,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in fp16_models
] + [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=demote_compile_config,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in quant_models
Expand Down
4 changes: 2 additions & 2 deletions build_tools/python/benchmark_suites/iree/riscv_benchmarks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def generate(
List[iree_definitions.E2EModelRunConfig]]:
"""Generates IREE compile and run configs."""
gen_configs = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.DEFAULT_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in self.MODELS
Expand Down Expand Up @@ -68,7 +68,7 @@ def generate(
List[iree_definitions.E2EModelRunConfig]]:
"""Generates IREE compile and run configs."""
gen_configs = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.DEFAULT_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in self.MODELS
Expand Down
2 changes: 1 addition & 1 deletion build_tools/python/benchmark_suites/iree/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def generate_e2e_model_run_configs(
"""Generates the run specs from the product of compile specs and run configs.
"""
return [
iree_definitions.E2EModelRunConfig(
iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=module_generation_config,
module_execution_config=module_execution_config,
target_device_spec=device_spec,
Expand Down
2 changes: 1 addition & 1 deletion build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def generate(
"""Generates IREE compile and run configs."""

gen_configs = [
iree_definitions.ModuleGenerationConfig(
iree_definitions.ModuleGenerationConfig.with_flag_generation(
compile_config=self.DEFAULT_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model)) for
model in [tflite_models.MOBILENET_V2, tflite_models.MOBILENET_V3SMALL]
Expand Down
Loading

0 comments on commit 0246bbf

Please sign in to comment.