Skip to content

Commit

Permalink
Give names to configs in e2e test framework (iree-org#12649)
Browse files Browse the repository at this point in the history
  • Loading branch information
@NatashaKnk
Jerry Wu authored and NatashaKnk committed Jul 6, 2023
1 parent a8e68a0 commit 0fcf874
Show file tree
Hide file tree
Showing 24 changed files with 313 additions and 174 deletions.
8 changes: 4 additions & 4 deletions build_tools/benchmarks/collect_compilation_statistics_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def test_get_module_map_from_compilation_benchmark_config(self):
entry_function="main",
input_types=["1xf32"])
imported_model_a = iree_definitions.ImportedModel.from_model(model_a)
compile_config_a = iree_definitions.CompileConfig(
compile_config_a = iree_definitions.CompileConfig.build(
id="config_a",
tags=["defaults"],
compile_targets=[
Expand All @@ -112,7 +112,7 @@ def test_get_module_map_from_compilation_benchmark_config(self):
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
])
compile_config_b = iree_definitions.CompileConfig(
compile_config_b = iree_definitions.CompileConfig.build(
id="config_b",
tags=["defaults"],
compile_targets=[
Expand All @@ -122,9 +122,9 @@ def test_get_module_map_from_compilation_benchmark_config(self):
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
])
gen_config_a = iree_definitions.ModuleGenerationConfig.with_flag_generation(
gen_config_a = iree_definitions.ModuleGenerationConfig.build(
imported_model=imported_model_a, compile_config=compile_config_a)
gen_config_b = iree_definitions.ModuleGenerationConfig.with_flag_generation(
gen_config_b = iree_definitions.ModuleGenerationConfig.build(
imported_model=imported_model_a, compile_config=compile_config_b)
benchmark_config = dict(generation_configs=serialization.serialize_and_pack(
[gen_config_a, gen_config_b]),
Expand Down
35 changes: 17 additions & 18 deletions build_tools/benchmarks/common/benchmark_suite_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,59 +147,58 @@ def test_load_from_run_configs(self):
source_url="",
entry_function="predict",
input_types=["1xf32"])
exec_config_a = iree_definitions.ModuleExecutionConfig(
exec_config_a = iree_definitions.ModuleExecutionConfig.build(
id="exec_a",
tags=["defaults"],
loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF,
driver=iree_definitions.RuntimeDriver.LOCAL_SYNC)
exec_config_b = iree_definitions.ModuleExecutionConfig(
exec_config_b = iree_definitions.ModuleExecutionConfig.build(
id="exec_b",
tags=["experimental"],
loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF,
driver=iree_definitions.RuntimeDriver.LOCAL_TASK)
device_spec_a = common_definitions.DeviceSpec(
device_spec_a = common_definitions.DeviceSpec.build(
id="dev_a",
device_name="a",
architecture=common_definitions.DeviceArchitecture.RV32_GENERIC,
host_environment=common_definitions.HostEnvironment.LINUX_X86_64,
device_parameters=[])
device_spec_b = common_definitions.DeviceSpec(
device_parameters=[],
tags=[])
device_spec_b = common_definitions.DeviceSpec.build(
id="dev_b",
device_name="b",
architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
host_environment=common_definitions.HostEnvironment.LINUX_X86_64,
device_parameters=[])
device_parameters=[],
tags=[])
compile_target = iree_definitions.CompileTarget(
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=iree_definitions.ModuleGenerationConfig.
with_flag_generation(
run_config_a = iree_definitions.E2EModelRunConfig.build(
module_generation_config=iree_definitions.ModuleGenerationConfig.build(
imported_model=iree_definitions.ImportedModel.from_model(
model_tflite),
compile_config=iree_definitions.CompileConfig(
compile_config=iree_definitions.CompileConfig.build(
id="1", tags=[], compile_targets=[compile_target])),
module_execution_config=exec_config_a,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=iree_definitions.ModuleGenerationConfig.
with_flag_generation(
run_config_b = iree_definitions.E2EModelRunConfig.build(
module_generation_config=iree_definitions.ModuleGenerationConfig.build(
imported_model=iree_definitions.ImportedModel.from_model(
model_tflite),
compile_config=iree_definitions.CompileConfig(
compile_config=iree_definitions.CompileConfig.build(
id="2", tags=[], compile_targets=[compile_target])),
module_execution_config=exec_config_b,
target_device_spec=device_spec_b,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation(
module_generation_config=iree_definitions.ModuleGenerationConfig.
with_flag_generation(
run_config_c = iree_definitions.E2EModelRunConfig.build(
module_generation_config=iree_definitions.ModuleGenerationConfig.build(
imported_model=iree_definitions.ImportedModel.from_model(model_tf),
compile_config=iree_definitions.CompileConfig(
compile_config=iree_definitions.CompileConfig.build(
id="3", tags=[], compile_targets=[compile_target])),
module_execution_config=exec_config_a,
target_device_spec=device_spec_a,
Expand Down
82 changes: 46 additions & 36 deletions build_tools/benchmarks/export_benchmark_config_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
source_url="",
entry_function="predict",
input_types=["1xf32"])
COMMON_GEN_CONFIG = iree_definitions.ModuleGenerationConfig.with_flag_generation(
COMMON_GEN_CONFIG = iree_definitions.ModuleGenerationConfig.build(
imported_model=iree_definitions.ImportedModel.from_model(COMMON_MODEL),
compile_config=iree_definitions.CompileConfig(
compile_config=iree_definitions.CompileConfig.build(
id="1",
tags=[],
compile_targets=[
Expand All @@ -30,7 +30,7 @@
RV64_GENERIC,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
]))
COMMON_EXEC_CONFIG = iree_definitions.ModuleExecutionConfig(
COMMON_EXEC_CONFIG = iree_definitions.ModuleExecutionConfig.build(
id="exec",
tags=[],
loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF,
Expand All @@ -40,34 +40,37 @@
class ExportBenchmarkConfigTest(unittest.TestCase):

def test_filter_and_group_run_configs_set_all_filters(self):
device_spec_a = common_definitions.DeviceSpec(
device_spec_a = common_definitions.DeviceSpec.build(
id="dev_a_cpu",
device_name="dev_a_cpu",
architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
device_spec_b = common_definitions.DeviceSpec(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
device_spec_b = common_definitions.DeviceSpec.build(
id="dev_a_gpu",
device_name="dev_a_gpu",
architecture=common_definitions.DeviceArchitecture.VALHALL_MALI,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
device_spec_c = common_definitions.DeviceSpec(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
device_spec_c = common_definitions.DeviceSpec.build(
id="dev_c",
device_name="dev_c",
architecture=common_definitions.DeviceArchitecture.CUDA_SM80,
host_environment=common_definitions.HostEnvironment.LINUX_X86_64)
matched_run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
host_environment=common_definitions.HostEnvironment.LINUX_X86_64,
tags=[])
matched_run_config_a = iree_definitions.E2EModelRunConfig.build(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
unmatched_run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
unmatched_run_config_b = iree_definitions.E2EModelRunConfig.build(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_b,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
matched_run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation(
matched_run_config_c = iree_definitions.E2EModelRunConfig.build(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_c,
Expand All @@ -91,34 +94,37 @@ def test_filter_and_group_run_configs_set_all_filters(self):
})

def test_filter_and_group_run_configs_include_all(self):
device_spec_a = common_definitions.DeviceSpec(
device_spec_a = common_definitions.DeviceSpec.build(
id="dev_a_cpu",
device_name="dev_a_cpu",
architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
device_spec_b = common_definitions.DeviceSpec(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
device_spec_b = common_definitions.DeviceSpec.build(
id="dev_a_gpu",
device_name="dev_a_gpu",
architecture=common_definitions.DeviceArchitecture.VALHALL_MALI,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
device_spec_c = common_definitions.DeviceSpec(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
device_spec_c = common_definitions.DeviceSpec.build(
id="dev_a_second_gpu",
device_name="dev_a_gpu",
architecture=common_definitions.DeviceArchitecture.ADRENO_GENERIC,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
run_config_a = iree_definitions.E2EModelRunConfig.build(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
run_config_b = iree_definitions.E2EModelRunConfig.build(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_b,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation(
run_config_c = iree_definitions.E2EModelRunConfig.build(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_c,
Expand All @@ -136,23 +142,25 @@ def test_filter_and_group_run_configs_include_all(self):
})

def test_filter_and_group_run_configs_set_target_device_names(self):
device_spec_a = common_definitions.DeviceSpec(
device_spec_a = common_definitions.DeviceSpec.build(
id="dev_a",
device_name="dev_a",
architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
device_spec_b = common_definitions.DeviceSpec(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
device_spec_b = common_definitions.DeviceSpec.build(
id="dev_b",
device_name="dev_b",
architecture=common_definitions.DeviceArchitecture.VALHALL_MALI,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
run_config_a = iree_definitions.E2EModelRunConfig.build(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
run_config_b = iree_definitions.E2EModelRunConfig.build(
module_generation_config=COMMON_GEN_CONFIG,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_b,
Expand Down Expand Up @@ -189,31 +197,33 @@ def test_filter_and_group_run_configs_set_preset_matchers(self):
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
target_abi=iree_definitions.TargetABI.LINUX_GNU)
compile_config = iree_definitions.CompileConfig(
compile_config = iree_definitions.CompileConfig.build(
id="1", tags=[], compile_targets=[compile_target])
small_gen_config = iree_definitions.ModuleGenerationConfig.with_flag_generation(
small_gen_config = iree_definitions.ModuleGenerationConfig.build(
imported_model=iree_definitions.ImportedModel.from_model(small_model),
compile_config=compile_config)
big_gen_config = iree_definitions.ModuleGenerationConfig.with_flag_generation(
big_gen_config = iree_definitions.ModuleGenerationConfig.build(
imported_model=iree_definitions.ImportedModel.from_model(big_model),
compile_config=compile_config)
device_spec_a = common_definitions.DeviceSpec(
device_spec_a = common_definitions.DeviceSpec.build(
id="dev_a",
device_name="dev_a",
architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
device_spec_b = common_definitions.DeviceSpec(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
device_spec_b = common_definitions.DeviceSpec.build(
id="dev_b",
device_name="dev_b",
architecture=common_definitions.DeviceArchitecture.VALHALL_MALI,
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A)
run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation(
host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
tags=[])
run_config_a = iree_definitions.E2EModelRunConfig.build(
module_generation_config=small_gen_config,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_a,
input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation(
run_config_b = iree_definitions.E2EModelRunConfig.build(
module_generation_config=big_gen_config,
module_execution_config=COMMON_EXEC_CONFIG,
target_device_spec=device_spec_b,
Expand Down
12 changes: 6 additions & 6 deletions build_tools/python/benchmark_suites/iree/adreno_benchmarks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,16 @@ class Android_Adreno_Benchmarks(object):
target_backend=iree_definitions.TargetBackend.VULKAN_SPIRV,
target_architecture=common_definitions.DeviceArchitecture.ADRENO_GENERIC,
target_abi=iree_definitions.TargetABI.VULKAN_ANDROID31)
DEFAULT_COMPILE_CONFIG = iree_definitions.CompileConfig(
DEFAULT_COMPILE_CONFIG = iree_definitions.CompileConfig.build(
id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_ADRENO_GENERIC_DEFAULTS,
tags=["default-flags"],
compile_targets=[ADRENO_GPU_COMPILE_TARGET])
FUSE_PADDING_COMPILE_CONFIG = iree_definitions.CompileConfig(
FUSE_PADDING_COMPILE_CONFIG = iree_definitions.CompileConfig.build(
id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_ADRENO_GENERIC_FUSE_PADDING,
tags=["experimental-flags", "fuse-padding"],
compile_targets=[ADRENO_GPU_COMPILE_TARGET],
extra_flags=["--iree-flow-enable-fuse-padding-into-linalg-consumer-ops"])
FUSE_PADDING_REPEATED_KERNEL_COMPILE_CONFIG = iree_definitions.CompileConfig(
FUSE_PADDING_REPEATED_KERNEL_COMPILE_CONFIG = iree_definitions.CompileConfig.build(
id=unique_ids.
IREE_COMPILE_CONFIG_ANDROID_ADRENO_GENERIC_FUSE_PADDING_REPEATED_KERNEL,
tags=["experimental-flags", "fuse-padding", "repeated-kernel"],
Expand All @@ -51,19 +51,19 @@ def generate(
tflite_models.MOBILENET_V3SMALL,
]
default_gen_configs = [
iree_definitions.ModuleGenerationConfig.with_flag_generation(
iree_definitions.ModuleGenerationConfig.build(
compile_config=self.DEFAULT_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in default_models
]
fuse_padding_gen_configs = [
iree_definitions.ModuleGenerationConfig.with_flag_generation(
iree_definitions.ModuleGenerationConfig.build(
compile_config=self.FUSE_PADDING_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in default_models
]
fuse_padding_repeated_kernel_gen_configs = [
iree_definitions.ModuleGenerationConfig.with_flag_generation(
iree_definitions.ModuleGenerationConfig.build(
compile_config=self.FUSE_PADDING_REPEATED_KERNEL_COMPILE_CONFIG,
imported_model=iree_definitions.ImportedModel.from_model(model))
for model in [
Expand Down
Loading

0 comments on commit 0fcf874

Please sign in to comment.