✨ Replace keys from benchmarking script

anomalib/utils/sweep/config.py

@@ -5,12 +5,48 @@
 
 import itertools
 import operator
+from collections.abc import Iterable, ValuesView
 from functools import reduce
-from typing import Any, Generator, List
+from typing import Any, Generator, List, Tuple
 
 from omegaconf import DictConfig
 
 
+def convert_to_tuple(values: ValuesView) -> List[Tuple]:
+    """Converts a ValuesView object to a list of tuples.
+
+    This is useful to get list of possible values for each parameter in the config and a tuple for values that are
+    are to be patched. Ideally this is useful when used with product.
+
+    Example:
+        >>> params = DictConfig({
+                "dataset.category": [
+                    "bottle",
+                    "cable",
+                ],
+                "dataset.image_size": 224,
+                "model_name": ["padim"],
+            })
+        >>> convert_to_tuple(params.values())
+        [('bottle', 'cable'), (224,), ('padim',)]
+        >>> list(itertools.product(*convert_to_tuple(params.values())))
+        [('bottle', 224, 'padim'), ('cable', 224, 'padim')]
+
+    Args:
+        values: ValuesView: ValuesView object to be converted to a list of tuples.
+
+    Returns:
+        List[Tuple]: List of tuples.
+    """
+    return_list = []
+    for value in values:
+        if isinstance(value, Iterable) and not isinstance(value, str):
+            return_list.append(tuple(value))
+        else:
+            return_list.append((value,))
+    return return_list
+
+
 def flatten_sweep_params(params_dict: DictConfig) -> DictConfig:
     """Flatten the nested parameters section of the config object.
 
@@ -63,21 +99,22 @@ def get_run_config(params_dict: DictConfig) -> Generator[DictConfig, None, None]
                 "child1": ['a', 'b', 'c'],
                 "child2": [1, 2, 3]
             },
-            "parent2":['model1', 'model2']
+            "parent2":['model1', 'model2'],
+            "parent3": 'replacement_value'
         })
         >>> for run_config in get_run_config(dummy_config):
         >>>    print(run_config)
-        {'parent1.child1': 'a', 'parent1.child2': 1, 'parent2': 'model1'}
-        {'parent1.child1': 'a', 'parent1.child2': 1, 'parent2': 'model2'}
-        {'parent1.child1': 'a', 'parent1.child2': 2, 'parent2': 'model1'}
+        {'parent1.child1': 'a', 'parent1.child2': 1, 'parent2': 'model1', 'parent3': 'replacement_value'}
+        {'parent1.child1': 'a', 'parent1.child2': 1, 'parent2': 'model2', 'parent3': 'replacement_value'}
+        {'parent1.child1': 'a', 'parent1.child2': 2, 'parent2': 'model1', 'parent3': 'replacement_value'}
         ...
 
     Yields:
         Generator[DictConfig]: Dictionary containing flattened keys
         and values for current run.
     """
     params = flatten_sweep_params(params_dict)
-    combinations = list(itertools.product(*params.values()))
+    combinations = list(itertools.product(*convert_to_tuple(params.values())))
     keys = params.keys()
     for combination in combinations:
         run_config = DictConfig({})

docs/source/tutorials/benchmarking.rst

@@ -44,6 +44,39 @@ This configuration computes the throughput and performance metrics on CPU and GP
     seed: 0
     image_size: 256
 
+Additionally, the keys can be replaced from the benchmarking script. To do this just use the value instead of passing an array. Taking the example of the folder dataset above, the configuration file can be modified as.
+
+.. code-block:: yaml
+
+  seed: 42
+  compute_openvino: false
+  hardware:
+    - cpu
+    - gpu
+  writer:
+    - comet
+    - wandb
+    - tensorboard
+  grid_search:
+    dataset:
+      name: hazelnut
+      format: folder
+      path: path/hazelnut_toy
+      normal_dir: good # name of the folder containing normal images.
+      abnormal_dir: colour # name of the folder containing abnormal images.
+      normal_test_dir: null
+      task: segmentation # classification or segmentation
+      mask: path/hazelnut_toy/mask/colour
+      extensions: .jpg
+      split_ratio: 0.2
+      category:
+        - colour
+        - crack
+      image_size: [128, 256]
+    model_name:
+      - padim
+      - stfpm
+
 By default, ``compute_openvino`` is set to ``False`` to support instances where OpenVINO requirements are not installed in the environment. Once installed, this flag can be set to ``True`` to get the throughput on OpenVINO optimized models. The ``writer`` parameter is optional and can be set to ``writer: []`` in case the user only requires a csv file without logging to each respective logger. It is a good practice to set a value of seed to ensure reproducibility across runs and thus, is set to a non-zero value by default.
 
 Once a configuration is decided, benchmarking can easily be performed by calling

tests/pre_merge/utils/sweep/__init__.py

@@ -0,0 +1,4 @@
+"""Test sweep utils."""
+
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0

tests/pre_merge/utils/sweep/test_config.py

@@ -0,0 +1,52 @@
+"""Test sweep config utils."""
+
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from omegaconf import DictConfig
+
+from anomalib.utils.sweep.config import get_run_config, set_in_nested_config
+
+
+class TestSweepConfig:
+    def test_get_run_config(self):
+        """Test whether the run config is returned correctly and patches the keys which have only one value."""
+        dummy_config = DictConfig(
+            {
+                "parent1": {"child1": ["a", "b"], "child2": [1, 2]},
+                "parent2": ["model1", "model2"],
+                "parent3": "replacement_value",
+            }
+        )
+        run_config = list(get_run_config(dummy_config))
+        expected_value = [
+            {"parent1.child1": "a", "parent1.child2": 1, "parent2": "model1", "parent3": "replacement_value"},
+            {"parent1.child1": "a", "parent1.child2": 1, "parent2": "model2", "parent3": "replacement_value"},
+            {"parent1.child1": "a", "parent1.child2": 2, "parent2": "model1", "parent3": "replacement_value"},
+            {"parent1.child1": "a", "parent1.child2": 2, "parent2": "model2", "parent3": "replacement_value"},
+            {"parent1.child1": "b", "parent1.child2": 1, "parent2": "model1", "parent3": "replacement_value"},
+            {"parent1.child1": "b", "parent1.child2": 1, "parent2": "model2", "parent3": "replacement_value"},
+            {"parent1.child1": "b", "parent1.child2": 2, "parent2": "model1", "parent3": "replacement_value"},
+            {"parent1.child1": "b", "parent1.child2": 2, "parent2": "model2", "parent3": "replacement_value"},
+        ]
+        assert run_config == expected_value
+
+    def set_in_nested_config(self):
+        dummy_config = DictConfig(
+            {"parent1": {"child1": ["a", "b", "c"], "child2": [1, 2, 3]}, "parent2": ["model1", "model2"]}
+        )
+
+        model_config = DictConfig(
+            {
+                "parent1": {
+                    "child1": "e",
+                    "child2": 4,
+                },
+                "parent3": False,
+            }
+        )
+
+        for run_config in get_run_config(dummy_config):
+            for param in run_config.keys():
+                set_in_nested_config(model_config, param.split("."), run_config[param])
+        assert model_config == {"parent1": {"child1": "a", "child2": 1}, "parent3": False, "parent2": "model1"}