✨ Replace keys from benchmarking script (#595)

* Add util to convert single value to tuple

* Update documentation

* Remove unused pytest import

* Address PR comments

* update text in documentation

Co-authored-by: Ashwin Vaidya <ashwinitinvaidya@gmail.com>

anomalib/utils/sweep/config.py

@@ -5,12 +5,48 @@
 
 import itertools
 import operator
+from collections.abc import Iterable, ValuesView
 from functools import reduce
-from typing import Any, Generator, List
+from typing import Any, Generator, List, Tuple
 
 from omegaconf import DictConfig
 
 
+def convert_to_tuple(values: ValuesView) -> List[Tuple]:
+    """Converts a ValuesView object to a list of tuples.
+
+    This is useful to get list of possible values for each parameter in the config and a tuple for values that are
+    are to be patched. Ideally this is useful when used with product.
+
+    Example:
+        >>> params = DictConfig({
+                "dataset.category": [
+                    "bottle",
+                    "cable",
+                ],
+                "dataset.image_size": 224,
+                "model_name": ["padim"],
+            })
+        >>> convert_to_tuple(params.values())
+        [('bottle', 'cable'), (224,), ('padim',)]
+        >>> list(itertools.product(*convert_to_tuple(params.values())))
+        [('bottle', 224, 'padim'), ('cable', 224, 'padim')]
+
+    Args:
+        values: ValuesView: ValuesView object to be converted to a list of tuples.
+
+    Returns:
+        List[Tuple]: List of tuples.
+    """
+    return_list = []
+    for value in values:
+        if isinstance(value, Iterable) and not isinstance(value, str):
+            return_list.append(tuple(value))
+        else:
+            return_list.append((value,))
+    return return_list
+
+
 def flatten_sweep_params(params_dict: DictConfig) -> DictConfig:
     """Flatten the nested parameters section of the config object.
 
@@ -63,21 +99,22 @@ def get_run_config(params_dict: DictConfig) -> Generator[DictConfig, None, None]
                 "child1": ['a', 'b', 'c'],
                 "child2": [1, 2, 3]
             },
-            "parent2":['model1', 'model2']
+            "parent2":['model1', 'model2'],
+            "parent3": 'replacement_value'
         })
         >>> for run_config in get_run_config(dummy_config):
         >>>    print(run_config)
-        {'parent1.child1': 'a', 'parent1.child2': 1, 'parent2': 'model1'}
-        {'parent1.child1': 'a', 'parent1.child2': 1, 'parent2': 'model2'}
-        {'parent1.child1': 'a', 'parent1.child2': 2, 'parent2': 'model1'}
+        {'parent1.child1': 'a', 'parent1.child2': 1, 'parent2': 'model1', 'parent3': 'replacement_value'}
+        {'parent1.child1': 'a', 'parent1.child2': 1, 'parent2': 'model2', 'parent3': 'replacement_value'}
+        {'parent1.child1': 'a', 'parent1.child2': 2, 'parent2': 'model1', 'parent3': 'replacement_value'}
         ...
 
     Yields:
         Generator[DictConfig]: Dictionary containing flattened keys
         and values for current run.
     """
     params = flatten_sweep_params(params_dict)
-    combinations = list(itertools.product(*params.values()))
+    combinations = list(itertools.product(*convert_to_tuple(params.values())))
     keys = params.keys()
     for combination in combinations:
         run_config = DictConfig({})

docs/source/tutorials/benchmarking.rst

@@ -44,6 +44,39 @@ This configuration computes the throughput and performance metrics on CPU and GP
     seed: 0
     image_size: 256
 
+Additionally, it is possible to pass a single value instead of an array for any specific parameter. This will overwrite the parameter in each of the model configs and thereby ensures that the parameter is kept constant between all runs in the sweep. For example, to ensure that the same dataset is used between runs the configuration file can be modified as shown below.
+
+.. code-block:: yaml
+
+  seed: 42
+  compute_openvino: false
+  hardware:
+    - cpu
+    - gpu
+  writer:
+    - comet
+    - wandb
+    - tensorboard
+  grid_search:
+    dataset:
+      name: hazelnut
+      format: folder
+      path: path/hazelnut_toy
+      normal_dir: good # name of the folder containing normal images.
+      abnormal_dir: colour # name of the folder containing abnormal images.
+      normal_test_dir: null
+      task: segmentation # classification or segmentation
+      mask: path/hazelnut_toy/mask/colour
+      extensions: .jpg
+      split_ratio: 0.2
+      category:
+        - colour
+        - crack
+      image_size: [128, 256]
+    model_name:
+      - padim
+      - stfpm
+
 By default, ``compute_openvino`` is set to ``False`` to support instances where OpenVINO requirements are not installed in the environment. Once installed, this flag can be set to ``True`` to get the throughput on OpenVINO optimized models. The ``writer`` parameter is optional and can be set to ``writer: []`` in case the user only requires a csv file without logging to each respective logger. It is a good practice to set a value of seed to ensure reproducibility across runs and thus, is set to a non-zero value by default.
 
 Once a configuration is decided, benchmarking can easily be performed by calling

tests/pre_merge/utils/sweep/__init__.py

@@ -0,0 +1,4 @@
+"""Test sweep utils."""
+
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0

tests/pre_merge/utils/sweep/test_config.py

@@ -0,0 +1,52 @@
+"""Test sweep config utils."""
+
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from omegaconf import DictConfig
+
+from anomalib.utils.sweep.config import get_run_config, set_in_nested_config
+
+
+class TestSweepConfig:
+    def test_get_run_config(self):
+        """Test whether the run config is returned correctly and patches the keys which have only one value."""
+        dummy_config = DictConfig(
+            {
+                "parent1": {"child1": ["a", "b"], "child2": [1, 2]},
+                "parent2": ["model1", "model2"],
+                "parent3": "replacement_value",
+            }
+        )
+        run_config = list(get_run_config(dummy_config))
+        expected_value = [
+            {"parent1.child1": "a", "parent1.child2": 1, "parent2": "model1", "parent3": "replacement_value"},
+            {"parent1.child1": "a", "parent1.child2": 1, "parent2": "model2", "parent3": "replacement_value"},
+            {"parent1.child1": "a", "parent1.child2": 2, "parent2": "model1", "parent3": "replacement_value"},
+            {"parent1.child1": "a", "parent1.child2": 2, "parent2": "model2", "parent3": "replacement_value"},
+            {"parent1.child1": "b", "parent1.child2": 1, "parent2": "model1", "parent3": "replacement_value"},
+            {"parent1.child1": "b", "parent1.child2": 1, "parent2": "model2", "parent3": "replacement_value"},
+            {"parent1.child1": "b", "parent1.child2": 2, "parent2": "model1", "parent3": "replacement_value"},
+            {"parent1.child1": "b", "parent1.child2": 2, "parent2": "model2", "parent3": "replacement_value"},
+        ]
+        assert run_config == expected_value
+
+    def set_in_nested_config(self):
+        dummy_config = DictConfig(
+            {"parent1": {"child1": ["a", "b", "c"], "child2": [1, 2, 3]}, "parent2": ["model1", "model2"]}
+        )
+
+        model_config = DictConfig(
+            {
+                "parent1": {
+                    "child1": "e",
+                    "child2": 4,
+                },
+                "parent3": False,
+            }
+        )
+
+        for run_config in get_run_config(dummy_config):
+            for param in run_config.keys():
+                set_in_nested_config(model_config, param.split("."), run_config[param])
+        assert model_config == {"parent1": {"child1": "a", "child2": 1}, "parent3": False, "parent2": "model1"}