🏷 Convert adaptive_threshold to Enum in configs (#637)

* Rename image/pixel_metrics_names to image/pixel_metrics

* Rename image/pixel_metrics_names to image/pixel_metrics

* Modified config files.

* Modify get_callbacks function for the old cli

* Create pre-processing-configuration callback

* Add new CLI configuration for the post-processing configuration

* Add options to normalization_method

* Address mypy issues

* Fix docstring

* Address codacy issues

* renamed adaptive: true to threshold_method: adaptive in config.yaml files

* Reorder threshold params in config.yaml

* Add backward compatibility to threshold configs

* renamed the new cli config files

* Rename threshold_method to method in config.yaml

* Convert names to Enum

* Fix tests

* Rename AdaptiveThreshold to AnomalyScoreThreshold

* Fixed import

* Rename fixed to manual

* Rename some left-over variables.

anomalib/config/config.py

@@ -156,7 +156,19 @@ def get_configurable_parameters(
 
     # thresholding
     if "metrics" in config.keys():
-        if "pixel_default" not in config.metrics.threshold.keys():
-            config.metrics.threshold.pixel_default = config.metrics.threshold.image_default
+        # NOTE: Deprecate this after v0.4.0.
+        if "adaptive" in config.metrics.threshold.keys():
+            warn("adaptive will be deprecated in favor of method in config.metrics.threshold in v0.4.0.")
+            config.metrics.threshold.method = "adaptive" if config.metrics.threshold.adaptive else "manual"
+        if "image_default" in config.metrics.threshold.keys():
+            warn("image_default will be deprecated in favor of manual_image in config.metrics.threshold in v0.4.0.")
+            config.metrics.threshold.manual_image = (
+                None if config.metrics.threshold.adaptive else config.metrics.threshold.image_default
+            )
+        if "pixel_default" in config.metrics.threshold.keys():
+            warn("pixel_default will be deprecated in favor of manual_pixel in config.metrics.threshold in v0.4.0.")
+            config.metrics.threshold.manual_pixel = (
+                None if config.metrics.threshold.adaptive else config.metrics.threshold.pixel_default
+            )
 
     return config

anomalib/models/cflow/config.yaml

@@ -43,9 +43,9 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 0
-    pixel_default: 0
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
+    manual_pixel: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/components/base/anomaly_module.py

@@ -13,10 +13,11 @@
 from torch import Tensor, nn
 from torchmetrics import Metric
 
+from anomalib.post_processing import ThresholdMethod
 from anomalib.utils.metrics import (
-    AdaptiveThreshold,
     AnomalibMetricCollection,
     AnomalyScoreDistribution,
+    AnomalyScoreThreshold,
     MinMax,
 )
 
@@ -38,10 +39,9 @@ def __init__(self):
         self.loss: Tensor
         self.callbacks: List[Callback]
 
-        self.adaptive_threshold: bool
-
-        self.image_threshold = AdaptiveThreshold().cpu()
-        self.pixel_threshold = AdaptiveThreshold().cpu()
+        self.threshold_method: ThresholdMethod
+        self.image_threshold = AnomalyScoreThreshold().cpu()
+        self.pixel_threshold = AnomalyScoreThreshold().cpu()
 
         self.normalization_metrics: Metric
 
@@ -115,7 +115,7 @@ def validation_epoch_end(self, outputs):
         Args:
           outputs: Batch of outputs from the validation step
         """
-        if self.adaptive_threshold:
+        if self.threshold_method == ThresholdMethod.ADAPTIVE:
             self._compute_adaptive_threshold(outputs)
         self._collect_outputs(self.image_metrics, self.pixel_metrics, outputs)
         self._log_metrics()

anomalib/models/dfkde/config.yaml

@@ -32,8 +32,8 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 0
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/dfm/config.yaml

@@ -32,8 +32,8 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 0
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/draem/config.yaml

@@ -40,9 +40,9 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 3
-    pixel_default: 3
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
+    manual_pixel: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/fastflow/config.yaml

@@ -43,9 +43,9 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 0
-    pixel_default: 0
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
+    manual_pixel: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/ganomaly/config.yaml

@@ -44,8 +44,8 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 0
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/padim/config.yaml

@@ -38,9 +38,9 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 3
-    pixel_default: 3
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
+    manual_pixel: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/patchcore/config.yaml

@@ -39,9 +39,9 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 0
-    pixel_default: 0
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
+    manual_pixel: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/reverse_distillation/config.yaml

@@ -47,9 +47,9 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 0
-    pixel_default: 0
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
+    manual_pixel: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/models/stfpm/config.yaml

@@ -45,9 +45,9 @@ metrics:
     - F1Score
     - AUROC
   threshold:
-    image_default: 0
-    pixel_default: 0
-    adaptive: true
+    method: adaptive #options: [adaptive, manual]
+    manual_image: null
+    manual_pixel: null
 
 visualization:
   show_images: False # show images on the screen

anomalib/post_processing/__init__.py

@@ -3,7 +3,9 @@
 # Copyright (C) 2022 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+from .normalization import NormalizationMethod
 from .post_process import (
+    ThresholdMethod,
     add_anomalous_label,
     add_normal_label,
     anomaly_map_to_color_map,
@@ -19,5 +21,7 @@
     "superimpose_anomaly_map",
     "compute_mask",
     "ImageResult",
+    "NormalizationMethod",
     "Visualizer",
+    "ThresholdMethod",
 ]

anomalib/post_processing/normalization/__init__.py

@@ -2,3 +2,13 @@
 
 # Copyright (C) 2022 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
+
+from enum import Enum
+
+
+class NormalizationMethod(str, Enum):
+    """Normalization method for normalization."""
+
+    CDF = "cdf"
+    MIN_MAX = "min_max"
+    NONE = "none"

anomalib/post_processing/post_process.py

@@ -5,13 +5,21 @@
 
 
 import math
+from enum import Enum
 from typing import Optional, Tuple
 
 import cv2
 import numpy as np
 from skimage import morphology
 
 
+class ThresholdMethod(str, Enum):
+    """Threshold method to apply post-processing to the output predictions."""
+
+    ADAPTIVE = "adaptive"
+    MANUAL = "manual"
+
+
 def add_label(
     image: np.ndarray,
     label_name: str,

anomalib/utils/callbacks/__init__.py

@@ -69,15 +69,15 @@ def get_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]:
 
     # Add post-processing configurations to AnomalyModule.
     image_threshold = (
-        config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None
+        config.metrics.threshold.manual_image if "manual_image" in config.metrics.threshold.keys() else None
     )
     pixel_threshold = (
-        config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None
+        config.metrics.threshold.manual_pixel if "manual_pixel" in config.metrics.threshold.keys() else None
     )
     post_processing_callback = PostProcessingConfigurationCallback(
-        adaptive_threshold=config.metrics.threshold.adaptive,
-        default_image_threshold=image_threshold,
-        default_pixel_threshold=pixel_threshold,
+        threshold_method=config.metrics.threshold.method,
+        manual_image_threshold=image_threshold,
+        manual_pixel_threshold=pixel_threshold,
     )
     callbacks.append(post_processing_callback)
 

anomalib/utils/callbacks/post_processing_configuration.py

@@ -12,6 +12,7 @@
 from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY
 
 from anomalib.models.components.base.anomaly_module import AnomalyModule
+from anomalib.post_processing import NormalizationMethod, ThresholdMethod
 
 logger = logging.getLogger(__name__)
 
@@ -23,29 +24,41 @@ class PostProcessingConfigurationCallback(Callback):
     """Post-Processing Configuration Callback.
 
     Args:
-        normalization_method(Optional[str]): Normalization method. <None, min_max, cdf>
-        adaptive_threshold (bool): Flag indicating whether threshold should be adaptive.
-        default_image_threshold (Optional[float]): Default image threshold value.
-        default_pixel_threshold (Optional[float]): Default pixel threshold value.
+        normalization_method(NormalizationMethod): Normalization method. <none, min_max, cdf>
+        threshold_method (ThresholdMethod): Flag indicating whether threshold should be manual or adaptive.
+        manual_image_threshold (Optional[float]): Default manual image threshold value.
+        manual_pixel_threshold (Optional[float]): Default manual pixel threshold value.
     """
 
     def __init__(
         self,
-        normalization_method: str = "min_max",
-        adaptive_threshold: bool = True,
-        default_image_threshold: Optional[float] = None,
-        default_pixel_threshold: Optional[float] = None,
+        normalization_method: NormalizationMethod = NormalizationMethod.MIN_MAX,
+        threshold_method: ThresholdMethod = ThresholdMethod.ADAPTIVE,
+        manual_image_threshold: Optional[float] = None,
+        manual_pixel_threshold: Optional[float] = None,
     ) -> None:
         super().__init__()
         self.normalization_method = normalization_method
 
-        assert (
-            adaptive_threshold or default_image_threshold is not None and default_pixel_threshold is not None
-        ), "Default thresholds must be specified when adaptive threshold is disabled."
+        if threshold_method == ThresholdMethod.ADAPTIVE and all(
+            i is not None for i in [manual_image_threshold, manual_pixel_threshold]
+        ):
+            raise ValueError(
+                "When `threshold_method` is set to `adaptive`, `manual_image_threshold` and `manual_pixel_threshold` "
+                "must not be set."
+            )
 
-        self.adaptive_threshold = adaptive_threshold
-        self.default_image_threshold = default_image_threshold
-        self.default_pixel_threshold = default_pixel_threshold
+        if threshold_method == ThresholdMethod.MANUAL and all(
+            i is None for i in [manual_image_threshold, manual_pixel_threshold]
+        ):
+            raise ValueError(
+                "When `threshold_method` is set to `manual`, `manual_image_threshold` and `manual_pixel_threshold` "
+                "must be set."
+            )
+
+        self.threshold_method = threshold_method
+        self.manual_image_threshold = manual_image_threshold
+        self.manual_pixel_threshold = manual_pixel_threshold
 
     # pylint: disable=unused-argument
     def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[str] = None) -> None:
@@ -57,7 +70,7 @@ def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[st
             stage (Optional[str], optional): fit, validate, test or predict. Defaults to None.
         """
         if isinstance(pl_module, AnomalyModule):
-            pl_module.adaptive_threshold = self.adaptive_threshold
-            if pl_module.adaptive_threshold is False:
-                pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu()
-                pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu()
+            pl_module.threshold_method = self.threshold_method
+            if pl_module.threshold_method == ThresholdMethod.MANUAL:
+                pl_module.image_threshold.value = torch.tensor(self.manual_image_threshold).cpu()
+                pl_module.pixel_threshold.value = torch.tensor(self.manual_pixel_threshold).cpu()

anomalib/utils/cli/cli.py

@@ -109,9 +109,9 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None:
         parser.set_defaults(
             {
                 "post_processing.normalization_method": "min_max",
-                "post_processing.adaptive_threshold": True,
-                "post_processing.default_image_threshold": None,
-                "post_processing.default_pixel_threshold": None,
+                "post_processing.threshold_method": "adaptive",
+                "post_processing.manual_image_threshold": None,
+                "post_processing.manual_pixel_threshold": None,
             }
         )
 

anomalib/utils/metrics/__init__.py

@@ -10,8 +10,8 @@
 import torchmetrics
 from omegaconf import DictConfig, ListConfig
 
-from .adaptive_threshold import AdaptiveThreshold
 from .anomaly_score_distribution import AnomalyScoreDistribution
+from .anomaly_score_threshold import AnomalyScoreThreshold
 from .aupr import AUPR
 from .aupro import AUPRO
 from .auroc import AUROC
@@ -20,7 +20,7 @@
 from .optimal_f1 import OptimalF1
 from .pro import PRO
 
-__all__ = ["AUROC", "AUPR", "AUPRO", "OptimalF1", "AdaptiveThreshold", "AnomalyScoreDistribution", "MinMax", "PRO"]
+__all__ = ["AUROC", "AUPR", "AUPRO", "OptimalF1", "AnomalyScoreThreshold", "AnomalyScoreDistribution", "MinMax", "PRO"]
 
 
 def get_metrics(config: Union[ListConfig, DictConfig]) -> Tuple[AnomalibMetricCollection, AnomalibMetricCollection]:

anomalib/utils/metrics/adaptive_threshold.py → anomalib/utils/metrics/anomaly_score_threshold.py

@@ -1,4 +1,4 @@
-"""Implementation of Optimal F1 score based on TorchMetrics."""
+"""Implementation of AnomalyScoreThreshold based on TorchMetrics."""
 
 # Copyright (C) 2022 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
@@ -7,11 +7,16 @@
 from torchmetrics import PrecisionRecallCurve
 
 
-class AdaptiveThreshold(PrecisionRecallCurve):
-    """Optimal F1 Metric.
+class AnomalyScoreThreshold(PrecisionRecallCurve):
+    """Anomaly Score Threshold.
 
-    Compute the optimal F1 score at the adaptive threshold, based on the F1 metric of the true labels and the
-    predicted anomaly scores.
+    This class computes/stores the threshold that determines the anomalous label
+    given anomaly scores. If the threshold method is ``manual``, the class only
+    stores the manual threshold values.
+
+    If the threshold method is ``adaptive``, the class initially computes the
+    adaptive threshold to find the optimal f1_score and stores the computed
+    adaptive threshold value.
     """
 
     def __init__(self, default_value: float = 0.5, **kwargs):