From 80f47b445a2f1a4aa30d8aa3ff0736d67d6475f5 Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 27 Sep 2022 08:03:42 -0700 Subject: [PATCH 01/21] Rename image/pixel_metrics_names to image/pixel_metrics --- anomalib/utils/callbacks/metrics_configuration.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/anomalib/utils/callbacks/metrics_configuration.py b/anomalib/utils/callbacks/metrics_configuration.py index bc91c23e83..7890a00579 100644 --- a/anomalib/utils/callbacks/metrics_configuration.py +++ b/anomalib/utils/callbacks/metrics_configuration.py @@ -30,8 +30,8 @@ def __init__( task: str = "segmentation", default_image_threshold: Optional[float] = None, default_pixel_threshold: Optional[float] = None, - image_metric_names: Optional[List[str]] = None, - pixel_metric_names: Optional[List[str]] = None, + image_metrics: Optional[List[str]] = None, + pixel_metrics: Optional[List[str]] = None, normalization_method: str = "min_max", ): """Create image and pixel-level AnomalibMetricsCollection. @@ -46,14 +46,14 @@ def __init__( adaptive_threshold (bool): Flag indicating whether threshold should be adaptive. default_image_threshold (Optional[float]): Default image threshold value. default_pixel_threshold (Optional[float]): Default pixel threshold value. - image_metric_names (Optional[List[str]]): List of image-level metrics. - pixel_metric_names (Optional[List[str]]): List of pixel-level metrics. + image_metrics (Optional[List[str]]): List of image-level metrics. + pixel_metrics (Optional[List[str]]): List of pixel-level metrics. normalization_method(Optional[str]): Normalization method. """ # TODO: https://github.com/openvinotoolkit/anomalib/issues/384 self.task = task - self.image_metric_names = image_metric_names - self.pixel_metric_names = pixel_metric_names + self.image_metric_names = image_metrics + self.pixel_metric_names = pixel_metrics # TODO: https://github.com/openvinotoolkit/anomalib/issues/384 # TODO: This is a workaround. normalization-method is actually not used in metrics. From 803941361ab188e254a44fccc73baa28d827edbb Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 27 Sep 2022 08:04:04 -0700 Subject: [PATCH 02/21] Rename image/pixel_metrics_names to image/pixel_metrics --- configs/model/cflow.yaml | 4 ++-- configs/model/dfkde.yaml | 2 +- configs/model/dfm.yaml | 4 ++-- configs/model/draem.yaml | 4 ++-- configs/model/fastflow.yaml | 4 ++-- configs/model/ganomaly.yaml | 2 +- configs/model/padim.yaml | 4 ++-- configs/model/patchcore.yaml | 4 ++-- configs/model/reverse_distillation.yaml | 4 ++-- configs/model/stfpm.yaml | 4 ++-- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/configs/model/cflow.yaml b/configs/model/cflow.yaml index 2f925071e1..40148b1bf6 100644 --- a/configs/model/cflow.yaml +++ b/configs/model/cflow.yaml @@ -41,10 +41,10 @@ metrics: adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC normalization_method: min_max diff --git a/configs/model/dfkde.yaml b/configs/model/dfkde.yaml index 4a43a344b9..fbfe42f371 100644 --- a/configs/model/dfkde.yaml +++ b/configs/model/dfkde.yaml @@ -35,7 +35,7 @@ model: metrics: adaptive_threshold: true default_image_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC diff --git a/configs/model/dfm.yaml b/configs/model/dfm.yaml index 2670510a16..6096d0148e 100644 --- a/configs/model/dfm.yaml +++ b/configs/model/dfm.yaml @@ -33,10 +33,10 @@ metrics: adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC normalization_method: min_max diff --git a/configs/model/draem.yaml b/configs/model/draem.yaml index 78d9defd8f..02016166fb 100644 --- a/configs/model/draem.yaml +++ b/configs/model/draem.yaml @@ -33,10 +33,10 @@ metrics: adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC normalization_method: min_max diff --git a/configs/model/fastflow.yaml b/configs/model/fastflow.yaml index 1a2217ab6f..c4f1c78e00 100644 --- a/configs/model/fastflow.yaml +++ b/configs/model/fastflow.yaml @@ -39,10 +39,10 @@ metrics: adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC normalization_method: min_max diff --git a/configs/model/ganomaly.yaml b/configs/model/ganomaly.yaml index 23ca106244..be0b4e7099 100644 --- a/configs/model/ganomaly.yaml +++ b/configs/model/ganomaly.yaml @@ -38,7 +38,7 @@ model: metrics: adaptive_threshold: true default_image_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC diff --git a/configs/model/padim.yaml b/configs/model/padim.yaml index 6b9871feb4..19c2e94c4b 100644 --- a/configs/model/padim.yaml +++ b/configs/model/padim.yaml @@ -36,10 +36,10 @@ metrics: adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC normalization_method: min_max diff --git a/configs/model/patchcore.yaml b/configs/model/patchcore.yaml index d357b06aca..ac84c01865 100644 --- a/configs/model/patchcore.yaml +++ b/configs/model/patchcore.yaml @@ -36,10 +36,10 @@ metrics: adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC normalization_method: min_max diff --git a/configs/model/reverse_distillation.yaml b/configs/model/reverse_distillation.yaml index f1276bcdd7..b29bf13b0a 100644 --- a/configs/model/reverse_distillation.yaml +++ b/configs/model/reverse_distillation.yaml @@ -39,10 +39,10 @@ metrics: adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC normalization_method: min_max diff --git a/configs/model/stfpm.yaml b/configs/model/stfpm.yaml index bd5ea0be43..a6b8ff7cc1 100644 --- a/configs/model/stfpm.yaml +++ b/configs/model/stfpm.yaml @@ -41,10 +41,10 @@ metrics: adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC normalization_method: min_max From 2e6b831912b9ac9e0d25741f9d5576e9550c207f Mon Sep 17 00:00:00 2001 From: Samet Date: Sun, 2 Oct 2022 23:31:59 -0700 Subject: [PATCH 03/21] Modified config files. --- configs/model/cflow.yaml | 8 +++++--- configs/model/dfkde.yaml | 8 ++++++-- configs/model/dfm.yaml | 8 +++++--- configs/model/draem.yaml | 8 +++++--- configs/model/fastflow.yaml | 8 +++++--- configs/model/ganomaly.yaml | 8 +++++++- configs/model/padim.yaml | 8 +++++--- configs/model/patchcore.yaml | 8 +++++--- configs/model/reverse_distillation.yaml | 8 +++++--- configs/model/stfpm.yaml | 8 +++++--- 10 files changed, 53 insertions(+), 27 deletions(-) diff --git a/configs/model/cflow.yaml b/configs/model/cflow.yaml index 40148b1bf6..eb31c9badf 100644 --- a/configs/model/cflow.yaml +++ b/configs/model/cflow.yaml @@ -37,21 +37,23 @@ model: permute_soft: false lr: 0.0001 -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/dfkde.yaml b/configs/model/dfkde.yaml index fbfe42f371..d9c946b8a9 100644 --- a/configs/model/dfkde.yaml +++ b/configs/model/dfkde.yaml @@ -32,16 +32,20 @@ model: threshold_steepness: 0.05 threshold_offset: 12 -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null + default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/dfm.yaml b/configs/model/dfm.yaml index 6096d0148e..4e1a7ce657 100644 --- a/configs/model/dfm.yaml +++ b/configs/model/dfm.yaml @@ -29,21 +29,23 @@ model: pca_level: 0.97 score_type: fre -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/draem.yaml b/configs/model/draem.yaml index 02016166fb..5dd083459d 100644 --- a/configs/model/draem.yaml +++ b/configs/model/draem.yaml @@ -29,21 +29,23 @@ optimizer: init_args: lr: 0.0001 -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/fastflow.yaml b/configs/model/fastflow.yaml index c4f1c78e00..39801e3acd 100644 --- a/configs/model/fastflow.yaml +++ b/configs/model/fastflow.yaml @@ -35,21 +35,23 @@ optimizer: lr: 0.001 weight_decay: 0.00001 -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/ganomaly.yaml b/configs/model/ganomaly.yaml index be0b4e7099..cfa8d291a6 100644 --- a/configs/model/ganomaly.yaml +++ b/configs/model/ganomaly.yaml @@ -35,6 +35,12 @@ model: beta1: 0.5 beta2: 0.999 +post_processing: + normalization_method: min_max + adaptive_threshold: true + default_image_threshold: null + default_pixel_threshold: null + metrics: adaptive_threshold: true default_image_threshold: null @@ -44,7 +50,7 @@ metrics: visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/padim.yaml b/configs/model/padim.yaml index 19c2e94c4b..ac21d1b005 100644 --- a/configs/model/padim.yaml +++ b/configs/model/padim.yaml @@ -32,21 +32,23 @@ model: - layer2 - layer3 -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/patchcore.yaml b/configs/model/patchcore.yaml index ac84c01865..65a0db3d50 100644 --- a/configs/model/patchcore.yaml +++ b/configs/model/patchcore.yaml @@ -32,21 +32,23 @@ model: coreset_sampling_ratio: 0.1 num_neighbors: 9 -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/reverse_distillation.yaml b/configs/model/reverse_distillation.yaml index b29bf13b0a..b36f08c2f1 100644 --- a/configs/model/reverse_distillation.yaml +++ b/configs/model/reverse_distillation.yaml @@ -35,21 +35,23 @@ model: beta1: 0.5 beta2: 0.99 -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/stfpm.yaml b/configs/model/stfpm.yaml index a6b8ff7cc1..5b17689cb5 100644 --- a/configs/model/stfpm.yaml +++ b/configs/model/stfpm.yaml @@ -37,21 +37,23 @@ optimizer: momentum: 0.9 weight_decay: 0.0001 -metrics: +post_processing: + normalization_method: min_max adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null + +metrics: image_metrics: - F1Score - AUROC pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] From 8dce79e0406a0010a776764edc424a68a82e6554 Mon Sep 17 00:00:00 2001 From: Samet Date: Sun, 2 Oct 2022 23:32:47 -0700 Subject: [PATCH 04/21] Modify get_callbacks function for the old cli --- anomalib/utils/callbacks/__init__.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/anomalib/utils/callbacks/__init__.py b/anomalib/utils/callbacks/__init__.py index 270648fb9d..15def2a1f5 100644 --- a/anomalib/utils/callbacks/__init__.py +++ b/anomalib/utils/callbacks/__init__.py @@ -19,19 +19,22 @@ from .metrics_configuration import MetricsConfigurationCallback from .min_max_normalization import MinMaxNormalizationCallback from .model_loader import LoadModelCallback +from .post_processing_configuration import PostProcessingConfigurationCallback from .tiler_configuration import TilerConfigurationCallback from .timer import TimerCallback from .visualizer import ImageVisualizerCallback, MetricVisualizerCallback __all__ = [ "CdfNormalizationCallback", + "GraphLogger", + "ImageVisualizerCallback", "LoadModelCallback", "MetricsConfigurationCallback", + "MetricVisualizerCallback", "MinMaxNormalizationCallback", + "PostProcessingConfigurationCallback", "TilerConfigurationCallback", "TimerCallback", - "ImageVisualizerCallback", - "MetricVisualizerCallback", ] @@ -64,20 +67,25 @@ def get_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]: callbacks.extend([checkpoint, TimerCallback()]) - # Add metric configuration to the model via MetricsConfigurationCallback - image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None - pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None + # Add post-processing configurations to AnomalyModule. image_threshold = ( config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None ) pixel_threshold = ( config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None ) + post_processing_callback = PostProcessingConfigurationCallback( + adaptive_threshold=config.metrics.threshold.adaptive, + default_image_threshold=image_threshold, + default_pixel_threshold=pixel_threshold, + ) + callbacks.append(post_processing_callback) + + # Add metric configuration to the model via MetricsConfigurationCallback + image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None + pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None metrics_callback = MetricsConfigurationCallback( - config.metrics.threshold.adaptive, config.dataset.task, - image_threshold, - pixel_threshold, image_metric_names, pixel_metric_names, ) @@ -172,7 +180,8 @@ def add_visualizer_callback(callbacks: List[Callback], config: Union[DictConfig, config.visualization.inputs_are_normalized = not config.model.normalization_method == "none" else: config.visualization.task = config.data.init_args.task - config.visualization.inputs_are_normalized = not config.metrics.normalization_method == "none" + config.visualization.inputs_are_normalized = not config.post_processing.normalization_method == "none" + if config.visualization.log_images or config.visualization.save_images or config.visualization.show_images: image_save_path = ( config.visualization.image_save_path From 508d4696a338c1d238f93d28fafa8775003d6d90 Mon Sep 17 00:00:00 2001 From: Samet Date: Sun, 2 Oct 2022 23:37:18 -0700 Subject: [PATCH 05/21] Create pre-processing-configuration callback --- .../utils/callbacks/metrics_configuration.py | 29 --------- .../post_processing_configuration.py | 64 +++++++++++++++++++ 2 files changed, 64 insertions(+), 29 deletions(-) create mode 100644 anomalib/utils/callbacks/post_processing_configuration.py diff --git a/anomalib/utils/callbacks/metrics_configuration.py b/anomalib/utils/callbacks/metrics_configuration.py index 7890a00579..2abc47f848 100644 --- a/anomalib/utils/callbacks/metrics_configuration.py +++ b/anomalib/utils/callbacks/metrics_configuration.py @@ -8,7 +8,6 @@ from typing import List, Optional import pytorch_lightning as pl -import torch from pytorch_lightning.callbacks import Callback from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY @@ -26,13 +25,9 @@ class MetricsConfigurationCallback(Callback): def __init__( self, - adaptive_threshold: bool, task: str = "segmentation", - default_image_threshold: Optional[float] = None, - default_pixel_threshold: Optional[float] = None, image_metrics: Optional[List[str]] = None, pixel_metrics: Optional[List[str]] = None, - normalization_method: str = "min_max", ): """Create image and pixel-level AnomalibMetricsCollection. @@ -43,31 +38,13 @@ def __init__( Args: task (str): Task type of the current run. - adaptive_threshold (bool): Flag indicating whether threshold should be adaptive. - default_image_threshold (Optional[float]): Default image threshold value. - default_pixel_threshold (Optional[float]): Default pixel threshold value. image_metrics (Optional[List[str]]): List of image-level metrics. pixel_metrics (Optional[List[str]]): List of pixel-level metrics. - normalization_method(Optional[str]): Normalization method. """ - # TODO: https://github.com/openvinotoolkit/anomalib/issues/384 self.task = task self.image_metric_names = image_metrics self.pixel_metric_names = pixel_metrics - # TODO: https://github.com/openvinotoolkit/anomalib/issues/384 - # TODO: This is a workaround. normalization-method is actually not used in metrics. - # It's only accessed from `before_instantiate` method in `AnomalibCLI` to configure - # its callback. - self.normalization_method = normalization_method - - assert ( - adaptive_threshold or default_image_threshold is not None and default_pixel_threshold is not None - ), "Default thresholds must be specified when adaptive threshold is disabled." - self.adaptive_threshold = adaptive_threshold - self.default_image_threshold = default_image_threshold - self.default_pixel_threshold = default_pixel_threshold - def setup( self, _trainer: pl.Trainer, @@ -97,12 +74,6 @@ def setup( pixel_metric_names = self.pixel_metric_names if isinstance(pl_module, AnomalyModule): - pl_module.adaptive_threshold = self.adaptive_threshold - if not self.adaptive_threshold: - # pylint: disable=not-callable - pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu() - pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu() - pl_module.image_metrics = metric_collection_from_names(image_metric_names, "image_") pl_module.pixel_metrics = metric_collection_from_names(pixel_metric_names, "pixel_") diff --git a/anomalib/utils/callbacks/post_processing_configuration.py b/anomalib/utils/callbacks/post_processing_configuration.py new file mode 100644 index 0000000000..5ce00c8f83 --- /dev/null +++ b/anomalib/utils/callbacks/post_processing_configuration.py @@ -0,0 +1,64 @@ +"""Post-Processing Configuration Callback.""" + +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +import logging +from typing import Optional + +import torch +from pytorch_lightning import Callback, LightningModule, Trainer +from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY + +from anomalib.models.components.base.anomaly_module import AnomalyModule + +logger = logging.getLogger(__name__) + +__all__ = ["PostProcessingConfigurationCallback"] + + +@CALLBACK_REGISTRY +class PostProcessingConfigurationCallback(Callback): + """Post-Processing Configuration Callback. + + Args: + normalization_method(Optional[str]): Normalization method. None: + super().__init__() + self.normalization_method = normalization_method + + assert ( + adaptive_threshold or default_image_threshold is not None and default_pixel_threshold is not None + ), "Default thresholds must be specified when adaptive threshold is disabled." + + self.adaptive_threshold = adaptive_threshold + self.default_image_threshold = default_image_threshold + self.default_pixel_threshold = default_pixel_threshold + + # pylint: disable=unused-argument + def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[str] = None) -> None: + """Setup post-processing configuration within Anomalib Model. + + Args: + trainer (Trainer): PyTorch Lightning Trainer + pl_module (LightningModule): Anomalib Model that inherits pl LightningModule. + stage (Optional[str], optional): fit, validate, test or predict. Defaults to None. + """ + if isinstance(pl_module, AnomalyModule): + pl_module.adaptive_threshold = self.adaptive_threshold + if pl_module.adaptive_threshold is False: + # pylint: disable=not-callable + pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu() + pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu() From e1a0671c9ac9715f9df7bb79b93ad40d76951ed6 Mon Sep 17 00:00:00 2001 From: Samet Date: Sun, 2 Oct 2022 23:38:42 -0700 Subject: [PATCH 06/21] Add new CLI configuration for the post-processing configuration --- anomalib/utils/cli/cli.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/anomalib/utils/cli/cli.py b/anomalib/utils/cli/cli.py index 48c1769163..7ce51a918a 100644 --- a/anomalib/utils/cli/cli.py +++ b/anomalib/utils/cli/cli.py @@ -26,6 +26,7 @@ MetricsConfigurationCallback, MinMaxNormalizationCallback, ModelCheckpoint, + PostProcessingConfigurationCallback, TilerConfigurationCallback, TimerCallback, add_visualizer_callback, @@ -90,7 +91,6 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None: Args: parser (LightningArgumentParser): Lightning Argument Parser. """ - # TODO: https://github.com/openvinotoolkit/anomalib/issues/19 # TODO: https://github.com/openvinotoolkit/anomalib/issues/20 parser.add_argument( "--export_mode", type=str, default="", help="Select export mode to ONNX or OpenVINO IR format." @@ -105,18 +105,24 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None: parser.add_lightning_class_args(TilerConfigurationCallback, "tiling") # type: ignore parser.set_defaults({"tiling.enable": False}) + parser.add_lightning_class_args(PostProcessingConfigurationCallback, "post_processing") # type: ignore + parser.set_defaults( + { + "post_processing.normalization_method": "min_max", + "post_processing.adaptive_threshold": True, + "post_processing.default_image_threshold": None, + "post_processing.default_pixel_threshold": None, + } + ) + # TODO: Assign these default values within the MetricsConfigurationCallback # - https://github.com/openvinotoolkit/anomalib/issues/384 parser.add_lightning_class_args(MetricsConfigurationCallback, "metrics") # type: ignore parser.set_defaults( { - "metrics.adaptive_threshold": True, "metrics.task": "segmentation", - "metrics.default_image_threshold": None, - "metrics.default_pixel_threshold": None, - "metrics.image_metric_names": ["F1Score", "AUROC"], - "metrics.pixel_metric_names": ["F1Score", "AUROC"], - "metrics.normalization_method": "min_max", + "metrics.image_metrics": ["F1Score", "AUROC"], + "metrics.pixel_metrics": ["F1Score", "AUROC"], } ) @@ -203,7 +209,7 @@ def __set_callbacks(self) -> None: # TODO: This could be set in PostProcessingConfiguration callback # - https://github.com/openvinotoolkit/anomalib/issues/384 # Normalization. - normalization = config.metrics.normalization_method + normalization = config.post_processing.normalization_method if normalization: if normalization == "min_max": callbacks.append(MinMaxNormalizationCallback()) From b9ae69650c07d107cb1c8fe4665623b793fc4b1e Mon Sep 17 00:00:00 2001 From: Samet Date: Sun, 2 Oct 2022 23:41:28 -0700 Subject: [PATCH 07/21] Add options to normalization_method --- configs/model/cflow.yaml | 2 +- configs/model/dfkde.yaml | 2 +- configs/model/dfm.yaml | 2 +- configs/model/draem.yaml | 2 +- configs/model/fastflow.yaml | 2 +- configs/model/ganomaly.yaml | 2 +- configs/model/padim.yaml | 2 +- configs/model/patchcore.yaml | 2 +- configs/model/reverse_distillation.yaml | 2 +- configs/model/stfpm.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/configs/model/cflow.yaml b/configs/model/cflow.yaml index eb31c9badf..672b101fa1 100644 --- a/configs/model/cflow.yaml +++ b/configs/model/cflow.yaml @@ -38,7 +38,7 @@ model: lr: 0.0001 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/dfkde.yaml b/configs/model/dfkde.yaml index d9c946b8a9..54ba782db0 100644 --- a/configs/model/dfkde.yaml +++ b/configs/model/dfkde.yaml @@ -33,7 +33,7 @@ model: threshold_offset: 12 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/dfm.yaml b/configs/model/dfm.yaml index 4e1a7ce657..69f7fd0a8d 100644 --- a/configs/model/dfm.yaml +++ b/configs/model/dfm.yaml @@ -30,7 +30,7 @@ model: score_type: fre post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/draem.yaml b/configs/model/draem.yaml index 5dd083459d..9cda0e1ac0 100644 --- a/configs/model/draem.yaml +++ b/configs/model/draem.yaml @@ -30,7 +30,7 @@ optimizer: lr: 0.0001 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/fastflow.yaml b/configs/model/fastflow.yaml index 39801e3acd..285be59ec4 100644 --- a/configs/model/fastflow.yaml +++ b/configs/model/fastflow.yaml @@ -36,7 +36,7 @@ optimizer: weight_decay: 0.00001 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/ganomaly.yaml b/configs/model/ganomaly.yaml index cfa8d291a6..3ba6708e5e 100644 --- a/configs/model/ganomaly.yaml +++ b/configs/model/ganomaly.yaml @@ -36,7 +36,7 @@ model: beta2: 0.999 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/padim.yaml b/configs/model/padim.yaml index ac21d1b005..013d0485d8 100644 --- a/configs/model/padim.yaml +++ b/configs/model/padim.yaml @@ -33,7 +33,7 @@ model: - layer3 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/patchcore.yaml b/configs/model/patchcore.yaml index 65a0db3d50..4e99bf931e 100644 --- a/configs/model/patchcore.yaml +++ b/configs/model/patchcore.yaml @@ -33,7 +33,7 @@ model: num_neighbors: 9 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/reverse_distillation.yaml b/configs/model/reverse_distillation.yaml index b36f08c2f1..f067b33846 100644 --- a/configs/model/reverse_distillation.yaml +++ b/configs/model/reverse_distillation.yaml @@ -36,7 +36,7 @@ model: beta2: 0.99 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null diff --git a/configs/model/stfpm.yaml b/configs/model/stfpm.yaml index 5b17689cb5..7fc8df2d7f 100644 --- a/configs/model/stfpm.yaml +++ b/configs/model/stfpm.yaml @@ -38,7 +38,7 @@ optimizer: weight_decay: 0.0001 post_processing: - normalization_method: min_max + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null From 00202c86dd09d39e885006832e630e7b2e1c9495 Mon Sep 17 00:00:00 2001 From: Samet Date: Mon, 3 Oct 2022 01:37:42 -0700 Subject: [PATCH 08/21] Address mypy issues --- anomalib/utils/sweep/helpers/callbacks.py | 52 ++++++++++++++++------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/anomalib/utils/sweep/helpers/callbacks.py b/anomalib/utils/sweep/helpers/callbacks.py index d4cf90676e..3f1057a5d5 100644 --- a/anomalib/utils/sweep/helpers/callbacks.py +++ b/anomalib/utils/sweep/helpers/callbacks.py @@ -9,7 +9,10 @@ from omegaconf import DictConfig, ListConfig from pytorch_lightning import Callback -from anomalib.utils.callbacks import MetricsConfigurationCallback +from anomalib.utils.callbacks import ( + MetricsConfigurationCallback, + PostProcessingConfigurationCallback, +) from anomalib.utils.callbacks.timer import TimerCallback @@ -24,21 +27,40 @@ def get_sweep_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback] """ callbacks: List[Callback] = [TimerCallback()] # Add metric configuration to the model via MetricsConfigurationCallback - image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None - pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None - image_threshold = ( - config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None - ) - pixel_threshold = ( - config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None + + # TODO: Remove this once the old CLI is deprecated. + if isinstance(config, DictConfig): + image_metrics = config.metrics.image if "image" in config.metrics.keys() else None + pixel_metrics = config.metrics.pixel if "pixel" in config.metrics.keys() else None + image_threshold = ( + config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None + ) + pixel_threshold = ( + config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None + ) + normalization_method = config.model.normalization_method + # NOTE: This is for the new anomalib CLI. + else: + image_metrics = config.metrics.image_metrics if "image_metrics" in config.metrics else None + pixel_metrics = config.metrics.pixel_metrics if "pixel_metrics" in config.metrics else None + image_threshold = ( + config.post_processing.default_image_threshold if "image_default" in config.post_processing.keys() else None + ) + pixel_threshold = ( + config.post_processing.default_pixel_threshold if "pixel_default" in config.post_processing.keys() else None + ) + normalization_method = config.post_processing.normalization_method + + post_processing_configuration_callback = PostProcessingConfigurationCallback( + normalization_method=normalization_method, + default_image_threshold=image_threshold, + default_pixel_threshold=pixel_threshold, ) - metrics_callback = MetricsConfigurationCallback( - config.metrics.threshold.adaptive, - image_threshold, - pixel_threshold, - image_metric_names, - pixel_metric_names, + callbacks.append(post_processing_configuration_callback) + + metrics_configuration_callback = MetricsConfigurationCallback( + task=config.dataset.task, image_metrics=image_metrics, pixel_metrics=pixel_metrics ) - callbacks.append(metrics_callback) + callbacks.append(metrics_configuration_callback) return callbacks From 54c9a397cce28e959c3708e6358d429411b560fc Mon Sep 17 00:00:00 2001 From: Samet Date: Mon, 3 Oct 2022 01:39:54 -0700 Subject: [PATCH 09/21] Fix docstring --- anomalib/utils/callbacks/post_processing_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anomalib/utils/callbacks/post_processing_configuration.py b/anomalib/utils/callbacks/post_processing_configuration.py index 5ce00c8f83..3d1d413d4b 100644 --- a/anomalib/utils/callbacks/post_processing_configuration.py +++ b/anomalib/utils/callbacks/post_processing_configuration.py @@ -23,7 +23,7 @@ class PostProcessingConfigurationCallback(Callback): """Post-Processing Configuration Callback. Args: - normalization_method(Optional[str]): Normalization method. adaptive_threshold (bool): Flag indicating whether threshold should be adaptive. default_image_threshold (Optional[float]): Default image threshold value. default_pixel_threshold (Optional[float]): Default pixel threshold value. From 838bd0b7474fe62b982966be6c571f0fff372c45 Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 4 Oct 2022 01:53:58 -0700 Subject: [PATCH 10/21] Address codacy issues --- anomalib/utils/callbacks/post_processing_configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/anomalib/utils/callbacks/post_processing_configuration.py b/anomalib/utils/callbacks/post_processing_configuration.py index 3d1d413d4b..9549a87fd9 100644 --- a/anomalib/utils/callbacks/post_processing_configuration.py +++ b/anomalib/utils/callbacks/post_processing_configuration.py @@ -59,6 +59,5 @@ def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[st if isinstance(pl_module, AnomalyModule): pl_module.adaptive_threshold = self.adaptive_threshold if pl_module.adaptive_threshold is False: - # pylint: disable=not-callable pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu() pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu() From 61352ae3cb82a9fe5d24f87483bea9e5ed22fdfd Mon Sep 17 00:00:00 2001 From: Samet Date: Mon, 17 Oct 2022 12:36:06 -0700 Subject: [PATCH 11/21] renamed adaptive: true to threshold_method: adaptive in config.yaml files --- anomalib/models/cflow/config.yaml | 2 +- anomalib/models/dfkde/config.yaml | 2 +- anomalib/models/dfm/config.yaml | 2 +- anomalib/models/draem/config.yaml | 2 +- anomalib/models/fastflow/config.yaml | 2 +- anomalib/models/ganomaly/config.yaml | 2 +- anomalib/models/padim/config.yaml | 2 +- anomalib/models/patchcore/config.yaml | 2 +- anomalib/models/reverse_distillation/config.yaml | 2 +- anomalib/models/stfpm/config.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/anomalib/models/cflow/config.yaml b/anomalib/models/cflow/config.yaml index be166c2417..b889256c60 100644 --- a/anomalib/models/cflow/config.yaml +++ b/anomalib/models/cflow/config.yaml @@ -45,7 +45,7 @@ metrics: threshold: image_default: 0 pixel_default: 0 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/dfkde/config.yaml b/anomalib/models/dfkde/config.yaml index 070f0c2456..b01b4f5619 100644 --- a/anomalib/models/dfkde/config.yaml +++ b/anomalib/models/dfkde/config.yaml @@ -33,7 +33,7 @@ metrics: - AUROC threshold: image_default: 0 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/dfm/config.yaml b/anomalib/models/dfm/config.yaml index 47db50fb4e..01836d07e0 100755 --- a/anomalib/models/dfm/config.yaml +++ b/anomalib/models/dfm/config.yaml @@ -33,7 +33,7 @@ metrics: - AUROC threshold: image_default: 0 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/draem/config.yaml b/anomalib/models/draem/config.yaml index 9f3326daa1..f572da74d2 100644 --- a/anomalib/models/draem/config.yaml +++ b/anomalib/models/draem/config.yaml @@ -42,7 +42,7 @@ metrics: threshold: image_default: 3 pixel_default: 3 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/fastflow/config.yaml b/anomalib/models/fastflow/config.yaml index b02f430fd9..fc2c333832 100644 --- a/anomalib/models/fastflow/config.yaml +++ b/anomalib/models/fastflow/config.yaml @@ -45,7 +45,7 @@ metrics: threshold: image_default: 0 pixel_default: 0 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/ganomaly/config.yaml b/anomalib/models/ganomaly/config.yaml index 2e5dfb6bba..8198675bdc 100644 --- a/anomalib/models/ganomaly/config.yaml +++ b/anomalib/models/ganomaly/config.yaml @@ -45,7 +45,7 @@ metrics: - AUROC threshold: image_default: 0 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/padim/config.yaml b/anomalib/models/padim/config.yaml index 92e66618dc..ed89e34dcc 100644 --- a/anomalib/models/padim/config.yaml +++ b/anomalib/models/padim/config.yaml @@ -40,7 +40,7 @@ metrics: threshold: image_default: 3 pixel_default: 3 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/patchcore/config.yaml b/anomalib/models/patchcore/config.yaml index 31567ad530..4c5b743595 100644 --- a/anomalib/models/patchcore/config.yaml +++ b/anomalib/models/patchcore/config.yaml @@ -41,7 +41,7 @@ metrics: threshold: image_default: 0 pixel_default: 0 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/reverse_distillation/config.yaml b/anomalib/models/reverse_distillation/config.yaml index d30f3baedf..f7be341d7e 100644 --- a/anomalib/models/reverse_distillation/config.yaml +++ b/anomalib/models/reverse_distillation/config.yaml @@ -49,7 +49,7 @@ metrics: threshold: image_default: 0 pixel_default: 0 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen diff --git a/anomalib/models/stfpm/config.yaml b/anomalib/models/stfpm/config.yaml index fe3637bf27..aa24dfb554 100644 --- a/anomalib/models/stfpm/config.yaml +++ b/anomalib/models/stfpm/config.yaml @@ -47,7 +47,7 @@ metrics: threshold: image_default: 0 pixel_default: 0 - adaptive: true + threshold_method: adaptive #options: [adaptive, fixed] visualization: show_images: False # show images on the screen From 6ce5f71fa6cc096977449d2ea388c59ec70747af Mon Sep 17 00:00:00 2001 From: Samet Date: Mon, 17 Oct 2022 12:41:25 -0700 Subject: [PATCH 12/21] Reorder threshold params in config.yaml --- anomalib/models/cflow/config.yaml | 4 ++-- anomalib/models/dfkde/config.yaml | 2 +- anomalib/models/dfm/config.yaml | 2 +- anomalib/models/draem/config.yaml | 4 ++-- anomalib/models/fastflow/config.yaml | 4 ++-- anomalib/models/ganomaly/config.yaml | 2 +- anomalib/models/padim/config.yaml | 4 ++-- anomalib/models/patchcore/config.yaml | 4 ++-- anomalib/models/reverse_distillation/config.yaml | 4 ++-- anomalib/models/stfpm/config.yaml | 4 ++-- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/anomalib/models/cflow/config.yaml b/anomalib/models/cflow/config.yaml index b889256c60..a773971f61 100644 --- a/anomalib/models/cflow/config.yaml +++ b/anomalib/models/cflow/config.yaml @@ -43,9 +43,9 @@ metrics: - F1Score - AUROC threshold: - image_default: 0 - pixel_default: 0 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null + fixed_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/dfkde/config.yaml b/anomalib/models/dfkde/config.yaml index b01b4f5619..f89fd8b63f 100644 --- a/anomalib/models/dfkde/config.yaml +++ b/anomalib/models/dfkde/config.yaml @@ -32,8 +32,8 @@ metrics: - F1Score - AUROC threshold: - image_default: 0 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/dfm/config.yaml b/anomalib/models/dfm/config.yaml index 01836d07e0..7e117b10a2 100755 --- a/anomalib/models/dfm/config.yaml +++ b/anomalib/models/dfm/config.yaml @@ -32,8 +32,8 @@ metrics: - F1Score - AUROC threshold: - image_default: 0 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/draem/config.yaml b/anomalib/models/draem/config.yaml index f572da74d2..9aadeb84c6 100644 --- a/anomalib/models/draem/config.yaml +++ b/anomalib/models/draem/config.yaml @@ -40,9 +40,9 @@ metrics: - F1Score - AUROC threshold: - image_default: 3 - pixel_default: 3 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null + fixed_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/fastflow/config.yaml b/anomalib/models/fastflow/config.yaml index fc2c333832..8a546addd3 100644 --- a/anomalib/models/fastflow/config.yaml +++ b/anomalib/models/fastflow/config.yaml @@ -43,9 +43,9 @@ metrics: - F1Score - AUROC threshold: - image_default: 0 - pixel_default: 0 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null + fixed_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/ganomaly/config.yaml b/anomalib/models/ganomaly/config.yaml index 8198675bdc..407b4e3cad 100644 --- a/anomalib/models/ganomaly/config.yaml +++ b/anomalib/models/ganomaly/config.yaml @@ -44,8 +44,8 @@ metrics: - F1Score - AUROC threshold: - image_default: 0 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/padim/config.yaml b/anomalib/models/padim/config.yaml index ed89e34dcc..5a67184a8d 100644 --- a/anomalib/models/padim/config.yaml +++ b/anomalib/models/padim/config.yaml @@ -38,9 +38,9 @@ metrics: - F1Score - AUROC threshold: - image_default: 3 - pixel_default: 3 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null + fixed_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/patchcore/config.yaml b/anomalib/models/patchcore/config.yaml index 4c5b743595..756f289460 100644 --- a/anomalib/models/patchcore/config.yaml +++ b/anomalib/models/patchcore/config.yaml @@ -39,9 +39,9 @@ metrics: - F1Score - AUROC threshold: - image_default: 0 - pixel_default: 0 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null + fixed_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/reverse_distillation/config.yaml b/anomalib/models/reverse_distillation/config.yaml index f7be341d7e..92c2f9f959 100644 --- a/anomalib/models/reverse_distillation/config.yaml +++ b/anomalib/models/reverse_distillation/config.yaml @@ -47,9 +47,9 @@ metrics: - F1Score - AUROC threshold: - image_default: 0 - pixel_default: 0 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null + fixed_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/stfpm/config.yaml b/anomalib/models/stfpm/config.yaml index aa24dfb554..104d62b429 100644 --- a/anomalib/models/stfpm/config.yaml +++ b/anomalib/models/stfpm/config.yaml @@ -45,9 +45,9 @@ metrics: - F1Score - AUROC threshold: - image_default: 0 - pixel_default: 0 threshold_method: adaptive #options: [adaptive, fixed] + fixed_image: null + fixed_pixel: null visualization: show_images: False # show images on the screen From c2349678acb25b90f8159dc9f454114a6d095751 Mon Sep 17 00:00:00 2001 From: Samet Date: Mon, 17 Oct 2022 12:56:37 -0700 Subject: [PATCH 13/21] Add backward compatibility to threshold configs --- anomalib/config/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/anomalib/config/config.py b/anomalib/config/config.py index 6312c1012f..358cc55c81 100644 --- a/anomalib/config/config.py +++ b/anomalib/config/config.py @@ -156,7 +156,14 @@ def get_configurable_parameters( # thresholding if "metrics" in config.keys(): + # NOTE: Deprecate this after v0.4.0. + if "adaptive" in config.metrics.threshold.keys(): + warn("adaptive will be deprecated in favor of method in config.metrics.threshold in v0.4.0.") + config.metrics.threshold.method = "adaptive" if config.metrics.threshold.adaptive else "fixed" + if "image_default" in config.metrics.threshold.keys(): + warn("image_default will be deprecated in favor of fixed_image in config.metrics.threshold in v0.4.0.") + config.metrics.threshold.fixed_image = config.metrics.threshold.image_default if "pixel_default" not in config.metrics.threshold.keys(): - config.metrics.threshold.pixel_default = config.metrics.threshold.image_default + config.metrics.threshold.fixed_pixel = config.metrics.threshold.pixel_default return config From 86156d1fcdd9127992d6cf4a492eb90e1d1a0a1b Mon Sep 17 00:00:00 2001 From: Samet Date: Mon, 17 Oct 2022 13:06:42 -0700 Subject: [PATCH 14/21] renamed the new cli config files --- configs/model/cflow.yaml | 6 +++--- configs/model/dfkde.yaml | 6 +++--- configs/model/dfm.yaml | 6 +++--- configs/model/draem.yaml | 6 +++--- configs/model/fastflow.yaml | 6 +++--- configs/model/ganomaly.yaml | 10 +++++----- configs/model/padim.yaml | 6 +++--- configs/model/patchcore.yaml | 6 +++--- configs/model/reverse_distillation.yaml | 6 +++--- configs/model/stfpm.yaml | 6 +++--- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/configs/model/cflow.yaml b/configs/model/cflow.yaml index 672b101fa1..60e0c15365 100644 --- a/configs/model/cflow.yaml +++ b/configs/model/cflow.yaml @@ -39,9 +39,9 @@ model: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/dfkde.yaml b/configs/model/dfkde.yaml index 54ba782db0..b8e813c8a3 100644 --- a/configs/model/dfkde.yaml +++ b/configs/model/dfkde.yaml @@ -34,9 +34,9 @@ model: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/dfm.yaml b/configs/model/dfm.yaml index 69f7fd0a8d..f09d17bd4a 100644 --- a/configs/model/dfm.yaml +++ b/configs/model/dfm.yaml @@ -31,9 +31,9 @@ model: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/draem.yaml b/configs/model/draem.yaml index 9cda0e1ac0..a5760a1d71 100644 --- a/configs/model/draem.yaml +++ b/configs/model/draem.yaml @@ -31,9 +31,9 @@ optimizer: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/fastflow.yaml b/configs/model/fastflow.yaml index 285be59ec4..2f89d732f5 100644 --- a/configs/model/fastflow.yaml +++ b/configs/model/fastflow.yaml @@ -37,9 +37,9 @@ optimizer: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/ganomaly.yaml b/configs/model/ganomaly.yaml index 3ba6708e5e..5fbf5e890d 100644 --- a/configs/model/ganomaly.yaml +++ b/configs/model/ganomaly.yaml @@ -37,13 +37,13 @@ model: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: - adaptive_threshold: true - default_image_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null image_metrics: - F1Score - AUROC diff --git a/configs/model/padim.yaml b/configs/model/padim.yaml index 013d0485d8..730c34fa69 100644 --- a/configs/model/padim.yaml +++ b/configs/model/padim.yaml @@ -34,9 +34,9 @@ model: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/patchcore.yaml b/configs/model/patchcore.yaml index 4e99bf931e..a9a9db46b1 100644 --- a/configs/model/patchcore.yaml +++ b/configs/model/patchcore.yaml @@ -34,9 +34,9 @@ model: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/reverse_distillation.yaml b/configs/model/reverse_distillation.yaml index f067b33846..e9a6a3d308 100644 --- a/configs/model/reverse_distillation.yaml +++ b/configs/model/reverse_distillation.yaml @@ -37,9 +37,9 @@ model: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/stfpm.yaml b/configs/model/stfpm.yaml index 7fc8df2d7f..38b8901d6b 100644 --- a/configs/model/stfpm.yaml +++ b/configs/model/stfpm.yaml @@ -39,9 +39,9 @@ optimizer: post_processing: normalization_method: min_max # - adaptive_threshold: true - default_image_threshold: null - default_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, fixed] + fixed_image_threshold: null + fixed_pixel_threshold: null metrics: image_metrics: From b2426e8fa73a3f8cc2c81cf19d1670c34470b99e Mon Sep 17 00:00:00 2001 From: Samet Date: Mon, 17 Oct 2022 22:20:18 -0700 Subject: [PATCH 15/21] Rename threshold_method to method in config.yaml --- anomalib/models/cflow/config.yaml | 2 +- anomalib/models/dfkde/config.yaml | 2 +- anomalib/models/dfm/config.yaml | 2 +- anomalib/models/draem/config.yaml | 2 +- anomalib/models/fastflow/config.yaml | 2 +- anomalib/models/ganomaly/config.yaml | 2 +- anomalib/models/padim/config.yaml | 2 +- anomalib/models/patchcore/config.yaml | 2 +- anomalib/models/reverse_distillation/config.yaml | 2 +- anomalib/models/stfpm/config.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/anomalib/models/cflow/config.yaml b/anomalib/models/cflow/config.yaml index a773971f61..5e676cd4a0 100644 --- a/anomalib/models/cflow/config.yaml +++ b/anomalib/models/cflow/config.yaml @@ -43,7 +43,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/dfkde/config.yaml b/anomalib/models/dfkde/config.yaml index f89fd8b63f..d4b53025f2 100644 --- a/anomalib/models/dfkde/config.yaml +++ b/anomalib/models/dfkde/config.yaml @@ -32,7 +32,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null visualization: diff --git a/anomalib/models/dfm/config.yaml b/anomalib/models/dfm/config.yaml index 7e117b10a2..be84b8ab3e 100755 --- a/anomalib/models/dfm/config.yaml +++ b/anomalib/models/dfm/config.yaml @@ -32,7 +32,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null visualization: diff --git a/anomalib/models/draem/config.yaml b/anomalib/models/draem/config.yaml index 9aadeb84c6..1c9159fc96 100644 --- a/anomalib/models/draem/config.yaml +++ b/anomalib/models/draem/config.yaml @@ -40,7 +40,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/fastflow/config.yaml b/anomalib/models/fastflow/config.yaml index 8a546addd3..7908887a5b 100644 --- a/anomalib/models/fastflow/config.yaml +++ b/anomalib/models/fastflow/config.yaml @@ -43,7 +43,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/ganomaly/config.yaml b/anomalib/models/ganomaly/config.yaml index 407b4e3cad..287993c1c0 100644 --- a/anomalib/models/ganomaly/config.yaml +++ b/anomalib/models/ganomaly/config.yaml @@ -44,7 +44,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null visualization: diff --git a/anomalib/models/padim/config.yaml b/anomalib/models/padim/config.yaml index 5a67184a8d..b0b182fd65 100644 --- a/anomalib/models/padim/config.yaml +++ b/anomalib/models/padim/config.yaml @@ -38,7 +38,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/patchcore/config.yaml b/anomalib/models/patchcore/config.yaml index 756f289460..306b19ea13 100644 --- a/anomalib/models/patchcore/config.yaml +++ b/anomalib/models/patchcore/config.yaml @@ -39,7 +39,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/reverse_distillation/config.yaml b/anomalib/models/reverse_distillation/config.yaml index 92c2f9f959..719f1fa758 100644 --- a/anomalib/models/reverse_distillation/config.yaml +++ b/anomalib/models/reverse_distillation/config.yaml @@ -47,7 +47,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/stfpm/config.yaml b/anomalib/models/stfpm/config.yaml index 104d62b429..24b15d3fb9 100644 --- a/anomalib/models/stfpm/config.yaml +++ b/anomalib/models/stfpm/config.yaml @@ -45,7 +45,7 @@ metrics: - F1Score - AUROC threshold: - threshold_method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, fixed] fixed_image: null fixed_pixel: null From 2756b8da1a026b132a0bfdf1b9f504e939b55770 Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 18 Oct 2022 01:54:10 -0700 Subject: [PATCH 16/21] Convert names to Enum --- anomalib/config/config.py | 11 +++-- .../models/components/base/anomaly_module.py | 6 +-- anomalib/post_processing/__init__.py | 4 ++ .../post_processing/normalization/__init__.py | 10 +++++ anomalib/post_processing/post_process.py | 8 ++++ anomalib/utils/callbacks/__init__.py | 14 +++---- .../post_processing_configuration.py | 40 ++++++++++--------- anomalib/utils/cli/cli.py | 6 +-- anomalib/utils/sweep/helpers/callbacks.py | 12 +++--- .../test_normalization_callback.py | 2 +- .../utils/metrics/test_adaptive_threshold.py | 2 +- 11 files changed, 71 insertions(+), 44 deletions(-) diff --git a/anomalib/config/config.py b/anomalib/config/config.py index 358cc55c81..a96085ba19 100644 --- a/anomalib/config/config.py +++ b/anomalib/config/config.py @@ -162,8 +162,13 @@ def get_configurable_parameters( config.metrics.threshold.method = "adaptive" if config.metrics.threshold.adaptive else "fixed" if "image_default" in config.metrics.threshold.keys(): warn("image_default will be deprecated in favor of fixed_image in config.metrics.threshold in v0.4.0.") - config.metrics.threshold.fixed_image = config.metrics.threshold.image_default - if "pixel_default" not in config.metrics.threshold.keys(): - config.metrics.threshold.fixed_pixel = config.metrics.threshold.pixel_default + config.metrics.threshold.fixed_image = ( + None if config.metrics.threshold.adaptive else config.metrics.threshold.image_default + ) + if "pixel_default" in config.metrics.threshold.keys(): + warn("pixel_default will be deprecated in favor of fixed_pixel in config.metrics.threshold in v0.4.0.") + config.metrics.threshold.fixed_pixel = ( + None if config.metrics.threshold.adaptive else config.metrics.threshold.pixel_default + ) return config diff --git a/anomalib/models/components/base/anomaly_module.py b/anomalib/models/components/base/anomaly_module.py index 16fabb9ef3..b575303c72 100644 --- a/anomalib/models/components/base/anomaly_module.py +++ b/anomalib/models/components/base/anomaly_module.py @@ -13,6 +13,7 @@ from torch import Tensor, nn from torchmetrics import Metric +from anomalib.post_processing import ThresholdMethod from anomalib.utils.metrics import ( AdaptiveThreshold, AnomalibMetricCollection, @@ -38,8 +39,7 @@ def __init__(self): self.loss: Tensor self.callbacks: List[Callback] - self.adaptive_threshold: bool - + self.threshold_method: ThresholdMethod self.image_threshold = AdaptiveThreshold().cpu() self.pixel_threshold = AdaptiveThreshold().cpu() @@ -115,7 +115,7 @@ def validation_epoch_end(self, outputs): Args: outputs: Batch of outputs from the validation step """ - if self.adaptive_threshold: + if self.threshold_method == ThresholdMethod.ADAPTIVE: self._compute_adaptive_threshold(outputs) self._collect_outputs(self.image_metrics, self.pixel_metrics, outputs) self._log_metrics() diff --git a/anomalib/post_processing/__init__.py b/anomalib/post_processing/__init__.py index dcdf1ab7de..17eb3f84aa 100644 --- a/anomalib/post_processing/__init__.py +++ b/anomalib/post_processing/__init__.py @@ -3,7 +3,9 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from .normalization import NormalizationMethod from .post_process import ( + ThresholdMethod, add_anomalous_label, add_normal_label, anomaly_map_to_color_map, @@ -19,5 +21,7 @@ "superimpose_anomaly_map", "compute_mask", "ImageResult", + "NormalizationMethod", "Visualizer", + "ThresholdMethod", ] diff --git a/anomalib/post_processing/normalization/__init__.py b/anomalib/post_processing/normalization/__init__.py index fb2c272cc2..b8a57f8077 100644 --- a/anomalib/post_processing/normalization/__init__.py +++ b/anomalib/post_processing/normalization/__init__.py @@ -2,3 +2,13 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 + +from enum import Enum + + +class NormalizationMethod(str, Enum): + """Normalization method for normalization.""" + + CDF = "cdf" + MIN_MAX = "min_max" + NONE = "none" diff --git a/anomalib/post_processing/post_process.py b/anomalib/post_processing/post_process.py index e8c4debbad..bdeb224ac8 100644 --- a/anomalib/post_processing/post_process.py +++ b/anomalib/post_processing/post_process.py @@ -5,6 +5,7 @@ import math +from enum import Enum from typing import Optional, Tuple import cv2 @@ -12,6 +13,13 @@ from skimage import morphology +class ThresholdMethod(str, Enum): + """Threshold method to apply post-processing to the output predictions.""" + + ADAPTIVE = "adaptive" + FIXED = "fixed" + + def add_label( image: np.ndarray, label_name: str, diff --git a/anomalib/utils/callbacks/__init__.py b/anomalib/utils/callbacks/__init__.py index 15def2a1f5..fa09e05669 100644 --- a/anomalib/utils/callbacks/__init__.py +++ b/anomalib/utils/callbacks/__init__.py @@ -68,16 +68,12 @@ def get_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]: callbacks.extend([checkpoint, TimerCallback()]) # Add post-processing configurations to AnomalyModule. - image_threshold = ( - config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None - ) - pixel_threshold = ( - config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None - ) + image_threshold = config.metrics.threshold.fixed_image if "fixed_image" in config.metrics.threshold.keys() else None + pixel_threshold = config.metrics.threshold.fixed_pixel if "fixed_pixel" in config.metrics.threshold.keys() else None post_processing_callback = PostProcessingConfigurationCallback( - adaptive_threshold=config.metrics.threshold.adaptive, - default_image_threshold=image_threshold, - default_pixel_threshold=pixel_threshold, + threshold_method=config.metrics.threshold.method, + fixed_image_threshold=image_threshold, + fixed_pixel_threshold=pixel_threshold, ) callbacks.append(post_processing_callback) diff --git a/anomalib/utils/callbacks/post_processing_configuration.py b/anomalib/utils/callbacks/post_processing_configuration.py index 9549a87fd9..5ff3fd418f 100644 --- a/anomalib/utils/callbacks/post_processing_configuration.py +++ b/anomalib/utils/callbacks/post_processing_configuration.py @@ -12,6 +12,7 @@ from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY from anomalib.models.components.base.anomaly_module import AnomalyModule +from anomalib.post_processing import NormalizationMethod, ThresholdMethod logger = logging.getLogger(__name__) @@ -23,29 +24,32 @@ class PostProcessingConfigurationCallback(Callback): """Post-Processing Configuration Callback. Args: - normalization_method(Optional[str]): Normalization method. - adaptive_threshold (bool): Flag indicating whether threshold should be adaptive. - default_image_threshold (Optional[float]): Default image threshold value. - default_pixel_threshold (Optional[float]): Default pixel threshold value. + normalization_method(NormalizationMethod): Normalization method. + threshold_method (ThresholdMethod): Flag indicating whether threshold should be fixed or adaptive. + fixed_image_threshold (Optional[float]): Default fixed image threshold value. + fixed_pixel_threshold (Optional[float]): Default fixed pixel threshold value. """ def __init__( self, - normalization_method: str = "min_max", - adaptive_threshold: bool = True, - default_image_threshold: Optional[float] = None, - default_pixel_threshold: Optional[float] = None, + normalization_method: NormalizationMethod = NormalizationMethod.MIN_MAX, + threshold_method: ThresholdMethod = ThresholdMethod.ADAPTIVE, + fixed_image_threshold: Optional[float] = None, + fixed_pixel_threshold: Optional[float] = None, ) -> None: super().__init__() self.normalization_method = normalization_method - assert ( - adaptive_threshold or default_image_threshold is not None and default_pixel_threshold is not None - ), "Default thresholds must be specified when adaptive threshold is disabled." + assert threshold_method == ThresholdMethod.ADAPTIVE and all( + i is None for i in [fixed_image_threshold, fixed_pixel_threshold] + ), ( + "Default thresholds must be specified when adaptive threshold is disabled. " + "When adaptive threshold is enabled, please set default image and pixel thresholds to None." + ) - self.adaptive_threshold = adaptive_threshold - self.default_image_threshold = default_image_threshold - self.default_pixel_threshold = default_pixel_threshold + self.threshold_method = threshold_method + self.fixed_image_threshold = fixed_image_threshold + self.fixed_pixel_threshold = fixed_pixel_threshold # pylint: disable=unused-argument def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[str] = None) -> None: @@ -57,7 +61,7 @@ def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[st stage (Optional[str], optional): fit, validate, test or predict. Defaults to None. """ if isinstance(pl_module, AnomalyModule): - pl_module.adaptive_threshold = self.adaptive_threshold - if pl_module.adaptive_threshold is False: - pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu() - pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu() + pl_module.threshold_method = self.threshold_method + if pl_module.threshold_method is False: + pl_module.image_threshold.value = torch.tensor(self.fixed_image_threshold).cpu() + pl_module.pixel_threshold.value = torch.tensor(self.fixed_pixel_threshold).cpu() diff --git a/anomalib/utils/cli/cli.py b/anomalib/utils/cli/cli.py index 7ce51a918a..a37bba5ec8 100644 --- a/anomalib/utils/cli/cli.py +++ b/anomalib/utils/cli/cli.py @@ -109,9 +109,9 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None: parser.set_defaults( { "post_processing.normalization_method": "min_max", - "post_processing.adaptive_threshold": True, - "post_processing.default_image_threshold": None, - "post_processing.default_pixel_threshold": None, + "post_processing.threshold_method": "adaptive", + "post_processing.fixed_image_threshold": None, + "post_processing.fixed_pixel_threshold": None, } ) diff --git a/anomalib/utils/sweep/helpers/callbacks.py b/anomalib/utils/sweep/helpers/callbacks.py index 3f1057a5d5..3085518d16 100644 --- a/anomalib/utils/sweep/helpers/callbacks.py +++ b/anomalib/utils/sweep/helpers/callbacks.py @@ -33,10 +33,10 @@ def get_sweep_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback] image_metrics = config.metrics.image if "image" in config.metrics.keys() else None pixel_metrics = config.metrics.pixel if "pixel" in config.metrics.keys() else None image_threshold = ( - config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None + config.metrics.threshold.fixed_image if "fixed_image" in config.metrics.threshold.keys() else None ) pixel_threshold = ( - config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None + config.metrics.threshold.fixed_pixel if "fixed_pixel" in config.metrics.threshold.keys() else None ) normalization_method = config.model.normalization_method # NOTE: This is for the new anomalib CLI. @@ -44,17 +44,17 @@ def get_sweep_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback] image_metrics = config.metrics.image_metrics if "image_metrics" in config.metrics else None pixel_metrics = config.metrics.pixel_metrics if "pixel_metrics" in config.metrics else None image_threshold = ( - config.post_processing.default_image_threshold if "image_default" in config.post_processing.keys() else None + config.post_processing.fixed_image_threshold if "image_default" in config.post_processing.keys() else None ) pixel_threshold = ( - config.post_processing.default_pixel_threshold if "pixel_default" in config.post_processing.keys() else None + config.post_processing.fixed_pixel_threshold if "pixel_default" in config.post_processing.keys() else None ) normalization_method = config.post_processing.normalization_method post_processing_configuration_callback = PostProcessingConfigurationCallback( normalization_method=normalization_method, - default_image_threshold=image_threshold, - default_pixel_threshold=pixel_threshold, + fixed_image_threshold=image_threshold, + fixed_pixel_threshold=pixel_threshold, ) callbacks.append(post_processing_configuration_callback) diff --git a/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py b/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py index 6b2ddf6bf9..e7db2b48ee 100644 --- a/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py +++ b/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py @@ -23,7 +23,7 @@ def test_normalizer(path=get_dataset_path(), category="shapes"): config = get_configurable_parameters(config_path="anomalib/models/padim/config.yaml") config.dataset.path = path config.dataset.category = category - config.metrics.threshold.adaptive = True + config.metrics.threshold.method = "adaptive" config.project.log_images_to = [] config.metrics.image = ["F1Score", "AUROC"] diff --git a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py index 1a7eef5b61..e199ee68ab 100644 --- a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py +++ b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py @@ -40,7 +40,7 @@ def test_non_adaptive_threshold(): config = get_test_configurable_parameters(config_path="anomalib/models/padim/config.yaml") config.model.normalization_method = "none" - config.metrics.threshold.adaptive = False + config.metrics.threshold.method = "fixed" config.trainer.fast_dev_run = True config.metrics.image = ["F1Score"] config.metrics.pixel = ["F1Score"] From 429bad7d7211e03150d9ba346fb5ab0222598ada Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 18 Oct 2022 04:40:05 -0700 Subject: [PATCH 17/21] Fix tests --- .../post_processing_configuration.py | 21 +++++++++++++------ .../utils/metrics/test_adaptive_threshold.py | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/anomalib/utils/callbacks/post_processing_configuration.py b/anomalib/utils/callbacks/post_processing_configuration.py index 5ff3fd418f..18947468ad 100644 --- a/anomalib/utils/callbacks/post_processing_configuration.py +++ b/anomalib/utils/callbacks/post_processing_configuration.py @@ -40,12 +40,21 @@ def __init__( super().__init__() self.normalization_method = normalization_method - assert threshold_method == ThresholdMethod.ADAPTIVE and all( + if threshold_method == ThresholdMethod.ADAPTIVE and all( + i is not None for i in [fixed_image_threshold, fixed_pixel_threshold] + ): + raise ValueError( + "When `threshold_method` is set to `adaptive`, `fixed_image_threshold` and `fixed_pixel_threshold` " + "must not be set." + ) + + if threshold_method == ThresholdMethod.FIXED and all( i is None for i in [fixed_image_threshold, fixed_pixel_threshold] - ), ( - "Default thresholds must be specified when adaptive threshold is disabled. " - "When adaptive threshold is enabled, please set default image and pixel thresholds to None." - ) + ): + raise ValueError( + "When `threshold_method` is set to `fixed`, `fixed_image_threshold` and `fixed_pixel_threshold` " + "must be set." + ) self.threshold_method = threshold_method self.fixed_image_threshold = fixed_image_threshold @@ -62,6 +71,6 @@ def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[st """ if isinstance(pl_module, AnomalyModule): pl_module.threshold_method = self.threshold_method - if pl_module.threshold_method is False: + if pl_module.threshold_method == ThresholdMethod.FIXED: pl_module.image_threshold.value = torch.tensor(self.fixed_image_threshold).cpu() pl_module.pixel_threshold.value = torch.tensor(self.fixed_pixel_threshold).cpu() diff --git a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py index e199ee68ab..edf125ebf6 100644 --- a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py +++ b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py @@ -47,8 +47,8 @@ def test_non_adaptive_threshold(): image_threshold = random.random() pixel_threshold = random.random() - config.metrics.threshold.image_default = image_threshold - config.metrics.threshold.pixel_default = pixel_threshold + config.metrics.threshold.fixed_image = image_threshold + config.metrics.threshold.fixed_pixel = pixel_threshold model = get_model(config) datamodule = get_datamodule(config) From c0c0a491d5335caad8291dbbb2044b134e909e37 Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 18 Oct 2022 04:54:05 -0700 Subject: [PATCH 18/21] Rename AdaptiveThreshold to AnomalyScoreThreshold --- anomalib/models/components/base/anomaly_module.py | 6 +++--- anomalib/utils/metrics/__init__.py | 4 ++-- ...ve_threshold.py => anomaly_score_threshold.py} | 15 ++++++++++----- .../export_callback/dummy_lightning_model.py | 10 +++++++--- .../utils/metrics/test_adaptive_threshold.py | 4 ++-- 5 files changed, 24 insertions(+), 15 deletions(-) rename anomalib/utils/metrics/{adaptive_threshold.py => anomaly_score_threshold.py} (71%) diff --git a/anomalib/models/components/base/anomaly_module.py b/anomalib/models/components/base/anomaly_module.py index b575303c72..b6127a6dbd 100644 --- a/anomalib/models/components/base/anomaly_module.py +++ b/anomalib/models/components/base/anomaly_module.py @@ -15,9 +15,9 @@ from anomalib.post_processing import ThresholdMethod from anomalib.utils.metrics import ( - AdaptiveThreshold, AnomalibMetricCollection, AnomalyScoreDistribution, + AnomalyScoreThreshold, MinMax, ) @@ -40,8 +40,8 @@ def __init__(self): self.callbacks: List[Callback] self.threshold_method: ThresholdMethod - self.image_threshold = AdaptiveThreshold().cpu() - self.pixel_threshold = AdaptiveThreshold().cpu() + self.image_threshold = AnomalyScoreThreshold().cpu() + self.pixel_threshold = AnomalyScoreThreshold().cpu() self.normalization_metrics: Metric diff --git a/anomalib/utils/metrics/__init__.py b/anomalib/utils/metrics/__init__.py index c0c44d4e4d..36cf0d5781 100644 --- a/anomalib/utils/metrics/__init__.py +++ b/anomalib/utils/metrics/__init__.py @@ -10,7 +10,7 @@ import torchmetrics from omegaconf import DictConfig, ListConfig -from .adaptive_threshold import AdaptiveThreshold +from .adaptive_threshold import AnomalyScoreThreshold from .anomaly_score_distribution import AnomalyScoreDistribution from .aupr import AUPR from .aupro import AUPRO @@ -20,7 +20,7 @@ from .optimal_f1 import OptimalF1 from .pro import PRO -__all__ = ["AUROC", "AUPR", "AUPRO", "OptimalF1", "AdaptiveThreshold", "AnomalyScoreDistribution", "MinMax", "PRO"] +__all__ = ["AUROC", "AUPR", "AUPRO", "OptimalF1", "AnomalyScoreThreshold", "AnomalyScoreDistribution", "MinMax", "PRO"] def get_metrics(config: Union[ListConfig, DictConfig]) -> Tuple[AnomalibMetricCollection, AnomalibMetricCollection]: diff --git a/anomalib/utils/metrics/adaptive_threshold.py b/anomalib/utils/metrics/anomaly_score_threshold.py similarity index 71% rename from anomalib/utils/metrics/adaptive_threshold.py rename to anomalib/utils/metrics/anomaly_score_threshold.py index fd112433f1..1375958264 100644 --- a/anomalib/utils/metrics/adaptive_threshold.py +++ b/anomalib/utils/metrics/anomaly_score_threshold.py @@ -1,4 +1,4 @@ -"""Implementation of Optimal F1 score based on TorchMetrics.""" +"""Implementation of AnomalyScoreThreshold based on TorchMetrics.""" # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 @@ -7,11 +7,16 @@ from torchmetrics import PrecisionRecallCurve -class AdaptiveThreshold(PrecisionRecallCurve): - """Optimal F1 Metric. +class AnomalyScoreThreshold(PrecisionRecallCurve): + """Anomaly Score Threshold. - Compute the optimal F1 score at the adaptive threshold, based on the F1 metric of the true labels and the - predicted anomaly scores. + This class computes/stores the threshold that determines the anomalous label + given anomaly scores. If the threshold method is ``fixed``, the class only + stores the fixed threshold values. + + If the threshold method is ``adaptive``, the class initially computes the + adaptive threshold to find the optimal f1_score and stores the computed + adaptive threshold value. """ def __init__(self, default_value: float = 0.5, **kwargs): diff --git a/tests/pre_merge/utils/callbacks/export_callback/dummy_lightning_model.py b/tests/pre_merge/utils/callbacks/export_callback/dummy_lightning_model.py index 0a6e4c9a44..c9e0929181 100644 --- a/tests/pre_merge/utils/callbacks/export_callback/dummy_lightning_model.py +++ b/tests/pre_merge/utils/callbacks/export_callback/dummy_lightning_model.py @@ -9,7 +9,11 @@ from torchvision.datasets import FakeData from anomalib.utils.callbacks import ImageVisualizerCallback -from anomalib.utils.metrics import AdaptiveThreshold, AnomalyScoreDistribution, MinMax +from anomalib.utils.metrics import ( + AnomalyScoreDistribution, + AnomalyScoreThreshold, + MinMax, +) class FakeDataModule(pl.LightningDataModule): @@ -84,8 +88,8 @@ def __init__(self, hparams: Union[DictConfig, ListConfig]): ) ] # test if this is removed - self.image_threshold = AdaptiveThreshold(hparams.model.threshold.image_default).cpu() - self.pixel_threshold = AdaptiveThreshold(hparams.model.threshold.pixel_default).cpu() + self.image_threshold = AnomalyScoreThreshold(hparams.model.threshold.image_default).cpu() + self.pixel_threshold = AnomalyScoreThreshold(hparams.model.threshold.pixel_default).cpu() self.training_distribution = AnomalyScoreDistribution().cpu() self.min_max = MinMax().cpu() diff --git a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py index edf125ebf6..96f3692d6a 100644 --- a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py +++ b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py @@ -11,7 +11,7 @@ from anomalib.data import get_datamodule from anomalib.models import get_model from anomalib.utils.callbacks import get_callbacks -from anomalib.utils.metrics import AdaptiveThreshold +from anomalib.utils.metrics import AnomalyScoreThreshold from tests.helpers.config import get_test_configurable_parameters @@ -25,7 +25,7 @@ def test_adaptive_threshold(labels, preds, target_threshold): """Test if the adaptive threshold computation returns the desired value.""" - adaptive_threshold = AdaptiveThreshold(default_value=0.5) + adaptive_threshold = AnomalyScoreThreshold(default_value=0.5) adaptive_threshold.update(preds, labels) threshold_value = adaptive_threshold.compute() From 26b1577bbcb863fa3ae5eccf1805f511c238a19a Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 18 Oct 2022 05:10:37 -0700 Subject: [PATCH 19/21] Fixed import --- anomalib/utils/metrics/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anomalib/utils/metrics/__init__.py b/anomalib/utils/metrics/__init__.py index 36cf0d5781..a31a94a559 100644 --- a/anomalib/utils/metrics/__init__.py +++ b/anomalib/utils/metrics/__init__.py @@ -10,8 +10,8 @@ import torchmetrics from omegaconf import DictConfig, ListConfig -from .adaptive_threshold import AnomalyScoreThreshold from .anomaly_score_distribution import AnomalyScoreDistribution +from .anomaly_score_threshold import AnomalyScoreThreshold from .aupr import AUPR from .aupro import AUPRO from .auroc import AUROC From 332e73728ef3381c01675ae0013425bfe51a9409 Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 18 Oct 2022 06:40:52 -0700 Subject: [PATCH 20/21] Rename fixed to manual --- anomalib/config/config.py | 2 +- anomalib/models/cflow/config.yaml | 2 +- anomalib/models/dfkde/config.yaml | 2 +- anomalib/models/dfm/config.yaml | 2 +- anomalib/models/draem/config.yaml | 2 +- anomalib/models/fastflow/config.yaml | 2 +- anomalib/models/ganomaly/config.yaml | 2 +- anomalib/models/padim/config.yaml | 2 +- anomalib/models/patchcore/config.yaml | 2 +- .../models/reverse_distillation/config.yaml | 2 +- anomalib/models/stfpm/config.yaml | 2 +- anomalib/post_processing/post_process.py | 2 +- anomalib/utils/callbacks/__init__.py | 4 +-- .../post_processing_configuration.py | 30 +++++++++---------- anomalib/utils/cli/cli.py | 4 +-- .../utils/metrics/anomaly_score_threshold.py | 4 +-- anomalib/utils/sweep/helpers/callbacks.py | 8 ++--- configs/model/cflow.yaml | 6 ++-- configs/model/dfkde.yaml | 6 ++-- configs/model/dfm.yaml | 6 ++-- configs/model/draem.yaml | 6 ++-- configs/model/fastflow.yaml | 6 ++-- configs/model/ganomaly.yaml | 10 +++---- configs/model/padim.yaml | 6 ++-- configs/model/patchcore.yaml | 6 ++-- configs/model/reverse_distillation.yaml | 6 ++-- configs/model/stfpm.yaml | 6 ++-- .../utils/metrics/test_adaptive_threshold.py | 6 ++-- 28 files changed, 72 insertions(+), 72 deletions(-) diff --git a/anomalib/config/config.py b/anomalib/config/config.py index a96085ba19..da93a5f912 100644 --- a/anomalib/config/config.py +++ b/anomalib/config/config.py @@ -159,7 +159,7 @@ def get_configurable_parameters( # NOTE: Deprecate this after v0.4.0. if "adaptive" in config.metrics.threshold.keys(): warn("adaptive will be deprecated in favor of method in config.metrics.threshold in v0.4.0.") - config.metrics.threshold.method = "adaptive" if config.metrics.threshold.adaptive else "fixed" + config.metrics.threshold.method = "adaptive" if config.metrics.threshold.adaptive else "manual" if "image_default" in config.metrics.threshold.keys(): warn("image_default will be deprecated in favor of fixed_image in config.metrics.threshold in v0.4.0.") config.metrics.threshold.fixed_image = ( diff --git a/anomalib/models/cflow/config.yaml b/anomalib/models/cflow/config.yaml index 5e676cd4a0..47472bf846 100644 --- a/anomalib/models/cflow/config.yaml +++ b/anomalib/models/cflow/config.yaml @@ -43,7 +43,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/dfkde/config.yaml b/anomalib/models/dfkde/config.yaml index d4b53025f2..5a434b4b99 100644 --- a/anomalib/models/dfkde/config.yaml +++ b/anomalib/models/dfkde/config.yaml @@ -32,7 +32,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null visualization: diff --git a/anomalib/models/dfm/config.yaml b/anomalib/models/dfm/config.yaml index be84b8ab3e..19e0b8b5e0 100755 --- a/anomalib/models/dfm/config.yaml +++ b/anomalib/models/dfm/config.yaml @@ -32,7 +32,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null visualization: diff --git a/anomalib/models/draem/config.yaml b/anomalib/models/draem/config.yaml index 1c9159fc96..7a357ac60e 100644 --- a/anomalib/models/draem/config.yaml +++ b/anomalib/models/draem/config.yaml @@ -40,7 +40,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/fastflow/config.yaml b/anomalib/models/fastflow/config.yaml index 7908887a5b..170b164158 100644 --- a/anomalib/models/fastflow/config.yaml +++ b/anomalib/models/fastflow/config.yaml @@ -43,7 +43,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/ganomaly/config.yaml b/anomalib/models/ganomaly/config.yaml index 287993c1c0..5d2b8a5e61 100644 --- a/anomalib/models/ganomaly/config.yaml +++ b/anomalib/models/ganomaly/config.yaml @@ -44,7 +44,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null visualization: diff --git a/anomalib/models/padim/config.yaml b/anomalib/models/padim/config.yaml index b0b182fd65..d219f23485 100644 --- a/anomalib/models/padim/config.yaml +++ b/anomalib/models/padim/config.yaml @@ -38,7 +38,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/patchcore/config.yaml b/anomalib/models/patchcore/config.yaml index 306b19ea13..babc5fb523 100644 --- a/anomalib/models/patchcore/config.yaml +++ b/anomalib/models/patchcore/config.yaml @@ -39,7 +39,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/reverse_distillation/config.yaml b/anomalib/models/reverse_distillation/config.yaml index 719f1fa758..350a5bd7fd 100644 --- a/anomalib/models/reverse_distillation/config.yaml +++ b/anomalib/models/reverse_distillation/config.yaml @@ -47,7 +47,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null fixed_pixel: null diff --git a/anomalib/models/stfpm/config.yaml b/anomalib/models/stfpm/config.yaml index 24b15d3fb9..5a625bdb36 100644 --- a/anomalib/models/stfpm/config.yaml +++ b/anomalib/models/stfpm/config.yaml @@ -45,7 +45,7 @@ metrics: - F1Score - AUROC threshold: - method: adaptive #options: [adaptive, fixed] + method: adaptive #options: [adaptive, manual] fixed_image: null fixed_pixel: null diff --git a/anomalib/post_processing/post_process.py b/anomalib/post_processing/post_process.py index bdeb224ac8..000d1ac53b 100644 --- a/anomalib/post_processing/post_process.py +++ b/anomalib/post_processing/post_process.py @@ -17,7 +17,7 @@ class ThresholdMethod(str, Enum): """Threshold method to apply post-processing to the output predictions.""" ADAPTIVE = "adaptive" - FIXED = "fixed" + MANUAL = "manual" def add_label( diff --git a/anomalib/utils/callbacks/__init__.py b/anomalib/utils/callbacks/__init__.py index fa09e05669..fee4fa0305 100644 --- a/anomalib/utils/callbacks/__init__.py +++ b/anomalib/utils/callbacks/__init__.py @@ -72,8 +72,8 @@ def get_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]: pixel_threshold = config.metrics.threshold.fixed_pixel if "fixed_pixel" in config.metrics.threshold.keys() else None post_processing_callback = PostProcessingConfigurationCallback( threshold_method=config.metrics.threshold.method, - fixed_image_threshold=image_threshold, - fixed_pixel_threshold=pixel_threshold, + manual_image_threshold=image_threshold, + manual_pixel_threshold=pixel_threshold, ) callbacks.append(post_processing_callback) diff --git a/anomalib/utils/callbacks/post_processing_configuration.py b/anomalib/utils/callbacks/post_processing_configuration.py index 18947468ad..9437d3ba2e 100644 --- a/anomalib/utils/callbacks/post_processing_configuration.py +++ b/anomalib/utils/callbacks/post_processing_configuration.py @@ -25,40 +25,40 @@ class PostProcessingConfigurationCallback(Callback): Args: normalization_method(NormalizationMethod): Normalization method. - threshold_method (ThresholdMethod): Flag indicating whether threshold should be fixed or adaptive. - fixed_image_threshold (Optional[float]): Default fixed image threshold value. - fixed_pixel_threshold (Optional[float]): Default fixed pixel threshold value. + threshold_method (ThresholdMethod): Flag indicating whether threshold should be manual or adaptive. + manual_image_threshold (Optional[float]): Default manual image threshold value. + manual_pixel_threshold (Optional[float]): Default manual pixel threshold value. """ def __init__( self, normalization_method: NormalizationMethod = NormalizationMethod.MIN_MAX, threshold_method: ThresholdMethod = ThresholdMethod.ADAPTIVE, - fixed_image_threshold: Optional[float] = None, - fixed_pixel_threshold: Optional[float] = None, + manual_image_threshold: Optional[float] = None, + manual_pixel_threshold: Optional[float] = None, ) -> None: super().__init__() self.normalization_method = normalization_method if threshold_method == ThresholdMethod.ADAPTIVE and all( - i is not None for i in [fixed_image_threshold, fixed_pixel_threshold] + i is not None for i in [manual_image_threshold, manual_pixel_threshold] ): raise ValueError( - "When `threshold_method` is set to `adaptive`, `fixed_image_threshold` and `fixed_pixel_threshold` " + "When `threshold_method` is set to `adaptive`, `manual_image_threshold` and `manual_pixel_threshold` " "must not be set." ) - if threshold_method == ThresholdMethod.FIXED and all( - i is None for i in [fixed_image_threshold, fixed_pixel_threshold] + if threshold_method == ThresholdMethod.MANUAL and all( + i is None for i in [manual_image_threshold, manual_pixel_threshold] ): raise ValueError( - "When `threshold_method` is set to `fixed`, `fixed_image_threshold` and `fixed_pixel_threshold` " + "When `threshold_method` is set to `manual`, `manual_image_threshold` and `manual_pixel_threshold` " "must be set." ) self.threshold_method = threshold_method - self.fixed_image_threshold = fixed_image_threshold - self.fixed_pixel_threshold = fixed_pixel_threshold + self.manual_image_threshold = manual_image_threshold + self.manual_pixel_threshold = manual_pixel_threshold # pylint: disable=unused-argument def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[str] = None) -> None: @@ -71,6 +71,6 @@ def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[st """ if isinstance(pl_module, AnomalyModule): pl_module.threshold_method = self.threshold_method - if pl_module.threshold_method == ThresholdMethod.FIXED: - pl_module.image_threshold.value = torch.tensor(self.fixed_image_threshold).cpu() - pl_module.pixel_threshold.value = torch.tensor(self.fixed_pixel_threshold).cpu() + if pl_module.threshold_method == ThresholdMethod.MANUAL: + pl_module.image_threshold.value = torch.tensor(self.manual_image_threshold).cpu() + pl_module.pixel_threshold.value = torch.tensor(self.manual_pixel_threshold).cpu() diff --git a/anomalib/utils/cli/cli.py b/anomalib/utils/cli/cli.py index a37bba5ec8..14a0bc8470 100644 --- a/anomalib/utils/cli/cli.py +++ b/anomalib/utils/cli/cli.py @@ -110,8 +110,8 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None: { "post_processing.normalization_method": "min_max", "post_processing.threshold_method": "adaptive", - "post_processing.fixed_image_threshold": None, - "post_processing.fixed_pixel_threshold": None, + "post_processing.manual_image_threshold": None, + "post_processing.manual_pixel_threshold": None, } ) diff --git a/anomalib/utils/metrics/anomaly_score_threshold.py b/anomalib/utils/metrics/anomaly_score_threshold.py index 1375958264..36709b55e0 100644 --- a/anomalib/utils/metrics/anomaly_score_threshold.py +++ b/anomalib/utils/metrics/anomaly_score_threshold.py @@ -11,8 +11,8 @@ class AnomalyScoreThreshold(PrecisionRecallCurve): """Anomaly Score Threshold. This class computes/stores the threshold that determines the anomalous label - given anomaly scores. If the threshold method is ``fixed``, the class only - stores the fixed threshold values. + given anomaly scores. If the threshold method is ``manual``, the class only + stores the manual threshold values. If the threshold method is ``adaptive``, the class initially computes the adaptive threshold to find the optimal f1_score and stores the computed diff --git a/anomalib/utils/sweep/helpers/callbacks.py b/anomalib/utils/sweep/helpers/callbacks.py index 3085518d16..2595567334 100644 --- a/anomalib/utils/sweep/helpers/callbacks.py +++ b/anomalib/utils/sweep/helpers/callbacks.py @@ -44,17 +44,17 @@ def get_sweep_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback] image_metrics = config.metrics.image_metrics if "image_metrics" in config.metrics else None pixel_metrics = config.metrics.pixel_metrics if "pixel_metrics" in config.metrics else None image_threshold = ( - config.post_processing.fixed_image_threshold if "image_default" in config.post_processing.keys() else None + config.post_processing.manual_image_threshold if "image_default" in config.post_processing.keys() else None ) pixel_threshold = ( - config.post_processing.fixed_pixel_threshold if "pixel_default" in config.post_processing.keys() else None + config.post_processing.manual_pixel_threshold if "pixel_default" in config.post_processing.keys() else None ) normalization_method = config.post_processing.normalization_method post_processing_configuration_callback = PostProcessingConfigurationCallback( normalization_method=normalization_method, - fixed_image_threshold=image_threshold, - fixed_pixel_threshold=pixel_threshold, + manual_image_threshold=image_threshold, + manual_pixel_threshold=pixel_threshold, ) callbacks.append(post_processing_configuration_callback) diff --git a/configs/model/cflow.yaml b/configs/model/cflow.yaml index 60e0c15365..8290475400 100644 --- a/configs/model/cflow.yaml +++ b/configs/model/cflow.yaml @@ -39,9 +39,9 @@ model: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/dfkde.yaml b/configs/model/dfkde.yaml index b8e813c8a3..86a427042f 100644 --- a/configs/model/dfkde.yaml +++ b/configs/model/dfkde.yaml @@ -34,9 +34,9 @@ model: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/dfm.yaml b/configs/model/dfm.yaml index f09d17bd4a..75d994895d 100644 --- a/configs/model/dfm.yaml +++ b/configs/model/dfm.yaml @@ -31,9 +31,9 @@ model: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/draem.yaml b/configs/model/draem.yaml index a5760a1d71..0e428e076e 100644 --- a/configs/model/draem.yaml +++ b/configs/model/draem.yaml @@ -31,9 +31,9 @@ optimizer: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/fastflow.yaml b/configs/model/fastflow.yaml index 2f89d732f5..23b5aa3b65 100644 --- a/configs/model/fastflow.yaml +++ b/configs/model/fastflow.yaml @@ -37,9 +37,9 @@ optimizer: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/ganomaly.yaml b/configs/model/ganomaly.yaml index 5fbf5e890d..a8e3a70dbd 100644 --- a/configs/model/ganomaly.yaml +++ b/configs/model/ganomaly.yaml @@ -37,13 +37,13 @@ model: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null image_metrics: - F1Score - AUROC diff --git a/configs/model/padim.yaml b/configs/model/padim.yaml index 730c34fa69..b1c8e3f876 100644 --- a/configs/model/padim.yaml +++ b/configs/model/padim.yaml @@ -34,9 +34,9 @@ model: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/patchcore.yaml b/configs/model/patchcore.yaml index a9a9db46b1..b3d3f54f6b 100644 --- a/configs/model/patchcore.yaml +++ b/configs/model/patchcore.yaml @@ -34,9 +34,9 @@ model: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/reverse_distillation.yaml b/configs/model/reverse_distillation.yaml index e9a6a3d308..cf1abd1f07 100644 --- a/configs/model/reverse_distillation.yaml +++ b/configs/model/reverse_distillation.yaml @@ -37,9 +37,9 @@ model: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/configs/model/stfpm.yaml b/configs/model/stfpm.yaml index 38b8901d6b..b9b2914643 100644 --- a/configs/model/stfpm.yaml +++ b/configs/model/stfpm.yaml @@ -39,9 +39,9 @@ optimizer: post_processing: normalization_method: min_max # - threshold_method: adaptive # options: [adaptive, fixed] - fixed_image_threshold: null - fixed_pixel_threshold: null + threshold_method: adaptive # options: [adaptive, manual] + manual_image_threshold: null + manual_pixel_threshold: null metrics: image_metrics: diff --git a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py index 96f3692d6a..5f698f93b1 100644 --- a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py +++ b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py @@ -32,15 +32,15 @@ def test_adaptive_threshold(labels, preds, target_threshold): assert threshold_value == target_threshold -def test_non_adaptive_threshold(): +def test_manual_threshold(): """ - Test if the non-adaptive threshold gets used in the F1 score computation when + Test if the manual threshold gets used in the F1 score computation when adaptive thresholding is disabled and no normalization is used. """ config = get_test_configurable_parameters(config_path="anomalib/models/padim/config.yaml") config.model.normalization_method = "none" - config.metrics.threshold.method = "fixed" + config.metrics.threshold.method = "manual" config.trainer.fast_dev_run = True config.metrics.image = ["F1Score"] config.metrics.pixel = ["F1Score"] From d0be1116d99db7c6e3621737cb0ace07e08e04cd Mon Sep 17 00:00:00 2001 From: Samet Date: Tue, 18 Oct 2022 06:51:40 -0700 Subject: [PATCH 21/21] Rename some left-over variables. --- anomalib/config/config.py | 8 ++++---- anomalib/models/cflow/config.yaml | 4 ++-- anomalib/models/dfkde/config.yaml | 2 +- anomalib/models/dfm/config.yaml | 2 +- anomalib/models/draem/config.yaml | 4 ++-- anomalib/models/fastflow/config.yaml | 4 ++-- anomalib/models/ganomaly/config.yaml | 2 +- anomalib/models/padim/config.yaml | 4 ++-- anomalib/models/patchcore/config.yaml | 4 ++-- anomalib/models/reverse_distillation/config.yaml | 4 ++-- anomalib/models/stfpm/config.yaml | 4 ++-- anomalib/utils/callbacks/__init__.py | 8 ++++++-- anomalib/utils/sweep/helpers/callbacks.py | 4 ++-- tests/pre_merge/utils/metrics/test_adaptive_threshold.py | 4 ++-- 14 files changed, 31 insertions(+), 27 deletions(-) diff --git a/anomalib/config/config.py b/anomalib/config/config.py index da93a5f912..65adf5d292 100644 --- a/anomalib/config/config.py +++ b/anomalib/config/config.py @@ -161,13 +161,13 @@ def get_configurable_parameters( warn("adaptive will be deprecated in favor of method in config.metrics.threshold in v0.4.0.") config.metrics.threshold.method = "adaptive" if config.metrics.threshold.adaptive else "manual" if "image_default" in config.metrics.threshold.keys(): - warn("image_default will be deprecated in favor of fixed_image in config.metrics.threshold in v0.4.0.") - config.metrics.threshold.fixed_image = ( + warn("image_default will be deprecated in favor of manual_image in config.metrics.threshold in v0.4.0.") + config.metrics.threshold.manual_image = ( None if config.metrics.threshold.adaptive else config.metrics.threshold.image_default ) if "pixel_default" in config.metrics.threshold.keys(): - warn("pixel_default will be deprecated in favor of fixed_pixel in config.metrics.threshold in v0.4.0.") - config.metrics.threshold.fixed_pixel = ( + warn("pixel_default will be deprecated in favor of manual_pixel in config.metrics.threshold in v0.4.0.") + config.metrics.threshold.manual_pixel = ( None if config.metrics.threshold.adaptive else config.metrics.threshold.pixel_default ) diff --git a/anomalib/models/cflow/config.yaml b/anomalib/models/cflow/config.yaml index 47472bf846..4337538445 100644 --- a/anomalib/models/cflow/config.yaml +++ b/anomalib/models/cflow/config.yaml @@ -44,8 +44,8 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null - fixed_pixel: null + manual_image: null + manual_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/dfkde/config.yaml b/anomalib/models/dfkde/config.yaml index 5a434b4b99..d98bef0fec 100644 --- a/anomalib/models/dfkde/config.yaml +++ b/anomalib/models/dfkde/config.yaml @@ -33,7 +33,7 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null + manual_image: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/dfm/config.yaml b/anomalib/models/dfm/config.yaml index 19e0b8b5e0..fc64d78bd5 100755 --- a/anomalib/models/dfm/config.yaml +++ b/anomalib/models/dfm/config.yaml @@ -33,7 +33,7 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null + manual_image: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/draem/config.yaml b/anomalib/models/draem/config.yaml index 7a357ac60e..f76e163a27 100644 --- a/anomalib/models/draem/config.yaml +++ b/anomalib/models/draem/config.yaml @@ -41,8 +41,8 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null - fixed_pixel: null + manual_image: null + manual_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/fastflow/config.yaml b/anomalib/models/fastflow/config.yaml index 170b164158..efe749afdd 100644 --- a/anomalib/models/fastflow/config.yaml +++ b/anomalib/models/fastflow/config.yaml @@ -44,8 +44,8 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null - fixed_pixel: null + manual_image: null + manual_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/ganomaly/config.yaml b/anomalib/models/ganomaly/config.yaml index 5d2b8a5e61..b511da3160 100644 --- a/anomalib/models/ganomaly/config.yaml +++ b/anomalib/models/ganomaly/config.yaml @@ -45,7 +45,7 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null + manual_image: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/padim/config.yaml b/anomalib/models/padim/config.yaml index d219f23485..63d7df90c5 100644 --- a/anomalib/models/padim/config.yaml +++ b/anomalib/models/padim/config.yaml @@ -39,8 +39,8 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null - fixed_pixel: null + manual_image: null + manual_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/patchcore/config.yaml b/anomalib/models/patchcore/config.yaml index babc5fb523..66a55f1e5c 100644 --- a/anomalib/models/patchcore/config.yaml +++ b/anomalib/models/patchcore/config.yaml @@ -40,8 +40,8 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null - fixed_pixel: null + manual_image: null + manual_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/reverse_distillation/config.yaml b/anomalib/models/reverse_distillation/config.yaml index 350a5bd7fd..696e76874b 100644 --- a/anomalib/models/reverse_distillation/config.yaml +++ b/anomalib/models/reverse_distillation/config.yaml @@ -48,8 +48,8 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null - fixed_pixel: null + manual_image: null + manual_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/models/stfpm/config.yaml b/anomalib/models/stfpm/config.yaml index 5a625bdb36..153e92b3db 100644 --- a/anomalib/models/stfpm/config.yaml +++ b/anomalib/models/stfpm/config.yaml @@ -46,8 +46,8 @@ metrics: - AUROC threshold: method: adaptive #options: [adaptive, manual] - fixed_image: null - fixed_pixel: null + manual_image: null + manual_pixel: null visualization: show_images: False # show images on the screen diff --git a/anomalib/utils/callbacks/__init__.py b/anomalib/utils/callbacks/__init__.py index fee4fa0305..8f9126d1b5 100644 --- a/anomalib/utils/callbacks/__init__.py +++ b/anomalib/utils/callbacks/__init__.py @@ -68,8 +68,12 @@ def get_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]: callbacks.extend([checkpoint, TimerCallback()]) # Add post-processing configurations to AnomalyModule. - image_threshold = config.metrics.threshold.fixed_image if "fixed_image" in config.metrics.threshold.keys() else None - pixel_threshold = config.metrics.threshold.fixed_pixel if "fixed_pixel" in config.metrics.threshold.keys() else None + image_threshold = ( + config.metrics.threshold.manual_image if "manual_image" in config.metrics.threshold.keys() else None + ) + pixel_threshold = ( + config.metrics.threshold.manual_pixel if "manual_pixel" in config.metrics.threshold.keys() else None + ) post_processing_callback = PostProcessingConfigurationCallback( threshold_method=config.metrics.threshold.method, manual_image_threshold=image_threshold, diff --git a/anomalib/utils/sweep/helpers/callbacks.py b/anomalib/utils/sweep/helpers/callbacks.py index 2595567334..62dacda2d1 100644 --- a/anomalib/utils/sweep/helpers/callbacks.py +++ b/anomalib/utils/sweep/helpers/callbacks.py @@ -33,10 +33,10 @@ def get_sweep_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback] image_metrics = config.metrics.image if "image" in config.metrics.keys() else None pixel_metrics = config.metrics.pixel if "pixel" in config.metrics.keys() else None image_threshold = ( - config.metrics.threshold.fixed_image if "fixed_image" in config.metrics.threshold.keys() else None + config.metrics.threshold.manual_image if "manual_image" in config.metrics.threshold.keys() else None ) pixel_threshold = ( - config.metrics.threshold.fixed_pixel if "fixed_pixel" in config.metrics.threshold.keys() else None + config.metrics.threshold.manual_pixel if "manual_pixel" in config.metrics.threshold.keys() else None ) normalization_method = config.model.normalization_method # NOTE: This is for the new anomalib CLI. diff --git a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py index 5f698f93b1..eb9669d23b 100644 --- a/tests/pre_merge/utils/metrics/test_adaptive_threshold.py +++ b/tests/pre_merge/utils/metrics/test_adaptive_threshold.py @@ -47,8 +47,8 @@ def test_manual_threshold(): image_threshold = random.random() pixel_threshold = random.random() - config.metrics.threshold.fixed_image = image_threshold - config.metrics.threshold.fixed_pixel = pixel_threshold + config.metrics.threshold.manual_image = image_threshold + config.metrics.threshold.manual_pixel = pixel_threshold model = get_model(config) datamodule = get_datamodule(config)