diff --git a/anomalib/utils/callbacks/__init__.py b/anomalib/utils/callbacks/__init__.py index 270648fb9d..15def2a1f5 100644 --- a/anomalib/utils/callbacks/__init__.py +++ b/anomalib/utils/callbacks/__init__.py @@ -19,19 +19,22 @@ from .metrics_configuration import MetricsConfigurationCallback from .min_max_normalization import MinMaxNormalizationCallback from .model_loader import LoadModelCallback +from .post_processing_configuration import PostProcessingConfigurationCallback from .tiler_configuration import TilerConfigurationCallback from .timer import TimerCallback from .visualizer import ImageVisualizerCallback, MetricVisualizerCallback __all__ = [ "CdfNormalizationCallback", + "GraphLogger", + "ImageVisualizerCallback", "LoadModelCallback", "MetricsConfigurationCallback", + "MetricVisualizerCallback", "MinMaxNormalizationCallback", + "PostProcessingConfigurationCallback", "TilerConfigurationCallback", "TimerCallback", - "ImageVisualizerCallback", - "MetricVisualizerCallback", ] @@ -64,20 +67,25 @@ def get_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]: callbacks.extend([checkpoint, TimerCallback()]) - # Add metric configuration to the model via MetricsConfigurationCallback - image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None - pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None + # Add post-processing configurations to AnomalyModule. image_threshold = ( config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None ) pixel_threshold = ( config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None ) + post_processing_callback = PostProcessingConfigurationCallback( + adaptive_threshold=config.metrics.threshold.adaptive, + default_image_threshold=image_threshold, + default_pixel_threshold=pixel_threshold, + ) + callbacks.append(post_processing_callback) + + # Add metric configuration to the model via MetricsConfigurationCallback + image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None + pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None metrics_callback = MetricsConfigurationCallback( - config.metrics.threshold.adaptive, config.dataset.task, - image_threshold, - pixel_threshold, image_metric_names, pixel_metric_names, ) @@ -172,7 +180,8 @@ def add_visualizer_callback(callbacks: List[Callback], config: Union[DictConfig, config.visualization.inputs_are_normalized = not config.model.normalization_method == "none" else: config.visualization.task = config.data.init_args.task - config.visualization.inputs_are_normalized = not config.metrics.normalization_method == "none" + config.visualization.inputs_are_normalized = not config.post_processing.normalization_method == "none" + if config.visualization.log_images or config.visualization.save_images or config.visualization.show_images: image_save_path = ( config.visualization.image_save_path diff --git a/anomalib/utils/callbacks/metrics_configuration.py b/anomalib/utils/callbacks/metrics_configuration.py index bc91c23e83..2abc47f848 100644 --- a/anomalib/utils/callbacks/metrics_configuration.py +++ b/anomalib/utils/callbacks/metrics_configuration.py @@ -8,7 +8,6 @@ from typing import List, Optional import pytorch_lightning as pl -import torch from pytorch_lightning.callbacks import Callback from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY @@ -26,13 +25,9 @@ class MetricsConfigurationCallback(Callback): def __init__( self, - adaptive_threshold: bool, task: str = "segmentation", - default_image_threshold: Optional[float] = None, - default_pixel_threshold: Optional[float] = None, - image_metric_names: Optional[List[str]] = None, - pixel_metric_names: Optional[List[str]] = None, - normalization_method: str = "min_max", + image_metrics: Optional[List[str]] = None, + pixel_metrics: Optional[List[str]] = None, ): """Create image and pixel-level AnomalibMetricsCollection. @@ -43,30 +38,12 @@ def __init__( Args: task (str): Task type of the current run. - adaptive_threshold (bool): Flag indicating whether threshold should be adaptive. - default_image_threshold (Optional[float]): Default image threshold value. - default_pixel_threshold (Optional[float]): Default pixel threshold value. - image_metric_names (Optional[List[str]]): List of image-level metrics. - pixel_metric_names (Optional[List[str]]): List of pixel-level metrics. - normalization_method(Optional[str]): Normalization method. + image_metrics (Optional[List[str]]): List of image-level metrics. + pixel_metrics (Optional[List[str]]): List of pixel-level metrics. """ - # TODO: https://github.com/openvinotoolkit/anomalib/issues/384 self.task = task - self.image_metric_names = image_metric_names - self.pixel_metric_names = pixel_metric_names - - # TODO: https://github.com/openvinotoolkit/anomalib/issues/384 - # TODO: This is a workaround. normalization-method is actually not used in metrics. - # It's only accessed from `before_instantiate` method in `AnomalibCLI` to configure - # its callback. - self.normalization_method = normalization_method - - assert ( - adaptive_threshold or default_image_threshold is not None and default_pixel_threshold is not None - ), "Default thresholds must be specified when adaptive threshold is disabled." - self.adaptive_threshold = adaptive_threshold - self.default_image_threshold = default_image_threshold - self.default_pixel_threshold = default_pixel_threshold + self.image_metric_names = image_metrics + self.pixel_metric_names = pixel_metrics def setup( self, @@ -97,12 +74,6 @@ def setup( pixel_metric_names = self.pixel_metric_names if isinstance(pl_module, AnomalyModule): - pl_module.adaptive_threshold = self.adaptive_threshold - if not self.adaptive_threshold: - # pylint: disable=not-callable - pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu() - pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu() - pl_module.image_metrics = metric_collection_from_names(image_metric_names, "image_") pl_module.pixel_metrics = metric_collection_from_names(pixel_metric_names, "pixel_") diff --git a/anomalib/utils/callbacks/post_processing_configuration.py b/anomalib/utils/callbacks/post_processing_configuration.py new file mode 100644 index 0000000000..9549a87fd9 --- /dev/null +++ b/anomalib/utils/callbacks/post_processing_configuration.py @@ -0,0 +1,63 @@ +"""Post-Processing Configuration Callback.""" + +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +import logging +from typing import Optional + +import torch +from pytorch_lightning import Callback, LightningModule, Trainer +from pytorch_lightning.utilities.cli import CALLBACK_REGISTRY + +from anomalib.models.components.base.anomaly_module import AnomalyModule + +logger = logging.getLogger(__name__) + +__all__ = ["PostProcessingConfigurationCallback"] + + +@CALLBACK_REGISTRY +class PostProcessingConfigurationCallback(Callback): + """Post-Processing Configuration Callback. + + Args: + normalization_method(Optional[str]): Normalization method. + adaptive_threshold (bool): Flag indicating whether threshold should be adaptive. + default_image_threshold (Optional[float]): Default image threshold value. + default_pixel_threshold (Optional[float]): Default pixel threshold value. + """ + + def __init__( + self, + normalization_method: str = "min_max", + adaptive_threshold: bool = True, + default_image_threshold: Optional[float] = None, + default_pixel_threshold: Optional[float] = None, + ) -> None: + super().__init__() + self.normalization_method = normalization_method + + assert ( + adaptive_threshold or default_image_threshold is not None and default_pixel_threshold is not None + ), "Default thresholds must be specified when adaptive threshold is disabled." + + self.adaptive_threshold = adaptive_threshold + self.default_image_threshold = default_image_threshold + self.default_pixel_threshold = default_pixel_threshold + + # pylint: disable=unused-argument + def setup(self, trainer: Trainer, pl_module: LightningModule, stage: Optional[str] = None) -> None: + """Setup post-processing configuration within Anomalib Model. + + Args: + trainer (Trainer): PyTorch Lightning Trainer + pl_module (LightningModule): Anomalib Model that inherits pl LightningModule. + stage (Optional[str], optional): fit, validate, test or predict. Defaults to None. + """ + if isinstance(pl_module, AnomalyModule): + pl_module.adaptive_threshold = self.adaptive_threshold + if pl_module.adaptive_threshold is False: + pl_module.image_threshold.value = torch.tensor(self.default_image_threshold).cpu() + pl_module.pixel_threshold.value = torch.tensor(self.default_pixel_threshold).cpu() diff --git a/anomalib/utils/cli/cli.py b/anomalib/utils/cli/cli.py index 48c1769163..7ce51a918a 100644 --- a/anomalib/utils/cli/cli.py +++ b/anomalib/utils/cli/cli.py @@ -26,6 +26,7 @@ MetricsConfigurationCallback, MinMaxNormalizationCallback, ModelCheckpoint, + PostProcessingConfigurationCallback, TilerConfigurationCallback, TimerCallback, add_visualizer_callback, @@ -90,7 +91,6 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None: Args: parser (LightningArgumentParser): Lightning Argument Parser. """ - # TODO: https://github.com/openvinotoolkit/anomalib/issues/19 # TODO: https://github.com/openvinotoolkit/anomalib/issues/20 parser.add_argument( "--export_mode", type=str, default="", help="Select export mode to ONNX or OpenVINO IR format." @@ -105,18 +105,24 @@ def add_arguments_to_parser(self, parser: LightningArgumentParser) -> None: parser.add_lightning_class_args(TilerConfigurationCallback, "tiling") # type: ignore parser.set_defaults({"tiling.enable": False}) + parser.add_lightning_class_args(PostProcessingConfigurationCallback, "post_processing") # type: ignore + parser.set_defaults( + { + "post_processing.normalization_method": "min_max", + "post_processing.adaptive_threshold": True, + "post_processing.default_image_threshold": None, + "post_processing.default_pixel_threshold": None, + } + ) + # TODO: Assign these default values within the MetricsConfigurationCallback # - https://github.com/openvinotoolkit/anomalib/issues/384 parser.add_lightning_class_args(MetricsConfigurationCallback, "metrics") # type: ignore parser.set_defaults( { - "metrics.adaptive_threshold": True, "metrics.task": "segmentation", - "metrics.default_image_threshold": None, - "metrics.default_pixel_threshold": None, - "metrics.image_metric_names": ["F1Score", "AUROC"], - "metrics.pixel_metric_names": ["F1Score", "AUROC"], - "metrics.normalization_method": "min_max", + "metrics.image_metrics": ["F1Score", "AUROC"], + "metrics.pixel_metrics": ["F1Score", "AUROC"], } ) @@ -203,7 +209,7 @@ def __set_callbacks(self) -> None: # TODO: This could be set in PostProcessingConfiguration callback # - https://github.com/openvinotoolkit/anomalib/issues/384 # Normalization. - normalization = config.metrics.normalization_method + normalization = config.post_processing.normalization_method if normalization: if normalization == "min_max": callbacks.append(MinMaxNormalizationCallback()) diff --git a/anomalib/utils/sweep/helpers/callbacks.py b/anomalib/utils/sweep/helpers/callbacks.py index 975224b9b7..3f1057a5d5 100644 --- a/anomalib/utils/sweep/helpers/callbacks.py +++ b/anomalib/utils/sweep/helpers/callbacks.py @@ -9,7 +9,10 @@ from omegaconf import DictConfig, ListConfig from pytorch_lightning import Callback -from anomalib.utils.callbacks import MetricsConfigurationCallback +from anomalib.utils.callbacks import ( + MetricsConfigurationCallback, + PostProcessingConfigurationCallback, +) from anomalib.utils.callbacks.timer import TimerCallback @@ -24,23 +27,40 @@ def get_sweep_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback] """ callbacks: List[Callback] = [TimerCallback()] # Add metric configuration to the model via MetricsConfigurationCallback - image_metric_names = config.metrics.image if "image" in config.metrics.keys() else None - pixel_metric_names = config.metrics.pixel if "pixel" in config.metrics.keys() else None - image_threshold = ( - config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None - ) - pixel_threshold = ( - config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None - ) - metrics_callback = MetricsConfigurationCallback( - adaptive_threshold=config.metrics.threshold.adaptive, - task=config.dataset.task, + + # TODO: Remove this once the old CLI is deprecated. + if isinstance(config, DictConfig): + image_metrics = config.metrics.image if "image" in config.metrics.keys() else None + pixel_metrics = config.metrics.pixel if "pixel" in config.metrics.keys() else None + image_threshold = ( + config.metrics.threshold.image_default if "image_default" in config.metrics.threshold.keys() else None + ) + pixel_threshold = ( + config.metrics.threshold.pixel_default if "pixel_default" in config.metrics.threshold.keys() else None + ) + normalization_method = config.model.normalization_method + # NOTE: This is for the new anomalib CLI. + else: + image_metrics = config.metrics.image_metrics if "image_metrics" in config.metrics else None + pixel_metrics = config.metrics.pixel_metrics if "pixel_metrics" in config.metrics else None + image_threshold = ( + config.post_processing.default_image_threshold if "image_default" in config.post_processing.keys() else None + ) + pixel_threshold = ( + config.post_processing.default_pixel_threshold if "pixel_default" in config.post_processing.keys() else None + ) + normalization_method = config.post_processing.normalization_method + + post_processing_configuration_callback = PostProcessingConfigurationCallback( + normalization_method=normalization_method, default_image_threshold=image_threshold, default_pixel_threshold=pixel_threshold, - image_metric_names=image_metric_names, - pixel_metric_names=pixel_metric_names, - normalization_method=config.model.normalization_method, ) - callbacks.append(metrics_callback) + callbacks.append(post_processing_configuration_callback) + + metrics_configuration_callback = MetricsConfigurationCallback( + task=config.dataset.task, image_metrics=image_metrics, pixel_metrics=pixel_metrics + ) + callbacks.append(metrics_configuration_callback) return callbacks diff --git a/configs/model/cflow.yaml b/configs/model/cflow.yaml index 2f925071e1..672b101fa1 100644 --- a/configs/model/cflow.yaml +++ b/configs/model/cflow.yaml @@ -37,21 +37,23 @@ model: permute_soft: false lr: 0.0001 -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + +metrics: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/dfkde.yaml b/configs/model/dfkde.yaml index 4a43a344b9..54ba782db0 100644 --- a/configs/model/dfkde.yaml +++ b/configs/model/dfkde.yaml @@ -32,16 +32,20 @@ model: threshold_steepness: 0.05 threshold_offset: 12 -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null - image_metric_names: + default_pixel_threshold: null + +metrics: + image_metrics: - F1Score - AUROC visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/dfm.yaml b/configs/model/dfm.yaml index 2670510a16..69f7fd0a8d 100644 --- a/configs/model/dfm.yaml +++ b/configs/model/dfm.yaml @@ -29,21 +29,23 @@ model: pca_level: 0.97 score_type: fre -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + +metrics: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/draem.yaml b/configs/model/draem.yaml index 78d9defd8f..9cda0e1ac0 100644 --- a/configs/model/draem.yaml +++ b/configs/model/draem.yaml @@ -29,21 +29,23 @@ optimizer: init_args: lr: 0.0001 -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + +metrics: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/fastflow.yaml b/configs/model/fastflow.yaml index 1a2217ab6f..285be59ec4 100644 --- a/configs/model/fastflow.yaml +++ b/configs/model/fastflow.yaml @@ -35,21 +35,23 @@ optimizer: lr: 0.001 weight_decay: 0.00001 -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + +metrics: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/ganomaly.yaml b/configs/model/ganomaly.yaml index 23ca106244..3ba6708e5e 100644 --- a/configs/model/ganomaly.yaml +++ b/configs/model/ganomaly.yaml @@ -35,16 +35,22 @@ model: beta1: 0.5 beta2: 0.999 +post_processing: + normalization_method: min_max # + adaptive_threshold: true + default_image_threshold: null + default_pixel_threshold: null + metrics: adaptive_threshold: true default_image_threshold: null - image_metric_names: + image_metrics: - F1Score - AUROC visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/padim.yaml b/configs/model/padim.yaml index 6b9871feb4..013d0485d8 100644 --- a/configs/model/padim.yaml +++ b/configs/model/padim.yaml @@ -32,21 +32,23 @@ model: - layer2 - layer3 -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + +metrics: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/patchcore.yaml b/configs/model/patchcore.yaml index d357b06aca..4e99bf931e 100644 --- a/configs/model/patchcore.yaml +++ b/configs/model/patchcore.yaml @@ -32,21 +32,23 @@ model: coreset_sampling_ratio: 0.1 num_neighbors: 9 -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + +metrics: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/reverse_distillation.yaml b/configs/model/reverse_distillation.yaml index f1276bcdd7..f067b33846 100644 --- a/configs/model/reverse_distillation.yaml +++ b/configs/model/reverse_distillation.yaml @@ -35,21 +35,23 @@ model: beta1: 0.5 beta2: 0.99 -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + +metrics: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"] diff --git a/configs/model/stfpm.yaml b/configs/model/stfpm.yaml index bd5ea0be43..7fc8df2d7f 100644 --- a/configs/model/stfpm.yaml +++ b/configs/model/stfpm.yaml @@ -37,21 +37,23 @@ optimizer: momentum: 0.9 weight_decay: 0.0001 -metrics: +post_processing: + normalization_method: min_max # adaptive_threshold: true default_image_threshold: null default_pixel_threshold: null - image_metric_names: + +metrics: + image_metrics: - F1Score - AUROC - pixel_metric_names: + pixel_metrics: - F1Score - AUROC - normalization_method: min_max visualization: show_images: False # show images on the screen - save_images: False # save images to the file system + save_images: True # save images to the file system log_images: False # log images to the available loggers (if any) mode: full # options: ["full", "simple"]