Patchcore validation and test error!

[1chimaruGin]

Describe the bug

• Value Error: No samples to concatenate
  How can I fix this?
  I trained on my custom dataset. with the following config.

dataset:
    name:  # Dataset name
    format: folder # Dataset type
    path:  # Dataset train path
    normal_dir: OK # name of the folder containing normal images.
    abnormal_dir: NG # name of the folder containing abnormal images.
    normal_test_dir: null # name of the folder containing normal test images.
    normal: OK
    abnormal: NG 
    split_ratio: 0.2
    seed: 47
    task: classification # options: [segmentation, classification]
    mask: null
    extensions: null
    image_size: 256
    train_batch_size: 32
    test_batch_size: 1  
    num_workers: 1 # 36
    transform_config:
        train: null 
        val: null
    create_validation_set: false
    tiling:
        apply: true
        tile_size: 24
        stride: 2
        remove_border_count: 0
        use_random_tiling: False
        random_tile_count: 16

model:
  name: patchcore
  backbone: wide_resnet50_2
  pre_trained: true
  layers:
    - layer2
    - layer3
  coreset_sampling_ratio: 0.1
  num_neighbors: 9
  normalization_method: min_max # options: [null, min_max, cdf]

metrics:
  image:
    - F1Score
    - AUROC
  pixel:
    - F1Score
    - AUROC
  threshold:
    image_default: 0
    pixel_default: 0
    adaptive: true

visualization:
  show_images: False # show images on the screen
  save_images: True # save images to the file system
  log_images: True # log images to the available loggers (if any)
  image_save_path: null # path to which images will be saved
  mode: full # options: ["full", "simple"]

project:
  seed: 0
  path: ./results

logging:
  logger: [] # options: [tensorboard, wandb, csv] or combinations.
  log_graph: false # Logs the model graph to respective logger.

# PL Trainer Args. Don't add extra parameter here.
trainer:
  accelerator: auto # <"cpu", "gpu", "tpu", "ipu", "hpu", "auto">
  amp_backend: native
  auto_lr_find: false
  auto_scale_batch_size: false
  auto_select_gpus: false
  benchmark: false
  check_val_every_n_epoch: 1 # Don't validate before extracting features.
  default_root_dir: null
  detect_anomaly: false
  deterministic: false
  devices: 1
  enable_checkpointing: true
  enable_model_summary: true
  enable_progress_bar: true
  fast_dev_run: false
  gpus: null # Set automatically
  gradient_clip_val: 0
  ipus: null
  limit_predict_batches: 1.0
  limit_test_batches: 1.0
  limit_train_batches: 1.0
  limit_val_batches: 1.0
  log_every_n_steps: 50
  log_gpu_memory: null
  max_epochs: 1
  max_steps: -1
  max_time: null
  min_epochs: null
  min_steps: null
  move_metrics_to_cpu: false
  multiple_trainloader_mode: max_size_cycle
  num_nodes: 1
  num_processes: null
  num_sanity_val_steps: 0
  overfit_batches: 0.0
  plugins: null
  precision: 32
  profiler: null
  reload_dataloaders_every_n_epochs: 0
  replace_sampler_ddp: true
  strategy: null
  sync_batchnorm: false
  tpu_cores: null
  track_grad_norm: -1
  val_check_interval: 1.0 # Don't validate before extracting features.

Error

Testing DataLoader 0: 100%
40/40 [00:05<00:00, 7.94it/s]
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Input In [16], in <cell line: 6>()
      2 load_model_callback = LoadModelCallback(
      3     weights_path=trainer.checkpoint_callback.best_model_path
      4 )
      5 trainer.callbacks.insert(0, load_model_callback)
----> 6 trainer.test(model=model, datamodule=datamodule)

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:938, in Trainer.test(self, model, dataloaders, ckpt_path, verbose, datamodule)
    912 r"""
    913 Perform one evaluation epoch over the test set.
    914 It's separated from fit to make sure you never run on your test set until you want to.
   (...)
    935     The length of the list corresponds to the number of test dataloaders used.
    936 """
    937 self.strategy.model = model or self.lightning_module
--> 938 return self._call_and_handle_interrupt(self._test_impl, model, dataloaders, ckpt_path, verbose, datamodule)

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:723, in Trainer._call_and_handle_interrupt(self, trainer_fn, *args, **kwargs)
    721         return self.strategy.launcher.launch(trainer_fn, *args, trainer=self, **kwargs)
    722     else:
--> 723         return trainer_fn(*args, **kwargs)
    724 # TODO: treat KeyboardInterrupt as BaseException (delete the code below) in v1.7
    725 except KeyboardInterrupt as exception:

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:985, in Trainer._test_impl(self, model, dataloaders, ckpt_path, verbose, datamodule)
    982 self._tested_ckpt_path = self.ckpt_path  # TODO: remove in v1.8
    984 # run test
--> 985 results = self._run(model, ckpt_path=self.ckpt_path)
    987 assert self.state.stopped
    988 self.testing = False

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1236, in Trainer._run(self, model, ckpt_path)
   1232 self._checkpoint_connector.restore_training_state()
   1234 self._checkpoint_connector.resume_end()
-> 1236 results = self._run_stage()
   1238 log.detail(f"{self.__class__.__name__}: trainer tearing down")
   1239 self._teardown()

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1320, in Trainer._run_stage(self)
   1317 self.strategy.dispatch(self)
   1319 if self.evaluating:
-> 1320     return self._run_evaluate()
   1321 if self.predicting:
   1322     return self._run_predict()

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1365, in Trainer._run_evaluate(self)
   1362 self._evaluation_loop.trainer = self
   1364 with self.profiler.profile(f"run_{self.state.stage}_evaluation"), torch.no_grad():
-> 1365     eval_loop_results = self._evaluation_loop.run()
   1367 # remove the tensors from the eval results
   1368 for result in eval_loop_results:

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/loops/base.py:211, in Loop.run(self, *args, **kwargs)
    208         break
    209 self._restarting = False
--> 211 output = self.on_run_end()
    212 return output

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py:201, in EvaluationLoop.on_run_end(self)
    198 self.trainer._logger_connector.log_eval_end_metrics(all_logged_outputs)
    200 # hook
--> 201 self._on_evaluation_end()
    203 # enable train mode again
    204 self._on_evaluation_model_train()

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py:273, in EvaluationLoop._on_evaluation_end(self, *args, **kwargs)
    271 """Runs ``on_{validation/test}_end`` hook."""
    272 if self.trainer.testing:
--> 273     self.trainer._call_callback_hooks("on_test_end", *args, **kwargs)
    274     self.trainer._call_lightning_module_hook("on_test_end", *args, **kwargs)
    275     self.trainer._call_strategy_hook("on_test_end", *args, **kwargs)

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1636, in Trainer._call_callback_hooks(self, hook_name, *args, **kwargs)
   1634         if callable(fn):
   1635             with self.profiler.profile(f"[Callback]{callback.state_key}.{hook_name}"):
-> 1636                 fn(self, self.lightning_module, *args, **kwargs)
   1638 if pl_module:
   1639     # restore current_fx when nested context
   1640     pl_module._current_fx_name = prev_fx_name

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/anomalib/utils/callbacks/visualizer/visualizer_metric.py:53, in MetricVisualizerCallback.on_test_end(self, trainer, pl_module)
     50 for metric in metrics.values():
     51     # `generate_figure` needs to be defined for every metric that should be plotted automatically
     52     if hasattr(metric, "generate_figure"):
---> 53         fig, log_name = metric.generate_figure()
     54         file_name = f"{metrics.prefix}{log_name}"
     55         if self.log_images:

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/anomalib/utils/metrics/auroc.py:59, in AUROC.generate_figure(self)
     53 def generate_figure(self) -> Tuple[Figure, str]:
     54     """Generate a figure containing the ROC curve, the baseline and the AUROC.
     55 
     56     Returns:
     57         Tuple[Figure, str]: Tuple containing both the figure and the figure title to be used for logging
     58     """
---> 59     fpr, tpr = self._compute()
     60     auroc = self.compute()
     62     xlim = (0.0, 1.0)

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/anomalib/utils/metrics/auroc.py:50, in AUROC._compute(self)
     48 tpr: Tensor
     49 fpr: Tensor
---> 50 fpr, tpr, _thresholds = super().compute()
     51 return (fpr, tpr)

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/torchmetrics/classification/roc.py:154, in ROC.compute(self)
    141 def compute(self) -> Union[Tuple[Tensor, Tensor, Tensor], Tuple[List[Tensor], List[Tensor], List[Tensor]]]:
    142     """Compute the receiver operating characteristic.
    143 
    144     Returns:
   (...)
    152             thresholds used for computing false- and true-positive rates
    153     """
--> 154     preds = dim_zero_cat(self.preds)
    155     target = dim_zero_cat(self.target)
    156     if not self.num_classes:

File ~/anaconda3/envs/cuda/lib/python3.8/site-packages/torchmetrics/utilities/data.py:41, in dim_zero_cat(x)
     39 x = [y.unsqueeze(0) if y.numel() == 1 and y.ndim == 0 else y for y in x]
     40 if not x:  # empty list
---> 41     raise ValueError("No samples to concatenate")
     42 return torch.cat(x, dim=0)

ValueError: No samples to concatenate

[Junm0ri]

@1chimaruGin

metrics:
  image:
    - F1Score
    - AUROC
  threshold:
    image_default: 0
    adaptive: true

In my case, Editing the metrics section like above has worked.
(That means, you have to remove sections related to pixel when your task is not segmentation.)

You can check this issue

[1chimaruGin]

@Junm0ri Thanks! It's work.😊