Merge remote-tracking branch 'origin/main' into feature/damian/openpi…

…fpaf_val

src/deepsparse/yolact/pipelines.py

@@ -145,7 +145,7 @@ def process_inputs(
         if not isinstance(images, list):
             images = [images]
 
-        image_batch = list(self.executor.map(self._preprocess_image, inputs.images))
+        image_batch = list(self.executor.map(self._preprocess_image, images))
 
         original_image_shapes = None
         if image_batch and isinstance(image_batch[0], tuple):
@@ -220,12 +220,7 @@ def process_engine_outputs(
             confidence_single_image,
         ) in enumerate(zip(boxes, masks, confidence)):
 
-            original_image_shape = (
-                original_image_shapes[batch_idx]
-                if batch_idx < len(original_image_shapes)
-                else None
-            )
-
+            original_image_shape = original_image_shapes[0]
             decoded_boxes = decode(boxes_single_image, priors)
 
             results = detect(

src/deepsparse/yolact/utils/utils.py

@@ -78,6 +78,7 @@ def preprocess_array(
     :return: preprocessed numpy array (B, C, D, D); where (D,D) is image size expected
         by the network. It is a contiguous array with RGB channel order.
     """
+
     is_uint8 = image.dtype == numpy.uint8
 
     # put channel last to be compatible with cv2.resize
@@ -94,11 +95,13 @@ def preprocess_array(
         if image.shape[:2] != input_image_size:
             image = cv2.resize(image, input_image_size)
         image = numpy.expand_dims(image, 0)
+
     # put channel "first"
     image = image.transpose(0, 3, 1, 2)
-    # if uint8 image, convert to float32 and normalize
+    # convert to float32
+    image = image.astype(numpy.float32)
+    # if image is uint8, convert to [0,1] range
     if is_uint8:
-        image = image.astype(numpy.float32)
         image /= 255
     return numpy.ascontiguousarray(image), original_image_shape
 

src/deepsparse/yolov8/utils/validation/__init__.py

@@ -14,3 +14,5 @@
 
 # flake8: noqa
 from .detection_validator import *
+from .helpers import *
+from .segmentation_validator import *

src/deepsparse/yolov8/utils/validation/deepsparse_validator.py

@@ -1,32 +1,13 @@
-# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# neuralmagic: no copyright
 # flake8: noqa
 
 import json
-from collections import defaultdict
-from pathlib import Path
-from typing import Dict, List
+from typing import Dict
 
 from tqdm import tqdm
 
-import torch
-from deepsparse import Pipeline
-from deepsparse.yolo import YOLOOutput
-from ultralytics.yolo.cfg import get_cfg
+from deepsparse.yolov8.utils.validation.helpers import schema_to_tensor
 from ultralytics.yolo.data.utils import check_det_dataset
-from ultralytics.yolo.engine.validator import BaseValidator
 from ultralytics.yolo.utils import (
     DEFAULT_CFG,
     LOGGER,
@@ -35,119 +16,22 @@
     TQDM_BAR_FORMAT,
     callbacks,
 )
-from ultralytics.yolo.utils.files import increment_path
 from ultralytics.yolo.utils.ops import Profile
 from ultralytics.yolo.utils.torch_utils import select_device, smart_inference_mode
 
 
 __all__ = ["DeepSparseValidator"]
 
 
-def schema_to_tensor(pipeline_outputs: YOLOOutput, device: str) -> List[torch.Tensor]:
-    """
-    Transform the YOLOOutput to the format expected by the validation code.
-
-    :param pipeline_outputs: YOLOOutput from the pipeline
-    :param device: device to move the tensors to
-    :return list of tensor with the format [x1, y1, x2, y2, confidence, class]
-    """
-
-    preds = []
-
-    for boxes, labels, confidence in zip(
-        pipeline_outputs.boxes, pipeline_outputs.labels, pipeline_outputs.scores
-    ):
-
-        boxes = torch.tensor(boxes)
-
-        # map labels to integers and reshape for concatenation
-        labels = list(map(int, list(map(float, labels))))
-        labels = torch.tensor(labels).view(-1, 1)
-
-        # reshape for concatenation
-        scores = torch.tensor(confidence).view(-1, 1)
-        # concatenate and append to preds
-        preds.append(torch.cat([boxes, scores, labels], axis=1).to(device))
-    return preds
-
-
-# adapted from ULTRALYTICS GITHUB:
-# https://github.com/ultralytics/ultralytics/blob/main/ultralytics/yolo/engine/validator.py
-# the appropriate edits are marked with # deepsparse edit: <edit comment>
-
-
-class DeepSparseValidator(BaseValidator):  # deepsparse edit: overwriting BaseValidator
-    """
-    A DeepSparseValidator class for creating validators for
-    YOLOv8 Deepsparse pipeline.
-
-    Attributes:
-        pipeline (Pipeline): DeepSparse Pipeline to be evaluated
-        dataloader (DataLoader): Dataloader to use for validation.
-        pbar (tqdm): Progress bar to update during validation.
-        logger (logging.Logger): Logger to use for validation.
-        args (SimpleNamespace): Configuration for the validator.
-        model (nn.Module): Model to validate.
-        data (dict): Data dictionary.
-        device (torch.device): Device to use for validation.
-        batch_i (int): Current batch index.
-        training (bool): Whether the model is in training mode.
-        speed (float): Batch processing speed in seconds.
-        jdict (dict): Dictionary to store validation results.
-        save_dir (Path): Directory to save results.
-    """
-
-    def __init__(
-        self,
-        pipeline: Pipeline,  # deepsparse edit: added pipeline
-        dataloader=None,
-        save_dir=None,
-        pbar=None,
-        logger=None,
-        args=None,
-    ):
-        """
-        Initializes a DeepSparseValidator instance.
-        Args:
-            pipeline (Pipeline): DeepSparse Pipeline to be evaluated
-            dataloader (torch.utils.data.DataLoader): Dataloader to be used for validation.
-            save_dir (Path): Directory to save results.
-            pbar (tqdm.tqdm): Progress bar for displaying progress.
-            logger (logging.Logger): Logger to log messages.
-            args (SimpleNamespace): Configuration for the validator.
-        """
-        self.pipeline = pipeline  # deepsparse edit: added pipeline
-        self.dataloader = dataloader
-        self.pbar = pbar
-        self.logger = logger or LOGGER
-        self.args = args or get_cfg(DEFAULT_CFG)
-        self.model = None
-        self.data = None
-        self.device = None
-        self.batch_i = None
+class DeepSparseValidator:
+    def __init__(self, pipeline):
+        self.pipeline = pipeline
         self.training = False
-        self.speed = None
-        self.jdict = None
-
-        project = self.args.project or Path(SETTINGS["runs_dir"]) / self.args.task
-        name = self.args.name or f"{self.args.mode}"
-        self.save_dir = save_dir or increment_path(
-            Path(project) / name,
-            exist_ok=self.args.exist_ok if RANK in {-1, 0} else True,
-        )
-        (self.save_dir / "labels" if self.args.save_txt else self.save_dir).mkdir(
-            parents=True, exist_ok=True
-        )
-
-        if self.args.conf is None:
-            self.args.conf = 0.001  # default conf=0.001
-
-        self.callbacks = defaultdict(list, callbacks.default_callbacks)  # add callbacks
 
     @smart_inference_mode()
     # deepsparse edit: replaced arguments `trainer` and `model`
-    # with `stride` and `classes`
-    def __call__(self, stride: int, classes: Dict[int, str]):
+    # `classes`
+    def __call__(self, classes: Dict[int, str]):
         """
         Supports validation of a pre-trained model if passed or a model being trained
         if trainer is passed (trainer gets priority).
@@ -167,40 +51,34 @@ def __call__(self, stride: int, classes: Dict[int, str]):
         self.dataloader = self.dataloader or self.get_dataloader(
             self.data.get("val") or self.data.set("test"), self.args.batch
         )
-        # deepsparse edit: left only profiler for inference, removed the redundant
-        # profilers for pre-process, loss and post-process
-        dt = Profile()
+
+        dt = Profile(), Profile(), Profile(), Profile()
         n_batches = len(self.dataloader)
         desc = self.get_desc()
         # NOTE: keeping `not self.training` in tqdm will eliminate pbar after segmentation evaluation during training,
         # which may affect classification task since this arg is in yolov5/classify/val.py.
         # bar = tqdm(self.dataloader, desc, n_batches, not self.training, bar_format=TQDM_BAR_FORMAT)
         bar = tqdm(self.dataloader, desc, n_batches, bar_format=TQDM_BAR_FORMAT)
-        # deepsparse edit: replaced argument `model` with `classes`
         self.init_metrics(classes=classes)
         self.jdict = []  # empty before each val
         for batch_i, batch in enumerate(bar):
             self.run_callbacks("on_val_batch_start")
             self.batch_i = batch_i
+            # pre-process
+            with dt[0]:
+                batch = self.preprocess(batch)
 
-            # deepsparse edit:
-            # - removed the redundant pre-process function
-            # - removed the redundant loss computation
-            # - removed the redundant post-process function
-
-            # deepsparse edit: replaced the inference model with the DeepSparse pipeline
             # inference
-            with dt:
+            with dt[1]:
                 outputs = self.pipeline(
-                    images=[x.cpu().numpy() for x in batch["img"]],
-                    iou_thres=self.args.iou,
-                    conf_thres=self.args.conf,
-                    multi_label=True,
+                    images=[x.cpu().numpy() * 255 for x in batch["img"]],
+                    return_intermediate_outputs=True,
                 )
             preds = schema_to_tensor(pipeline_outputs=outputs, device=self.device)
-            batch["bboxes"] = batch["bboxes"].to(self.device)
-            batch["cls"] = batch["cls"].to(self.device)
-            batch["batch_idx"] = batch["batch_idx"].to(self.device)
+
+            # pre-process predictions
+            with dt[3]:
+                preds = self.postprocess(preds)
 
             self.update_metrics(preds, batch)
             if self.args.plots and batch_i < 3:
@@ -211,13 +89,15 @@ def __call__(self, stride: int, classes: Dict[int, str]):
         stats = self.get_stats()
         self.check_stats(stats)
         self.print_results()
-        self.speed = dt.t / len(self.dataloader.dataset) * 1e3  # speeds per image
+        self.speed = tuple(
+            x.t / len(self.dataloader.dataset) * 1e3 for x in dt
+        )  # speeds per image
         self.run_callbacks("on_val_end")
 
-        # deepsparse_edit: changed the string formatting to match the
-        # removed profilers
-        self.logger.info("Speed: %.1fms inference per image" % self.speed)
-
+        self.logger.info(
+            "Speed: %.1fms pre-process, %.1fms inference, %.1fms loss, %.1fms post-process per image"
+            % self.speed
+        )
         if self.args.save_json and self.jdict:
             with open(str(self.save_dir / "predictions.json"), "w") as f:
                 self.logger.info(f"Saving {f.name}...")

src/deepsparse/yolov8/utils/validation/detection_validator.py

@@ -3,7 +3,6 @@
 
 import os
 
-import torch
 from deepsparse import Pipeline
 from deepsparse.yolov8.utils.validation.deepsparse_validator import DeepSparseValidator
 from ultralytics.yolo.utils import ops
@@ -13,6 +12,7 @@
 
 __all__ = ["DeepSparseDetectionValidator"]
 
+
 # adapted from ULTRALYTICS GITHUB:
 # https://github.com/ultralytics/ultralytics/blob/main/ultralytics/yolo/v8/detect/val.py
 # the appropriate edits are marked with # deepsparse edit: <edit comment>
@@ -26,13 +26,8 @@ def __init__(
         logger=None,
         args=None,
     ):
-        super().__init__(pipeline, dataloader, save_dir, pbar, logger, args)
-        self.args.task = "detect"
-        self.is_coco = False
-        self.class_map = None
-        self.metrics = DetMetrics(save_dir=self.save_dir)
-        self.iouv = torch.linspace(0.5, 0.95, 10)  # iou vector for mAP@0.5:0.95
-        self.niou = self.iouv.numel()
+        DetectionValidator.__init__(self, dataloader, save_dir, pbar, logger, args)
+        DeepSparseValidator.__init__(self, pipeline)
 
     # deepsparse edit: replaced argument `model` with `classes`
     def init_metrics(self, classes):

src/deepsparse/yolov8/utils/validation/helpers.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import os
+import warnings
+from typing import List, Optional, Union
+
+import torch
+from deepsparse.yolo import YOLOOutput as YOLODetOutput
+from deepsparse.yolov8.schemas import YOLOSegOutput
+
+
+__all__ = ["schema_to_tensor", "check_coco128_segmentation"]
+
+
+def schema_to_tensor(
+    pipeline_outputs: Union[YOLOSegOutput, YOLODetOutput], device: str
+) -> List[Union[torch.Tensor, None, List[Optional[torch.Tensor]]]]:
+    """
+    Extract the intermediate outputs from the output schema.
+
+    :param pipeline_outputs: YOLO output schema
+    :param device: device to move the tensors to
+    :return
+        if segmentation,
+            return [output, [None, None, mask_protos]]
+        if detection,
+            return [output, None]
+        padding with Nones to match the input expected by the
+        postprocess function in the validation script
+    """
+
+    preds = pipeline_outputs.intermediate_outputs
+    if isinstance(preds, tuple):
+        output, mask_protos = preds
+        output = torch.from_numpy(output).to(device)
+        mask_protos = torch.from_numpy(mask_protos).to(device)
+        return [output, [None, None, mask_protos]]
+    else:
+        output = torch.from_numpy(preds).to(device)
+        return [output, None]
+
+
+def check_coco128_segmentation(args: argparse.Namespace):
+    """
+    Checks if the argument 'data' is coco128.yaml and if so,
+    replaces it with coco128-seg.yaml.
+
+    :param args: arguments to check
+    :return: the updated arguments
+    """
+    if args.data == "coco128.yaml":
+        dataset_name, dataset_extension = os.path.splitext(args.data)
+        dataset_yaml = dataset_name + "-seg" + dataset_extension
+        warnings.warn(
+            f"Dataset yaml {dataset_yaml} is not supported for segmentation. "
+            f"Attempting to use {dataset_yaml} instead."
+        )
+        args.data = dataset_yaml
+    return args