[OpenPifPaf] ONNX Evaluation Pipeline

setup.py

@@ -133,8 +133,10 @@ def _parse_requirements_file(file_path):
     "opencv-python<=4.6.0.66",
 ]
 _openpifpaf_integration_deps = [
-    "openpifpaf==0.13.6",
+    "openpifpaf==0.13.11",
     "opencv-python<=4.6.0.66",
+    "pycocotools >=2.0.6",
+    "scipy==1.10.1",
 ]
 _yolov8_integration_deps = _yolo_integration_deps + ["ultralytics==8.0.30"]
 
@@ -277,6 +279,7 @@ def _setup_entry_points() -> Dict:
             "deepsparse.yolov8.annotate=deepsparse.yolov8.annotate:main",
             "deepsparse.yolov8.eval=deepsparse.yolov8.validation:main",
             "deepsparse.pose_estimation.annotate=deepsparse.open_pif_paf.annotate:main",
+            "deepsparse.pose_estimation.eval=deepsparse.open_pif_paf.validation:main",
             "deepsparse.image_classification.annotate=deepsparse.image_classification.annotate:main",  # noqa E501
             "deepsparse.instance_segmentation.annotate=deepsparse.yolact.annotate:main",
             f"deepsparse.image_classification.eval={ic_eval}",

src/deepsparse/open_pif_paf/README.md

@@ -10,14 +10,70 @@ DeepSparse pipeline for OpenPifPaf
 ```python
 from deepsparse import Pipeline
 
-model_path: str = ... # path to open_pif_paf model
-pipeline = Pipeline.create(task="open_pif_paf", batch_size=1, model_path=model_path)
+model_path: str = ... # path to open_pif_paf model (SparseZoo stub or onnx model)
+pipeline = Pipeline.create(task="open_pif_paf", model_path=model_path)
 predictions = pipeline(images=['dancers.jpg'])
 # predictions have attributes `data', 'keypoints', 'scores', 'skeletons'
 predictions[0].scores
 >> scores=[0.8542259724243828, 0.7930507659912109]
 ```
-# predictions have attributes `data', 'keypoints', 'scores', 'skeletons'
+### Output CifCaf fields
+Alternatively, instead of returning the detected poses, it is possible to return the intermediate output - the CifCaf fields.
+This is the representation returned directly by the neural network, but not yet processed by the matching algorithm
+
+```python
+...
+pipeline = Pipeline.create(task="open_pif_paf", model_path=model_path,  return_cifcaf_fields=True)
+predictions = pipeline(images=['dancers.jpg'])
+predictions.fields
+```
+
+## Validation script:
+This paragraph describes how to run validation of the ONNX model/SparseZoo stub
+
+### Dataset
+For evaluation, you need to download the dataset. The [Open Pif Paf documentation](https://openpifpaf.github.io/) describes 
+thoroughly how to prepare different datasets for validation. This is the example for `crowdpose` dataset:
+
+```bash
+mkdir data-crowdpose
+cd data-crowdpose
+# download links here: https://github.com/Jeff-sjtu/CrowdPose
+unzip annotations.zip
+unzip images.zip
+# Now you can use the standard openpifpaf.train and openpifpaf.eval 
+# commands as documented in Training with --dataset=crowdpose.
+```
+### Create an ONNX model:
+
+```bash
+python3 -m openpifpaf.export_onnx --input-width 641 --input-height 641
+```
+
+### Validation command
+Once the dataset has been downloaded, run the command:
+```bash
+deepsparse.pose_estimation.eval --model-path openpifpaf-resnet50.onnx  --dataset cocokp --image_size 641
+```
+
+This should result in the evaluation output similar to this:
+```bash
+...
+ Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets= 20 ] = 0.502
+ Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets= 20 ] = 0.732
+ Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets= 20 ] = 0.523
+ Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.429
+ Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.605
+ Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 20 ] = 0.534
+ Average Recall     (AR) @[ IoU=0.50      | area=   all | maxDets= 20 ] = 0.744
+ Average Recall     (AR) @[ IoU=0.75      | area=   all | maxDets= 20 ] = 0.554
+ Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.457
+ Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.643
+...
+````
+
+
+### Expected output:
 
 ## The necessity of external OpenPifPaf helper function 
 <img width="678" alt="image" src="https://user-images.githubusercontent.com/97082108/203295520-42fa325f-8a94-4241-af6f-75938ef26b14.png">

src/deepsparse/open_pif_paf/__init__.py

@@ -11,3 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# flake8: noqa
+from .utils import *

src/deepsparse/open_pif_paf/pipelines.py

@@ -22,7 +22,11 @@
 
 import cv2
 import torch
-from deepsparse.open_pif_paf.schemas import OpenPifPafInput, OpenPifPafOutput
+from deepsparse.open_pif_paf.schemas import (
+    OpenPifPafFields,
+    OpenPifPafInput,
+    OpenPifPafOutput,
+)
 from deepsparse.pipeline import Pipeline
 from deepsparse.yolact.utils import preprocess_array
 from openpifpaf import decoder, network
@@ -57,12 +61,19 @@ class OpenPifPafPipeline(Pipeline):
     """
 
     def __init__(
-        self, *, image_size: Union[int, Tuple[int, int]] = (384, 384), **kwargs
+        self,
+        *,
+        image_size: Union[int, Tuple[int, int]] = (384, 384),
+        return_cifcaf_fields: bool = False,
+        **kwargs,
     ):
         super().__init__(**kwargs)
         self._image_size = (
             image_size if isinstance(image_size, Tuple) else (image_size, image_size)
         )
+        # whether to return the cif and caf fields or the
+        # complete decoded output
+        self.return_cifcaf_fields = return_cifcaf_fields
         # necessary openpifpaf dependencies for now
         model_cpu, _ = network.Factory().factory(head_metas=None)
         self.processor = decoder.factory(model_cpu.head_metas)
@@ -79,7 +90,7 @@ def output_schema(self) -> Type[OpenPifPafOutput]:
         """
         :return: pydantic model class that outputs to this pipeline must comply to
         """
-        return OpenPifPafOutput
+        return OpenPifPafOutput if not self.return_cifcaf_fields else OpenPifPafFields
 
     def setup_onnx_file_path(self) -> str:
         """
@@ -100,9 +111,18 @@ def process_inputs(self, inputs: OpenPifPafInput) -> List[numpy.ndarray]:
 
         image_batch = list(self.executor.map(self._preprocess_image, images))
 
+        original_image_shapes = None
+        if image_batch and isinstance(image_batch[0], tuple):
+            # splits image batch is of format:
+            #  [(preprocesses_img, original_image_shape), ...] into separate lists
+            image_batch, original_image_shapes = list(map(list, zip(*image_batch)))
+
         image_batch = numpy.concatenate(image_batch, axis=0)
 
-        return [image_batch]
+        postprocessing_kwargs = dict(
+            original_image_shapes=original_image_shapes,
+        )
+        return [image_batch], postprocessing_kwargs
 
     def process_engine_outputs(
         self, fields: List[numpy.ndarray], **kwargs
@@ -114,6 +134,11 @@ def process_engine_outputs(
         :return: Outputs of engine post-processed into an object in the `output_schema`
             format of this pipeline
         """
+        if self.return_cifcaf_fields:
+            batch_fields = []
+            for cif, caf in zip(*fields):
+                batch_fields.append([cif, caf])
+            return OpenPifPafFields(fields=batch_fields)
 
         data_batch, skeletons_batch, scores_batch, keypoints_batch = [], [], [], []
 

src/deepsparse/open_pif_paf/schemas.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import List, Tuple
+from typing import Iterable, List, TextIO, Tuple
 
+import numpy
+from PIL import Image
 from pydantic import BaseModel, Field
 
 from deepsparse.pipelines.computer_vision import ComputerVisionSchema
@@ -22,6 +24,7 @@
 __all__ = [
     "OpenPifPafInput",
     "OpenPifPafOutput",
+    "OpenPifPafFields",
 ]
 
 
@@ -33,6 +36,50 @@ class OpenPifPafInput(ComputerVisionSchema):
     pass
 
 
+class OpenPifPafFields(BaseModel):
+    """
+    Open Pif Paf is composed of two stages:
+     - Computing Cif/Caf fields using a parametrized model
+     - Applying a matching algorithm to obtain the final pose
+        predictions
+    In some cases (e.g. for validation), it may be useful to
+    obtain the Cif/Caf fields as output.
+    """
+
+    fields: List[List[numpy.ndarray]] = Field(
+        description="Cif/Caf fields returned by the network. "
+        "The outer list is the batch dimension, while the second "
+        "list contains two numpy arrays: Cif and Caf field values. "
+    )
+
+    @classmethod
+    def from_files(
+        cls, files: Iterable[TextIO], *args, from_server: bool = False, **kwargs
+    ) -> "OpenPifPafFields":
+        """
+        :param files: Iterable of file pointers to create OpenPifPafFields from
+        :param kwargs: extra keyword args to pass to OpenPifPafFields constructor
+        :return: OpenPifPafFields constructed from files
+        """
+        if "images" in kwargs:
+            raise ValueError(
+                f"argument 'images' cannot be specified in {cls.__name__} when "
+                "constructing from file(s)"
+            )
+        if from_server:
+            raise ValueError(
+                "Cannot construct OpenPifPafFields from server. This will create"
+                "numpy arrays that are not serializable."
+            )
+
+        files_numpy = [numpy.array(Image.open(file)) for file in files]
+        input_schema = cls(*args, images=files_numpy, **kwargs)
+        return input_schema
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
 class OpenPifPafOutput(BaseModel):
     """
     Output model for Open Pif Paf

src/deepsparse/open_pif_paf/utils/__init__.py

@@ -13,3 +13,4 @@
 # limitations under the License.
 # flake8: noqa
 from .annotate import *
+from .validation import *

src/deepsparse/open_pif_paf/utils/validation/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# flake8: noqa
+from .cli import *
+from .deepsparse_evaluator import *

src/deepsparse/open_pif_paf/utils/validation/cli.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+
+import torch
+from deepsparse.open_pif_paf.utils.validation.deepsparse_evaluator import (
+    DeepSparseEvaluator,
+)
+from deepsparse.open_pif_paf.utils.validation.deepsparse_predictor import (
+    DeepSparsePredictor,
+)
+from openpifpaf import __version__, datasets, decoder, logger, network, show, visualizer
+from openpifpaf.eval import CustomFormatter
+
+
+LOG = logging.getLogger(__name__)
+
+__all__ = ["cli"]
+
+
+# adapted from OPENPIFPAF GITHUB:
+# https://github.com/openpifpaf/openpifpaf/blob/main/src/openpifpaf/eval.py
+# the appropriate edits are marked with # deepsparse edit: <edit comment>
+def cli():
+    parser = argparse.ArgumentParser(
+        prog="python3 -m openpifpaf.eval",
+        usage="%(prog)s [options]",
+        description=__doc__,
+        formatter_class=CustomFormatter,
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="OpenPifPaf {version}".format(version=__version__),
+    )
+
+    datasets.cli(parser)
+    decoder.cli(parser)
+    logger.cli(parser)
+    network.Factory.cli(parser)
+    DeepSparsePredictor.cli(parser, skip_batch_size=True, skip_loader_workers=True)
+    show.cli(parser)
+    visualizer.cli(parser)
+    DeepSparseEvaluator.cli(parser)
+
+    parser.add_argument("--disable-cuda", action="store_true", help="disable CUDA")
+    parser.add_argument(
+        "--output", default=None, help="output filename without file extension"
+    )
+    parser.add_argument(
+        "--watch",
+        default=False,
+        const=60,
+        nargs="?",
+        type=int,
+        help=(
+            "Watch a directory for new checkpoint files. "
+            "Optionally specify the number of seconds between checks."
+        ),
+    )
+
+    # deepsparse edit: replace the parse_args call with parse_known_args
+    args, _ = parser.parse_known_args()
+
+    # add args.device
+    args.device = torch.device("cpu")
+    args.pin_memory = False
+    if not args.disable_cuda and torch.cuda.is_available():
+        args.device = torch.device("cuda")
+        args.pin_memory = True
+    LOG.debug("neural network device: %s", args.device)
+
+    datasets.configure(args)
+    decoder.configure(args)
+    network.Factory.configure(args)
+    DeepSparsePredictor.configure(args)
+    show.configure(args)
+    visualizer.configure(args)
+    DeepSparseEvaluator.configure(args)
+
+    return args