Drop python3.7 and update CI (#28)

* Drop python3.7 support due to EOL, update formatting * Fix mio5_params.mat_struct deprecation * Disable image generator tests as the model is not available anymore * Fix deprecation warnings in numpy and attrs, fix mask comparison * Remove py3.12 from CI as tf doesn't support * Update changelog * Bump checked os versions on CI
cvat-ai · Nov 20, 2023 · f8a24d4 · f8a24d4
1 parent 8a14a99
commit f8a24d4
Show file tree

Hide file tree

Showing 13 changed files with 37 additions and 23 deletions.
diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml
@@ -17,8 +17,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        os: ['macos-12', 'ubuntu-22.04', 'windows-2022']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
     name: build and test (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -74,6 +74,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (<https://github.com/openvinotoolkit/datumaro/pull/680>)
 - Export of masks with background class with id != 0 in the VOC, KITTI and Cityscapes formats
   (<https://github.com/cvat-ai/datumaro/pull/9>, <https://github.com/cvat-ai/datumaro/pull/16>)
+- Missing comparison of the base class attributes in the `Mask` class
+  (<https://github.com/cvat-ai/datumaro/pull/28>)
 
 ### Security
 - TBD

diff --git a/datumaro/components/annotation.py b/datumaro/components/annotation.py
@@ -320,12 +320,16 @@ def paint(self, colormap: Colormap) -> np.ndarray:
         return paint_mask(self.as_class_mask(), colormap)
 
     def __eq__(self, other):
-        if not super().__eq__(other):
-            return False
         if not isinstance(other, __class__):
             return False
+
+        parent_keys = [f.name for f in attr.fields(Annotation)]
+        self_parent_fields = {k: v for k, v in self.as_dict().items() if k in parent_keys}
+        other_parent_fields = {k: v for k, v in other.as_dict().items() if k in parent_keys}
+
         return (
-            (self.label == other.label)
+            (self_parent_fields == other_parent_fields)
+            and (self.label == other.label)
             and (self.z_order == other.z_order)
             and (np.array_equal(self.image, other.image))
         )

diff --git a/datumaro/plugins/coco_format/converter.py b/datumaro/plugins/coco_format/converter.py
@@ -309,7 +309,7 @@ def convert_instance(self, instance, item):
                 rles = [rles]
             else:
                 rles = mask_utils.merge(rles)
-            area = mask_utils.area(rles)
+            area = mask_utils.area(rles).item()
         else:
             _, _, w, h = bbox
             segmentation = []

diff --git a/datumaro/plugins/mpii_format/mpii_mat.py b/datumaro/plugins/mpii_format/mpii_mat.py
@@ -4,7 +4,9 @@
 
 import os.path as osp
 
+import scipy
 import scipy.io as spio
+from packaging.version import Version
 
 from datumaro.components.annotation import Bbox, LabelCategories, Points, PointsCategories
 from datumaro.components.extractor import AnnotationType, DatasetItem, Importer, SourceExtractor
@@ -13,6 +15,11 @@
 
 from .format import MPII_POINTS_JOINTS, MPII_POINTS_LABELS
 
+if Version(scipy.__version__) >= Version("1.8.0rc1"):
+    mat_type = spio.matlab.mat_struct
+else:
+    mat_type = spio.matlab.mio5_params.mat_struct
+
 
 class MpiiExtractor(SourceExtractor):
     def __init__(self, path):
@@ -44,11 +51,11 @@ def _load_items(self, path):
             group_num = 1
 
             image = getattr(item, "image", "")
-            if isinstance(image, spio.matlab.mio5_params.mat_struct):
+            if isinstance(image, mat_type):
                 image = getattr(image, "name", "")
 
             anno_values = getattr(item, "annorect", [])
-            if isinstance(anno_values, spio.matlab.mio5_params.mat_struct):
+            if isinstance(anno_values, mat_type):
                 anno_values = [anno_values]
 
             for val in anno_values:
@@ -65,12 +72,12 @@ def _load_items(self, path):
                     attributes["scale"] = scale
 
                 objpos = getattr(val, "objpos", None)
-                if isinstance(objpos, spio.matlab.mio5_params.mat_struct):
+                if isinstance(objpos, mat_type):
                     attributes["center"] = [getattr(objpos, "x", 0), getattr(objpos, "y", 0)]
 
                 annopoints = getattr(val, "annopoints", None)
-                if isinstance(annopoints, spio.matlab.mio5_params.mat_struct) and not isinstance(
-                    getattr(annopoints, "point"), spio.matlab.mio5_params.mat_struct
+                if isinstance(annopoints, mat_type) and not isinstance(
+                    getattr(annopoints, "point"), mat_type
                 ):
                     for point in getattr(annopoints, "point"):
                         point_id = getattr(point, "id")

diff --git a/datumaro/util/annotation_util.py b/datumaro/util/annotation_util.py
@@ -157,7 +157,7 @@ def _to_rle(ann):
 
         a = _to_rle(a)
         b = _to_rle(b)
-    return float(mask_utils.iou(a, b, [not is_bbox]))
+    return float(mask_utils.iou(a, b, [not is_bbox]).item())
 
 
 def PDJ(a, b, eps=None, ratio=0.05, bbox=None):

diff --git a/datumaro/util/mask_tools.py b/datumaro/util/mask_tools.py
@@ -101,7 +101,7 @@ def paint_mask(mask, colormap=None):
     if callable(colormap):
         map_fn = colormap
     else:
-        map_fn = lambda c: colormap.get(c, (-1, -1, -1))
+        map_fn = lambda c: colormap.get(c, (0, 0, 0))
     palette = np.array([map_fn(c)[::-1] for c in range(256)], dtype=np.uint8)
 
     mask = mask.astype(np.uint8)
@@ -117,7 +117,7 @@ def remap_mask(mask, map_fn):
     """
     check_is_mask(mask)
 
-    return np.array([map_fn(c) for c in range(256)], dtype=np.uint8)[mask]
+    return np.array([max(0, map_fn(c)) for c in range(256)], dtype=np.uint8)[mask]
 
 
 def make_index_mask(binary_mask, index, dtype=None):

diff --git a/requirements-core.txt b/requirements-core.txt
@@ -21,7 +21,7 @@ tensorboardX>=1.8,!=2.3
 
 # Builtin plugin dependencies
 
-# NDR
+# NDR, matlab format
 scipy
 
 # Image generator

diff --git a/requirements-default.txt b/requirements-default.txt
@@ -1,4 +1,3 @@
 dvc>=2.7.0,<3.0.0
-fsspec<2023.1; python_version == '3.7' # remove after 3.7 EOL. https://stackoverflow.com/a/75197382
 
 GitPython>=3.1.18,!=3.1.25 # https://github.com/openvinotoolkit/datumaro/issues/612
diff --git a/setup.py b/setup.py
@@ -69,7 +69,7 @@ def parse_requirements(filename=CORE_REQUIREMENTS_FILE):
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=CORE_REQUIREMENTS,
     extras_require={
         "tf": ["tensorflow"],

diff --git a/tests/cli/test_generator.py b/tests/cli/test_generator.py
@@ -11,6 +11,7 @@
 from ..requirements import Requirements, mark_requirement
 
 
+# TODO: update links
 @pytest.mark.xfail(reason="Cannot download the model file from the source")
 class ImageGeneratorTest(TestCase):
     def check_images_shape(self, img_dir, expected_shape):

diff --git a/tests/test_coco_format.py b/tests/test_coco_format.py
@@ -737,7 +737,7 @@ def test_can_import_stuff(self):
                             label=0,
                             id=7,
                             group=7,
-                            attributes={"is_crowd": False},
+                            attributes={"is_crowd": True},
                         ),
                     ],
                 ),
@@ -752,7 +752,7 @@ def test_can_import_stuff(self):
                             label=1,
                             id=2,
                             group=2,
-                            attributes={"is_crowd": False},
+                            attributes={"is_crowd": True},
                         ),
                     ],
                 ),
@@ -791,7 +791,7 @@ def test_can_import_stuff_with_any_annotation_filename(self):
                             label=0,
                             id=7,
                             group=7,
-                            attributes={"is_crowd": False},
+                            attributes={"is_crowd": True},
                         ),
                     ],
                 ),
@@ -1315,7 +1315,7 @@ def test_can_save_and_load_stuff(self):
                             np.array(
                                 [[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 1, 1], [0, 0, 0, 0]],
                             ),
-                            attributes={"is_crowd": False},
+                            attributes={"is_crowd": True},
                             label=4,
                             group=3,
                             id=3,
@@ -1332,7 +1332,7 @@ def test_can_save_and_load_stuff(self):
                             np.array(
                                 [[0, 0, 0, 0], [1, 1, 1, 0], [1, 1, 0, 0], [0, 0, 0, 0]],
                             ),
-                            attributes={"is_crowd": False},
+                            attributes={"is_crowd": True},
                             label=4,
                             group=3,
                             id=3,
@@ -2252,7 +2252,7 @@ def test_can_save_and_load_grouped_masks_and_polygons(self):
                             ),
                             attributes={"is_crowd": True},
                             label=0,
-                            id=0,
+                            id=1,
                             group=1,
                         ),
                     ],

diff --git a/tests/test_fractal_image_generator.py b/tests/test_fractal_image_generator.py
@@ -12,6 +12,7 @@
 from .requirements import Requirements, mark_requirement
 
 
+# TODO: update links
 @pytest.mark.xfail(reason="Cannot download the model file from the source")
 class FractalImageGeneratorTest(TestCase):
     @mark_requirement(Requirements.DATUM_677)