Update Datumaro dependency to 1.10 (#3411)

* Update formats wrt Datumaro Transforms API changes

* Add mismatching key output in tests

* Strip label dir in vggface2 import

* Update dependency

* update patch

cvat/apps/dataset_manager/bindings.py

@@ -624,11 +624,15 @@ def match_dm_item(item, task_data, root_hint=None):
     return frame_number
 
 def find_dataset_root(dm_dataset, task_data):
-    longest_path = max(dm_dataset, key=lambda x: len(Path(x.id).parts)).id
+    longest_path = max(dm_dataset, key=lambda x: len(Path(x.id).parts),
+        default=None)
+    if longest_path is None:
+        return None
+    longest_path = longest_path.id
+
     longest_match = task_data.match_frame_fuzzy(longest_path)
     if longest_match is None:
         return None
-
     longest_match = osp.dirname(task_data.frame_info[longest_match]['path'])
     prefix = longest_match[:-len(osp.dirname(longest_path)) or None]
     if prefix.endswith('/'):

cvat/apps/dataset_manager/formats/datumaro/__init__.py

@@ -41,9 +41,11 @@ class DatumaroProjectExporter:
                 'height': frame['height'],
             })
 
-        with open(osp.join(save_dir, 'config.json'), 'w') as config_file:
+        with open(osp.join(save_dir, 'config.json'),
+                'w', encoding='utf-8') as config_file:
             json.dump(config, config_file)
-        with open(osp.join(save_dir, 'images_meta.json'), 'w') as images_file:
+        with open(osp.join(save_dir, 'images_meta.json'),
+                'w', encoding='utf-8') as images_file:
             json.dump(images_meta, images_file)
 
     def _export(self, task_data, save_dir, save_images=False):

cvat/apps/dataset_manager/formats/datumaro/export_templates/plugins/cvat_rest_api_task_images.py

@@ -1,5 +1,4 @@
-
+# Copyright (C) 2020-2021 Intel Corporation
-# Copyright (C) 2020 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -70,12 +69,14 @@ class cvat_rest_api_task_images(SourceExtractor):
         self._local_dir = local_dir
         self._cache_dir = osp.join(local_dir, 'images')
 
-        with open(osp.join(url, 'config.json'), 'r') as config_file:
+        with open(osp.join(url, 'config.json'),
+                'r', encoding='utf-8') as config_file:
             config = json.load(config_file)
             config = Config(config, schema=CONFIG_SCHEMA)
         self._config = config
 
-        with open(osp.join(url, 'images_meta.json'), 'r') as images_file:
+        with open(osp.join(url, 'images_meta.json'),
+                'r', encoding='utf-8') as images_file:
             images_meta = json.load(images_file)
             image_list = images_meta['images']
 

cvat/apps/dataset_manager/formats/icdar.py

@@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory
 
 from datumaro.components.dataset import Dataset
 from datumaro.components.extractor import (AnnotationType, Caption, Label,
-    LabelCategories, Transform)
+    LabelCategories, ItemTransform)
 
 from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor,
     import_dm_annotations)
@@ -16,7 +16,7 @@ from cvat.apps.dataset_manager.util import make_zip_archive
 from .registry import dm_env, exporter, importer
 
 
-class AddLabelToAnns(Transform):
+class AddLabelToAnns(ItemTransform):
     def __init__(self, extractor, label):
         super().__init__(extractor)
 
@@ -39,7 +39,7 @@ class AddLabelToAnns(Transform):
                 ann.label = self._label
         return item.wrap(annotations=annotations)
 
-class CaptionToLabel(Transform):
+class CaptionToLabel(ItemTransform):
     def __init__(self, extractor, label):
         super().__init__(extractor)
 
@@ -64,7 +64,7 @@ class CaptionToLabel(Transform):
             annotations.remove(ann)
         return item.wrap(annotations=annotations)
 
-class LabelToCaption(Transform):
+class LabelToCaption(ItemTransform):
     def transform_item(self, item):
         annotations = item.annotations
         anns = [p for p in annotations

cvat/apps/dataset_manager/formats/market1501.py

@@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory
 
 from datumaro.components.dataset import Dataset
 from datumaro.components.extractor import (AnnotationType, Label,
-    LabelCategories, Transform)
+    LabelCategories, ItemTransform)
 
 from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor,
     import_dm_annotations)
@@ -15,7 +15,7 @@ from cvat.apps.dataset_manager.util import make_zip_archive
 
 from .registry import dm_env, exporter, importer
 
-class AttrToLabelAttr(Transform):
+class AttrToLabelAttr(ItemTransform):
     def __init__(self, extractor, label):
         super().__init__(extractor)
 
@@ -31,13 +31,14 @@ class AttrToLabelAttr(Transform):
         return self._categories
 
     def transform_item(self, item):
-        annotations = item.annotations
+        annotations = list(item.annotations)
+        attributes = item.attributes
         if item.attributes:
             annotations.append(Label(self._label, attributes=item.attributes))
-            item.attributes = {}
+            attributes = {}
-        return item.wrap(annotations=annotations)
+        return item.wrap(annotations=annotations, attributes=attributes)
 
-class LabelAttrToAttr(Transform):
+class LabelAttrToAttr(ItemTransform):
     def __init__(self, extractor, label):
         super().__init__(extractor)
 
@@ -46,8 +47,8 @@ class LabelAttrToAttr(Transform):
         self._label = label_cat.find(label)[0]
 
     def transform_item(self, item):
-        annotations = item.annotations
+        annotations = list(item.annotations)
-        attributes = item.attributes
+        attributes = dict(item.attributes)
         if self._label != None:
             labels = [ann for ann in annotations
                 if ann.type == AnnotationType.label \

cvat/apps/dataset_manager/formats/mots.py

@@ -5,7 +5,7 @@
 from tempfile import TemporaryDirectory
 
 from datumaro.components.dataset import Dataset
-from datumaro.components.extractor import AnnotationType, Transform
+from datumaro.components.extractor import AnnotationType, ItemTransform
 from pyunpack import Archive
 
 from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor,
@@ -15,7 +15,7 @@ from cvat.apps.dataset_manager.util import make_zip_archive
 from .registry import dm_env, exporter, importer
 
 
-class KeepTracks(Transform):
+class KeepTracks(ItemTransform):
     def transform_item(self, item):
         return item.wrap(annotations=[a for a in item.annotations
             if 'track_id' in a.attributes])

cvat/apps/dataset_manager/formats/registry.py

@@ -82,6 +82,14 @@ def make_importer(name):
 def make_exporter(name):
     return EXPORT_FORMATS[name]()
 
+
+# Add checking for TF availability to avoid CVAT sever instance / interpreter
+# crash and provide a meaningful diagnistic message in the case of AVX
+# instructions unavailability:
+# https://github.com/openvinotoolkit/cvat/pull/1567
+import datumaro.util.tf_util as TF
+TF.enable_tf_check = True
+
 # pylint: disable=unused-import
 import cvat.apps.dataset_manager.formats.coco
 import cvat.apps.dataset_manager.formats.cvat

cvat/apps/dataset_manager/formats/vggface2.py

@@ -29,4 +29,5 @@ def _import(src_file, task_data):
         zipfile.ZipFile(src_file).extractall(tmp_dir)
 
         dataset = Dataset.import_from(tmp_dir, 'vgg_face2', env=dm_env)
+        dataset.transform('rename', r"|([^/]+/)?(.+)|\2|")
         import_dm_annotations(dataset, task_data)

cvat/apps/engine/tests/test_rest_api.py

@@ -3331,9 +3331,13 @@ class TaskDataAPITestCase(APITestCase):
         response = self._create_task(None, data)
         self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
 
-def compare_objects(self, obj1, obj2, ignore_keys, fp_tolerance=.001):
+def compare_objects(self, obj1, obj2, ignore_keys, fp_tolerance=.001,
+        current_key=None):
+    key_info = "{}: ".format(current_key) if current_key else ""
+
     if isinstance(obj1, dict):
-        self.assertTrue(isinstance(obj2, dict), "{} != {}".format(obj1, obj2))
+        self.assertTrue(isinstance(obj2, dict),
+            "{}{} != {}".format(key_info, obj1, obj2))
         for k, v1 in obj1.items():
             if k in ignore_keys:
                 continue
@@ -3342,17 +3346,20 @@ def compare_objects(self, obj1, obj2, ignore_keys, fp_tolerance=.001):
                 key = lambda a: a['spec_id'] if 'spec_id' in a else a['id']
                 v1.sort(key=key)
                 v2.sort(key=key)
-            compare_objects(self, v1, v2, ignore_keys)
+            compare_objects(self, v1, v2, ignore_keys, current_key=k)
     elif isinstance(obj1, list):
-        self.assertTrue(isinstance(obj2, list), "{} != {}".format(obj1, obj2))
+        self.assertTrue(isinstance(obj2, list),
-        self.assertEqual(len(obj1), len(obj2), "{} != {}".format(obj1, obj2))
+            "{}{} != {}".format(key_info, obj1, obj2))
+        self.assertEqual(len(obj1), len(obj2),
+            "{}{} != {}".format(key_info, obj1, obj2))
         for v1, v2 in zip(obj1, obj2):
-            compare_objects(self, v1, v2, ignore_keys)
+            compare_objects(self, v1, v2, ignore_keys, current_key=current_key)
     else:
         if isinstance(obj1, float) or isinstance(obj2, float):
-            self.assertAlmostEqual(obj1, obj2, delta=fp_tolerance)
+            self.assertAlmostEqual(obj1, obj2, delta=fp_tolerance,
+                msg=current_key)
         else:
-            self.assertEqual(obj1, obj2)
+            self.assertEqual(obj1, obj2, msg=current_key)
 
 class JobAnnotationAPITestCase(APITestCase):
     def setUp(self):

cvat/requirements/base.txt

@@ -52,4 +52,4 @@ azure-storage-blob==12.8.1
 # --no-binary=pycocotools: workaround for binary incompatibility on numpy 1.20
 # of pycocotools and tensorflow 2.4.1
 # when pycocotools is installed by wheel in python 3.8+
-datumaro==0.1.9 --no-binary=datumaro --no-binary=pycocotools
+datumaro==0.1.10.1 --no-binary=datumaro --no-binary=pycocotools