Fix an issue with Imagenet dataset import (#4861)

3 years ago · 2311b10b80
parent 469217cc55
commit 2311b10b80
10 changed files with 119 additions and 13 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -138,6 +138,7 @@ Skeleton (<https://github.com/cvat-ai/cvat/pull/1>), (<https://github.com/opencv
 - Fix build dependencies for Siammask (<https://github.com/openvinotoolkit/cvat/pull/4486>)
 - Bug: Exif orientation information handled incorrectly (<https://github.com/openvinotoolkit/cvat/pull/4529>)
 - Fixed build of retinanet function image (<https://github.com/cvat-ai/cvat/pull/54>)
+- Bug: Import dataset of Imagenet format fail (<https://github.com/opencv/cvat/issues/4850>)

 ## \[2.0.0] - 2022-03-04
 ### Added
--- a/cvat/apps/dataset_manager/bindings.py
+++ b/cvat/apps/dataset_manager/bindings.py
@ -1885,17 +1885,24 @@ def load_dataset_data(project_annotation, dataset: dm.Dataset, project_data):
            'data_root': dataset.data_path + osp.sep,
        }

+        root_paths = set()
        for dataset_item in subset_dataset:
            if dataset_item.image and dataset_item.image.has_data:
                dataset_files['media'].append(dataset_item.image.path)
+                data_root = dataset_item.image.path.rsplit(dataset_item.id, 1)
+                if len(data_root) == 2:
+                    root_paths.add(data_root[0])
            elif dataset_item.point_cloud:
                dataset_files['media'].append(dataset_item.point_cloud)
+                data_root = dataset_item.point_cloud.rsplit(dataset_item.id, 1)
+                if len(data_root) == 2:
+                    root_paths.add(data_root[0])
+
            if isinstance(dataset_item.related_images, list):
                dataset_files['media'] += \
                    list(map(lambda ri: ri.path, dataset_item.related_images))

-        shortes_path = min(dataset_files['media'], key=lambda x: len(Path(x).parts), default=None)
-        if shortes_path is not None:
-            dataset_files['data_root'] = str(Path(shortes_path).parent.absolute()) + osp.sep
+        if len(root_paths):
+            dataset_files['data_root'] = osp.commonpath(root_paths) + osp.sep

        project_annotation.add_task(task_fields, dataset_files, project_data)
--- a/tests/python/rest_api/test_projects.py
+++ b/tests/python/rest_api/test_projects.py
@ -458,6 +458,23 @@ class TestImportExportDatasetProject:
            )
            assert response.status == HTTPStatus.ACCEPTED

+    def test_can_import_export_dataset_with_imagenet_format(self):
+        # https://github.com/opencv/cvat/issues/4850
+        username = "admin1"
+        format_name = "ImageNet 1.0"
+        project_id = 4
+
+        response = self._test_export_project(username, project_id, format_name)
+
+        tmp_file = io.BytesIO(response.data)
+        tmp_file.name = "dataset.zip"
+
+        import_data = {
+            "dataset_file": tmp_file,
+        }
+
+        self._test_import_project(username, project_id, format_name, import_data)
+

@pytest.mark.usefixtures("restore_db_per_function")
 class TestPatchProjectLabel:
--- a/tests/python/shared/assets/annotations.json
+++ b/tests/python/shared/assets/annotations.json
@ -361,7 +361,24 @@
          "z_order": 0
        }
      ],
-      "tags": [],
+      "tags": [
+        {
+          "attributes": [],
+          "frame": 2,
+          "group": 0,
+          "id": 1,
+          "label_id": 17,
+          "source": "manual"
+        },
+        {
+          "attributes": [],
+          "frame": 3,
+          "group": 0,
+          "id": 2,
+          "label_id": 16,
+          "source": "manual"
+        }
+      ],
      "tracks": [],
      "version": 0
    },
@ -1003,7 +1020,24 @@
          "z_order": 0
        }
      ],
-      "tags": [],
+      "tags": [
+        {
+          "attributes": [],
+          "frame": 2,
+          "group": 0,
+          "id": 1,
+          "label_id": 17,
+          "source": "manual"
+        },
+        {
+          "attributes": [],
+          "frame": 3,
+          "group": 0,
+          "id": 2,
+          "label_id": 16,
+          "source": "manual"
+        }
+      ],
      "tracks": [],
      "version": 0
    },
--- a/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2
+++ b/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2
--- a/tests/python/shared/assets/cvat_db/data.json
+++ b/tests/python/shared/assets/cvat_db/data.json
@ -36,7 +36,7 @@
  "pk": 1,
  "fields": {
    "password": "pbkdf2_sha256$260000$DevmxlmLwciP1P6sZs2Qag$U9DFtjTWx96Sk95qY6UXVcvpdQEP2LcoFBftk5D2RKY=",
-    "last_login": "2022-09-29T08:00:09.733Z",
+    "last_login": "2022-10-17T17:09:16.903Z",
    "is_superuser": true,
    "username": "admin1",
    "first_name": "Admin",
@ -487,6 +487,14 @@
    "expire_date": "2022-03-02T06:24:53.914Z"
  }
 },
+{
+  "model": "sessions.session",
+  "pk": "d3rhbsiax4etv2gtdkl6j883hvlz21rc",
+  "fields": {
+    "session_data": ".eJxVjMsOwiAQRf-FtSEMDAIu3fcbyPBQqgaS0q6M_y5NutDtOefeN_O0rcVvPS9-TuzCgJ1-WaD4zHUX6UH13nhsdV3mwPeEH7bzqaX8uh7t30GhXsYaQYNDpYMULqECS9IY0uIGxmYUQdoYMCpIeRgYiXVaDXoWgkImZJ8vqTk2zg:1okTc4:kq0YGX6_uyDwCa3VLl1bcp38oBfM6XfHIk0oE4XzpPw",
+    "expire_date": "2022-10-31T17:09:16.915Z"
+  }
+},
 {
  "model": "sessions.session",
  "pk": "dpaw6pntyqwr5l6qjv6zq3yoajp301be",
@ -2242,7 +2250,7 @@
    "assignee": null,
    "bug_tracker": "",
    "created_date": "2022-06-08T08:32:45.521Z",
-    "updated_date": "2022-09-28T12:26:54.279Z",
+    "updated_date": "2022-10-17T17:09:36.526Z",
    "status": "annotation",
    "organization": 2,
    "source_storage": null,
@ -2532,7 +2540,7 @@
    "assignee": null,
    "bug_tracker": "",
    "created_date": "2022-06-08T08:33:06.505Z",
-    "updated_date": "2022-06-08T08:33:20.808Z",
+    "updated_date": "2022-10-17T17:09:36.570Z",
    "overlap": 0,
    "segment_size": 5,
    "status": "annotation",
@ -3574,7 +3582,7 @@
  "fields": {
    "segment": 17,
    "assignee": null,
-    "updated_date": "2022-06-22T09:18:45.296Z",
+    "updated_date": "2022-10-17T17:09:36.571Z",
    "status": "annotation",
    "stage": "annotation",
    "state": "in progress"
@ -4726,6 +4734,45 @@
    "job": 18
  }
 },
+{
+  "model": "engine.jobcommit",
+  "pk": 73,
+  "fields": {
+    "scope": "create",
+    "owner": [
+      "admin1"
+    ],
+    "timestamp": "2022-10-17T17:09:36.573Z",
+    "data": {
+      "stage": "annotation",
+      "state": "in progress",
+      "assignee": null
+    },
+    "job": 17
+  }
+},
+{
+  "model": "engine.labeledimage",
+  "pk": 1,
+  "fields": {
+    "job": 17,
+    "label": 17,
+    "frame": 2,
+    "group": 0,
+    "source": "manual"
+  }
+},
+{
+  "model": "engine.labeledimage",
+  "pk": 2,
+  "fields": {
+    "job": 17,
+    "label": 16,
+    "frame": 3,
+    "group": 0,
+    "source": "manual"
+  }
+},
 {
  "model": "engine.labeledshape",
  "pk": 1,
--- a/tests/python/shared/assets/jobs.json
+++ b/tests/python/shared/assets/jobs.json
@ -235,7 +235,7 @@
      "status": "annotation",
      "stop_frame": 4,
      "task_id": 13,
-      "updated_date": "2022-06-22T09:18:45.296000Z",
+      "updated_date": "2022-10-17T17:09:36.571000Z",
      "url": "http://localhost:8080/api/jobs/17"
    },
    {
--- a/tests/python/shared/assets/projects.json
+++ b/tests/python/shared/assets/projects.json
@ -353,7 +353,7 @@
      "tasks": [
        13
      ],
-      "updated_date": "2022-09-28T12:26:54.279000Z",
+      "updated_date": "2022-10-17T17:09:36.526000Z",
      "url": "http://localhost:8080/api/projects/4"
    },
    {
--- a/tests/python/shared/assets/tasks.json
+++ b/tests/python/shared/assets/tasks.json
@ -303,7 +303,7 @@
      "status": "annotation",
      "subset": "",
      "target_storage": null,
-      "updated_date": "2022-06-08T08:33:20.808000Z",
+      "updated_date": "2022-10-17T17:09:36.570000Z",
      "url": "http://localhost:8080/api/tasks/13"
    },
    {
--- a/tests/python/shared/assets/users.json
+++ b/tests/python/shared/assets/users.json
@ -310,7 +310,7 @@
      "is_active": true,
      "is_staff": true,
      "is_superuser": true,
-      "last_login": "2022-09-29T08:00:09.733000Z",
+      "last_login": "2022-10-17T17:09:16.903140Z",
      "last_name": "First",
      "url": "http://localhost:8080/api/users/1",
      "username": "admin1"