Display more user understandable exception message (#3721)

* fix

* Update CHANGELOG

Co-authored-by: Nikita Manovich <nikita.manovich@intel.com>

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Fixed JSON transform issues in network requests (<https://github.com/openvinotoolkit/cvat/pull/3706>)
+- Display a more user-friendly exception message (<https://github.com/openvinotoolkit/cvat/pull/3721>)
 - Exception `DataCloneError: The object could not be cloned` (<https://github.com/openvinotoolkit/cvat/pull/3733>)
 
 ### Security

cvat/apps/engine/task.py

@@ -258,7 +258,8 @@ def _create_thread(tid, data, isImport=False):
             # prepare task manifest file from cloud storage manifest file
             manifest = ImageManifestManager(db_data.get_manifest_path())
             cloud_storage_manifest = ImageManifestManager(
-                os.path.join(db_data.cloud_storage.get_storage_dirname(), manifest_file[0])
+                os.path.join(db_data.cloud_storage.get_storage_dirname(), manifest_file[0]),
+                db_data.cloud_storage.get_storage_dirname()
             )
             cloud_storage_manifest.set_index()
             media_files = sorted(media['image'])

cvat/apps/engine/views.py

@@ -1349,7 +1349,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS
             if not os.path.exists(full_manifest_path) or \
                     datetime.utcfromtimestamp(os.path.getmtime(full_manifest_path)).replace(tzinfo=pytz.UTC) < storage.get_file_last_modified(manifest_path):
                 storage.download_file(manifest_path, full_manifest_path)
-            manifest = ImageManifestManager(full_manifest_path)
+            manifest = ImageManifestManager(full_manifest_path, db_storage.get_storage_dirname())
             # need to update index
             manifest.set_index()
             manifest_files = manifest.data
@@ -1406,7 +1406,10 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS
                     if not os.path.exists(full_manifest_path) or \
                             datetime.utcfromtimestamp(os.path.getmtime(full_manifest_path)).replace(tzinfo=pytz.UTC) < storage.get_file_last_modified(manifest_model.filename):
                         storage.download_file(manifest_model.filename, full_manifest_path)
-                    manifest = ImageManifestManager(os.path.join(db_storage.get_storage_dirname(), manifest_model.filename))
+                    manifest = ImageManifestManager(
+                        os.path.join(db_storage.get_storage_dirname(), manifest_model.filename),
+                        db_storage.get_storage_dirname()
+                    )
                     # need to update index
                     manifest.set_index()
                     if not len(manifest):

utils/dataset_manifest/core.py

@@ -182,23 +182,19 @@ class _Manifest:
     FILE_NAME = 'manifest.jsonl'
     VERSION = '1.1'
 
-    def __init__(self, path, is_created=False):
+    def __init__(self, path, upload_dir=None):
         assert path, 'A path to manifest file not found'
         self._path = os.path.join(path, self.FILE_NAME) if os.path.isdir(path) else path
-        self._is_created = is_created
+        self._upload_dir = upload_dir
 
     @property
     def path(self):
         return self._path
 
     @property
-    def is_created(self):
-        return self._is_created
-
-    @is_created.setter
-    def is_created(self, value):
-        assert isinstance(value, bool)
-        self._is_created = value
+    def name(self):
+        return os.path.basename(self._path) if not self._upload_dir \
+            else os.path.relpath(self._path, self._upload_dir)
 
 # Needed for faster iteration over the manifest file, will be generated to work inside CVAT
 # and will not be generated when manually creating a manifest
@@ -266,8 +262,14 @@ class _ManifestManager(ABC):
         'version' : 1,
         'type': 2,
     }
-    def __init__(self, path, *args, **kwargs):
-        self._manifest = _Manifest(path)
+
+    def _json_item_is_valid(self, **state):
+        for item in self._requared_item_attributes:
+            if state.get(item, None) is None:
+                raise Exception(f"Invalid '{self.manifest.name} file structure': '{item}' is required, but not found")
+
+    def __init__(self, path, upload_dir=None, *args, **kwargs):
+        self._manifest = _Manifest(path, upload_dir)
         self._index = _Index(os.path.dirname(self._manifest.path))
 
     def _parse_line(self, line):
@@ -284,7 +286,9 @@ class _ManifestManager(ABC):
                 offset = self._index[line]
                 manifest_file.seek(offset)
                 properties = manifest_file.readline()
-                return json.loads(properties)
+                parsed_properties = json.loads(properties)
+                self._json_item_is_valid(**parsed_properties)
+                return parsed_properties
 
     def init_index(self):
         if os.path.exists(self._index.path):
@@ -317,7 +321,9 @@ class _ManifestManager(ABC):
             while line:
                 if not line.strip():
                     continue
-                yield (image_number, json.loads(line))
+                parsed_properties = json.loads(line)
+                self._json_item_is_valid(**parsed_properties)
+                yield (image_number, parsed_properties)
                 image_number += 1
                 line = manifest_file.readline()
 
@@ -347,6 +353,8 @@ class _ManifestManager(ABC):
         pass
 
 class VideoManifestManager(_ManifestManager):
+    _requared_item_attributes = {'number', 'pts'}
+
     def __init__(self, manifest_path):
         super().__init__(manifest_path)
         setattr(self._manifest, 'TYPE', 'video')
@@ -375,7 +383,6 @@ class VideoManifestManager(_ManifestManager):
                     'checksum': item[2]
                 }, separators=(',', ':'))
                 manifest_file.write(f"{json_item}\n")
-        self._manifest.is_created = True
 
     def partial_update(self, number, properties):
         pass
@@ -461,8 +468,10 @@ class VideoManifestValidator(VideoManifestManager):
                 return
 
 class ImageManifestManager(_ManifestManager):
-    def __init__(self, manifest_path):
-        super().__init__(manifest_path)
+    _requared_item_attributes = {'name', 'extension'}
+
+    def __init__(self, manifest_path, upload_dir=None):
+        super().__init__(manifest_path, upload_dir)
         setattr(self._manifest, 'TYPE', 'images')
 
     def create(self, content, **kwargs):
@@ -481,7 +490,6 @@ class ImageManifestManager(_ManifestManager):
                     key: value for key, value in item.items()
                 }, separators=(',', ':'))
                 manifest_file.write(f"{json_item}\n")
-        self._manifest.is_created = True
 
     def partial_update(self, number, properties):
         pass