From 9551fea4cc4b416e2d4c4e281ec78b8257861d06 Mon Sep 17 00:00:00 2001 From: Maria Khrustaleva Date: Tue, 28 Sep 2021 10:10:21 +0300 Subject: [PATCH] Display more user understandable exception message (#3721) * fix * Update CHANGELOG Co-authored-by: Nikita Manovich --- CHANGELOG.md | 1 + cvat/apps/engine/task.py | 3 ++- cvat/apps/engine/views.py | 7 ++++-- utils/dataset_manifest/core.py | 42 ++++++++++++++++++++-------------- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddbde52f..5e57dcb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed JSON transform issues in network requests () +- Display a more user-friendly exception message () - Exception `DataCloneError: The object could not be cloned` () ### Security diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index e3cb293e..a4f1127b 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -258,7 +258,8 @@ def _create_thread(tid, data, isImport=False): # prepare task manifest file from cloud storage manifest file manifest = ImageManifestManager(db_data.get_manifest_path()) cloud_storage_manifest = ImageManifestManager( - os.path.join(db_data.cloud_storage.get_storage_dirname(), manifest_file[0]) + os.path.join(db_data.cloud_storage.get_storage_dirname(), manifest_file[0]), + db_data.cloud_storage.get_storage_dirname() ) cloud_storage_manifest.set_index() media_files = sorted(media['image']) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 61688290..920c7a15 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1349,7 +1349,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS if not os.path.exists(full_manifest_path) or \ datetime.utcfromtimestamp(os.path.getmtime(full_manifest_path)).replace(tzinfo=pytz.UTC) < storage.get_file_last_modified(manifest_path): storage.download_file(manifest_path, full_manifest_path) - manifest = ImageManifestManager(full_manifest_path) + manifest = ImageManifestManager(full_manifest_path, db_storage.get_storage_dirname()) # need to update index manifest.set_index() manifest_files = manifest.data @@ -1406,7 +1406,10 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS if not os.path.exists(full_manifest_path) or \ datetime.utcfromtimestamp(os.path.getmtime(full_manifest_path)).replace(tzinfo=pytz.UTC) < storage.get_file_last_modified(manifest_model.filename): storage.download_file(manifest_model.filename, full_manifest_path) - manifest = ImageManifestManager(os.path.join(db_storage.get_storage_dirname(), manifest_model.filename)) + manifest = ImageManifestManager( + os.path.join(db_storage.get_storage_dirname(), manifest_model.filename), + db_storage.get_storage_dirname() + ) # need to update index manifest.set_index() if not len(manifest): diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 02d09925..794fdf7e 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -182,23 +182,19 @@ class _Manifest: FILE_NAME = 'manifest.jsonl' VERSION = '1.1' - def __init__(self, path, is_created=False): + def __init__(self, path, upload_dir=None): assert path, 'A path to manifest file not found' self._path = os.path.join(path, self.FILE_NAME) if os.path.isdir(path) else path - self._is_created = is_created + self._upload_dir = upload_dir @property def path(self): return self._path @property - def is_created(self): - return self._is_created - - @is_created.setter - def is_created(self, value): - assert isinstance(value, bool) - self._is_created = value + def name(self): + return os.path.basename(self._path) if not self._upload_dir \ + else os.path.relpath(self._path, self._upload_dir) # Needed for faster iteration over the manifest file, will be generated to work inside CVAT # and will not be generated when manually creating a manifest @@ -266,8 +262,14 @@ class _ManifestManager(ABC): 'version' : 1, 'type': 2, } - def __init__(self, path, *args, **kwargs): - self._manifest = _Manifest(path) + + def _json_item_is_valid(self, **state): + for item in self._requared_item_attributes: + if state.get(item, None) is None: + raise Exception(f"Invalid '{self.manifest.name} file structure': '{item}' is required, but not found") + + def __init__(self, path, upload_dir=None, *args, **kwargs): + self._manifest = _Manifest(path, upload_dir) self._index = _Index(os.path.dirname(self._manifest.path)) def _parse_line(self, line): @@ -284,7 +286,9 @@ class _ManifestManager(ABC): offset = self._index[line] manifest_file.seek(offset) properties = manifest_file.readline() - return json.loads(properties) + parsed_properties = json.loads(properties) + self._json_item_is_valid(**parsed_properties) + return parsed_properties def init_index(self): if os.path.exists(self._index.path): @@ -317,7 +321,9 @@ class _ManifestManager(ABC): while line: if not line.strip(): continue - yield (image_number, json.loads(line)) + parsed_properties = json.loads(line) + self._json_item_is_valid(**parsed_properties) + yield (image_number, parsed_properties) image_number += 1 line = manifest_file.readline() @@ -347,6 +353,8 @@ class _ManifestManager(ABC): pass class VideoManifestManager(_ManifestManager): + _requared_item_attributes = {'number', 'pts'} + def __init__(self, manifest_path): super().__init__(manifest_path) setattr(self._manifest, 'TYPE', 'video') @@ -375,7 +383,6 @@ class VideoManifestManager(_ManifestManager): 'checksum': item[2] }, separators=(',', ':')) manifest_file.write(f"{json_item}\n") - self._manifest.is_created = True def partial_update(self, number, properties): pass @@ -461,8 +468,10 @@ class VideoManifestValidator(VideoManifestManager): return class ImageManifestManager(_ManifestManager): - def __init__(self, manifest_path): - super().__init__(manifest_path) + _requared_item_attributes = {'name', 'extension'} + + def __init__(self, manifest_path, upload_dir=None): + super().__init__(manifest_path, upload_dir) setattr(self._manifest, 'TYPE', 'images') def create(self, content, **kwargs): @@ -481,7 +490,6 @@ class ImageManifestManager(_ManifestManager): key: value for key, value in item.items() }, separators=(',', ':')) manifest_file.write(f"{json_item}\n") - self._manifest.is_created = True def partial_update(self, number, properties): pass