diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f9728f..3632f99b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed tus upload error over https () - Issues disappear when rescale a browser () - Auth token key is not returned when registering without email verification () +- Error in create project from backup for standard 3D annotation () ### Security - Updated ELK to 6.8.23 which uses log4j 2.17.1 () diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index a7693c67..3e644902 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2020 Intel Corporation +# Copyright (C) 2019-2022 Intel Corporation # # SPDX-License-Identifier: MIT @@ -136,6 +136,9 @@ class ImageListReader(IMediaReader): for i in range(self._start, self._stop, self._step): yield (self.get_image(i), self.get_path(i), i) + def __contains__(self, media_file): + return media_file in self._source_path + def filter(self, callback): source_path = list(filter(callback, self._source_path)) ImageListReader.__init__( @@ -172,14 +175,14 @@ class ImageListReader(IMediaReader): img = Image.open(self._source_path[i]) return img.width, img.height - def reconcile(self, source_files, step=1, start=0, stop=None, dimension=DimensionType.DIM_2D): + def reconcile(self, source_files, step=1, start=0, stop=None, dimension=DimensionType.DIM_2D, sorting_method=None): # FIXME ImageListReader.__init__(self, source_path=source_files, step=step, start=start, stop=stop, - sorting_method=self._sorting_method, + sorting_method=sorting_method if sorting_method else self._sorting_method, ) self._dimension = dimension @@ -328,13 +331,14 @@ class ZipReader(ImageListReader): else: # necessary for mime_type definition return self._source_path[i] - def reconcile(self, source_files, step=1, start=0, stop=None, dimension=DimensionType.DIM_2D): + def reconcile(self, source_files, step=1, start=0, stop=None, dimension=DimensionType.DIM_2D, sorting_method=None): super().reconcile( source_files=source_files, step=step, start=start, stop=stop, dimension=dimension, + sorting_method=sorting_method ) def extract(self): diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index d8e25ef7..5550dd23 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -1,5 +1,5 @@ -# Copyright (C) 2018-2021 Intel Corporation +# Copyright (C) 2018-2022 Intel Corporation # # SPDX-License-Identifier: MIT @@ -326,22 +326,6 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False): db_data.start_frame = 0 data['stop_frame'] = None db_data.frame_filter = '' - if isBackupRestore and media_type != 'video' and db_data.storage_method == models.StorageMethodChoice.CACHE: - # we should sort media_files according to the manifest content sequence - manifest = ImageManifestManager(db_data.get_manifest_path()) - manifest.set_index() - sorted_media_files = [] - for idx in range(len(media_files)): - properties = manifest[manifest_index(idx)] - image_name = properties.get('name', None) - image_extension = properties.get('extension', None) - - full_image_path = f"{image_name}{image_extension}" if image_name and image_extension else None - if full_image_path and full_image_path in media_files: - sorted_media_files.append(full_image_path) - media_files = sorted_media_files.copy() - del sorted_media_files - data['sorting_method'] = models.SortingMethod.PREDEFINED source_paths=[os.path.join(upload_dir, f) for f in media_files] if manifest_file and not isBackupRestore and data['sorting_method'] in {models.SortingMethod.RANDOM, models.SortingMethod.PREDEFINED}: raise Exception("It isn't supported to upload manifest file and use random sorting") @@ -368,8 +352,8 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False): extractor.extract() if db_data.storage == models.StorageChoice.LOCAL or \ - (db_data.storage == models.StorageChoice.SHARE and \ - isinstance(extractor, MEDIA_TYPES['zip']['extractor'])): + (db_data.storage == models.StorageChoice.SHARE and \ + isinstance(extractor, MEDIA_TYPES['zip']['extractor'])): validate_dimension.set_path(upload_dir) validate_dimension.validate() @@ -379,8 +363,15 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False): if validate_dimension.dimension == models.DimensionType.DIM_3D: db_task.dimension = models.DimensionType.DIM_3D + keys_of_related_files = validate_dimension.related_files.keys() + absolute_keys_of_related_files = [os.path.join(upload_dir, f) for f in keys_of_related_files] + # When a task is created, the sorting method can be random and in this case, reinitialization will be with correct sorting + # but when a task is restored from a backup, a random sorting is changed to predefined and we need to manually sort files + # in the correct order. + source_files = absolute_keys_of_related_files if not isBackupRestore else \ + [item for item in extractor.absolute_source_paths if item in absolute_keys_of_related_files] extractor.reconcile( - source_files=[os.path.join(upload_dir, f) for f in validate_dimension.related_files.keys()], + source_files=source_files, step=db_data.get_frame_step(), start=db_data.start_frame, stop=data['stop_frame'], @@ -392,6 +383,33 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False): extractor.filter(lambda x: not re.search(r'(^|{0})related_images{0}'.format(os.sep), x)) related_images = detect_related_images(extractor.absolute_source_paths, upload_dir) + if isBackupRestore and not isinstance(extractor, MEDIA_TYPES['video']['extractor']) and db_data.storage_method == models.StorageMethodChoice.CACHE and \ + db_data.sorting_method in {models.SortingMethod.RANDOM, models.SortingMethod.PREDEFINED} and validate_dimension.dimension != models.DimensionType.DIM_3D: + # we should sort media_files according to the manifest content sequence + # and we should do this in general after validation step for 3D data and after filtering from related_images + manifest = ImageManifestManager(db_data.get_manifest_path()) + manifest.set_index() + sorted_media_files = [] + + for idx in range(len(extractor.absolute_source_paths)): + properties = manifest[idx] + image_name = properties.get('name', None) + image_extension = properties.get('extension', None) + + full_image_path = os.path.join(upload_dir, f"{image_name}{image_extension}") if image_name and image_extension else None + if full_image_path and full_image_path in extractor: + sorted_media_files.append(full_image_path) + media_files = sorted_media_files.copy() + del sorted_media_files + data['sorting_method'] = models.SortingMethod.PREDEFINED + extractor.reconcile( + source_files=media_files, + step=db_data.get_frame_step(), + start=db_data.start_frame, + stop=data['stop_frame'], + sorting_method=data['sorting_method'], + ) + db_task.mode = task_mode db_data.compressed_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' and not data['use_zip_chunks'] else models.DataChoice.IMAGESET db_data.original_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' else models.DataChoice.IMAGESET diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 5939d115..8afb69d8 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2021 Intel Corporation +# Copyright (C) 2020-2022 Intel Corporation # # SPDX-License-Identifier: MIT @@ -2553,6 +2553,16 @@ class TaskImportExportAPITestCase(APITestCase): } ) + for sorting, _ in SortingMethod.choices(): + cls.media_data.append( + { + "image_quality": 75, + "server_files[0]": filename, + 'use_cache': True, + 'sorting_method': sorting, + } + ) + filename = os.path.join("videos", "test_video_1.mp4") path = os.path.join(settings.SHARE_ROOT, filename) os.makedirs(os.path.dirname(path)) @@ -2617,7 +2627,7 @@ class TaskImportExportAPITestCase(APITestCase): **use_cache_data, 'sorting_method': SortingMethod.RANDOM, }, - # predefined: test_1.jpg, test_2.jpg, test_10.jpg, test_2.jpg + # predefined: test_1.jpg, test_2.jpg, test_10.jpg, test_3.jpg { **use_cache_data, 'sorting_method': SortingMethod.PREDEFINED, diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index a6a3ac1d..d2e9da5f 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2021-2022 Intel Corporation # # SPDX-License-Identifier: MIT @@ -324,7 +324,7 @@ class _Index: def __getitem__(self, number): assert 0 <= number < len(self), \ - 'A invalid index number: {}\nMax: {}'.format(number, len(self)) + 'Invalid index number: {}\nMax: {}'.format(number, len(self) - 1) return self._index[number] def __len__(self): @@ -706,12 +706,15 @@ class _DatasetManifestStructureValidator(_BaseManifestValidator): raise ValueError('Incorrect name field') if not isinstance(_dict['extension'], str): raise ValueError('Incorrect extension field') - # width and height are required for 2d data - # FIXME for 3d when manual preparation of the manifest will be implemented - if not isinstance(_dict['width'], int): - raise ValueError('Incorrect width field') - if not isinstance(_dict['height'], int): - raise ValueError('Incorrect height field') + # FIXME + # Width and height are required for 2D data, but + # for 3D these parameters are not saved now. + # It is necessary to uncomment these restrictions when manual preparation for 3D data is implemented. + + # if not isinstance(_dict['width'], int): + # raise ValueError('Incorrect width field') + # if not isinstance(_dict['height'], int): + # raise ValueError('Incorrect height field') def is_manifest(full_manifest_path): return _is_video_manifest(full_manifest_path) or \ @@ -723,4 +726,4 @@ def _is_video_manifest(full_manifest_path): def _is_dataset_manifest(full_manifest_path): validator = _DatasetManifestStructureValidator(full_manifest_path) - return validator.validate() \ No newline at end of file + return validator.validate()