From 1a86efe65e5fbb4ea8e2455b9374eeff1520711d Mon Sep 17 00:00:00 2001 From: Maria Khrustaleva Date: Tue, 22 Jun 2021 22:41:22 +0300 Subject: [PATCH] Fixes cloud storage (#3336) * Fixes: - exception with getting content when specific_attributes='' - no deletion of temporary index for temporary manifest * Fix getting owner structure instead of id * Delete := * Fix creating task with cloud storage: preview && nonexistent field Co-authored-by: Nikita Manovich --- cvat/apps/engine/cache.py | 5 +++-- cvat/apps/engine/models.py | 6 +++--- cvat/apps/engine/serializers.py | 2 ++ cvat/apps/engine/task.py | 5 +++-- cvat/apps/engine/views.py | 12 +++++++----- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index d57861bc..75196ee8 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -91,7 +91,8 @@ class CacheInteraction: source_path = temp_file.name buf = cloud_storage_instance.download_fileobj(name) temp_file.write(buf.getvalue()) - if not (checksum := item.get('checksum', None)): + checksum = item.get('checksum', None) + if not checksum: slogger.glob.warning('A manifest file does not contain checksum for image {}'.format(item.get('name'))) if checksum and not md5_hash(source_path) == checksum: slogger.glob.warning('Hash sums of files {} do not match'.format(name)) @@ -103,7 +104,7 @@ class CacheInteraction: writer.save_as_chunk(images, buff) buff.seek(0) if db_data.storage == StorageChoice.CLOUD_STORAGE: - images = [image_path for image in images if os.path.exists((image_path := image[0]))] + images = [image[0] for image in images if os.path.exists(image[0])] for image_path in images: os.remove(image_path) return buff, mime_type diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index f88f748a..cbcc5f2d 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -608,8 +608,8 @@ class CloudStorage(models.Model): return os.path.join(self.get_storage_dirname(), "storage.log") def get_specific_attributes(self): - attributes = self.specific_attributes.split('&') + specific_attributes = self.specific_attributes return { item.split('=')[0].strip(): item.split('=')[1].strip() - for item in attributes - } if len(attributes) else dict() \ No newline at end of file + for item in specific_attributes.split('&') + } if specific_attributes else dict() \ No newline at end of file diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 519148f7..0c0b1130 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -772,9 +772,11 @@ class CombinedReviewSerializer(ReviewSerializer): return db_review class BaseCloudStorageSerializer(serializers.ModelSerializer): + owner = BasicUserSerializer(required=False) class Meta: model = models.CloudStorage exclude = ['credentials'] + read_only_fields = ('created_date', 'updated_date', 'owner') class CloudStorageSerializer(serializers.ModelSerializer): owner = BasicUserSerializer(required=False) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index a864bf14..a6f44848 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -243,7 +243,7 @@ def _create_thread(tid, data, isImport=False): credentials = Credentials() credentials.convert_from_db({ 'type': db_cloud_storage.credentials_type, - 'value': db_cloud_storage.value, + 'value': db_cloud_storage.credentials, }) details = { @@ -253,7 +253,8 @@ def _create_thread(tid, data, isImport=False): } cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details) cloud_storage_instance.download_file(manifest_file[0], db_data.get_manifest_path()) - cloud_storage_instance.download_file(media['image'][0], os.path.join(upload_dir, media['image'][0])) + first_sorted_media_image = sorted(media['image'])[0] + cloud_storage_instance.download_file(first_sorted_media_image, os.path.join(upload_dir, first_sorted_media_image)) av_scan_paths(upload_dir) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 430d1050..3adaa017 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -11,7 +11,7 @@ import traceback import uuid from datetime import datetime from distutils.util import strtobool -from tempfile import mkstemp, NamedTemporaryFile +from tempfile import mkstemp, TemporaryDirectory import cv2 from django.db.models.query import Prefetch @@ -1176,7 +1176,8 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS def get_queryset(self): queryset = super().get_queryset() - if (provider_type := self.request.query_params.get('provider_type', None)): + provider_type = self.request.query_params.get('provider_type', None) + if provider_type: if provider_type in CloudProviderChoice.list(): return queryset.filter(provider_type=provider_type) raise ValidationError('Unsupported type of cloud provider') @@ -1278,9 +1279,10 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS storage_files = storage.content manifest_path = request.query_params.get('manifest_path', 'manifest.jsonl') - with NamedTemporaryFile(mode='w+b', suffix='manifest', prefix='cvat') as tmp_manifest: - storage.download_file(manifest_path, tmp_manifest.name) - manifest = ImageManifestManager(tmp_manifest.name) + with TemporaryDirectory(suffix='manifest', prefix='cvat') as tmp_dir: + tmp_manifest_path = os.path.join(tmp_dir, 'manifest.jsonl') + storage.download_file(manifest_path, tmp_manifest_path) + manifest = ImageManifestManager(tmp_manifest_path) manifest.init_index() manifest_files = manifest.data content = {f:[] for f in set(storage_files) | set(manifest_files)}