Fixes cloud storage (#3336)

* Fixes:
- exception with getting content when specific_attributes=''
- no deletion of temporary index for  temporary manifest

* Fix getting owner structure instead of id

* Delete :=

* Fix creating task with cloud storage: preview && nonexistent field

Co-authored-by: Nikita Manovich <nikita.manovich@intel.com>

cvat/apps/engine/cache.py

@@ -91,7 +91,8 @@ class CacheInteraction:
                         source_path = temp_file.name
                         buf = cloud_storage_instance.download_fileobj(name)
                         temp_file.write(buf.getvalue())
-                        if not (checksum := item.get('checksum', None)):
+                        checksum = item.get('checksum', None)
+                        if not checksum:
                             slogger.glob.warning('A manifest file does not contain checksum for image {}'.format(item.get('name')))
                         if checksum and not md5_hash(source_path) == checksum:
                             slogger.glob.warning('Hash sums of files {} do not match'.format(name))
@@ -103,7 +104,7 @@ class CacheInteraction:
         writer.save_as_chunk(images, buff)
         buff.seek(0)
         if db_data.storage == StorageChoice.CLOUD_STORAGE:
-            images = [image_path for image in images if os.path.exists((image_path := image[0]))]
+            images = [image[0] for image in images if os.path.exists(image[0])]
             for image_path in images:
                 os.remove(image_path)
         return buff, mime_type

cvat/apps/engine/models.py

@@ -608,8 +608,8 @@ class CloudStorage(models.Model):
         return os.path.join(self.get_storage_dirname(), "storage.log")
 
     def get_specific_attributes(self):
-        attributes = self.specific_attributes.split('&')
+        specific_attributes = self.specific_attributes
         return {
             item.split('=')[0].strip(): item.split('=')[1].strip()
-                for item in attributes
+                for item in specific_attributes.split('&')
-        } if len(attributes) else dict()
+        } if specific_attributes else dict()

cvat/apps/engine/serializers.py

@@ -772,9 +772,11 @@ class CombinedReviewSerializer(ReviewSerializer):
         return db_review
 
 class BaseCloudStorageSerializer(serializers.ModelSerializer):
+    owner = BasicUserSerializer(required=False)
     class Meta:
         model = models.CloudStorage
         exclude = ['credentials']
+        read_only_fields = ('created_date', 'updated_date', 'owner')
 
 class CloudStorageSerializer(serializers.ModelSerializer):
     owner = BasicUserSerializer(required=False)

cvat/apps/engine/task.py

@@ -243,7 +243,7 @@ def _create_thread(tid, data, isImport=False):
             credentials = Credentials()
             credentials.convert_from_db({
                'type': db_cloud_storage.credentials_type,
-               'value': db_cloud_storage.value,
+               'value': db_cloud_storage.credentials,
             })
 
             details = {
@@ -253,7 +253,8 @@ def _create_thread(tid, data, isImport=False):
             }
             cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details)
             cloud_storage_instance.download_file(manifest_file[0], db_data.get_manifest_path())
-            cloud_storage_instance.download_file(media['image'][0], os.path.join(upload_dir, media['image'][0]))
+            first_sorted_media_image = sorted(media['image'])[0]
+            cloud_storage_instance.download_file(first_sorted_media_image, os.path.join(upload_dir, first_sorted_media_image))
 
     av_scan_paths(upload_dir)
 

cvat/apps/engine/views.py

@@ -11,7 +11,7 @@ import traceback
 import uuid
 from datetime import datetime
 from distutils.util import strtobool
-from tempfile import mkstemp, NamedTemporaryFile
+from tempfile import mkstemp, TemporaryDirectory
 
 import cv2
 from django.db.models.query import Prefetch
@@ -1176,7 +1176,8 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        if (provider_type := self.request.query_params.get('provider_type', None)):
+        provider_type = self.request.query_params.get('provider_type', None)
+        if provider_type:
             if provider_type in CloudProviderChoice.list():
                 return queryset.filter(provider_type=provider_type)
             raise ValidationError('Unsupported type of cloud provider')
@@ -1278,9 +1279,10 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS
             storage_files = storage.content
 
             manifest_path = request.query_params.get('manifest_path', 'manifest.jsonl')
-            with NamedTemporaryFile(mode='w+b', suffix='manifest', prefix='cvat') as tmp_manifest:
+            with TemporaryDirectory(suffix='manifest', prefix='cvat') as tmp_dir:
-                storage.download_file(manifest_path, tmp_manifest.name)
+                tmp_manifest_path = os.path.join(tmp_dir, 'manifest.jsonl')
-                manifest = ImageManifestManager(tmp_manifest.name)
+                storage.download_file(manifest_path, tmp_manifest_path)
+                manifest = ImageManifestManager(tmp_manifest_path)
                 manifest.init_index()
                 manifest_files = manifest.data
             content = {f:[] for f in set(storage_files) | set(manifest_files)}