main
Maya 5 years ago
parent 143fe28d39
commit d2e186a647

@ -10,7 +10,7 @@ from django.conf import settings
from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter,
Mpeg4CompressedChunkWriter, ZipChunkWriter, ZipCompressedChunkWriter) Mpeg4CompressedChunkWriter, ZipChunkWriter, ZipCompressedChunkWriter)
from cvat.apps.engine.models import DataChoice, UploadedDataStorageLocationChoice from cvat.apps.engine.models import DataChoice, StorageChoice
from cvat.apps.engine.prepare import PrepareInfo from cvat.apps.engine.prepare import PrepareInfo
from .log import slogger from .log import slogger
@ -44,9 +44,9 @@ class CacheInteraction:
images = [] images = []
buff = BytesIO() buff = BytesIO()
upload_dir = { upload_dir = {
UploadedDataStorageLocationChoice.LOCAL: db_data.get_upload_dirname(), StorageChoice.LOCAL: db_data.get_upload_dirname(),
UploadedDataStorageLocationChoice.SHARE: settings.SHARE_ROOT StorageChoice.SHARE: settings.SHARE_ROOT
}[db_data.uploaded_data_storage_location] }[db_data.storage]
try: try:
if os.path.exists(db_data.get_meta_path()): if os.path.exists(db_data.get_meta_path()):
source_path = os.path.join(upload_dir, db_data.video.path) source_path = os.path.join(upload_dir, db_data.video.path)

@ -13,10 +13,10 @@ class Migration(migrations.Migration):
operations = [ operations = [
migrations.AddField( migrations.AddField(
model_name='data', model_name='data',
name='uploaded_data_storage_location', name='storage',
field=models.CharField( field=models.CharField(
choices=[('local', 'LOCAL'), ('share', 'SHARE')], choices=[('local', 'LOCAL'), ('share', 'SHARE')],
default=cvat.apps.engine.models.UploadedDataStorageLocationChoice['LOCAL'], default=cvat.apps.engine.models.StorageChoice['LOCAL'],
max_length=15 max_length=15
), ),
), ),

@ -54,7 +54,7 @@ class StorageMethodChoice(str, Enum):
def __str__(self): def __str__(self):
return self.value return self.value
class UploadedDataStorageLocationChoice(str, Enum): class StorageChoice(str, Enum):
#AWS_S3 = 'aws_s3_bucket' #AWS_S3 = 'aws_s3_bucket'
LOCAL = 'local' LOCAL = 'local'
SHARE = 'share' SHARE = 'share'
@ -78,8 +78,7 @@ class Data(models.Model):
original_chunk_type = models.CharField(max_length=32, choices=DataChoice.choices(), original_chunk_type = models.CharField(max_length=32, choices=DataChoice.choices(),
default=DataChoice.IMAGESET) default=DataChoice.IMAGESET)
storage_method = models.CharField(max_length=15, choices=StorageMethodChoice.choices(), default=StorageMethodChoice.FILE_SYSTEM) storage_method = models.CharField(max_length=15, choices=StorageMethodChoice.choices(), default=StorageMethodChoice.FILE_SYSTEM)
uploaded_data_storage_location = models.CharField(max_length=15, choices=UploadedDataStorageLocationChoice.choices(), storage = models.CharField(max_length=15, choices=StorageChoice.choices(), default=StorageChoice.LOCAL)
default=UploadedDataStorageLocationChoice.LOCAL)
class Meta: class Meta:
default_permissions = () default_permissions = ()

@ -15,7 +15,7 @@ from urllib import parse as urlparse
from urllib import request as urlrequest from urllib import request as urlrequest
from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter
from cvat.apps.engine.models import DataChoice, StorageMethodChoice, UploadedDataStorageLocationChoice as LocationChoice from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice
from cvat.apps.engine.utils import av_scan_paths from cvat.apps.engine.utils import av_scan_paths
from cvat.apps.engine.prepare import prepare_meta from cvat.apps.engine.prepare import prepare_meta
@ -232,7 +232,7 @@ def _create_thread(tid, data):
"File with meta information can be uploaded if 'Use cache' option is also selected" "File with meta information can be uploaded if 'Use cache' option is also selected"
if data['server_files']: if data['server_files']:
if db_data.uploaded_data_storage_location == LocationChoice.LOCAL: if db_data.storage == StorageChoice.LOCAL:
_copy_data_from_share(data['server_files'], upload_dir) _copy_data_from_share(data['server_files'], upload_dir)
else: else:
upload_dir = settings.SHARE_ROOT upload_dir = settings.SHARE_ROOT
@ -251,7 +251,7 @@ def _create_thread(tid, data):
if extractor is not None: if extractor is not None:
raise Exception('Combined data types are not supported') raise Exception('Combined data types are not supported')
source_paths=[os.path.join(upload_dir, f) for f in media_files] source_paths=[os.path.join(upload_dir, f) for f in media_files]
if media_type in ('archive', 'zip') and db_data.uploaded_data_storage_location == LocationChoice.SHARE: if media_type in ('archive', 'zip') and db_data.storage == StorageChoice.SHARE:
source_paths.append(db_data.get_upload_dirname()) source_paths.append(db_data.get_upload_dirname())
extractor = MEDIA_TYPES[media_type]['extractor']( extractor = MEDIA_TYPES[media_type]['extractor'](
source_path=source_paths, source_path=source_paths,

@ -29,7 +29,7 @@ from rest_framework import status
from rest_framework.test import APIClient, APITestCase from rest_framework.test import APIClient, APITestCase
from cvat.apps.engine.models import (AttributeType, Data, Job, Project, from cvat.apps.engine.models import (AttributeType, Data, Job, Project,
Segment, StatusChoice, Task, StorageMethodChoice, UploadedDataStorageLocationChoice as StorageLocation) Segment, StatusChoice, Task, StorageMethodChoice, StorageChoice)
from cvat.apps.engine.prepare import prepare_meta, prepare_meta_for_upload from cvat.apps.engine.prepare import prepare_meta, prepare_meta_for_upload
def create_db_users(cls): def create_db_users(cls):
@ -1650,7 +1650,7 @@ class TaskDataAPITestCase(APITestCase):
def _test_api_v1_tasks_id_data_spec(self, user, spec, data, expected_compressed_type, expected_original_type, image_sizes, def _test_api_v1_tasks_id_data_spec(self, user, spec, data, expected_compressed_type, expected_original_type, image_sizes,
expected_storage_method=StorageMethodChoice.FILE_SYSTEM, expected_storage_method=StorageMethodChoice.FILE_SYSTEM,
expected_uploaded_data_location=StorageLocation.LOCAL): expected_uploaded_data_location=StorageChoice.LOCAL):
# create task # create task
response = self._create_task(user, spec) response = self._create_task(user, spec)
self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(response.status_code, status.HTTP_201_CREATED)
@ -1676,9 +1676,9 @@ class TaskDataAPITestCase(APITestCase):
self.assertEqual(len(image_sizes), task["size"]) self.assertEqual(len(image_sizes), task["size"])
db_data = Task.objects.get(pk=task_id).data db_data = Task.objects.get(pk=task_id).data
self.assertEqual(expected_storage_method, db_data.storage_method) self.assertEqual(expected_storage_method, db_data.storage_method)
self.assertEqual(expected_uploaded_data_location, db_data.uploaded_data_storage_location) self.assertEqual(expected_uploaded_data_location, db_data.storage)
# check if used share without copying inside and files doesn`t exist in ../raw/ # check if used share without copying inside and files doesn`t exist in ../raw/
if expected_uploaded_data_location is StorageLocation.SHARE: if expected_uploaded_data_location is StorageChoice.SHARE:
self.assertEqual(False, self.assertEqual(False,
os.path.exists(os.path.join(db_data.get_upload_dirname(), next(iter(data.values()))))) os.path.exists(os.path.join(db_data.get_upload_dirname(), next(iter(data.values())))))
@ -1798,12 +1798,12 @@ class TaskDataAPITestCase(APITestCase):
] ]
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes,
expected_uploaded_data_location=StorageLocation.SHARE) expected_uploaded_data_location=StorageChoice.SHARE)
task_spec.update([('name', 'my task #3')]) task_spec.update([('name', 'my task #3')])
task_data.update([('copy_data', True)]) task_data.update([('copy_data', True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET,
image_sizes, expected_uploaded_data_location=StorageLocation.LOCAL) image_sizes, expected_uploaded_data_location=StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "my video task #4", "name": "my video task #4",
@ -1839,12 +1839,12 @@ class TaskDataAPITestCase(APITestCase):
image_sizes = self._image_sizes[task_data["server_files[0]"]] image_sizes = self._image_sizes[task_data["server_files[0]"]]
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, image_sizes, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, image_sizes,
expected_uploaded_data_location=StorageLocation.SHARE) expected_uploaded_data_location=StorageChoice.SHARE)
task_spec.update([('name', 'my video task #6')]) task_spec.update([('name', 'my video task #6')])
task_data.update([('copy_data', True)]) task_data.update([('copy_data', True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO,
image_sizes, expected_uploaded_data_location=StorageLocation.LOCAL) image_sizes, expected_uploaded_data_location=StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "my video task without copying #7", "name": "my video task without copying #7",
@ -1862,12 +1862,12 @@ class TaskDataAPITestCase(APITestCase):
image_sizes = self._image_sizes[task_data["server_files[0]"]] image_sizes = self._image_sizes[task_data["server_files[0]"]]
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, image_sizes, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, image_sizes,
expected_uploaded_data_location=StorageLocation.SHARE) expected_uploaded_data_location=StorageChoice.SHARE)
task_spec.update([("name", "my video task #8")]) task_spec.update([("name", "my video task #8")])
task_data.update([("copy_data", True)]) task_data.update([("copy_data", True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO,
image_sizes, expected_uploaded_data_location=StorageLocation.LOCAL) image_sizes, expected_uploaded_data_location=StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "my video task without copying #9", "name": "my video task without copying #9",
@ -1887,12 +1887,12 @@ class TaskDataAPITestCase(APITestCase):
image_sizes = self._image_sizes[task_data["server_files[0]"]] image_sizes = self._image_sizes[task_data["server_files[0]"]]
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.VIDEO, image_sizes, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.VIDEO, image_sizes,
expected_uploaded_data_location=StorageLocation.SHARE) expected_uploaded_data_location=StorageChoice.SHARE)
task_spec.update([('name', 'my video task #10')]) task_spec.update([('name', 'my video task #10')])
task_data.update([('copy_data', True)]) task_data.update([('copy_data', True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.VIDEO, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.VIDEO,
image_sizes, expected_uploaded_data_location=StorageLocation.LOCAL) image_sizes, expected_uploaded_data_location=StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "my archive task without copying #11", "name": "my archive task without copying #11",
@ -1910,12 +1910,12 @@ class TaskDataAPITestCase(APITestCase):
image_sizes = self._image_sizes[task_data["server_files[0]"]] image_sizes = self._image_sizes[task_data["server_files[0]"]]
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes,
expected_uploaded_data_location=StorageLocation.SHARE) expected_uploaded_data_location=StorageChoice.SHARE)
task_spec.update([('name', 'my archive task #12')]) task_spec.update([('name', 'my archive task #12')])
task_data.update([('copy_data', True)]) task_data.update([('copy_data', True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET,
image_sizes, expected_uploaded_data_location=StorageLocation.LOCAL) image_sizes, expected_uploaded_data_location=StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "my archive task #13", "name": "my archive task #13",
@ -1953,12 +1953,12 @@ class TaskDataAPITestCase(APITestCase):
image_sizes = self._image_sizes[task_data["server_files[0]"]] image_sizes = self._image_sizes[task_data["server_files[0]"]]
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO,
self.ChunkType.VIDEO, image_sizes, StorageMethodChoice.CACHE, StorageLocation.SHARE) self.ChunkType.VIDEO, image_sizes, StorageMethodChoice.CACHE, StorageChoice.SHARE)
task_spec.update([('name', 'cached video task #15')]) task_spec.update([('name', 'cached video task #15')])
task_data.update([('copy_data', True)]) task_data.update([('copy_data', True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO,
image_sizes, StorageMethodChoice.CACHE, StorageLocation.LOCAL) image_sizes, StorageMethodChoice.CACHE, StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "cached images task without copying #16", "name": "cached images task without copying #16",
@ -1984,12 +1984,12 @@ class TaskDataAPITestCase(APITestCase):
] ]
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET,
self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE, StorageLocation.SHARE) self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE, StorageChoice.SHARE)
task_spec.update([('name', 'cached images task #17')]) task_spec.update([('name', 'cached images task #17')])
task_data.update([('copy_data', True)]) task_data.update([('copy_data', True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET,
image_sizes, StorageMethodChoice.CACHE, StorageLocation.LOCAL) image_sizes, StorageMethodChoice.CACHE, StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "my cached zip archive task without copying #18", "name": "my cached zip archive task without copying #18",
@ -2010,12 +2010,12 @@ class TaskDataAPITestCase(APITestCase):
image_sizes = self._image_sizes[task_data["server_files[0]"]] image_sizes = self._image_sizes[task_data["server_files[0]"]]
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET,
self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE, StorageLocation.SHARE) self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE, StorageChoice.SHARE)
task_spec.update([('name', 'my cached zip archive task #19')]) task_spec.update([('name', 'my cached zip archive task #19')])
task_data.update([('copy_data', True)]) task_data.update([('copy_data', True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET,
image_sizes, StorageMethodChoice.CACHE, StorageLocation.LOCAL) image_sizes, StorageMethodChoice.CACHE, StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "my cached pdf task #20", "name": "my cached pdf task #20",
@ -2083,12 +2083,12 @@ class TaskDataAPITestCase(APITestCase):
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO,
self.ChunkType.VIDEO, image_sizes, StorageMethodChoice.CACHE, self.ChunkType.VIDEO, image_sizes, StorageMethodChoice.CACHE,
StorageLocation.SHARE) StorageChoice.SHARE)
task_spec.update([('name', 'my video with meta info task #23')]) task_spec.update([('name', 'my video with meta info task #23')])
task_data.update([('copy_data', True)]) task_data.update([('copy_data', True)])
self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO,
image_sizes, StorageMethodChoice.CACHE, StorageLocation.LOCAL) image_sizes, StorageMethodChoice.CACHE, StorageChoice.LOCAL)
task_spec = { task_spec = {
"name": "my cached video task #14", "name": "my cached video task #14",

@ -37,7 +37,7 @@ from cvat.apps.authentication import auth
from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer
from cvat.apps.engine.frame_provider import FrameProvider from cvat.apps.engine.frame_provider import FrameProvider
from cvat.apps.engine.models import ( from cvat.apps.engine.models import (
Job, StatusChoice, Task, StorageMethodChoice, UploadedDataStorageLocationChoice Job, StatusChoice, Task, StorageMethodChoice, StorageChoice
) )
from cvat.apps.engine.serializers import ( from cvat.apps.engine.serializers import (
AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, AboutSerializer, AnnotationFileSerializer, BasicUserSerializer,
@ -402,8 +402,8 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet):
db_task.data.storage_method = StorageMethodChoice.CACHE db_task.data.storage_method = StorageMethodChoice.CACHE
db_task.data.save(update_fields=['storage_method']) db_task.data.save(update_fields=['storage_method'])
if data['server_files'] and data.get('copy_data') == False: if data['server_files'] and data.get('copy_data') == False:
db_task.data.uploaded_data_storage_location = UploadedDataStorageLocationChoice.SHARE db_task.data.storage = StorageChoice.SHARE
db_task.data.save(update_fields=['uploaded_data_storage_location']) db_task.data.save(update_fields=['storage'])
# if the value of stop_frame is 0, then inside the function we cannot know # if the value of stop_frame is 0, then inside the function we cannot know
# the value specified by the user or it's default value from the database # the value specified by the user or it's default value from the database
if 'stop_frame' not in serializer.validated_data: if 'stop_frame' not in serializer.validated_data:

Loading…
Cancel
Save