From fb8f5fffcb746a95ae4ce9848ad8988b318fc4b2 Mon Sep 17 00:00:00 2001 From: Maya Date: Thu, 5 Nov 2020 18:51:29 +0300 Subject: [PATCH] fix --- cvat/apps/engine/cache.py | 6 +++--- cvat/apps/engine/media_extractors.py | 17 +++++++++++------ cvat/apps/engine/prepare.py | 4 +++- cvat/apps/engine/task.py | 16 ++++++++-------- cvat/apps/engine/views.py | 1 + 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 5d7c01ee..4be31aec 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -39,7 +39,7 @@ class CacheInteraction: image_quality = 100 if writer_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality mime_type = 'video/mp4' if writer_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip' - extractor = writer_classes[quality](image_quality) + writer = writer_classes[quality](image_quality) images = [] buff = BytesIO() @@ -53,11 +53,11 @@ class CacheInteraction: meta = PrepareInfo(source_path=source_path, meta_path=db_data.get_meta_path()) for frame in meta.decode_needed_frames(chunk_number, db_data): images.append(frame) - extractor.save_as_chunk([(image, source_path, None) for image in images], buff) + writer.save_as_chunk([(image, source_path, None) for image in images], buff) else: with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file: images = [os.path.join(upload_dir, line.strip()) for line in dummy_file] - extractor.save_as_chunk([(image, image, None) for image in images], buff) + writer.save_as_chunk([(image, image, None) for image in images], buff) except FileNotFoundError as ex: slogger.glob.exception(f"{ex.strerror} {ex.filename}") diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 21430838..0a229b52 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -134,10 +134,12 @@ class DirectoryReader(ImageListReader): class ArchiveReader(DirectoryReader): def __init__(self, source_path, step=1, start=0, stop=None): self._archive_source = source_path[0] - Archive(self._archive_source).extractall(os.path.dirname(source_path[0])) - os.remove(self._archive_source) + extract_dir = source_path[1] if len(source_path) > 1 else os.path.dirname(source_path[0]) + Archive(self._archive_source).extractall(extract_dir) + if extract_dir == os.path.dirname(source_path[0]): + os.remove(self._archive_source) super().__init__( - source_path=[os.path.dirname(source_path[0])], + source_path=[extract_dir], step=step, start=start, stop=stop, @@ -177,6 +179,7 @@ class PdfReader(ImageListReader): class ZipReader(ImageListReader): def __init__(self, source_path, step=1, start=0, stop=None): self._zip_source = zipfile.ZipFile(source_path[0], mode='r') + self.extract_dir = source_path[1] if len(source_path) > 1 else None file_list = [f for f in self._zip_source.namelist() if get_mime(f) == 'image'] super().__init__(file_list, step, start, stop) @@ -196,13 +199,15 @@ class ZipReader(ImageListReader): def get_path(self, i): if self._zip_source.filename: - return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i]) + return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i]) \ + if not self.extract_dir else os.path.join(self.extract_dir, self._source_path[i]) else: # necessary for mime_type definition return self._source_path[i] def extract(self): - self._zip_source.extractall(os.path.dirname(self._zip_source.filename)) - os.remove(self._zip_source.filename) + self._zip_source.extractall(self.extract_dir if self.extract_dir else os.path.dirname(self._zip_source.filename)) + if not self.extract_dir: + os.remove(self._zip_source.filename) class VideoReader(IMediaReader): def __init__(self, source_path, step=1, start=0, stop=None): diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 9465b680..8c473148 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -162,8 +162,10 @@ class PrepareInfo(WorkWithVideo): class UploadedMeta(PrepareInfo): def __init__(self, **kwargs): super().__init__(**kwargs) + uploaded_meta = kwargs.get('uploaded_meta_path') + assert uploaded_meta is not None , 'No uploaded meta path' - with open(self.meta_path, 'r') as meta_file: + with open(uploaded_meta, 'r') as meta_file: lines = meta_file.read().strip().split('\n') self.frames = int(lines.pop()) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 8d5a0cb0..d6da9154 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -250,8 +250,11 @@ def _create_thread(tid, data): if media_files: if extractor is not None: raise Exception('Combined data types are not supported') + source_paths=[os.path.join(upload_dir, f) for f in media_files] + if media_type in ('archive', 'zip') and db_data.uploaded_data_storage_location == LocationChoice.SHARE: + source_paths.append(db_data.get_upload_dirname()) extractor = MEDIA_TYPES[media_type]['extractor']( - source_path=[os.path.join(upload_dir, f) for f in media_files], + source_path=source_paths, step=db_data.get_frame_step(), start=db_data.start_frame, stop=data['stop_frame'], @@ -305,14 +308,9 @@ def _create_thread(tid, data): if meta_info_file: try: from cvat.apps.engine.prepare import UploadedMeta - if os.path.split(meta_info_file[0])[0]: - if db_data.uploaded_data_storage_location is not LocationChoice.SHARE: - os.replace( - os.path.join(upload_dir, meta_info_file[0]), - db_data.get_meta_path() - ) meta_info = UploadedMeta(source_path=os.path.join(upload_dir, media_files[0]), - meta_path=db_data.get_meta_path()) + meta_path=db_data.get_meta_path(), + uploaded_meta=os.path.join(upload_dir, meta_info_file[0])) meta_info.check_seek_key_frames() meta_info.check_frames_numbers() meta_info.save_meta_info() @@ -325,6 +323,7 @@ def _create_thread(tid, data): meta_info, smooth_decoding = prepare_meta( media_file=media_files[0], upload_dir=upload_dir, + meta_dir=os.path.dirname(db_data.get_meta_path()), chunk_size=db_data.chunk_size ) assert smooth_decoding == True, 'Too few keyframes for smooth video decoding.' @@ -332,6 +331,7 @@ def _create_thread(tid, data): meta_info, smooth_decoding = prepare_meta( media_file=media_files[0], upload_dir=upload_dir, + meta_dir=os.path.dirname(db_data.get_meta_path()), chunk_size=db_data.chunk_size ) assert smooth_decoding == True, 'Too few keyframes for smooth video decoding.' diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 1edcea61..251d0d99 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -397,6 +397,7 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): data = {k:v for k, v in serializer.data.items()} data['use_zip_chunks'] = serializer.validated_data['use_zip_chunks'] data['use_cache'] = serializer.validated_data['use_cache'] + data['copy_data'] = serializer.validated_data['copy_data'] if data['use_cache']: db_task.data.storage_method = StorageMethodChoice.CACHE db_task.data.save(update_fields=['storage_method'])