From 8abadaff31fa921c3c0261cf40abc3d31ef73725 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 28 Jul 2020 14:44:21 +0300 Subject: [PATCH 01/20] Added cache integration settings --- cvat/settings/base.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 9155f2b3..3f161fbb 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -333,6 +333,9 @@ os.makedirs(DATA_ROOT, exist_ok=True) MEDIA_DATA_ROOT = os.path.join(DATA_ROOT, 'data') os.makedirs(MEDIA_DATA_ROOT, exist_ok=True) +CACHE_ROOT = os.path.join(DATA_ROOT, 'cache') +os.makedirs(CACHE_ROOT, exist_ok=True) + TASKS_ROOT = os.path.join(DATA_ROOT, 'tasks') os.makedirs(TASKS_ROOT, exist_ok=True) @@ -431,3 +434,17 @@ RESTRICTIONS = { 'engine.role.admin', ), } + +CACHES = { + 'default' : { + 'BACKEND' : 'diskcache.DjangoCache', + 'LOCATION' : CACHE_ROOT, + 'TIMEOUT' : None, + 'OPTIONS' : { + #'statistics' :True, + 'size_limit' : 2 ** 40, # 1 тб + } + } +} + +USE_CACHE = True From 08b195f05f9ec0df2cb834174f92a3a4bebe85cb Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 28 Jul 2020 14:47:24 +0300 Subject: [PATCH 02/20] Added preparation of meta information --- cvat/apps/engine/prepare.py | 76 +++++++++++++++++++++++++++++++++++++ cvat/apps/engine/task.py | 18 +++++++++ 2 files changed, 94 insertions(+) create mode 100644 cvat/apps/engine/prepare.py diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py new file mode 100644 index 00000000..18beaa80 --- /dev/null +++ b/cvat/apps/engine/prepare.py @@ -0,0 +1,76 @@ +import av + +class PrepareInfo: + + def __init__(self, source_path, meta_path): + self.source_path = source_path + self.meta_path = meta_path + + def _open_video_container(self, sourse_path, mode, options=None): + return av.open(sourse_path, mode=mode, options=options) + + def _close_video_container(self, container): + container.close() + + def _get_video_stream(self, container): + video_stream = next(stream for stream in container.streams if stream.type == 'video') + video_stream.thread_type = 'AUTO' + return video_stream + + #@get_execution_time + def save_meta_info(self): + container = self._open_video_container(self.source_path, mode='r') + video_stream = self._get_video_stream(container) + + with open(self.meta_path, 'w') as file: + frame_number = 0 + + for packet in container.demux(video_stream): + for frame in packet.decode(): + frame_number += 1 + if frame.key_frame: + file.write('{} {}\n'.format(frame_number, frame.pts)) + + self._close_video_container(container) + return frame_number# == task_size + + def get_nearest_left_key_frame(self, start_chunk_frame_number): + start_decode_frame_number = 0 + start_decode_timestamp = 0 + + with open(self.meta_path, 'r') as file: + for line in file: + frame_number, timestamp = line.strip().split(' ') + + #TODO: исправить если вдруг ключевой кадр окажется не первым + if int(frame_number) <= start_chunk_frame_number: + start_decode_frame_number = frame_number + start_decode_timestamp = timestamp + else: + break + + return int(start_decode_frame_number), int(start_decode_timestamp) + + def decode_needed_frames(self, chunk_number, chunk_size): + start_chunk_frame_number = (chunk_number - 1) * chunk_size + 1 + end_chunk_frame_number = start_chunk_frame_number + chunk_size #- 1 + start_decode_frame_number, start_decode_timestamp = self.get_nearest_left_key_frame(start_chunk_frame_number) + extra_frames = start_chunk_frame_number - start_decode_frame_number + + container = self._open_video_container(self.source_path, mode='r') + video_stream = self._get_video_stream(container) + container.seek(offset=start_decode_timestamp, stream=video_stream) + + frame_number = start_decode_frame_number - 1 + for packet in container.demux(video_stream): + for frame in packet.decode(): + frame_number += 1 + if frame_number < start_chunk_frame_number: + continue + elif frame_number < end_chunk_frame_number: + yield frame + else: + self._close_video_container(container) + return + + self._close_video_container(container) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 998bc4af..804ab1b4 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -24,6 +24,8 @@ from distutils.dir_util import copy_tree from . import models from .log import slogger +from .prepare import PrepareInfo +from diskcache import Cache ############################# Low Level server API @@ -230,6 +232,22 @@ def _create_thread(tid, data): job.meta['status'] = 'Media files are being extracted...' job.save_meta() + if settings.USE_CACHE: + for media_type, media_files in media.items(): + if media_files: + if task_mode == MEDIA_TYPES['video']['mode']: + meta_info = PrepareInfo(source_path=os.path.join(upload_dir, media_files[0]), + meta_path=os.path.join(upload_dir, 'meta_info.txt')) + meta_info.save_meta_info() + # else: + # with Cache(settings.CACHE_ROOT) as cache: + # counter_ = itertools.count(start=1) + + #TODO: chunk size + # for chunk_number, media_paths in itertools.groupby(media_files, lambda x: next(counter_) // db_data.chunk_size): + # cache.set('{}_{}'.format(tid, chunk_number), media_paths, tag='dummy') + #else: + db_images = [] extractor = None From 5fa50be1fad0b42c3d86ff6fd96ef21bd3810a72 Mon Sep 17 00:00:00 2001 From: Maya Date: Mon, 10 Aug 2020 09:54:56 +0300 Subject: [PATCH 03/20] Added most of video processing & cache implementation --- cvat/apps/engine/media_extractors.py | 62 ++++++++++++- cvat/apps/engine/models.py | 3 + cvat/apps/engine/prepare.py | 128 +++++++++++++++++++++++---- cvat/apps/engine/task.py | 128 +++++++++++++++++---------- cvat/apps/engine/views.py | 60 ++++++++++++- cvat/requirements/base.txt | 1 + 6 files changed, 315 insertions(+), 67 deletions(-) diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index dea14183..c2bc020f 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -285,6 +285,19 @@ class ZipChunkWriter(IChunkWriter): # and does not decode it to know img size. return [] + def save_as_chunk_to_buff(self, images, format_='jpeg'): + buff = io.BytesIO() + + with zipfile.ZipFile(buff, 'w') as zip_file: + for idx, image in enumerate(images): + arcname = '{:06d}.{}'.format(idx, format_) + if isinstance(image, av.VideoFrame): + zip_file.writestr(arcname, image.to_image().tobytes().getvalue()) + else: + zip_file.write(filename=image, arcname=arcname) + buff.seek(0) + return buff + class ZipCompressedChunkWriter(IChunkWriter): def save_as_chunk(self, images, chunk_path): image_sizes = [] @@ -297,20 +310,30 @@ class ZipCompressedChunkWriter(IChunkWriter): return image_sizes + def save_as_chunk_to_buff(self, images, format_='jpeg'): + buff = io.BytesIO() + with zipfile.ZipFile(buff, 'w') as zip_file: + for idx, image in enumerate(images): + (_, _, image_buf) = self._compress_image(image, self._image_quality) + arcname = '{:06d}.{}'.format(idx, format_) + zip_file.writestr(arcname, image_buf.getvalue()) + buff.seek(0) + return buff + class Mpeg4ChunkWriter(IChunkWriter): def __init__(self, _): super().__init__(17) self._output_fps = 25 @staticmethod - def _create_av_container(path, w, h, rate, options): + def _create_av_container(path, w, h, rate, options, f=None): # x264 requires width and height must be divisible by 2 for yuv420p if h % 2: h += 1 if w % 2: w += 1 - container = av.open(path, 'w') + container = av.open(path, 'w',format=f) video_stream = container.add_stream('libx264', rate=rate) video_stream.pix_fmt = "yuv420p" video_stream.width = w @@ -341,6 +364,41 @@ class Mpeg4ChunkWriter(IChunkWriter): output_container.close() return [(input_w, input_h)] + def save_as_chunk_to_buff(self, frames, format_): + if not frames: + raise Exception('no images to save') + + buff = io.BytesIO() + input_w = frames[0].width + input_h = frames[0].height + + output_container, output_v_stream = self._create_av_container( + path=buff, + w=input_w, + h=input_h, + rate=self._output_fps, + options={ + "crf": str(self._image_quality), + "preset": "ultrafast", + }, + f=format_, + ) + + for frame in frames: + # let libav set the correct pts and time_base + frame.pts = None + frame.time_base = None + + for packet in output_v_stream.encode(frame): + output_container.mux(packet) + + # Flush streams + for packet in output_v_stream.encode(): + output_container.mux(packet) + output_container.close() + buff.seek(0) + return buff + @staticmethod def _encode_images(images, container, stream): for frame, _, _ in images: diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index d4c46eb3..d2f8e26e 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -102,6 +102,9 @@ class Data(models.Model): def get_preview_path(self): return os.path.join(self.get_data_dirname(), 'preview.jpeg') + def get_meta_path(self): + return os.path.join(self.get_upload_dirname(), 'meta_info.txt') + class Video(models.Model): data = models.OneToOneField(Data, on_delete=models.CASCADE, related_name="video", null=True) path = models.CharField(max_length=1024, default='') diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 18beaa80..01d1c367 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -1,10 +1,11 @@ import av +import hashlib -class PrepareInfo: - - def __init__(self, source_path, meta_path): - self.source_path = source_path - self.meta_path = meta_path +class WorkWithVideo: + def __init__(self, **kwargs): + if not kwargs.get('source_path'): + raise Exeption('No sourse path') + self.source_path = kwargs.get('source_path') def _open_video_container(self, sourse_path, mode, options=None): return av.open(sourse_path, mode=mode, options=options) @@ -17,22 +18,114 @@ class PrepareInfo: video_stream.thread_type = 'AUTO' return video_stream - #@get_execution_time - def save_meta_info(self): + +class AnalyzeVideo(WorkWithVideo): + def check_type_first_frame(self): + container = self._open_video_container(self.source_path, mode='r') + video_stream = self._get_video_stream(container) + + for packet in container.demux(video_stream): + for frame in packet.decode(): + self._close_video_container(container) + assert frame.pict_type.name == 'I', 'First frame is not key frame' + return + + def check_video_timestamps_sequences(self): container = self._open_video_container(self.source_path, mode='r') video_stream = self._get_video_stream(container) - with open(self.meta_path, 'w') as file: - frame_number = 0 + frame_pts = -1 + frame_dts = -1 + for packet in container.demux(video_stream): + for frame in packet.decode(): + + if None not in [frame.pts, frame_pts] and frame.pts <= frame_pts: + self._close_video_container(container) + raise Exception('Invalid pts sequences') + + if None not in [frame.dts, frame_dts] and frame.dts <= frame_dts: + self._close_video_container(container) + raise Exception('Invalid dts sequences') + + frame_pts, frame_dts = frame.pts, frame.dts + self._close_video_container(container) + +# class Frame: +# def __init__(self, frame, frame_number=None): +# self.frame = frame +# if frame_number: +# self.frame_number = frame_number + +# def md5_hash(self): +# return hashlib.md5(self.frame.to_image().tobytes()).hexdigest() + +# def __eq__(self, image): +# return self.md5_hash(self) == image.md5_hash(image) and self.frame.pts == image.frame.pts + +# def __ne__(self, image): +# return md5_hash(self) != md5_hash(image) or self.frame.pts != image.frame.pts + +# def __len__(self): +# return (self.frame.width, self.frame.height) + - for packet in container.demux(video_stream): +def md5_hash(frame): + return hashlib.md5(frame.to_image().tobytes()).hexdigest() + +class PrepareInfo(WorkWithVideo): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + if not kwargs.get('meta_path'): + raise Exception('No meta path') + + self.meta_path = kwargs.get('meta_path') + self.key_frames = {} + self.frames = 0 + + def get_task_size(self): + return self.frames + + def check_seek_key_frames(self): + container = self._open_video_container(self.source_path, mode='r') + video_stream = self._get_video_stream(container) + + key_frames_copy = self.key_frames.copy() + + for _, key_frame in key_frames_copy.items(): + container.seek(offset=key_frame.pts, stream=video_stream) + flag = True + for packet in container.demux(video_stream): for frame in packet.decode(): - frame_number += 1 - if frame.key_frame: - file.write('{} {}\n'.format(frame_number, frame.pts)) + if md5_hash(frame) != md5_hash(key_frame) or frame.pts != key_frame.pts: + self.key_frames.pop(index) + flag = False + break + if not flag: + break + + if len(self.key_frames) == 0: #or self.frames // len(self.key_frames) > 300: + raise Exception('Too few keyframes') + + def save_key_frames(self): + container = self._open_video_container(self.source_path, mode='r') + video_stream = self._get_video_stream(container) + frame_number = 0 + for packet in container.demux(video_stream): + for frame in packet.decode(): + if frame.key_frame: + self.key_frames[frame_number] = frame + frame_number += 1 + + self.frames = frame_number self._close_video_container(container) - return frame_number# == task_size + + def save_meta_info(self): + with open(self.meta_path, 'w') as meta_file: + for index, frame in self.key_frames.items(): + meta_file.write('{} {}\n'.format(index, frame.pts)) def get_nearest_left_key_frame(self, start_chunk_frame_number): start_decode_frame_number = 0 @@ -52,10 +145,9 @@ class PrepareInfo: return int(start_decode_frame_number), int(start_decode_timestamp) def decode_needed_frames(self, chunk_number, chunk_size): - start_chunk_frame_number = (chunk_number - 1) * chunk_size + 1 - end_chunk_frame_number = start_chunk_frame_number + chunk_size #- 1 + start_chunk_frame_number = chunk_number * chunk_size + end_chunk_frame_number = start_chunk_frame_number + chunk_size start_decode_frame_number, start_decode_timestamp = self.get_nearest_left_key_frame(start_chunk_frame_number) - extra_frames = start_chunk_frame_number - start_decode_frame_number container = self._open_video_container(self.source_path, mode='r') video_stream = self._get_video_stream(container) @@ -73,4 +165,4 @@ class PrepareInfo: self._close_video_container(container) return - self._close_video_container(container) + self._close_video_container(container) \ No newline at end of file diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 804ab1b4..b468d4ef 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -24,7 +24,7 @@ from distutils.dir_util import copy_tree from . import models from .log import slogger -from .prepare import PrepareInfo +from .prepare import PrepareInfo, AnalyzeVideo from diskcache import Cache ############################# Low Level server API @@ -232,22 +232,6 @@ def _create_thread(tid, data): job.meta['status'] = 'Media files are being extracted...' job.save_meta() - if settings.USE_CACHE: - for media_type, media_files in media.items(): - if media_files: - if task_mode == MEDIA_TYPES['video']['mode']: - meta_info = PrepareInfo(source_path=os.path.join(upload_dir, media_files[0]), - meta_path=os.path.join(upload_dir, 'meta_info.txt')) - meta_info.save_meta_info() - # else: - # with Cache(settings.CACHE_ROOT) as cache: - # counter_ = itertools.count(start=1) - - #TODO: chunk size - # for chunk_number, media_paths in itertools.groupby(media_files, lambda x: next(counter_) // db_data.chunk_size): - # cache.set('{}_{}'.format(tid, chunk_number), media_paths, tag='dummy') - #else: - db_images = [] extractor = None @@ -294,37 +278,91 @@ def _create_thread(tid, data): else: db_data.chunk_size = 36 + #it's better to add the field to the Task model + video_suitable_on_the_fly_processing = True + video_path = "" video_size = (0, 0) - counter = itertools.count() - generator = itertools.groupby(extractor, lambda x: next(counter) // db_data.chunk_size) - for chunk_idx, chunk_data in generator: - chunk_data = list(chunk_data) - original_chunk_path = db_data.get_original_chunk_path(chunk_idx) - original_chunk_writer.save_as_chunk(chunk_data, original_chunk_path) - - compressed_chunk_path = db_data.get_compressed_chunk_path(chunk_idx) - img_sizes = compressed_chunk_writer.save_as_chunk(chunk_data, compressed_chunk_path) - - if db_task.mode == 'annotation': - db_images.extend([ - models.Image( - data=db_data, - path=os.path.relpath(data[1], upload_dir), - frame=data[2], - width=size[0], - height=size[1]) - - for data, size in zip(chunk_data, img_sizes) - ]) - else: - video_size = img_sizes[0] - video_path = chunk_data[0][1] + if settings.USE_CACHE: + for media_type, media_files in media.items(): + if media_files: + if task_mode == MEDIA_TYPES['video']['mode']: + try: + analizer = AnalyzeVideo(source_path=os.path.join(upload_dir, media_files[0])) + analizer.check_type_first_frame() + analizer.check_video_timestamps_sequences() + + meta_info = PrepareInfo(source_path=os.path.join(upload_dir, media_files[0]), + meta_path=os.path.join(upload_dir, 'meta_info.txt')) + meta_info.save_key_frames() + #meta_info.test_seek() + meta_info.check_seek_key_frames() + meta_info.save_meta_info() + + db_data.size = meta_info.get_task_size() + video_path = os.path.join(upload_dir, media_files[0]) + frame = meta_info.key_frames.get(next(iter(meta_info.key_frames))) + video_size = (frame.width, frame.height) + + except AssertionError as ex: + video_suitable_on_the_fly_processing = False + except Exception as ex: + video_suitable_on_the_fly_processing = False + + else:#images, TODO:archive + with Cache(settings.CACHE_ROOT) as cache: + counter_ = itertools.count() + + for chunk_number, media_paths in itertools.groupby(media_files, lambda x: next(counter_) // db_data.chunk_size): + media_paths = list(media_paths) + cache.set('{}_{}'.format(tid, chunk_number), [os.path.join(upload_dir, file_name) for file_name in media_paths], tag='dummy') + + img_sizes = [] + from PIL import Image + for media_path in media_paths: + img_sizes += [Image.open(os.path.join(upload_dir, media_path)).size] + db_data.size += len(media_paths) + db_images.extend([ + models.Image( + data=db_data, + path=data[1], + frame=data[0], + width=size[0], + height=size[1]) + for data, size in zip(enumerate(media_paths), img_sizes) + ]) + + + if db_task.mode == 'interpolation' and not video_suitable_on_the_fly_processing or not settings.USE_CACHE: + counter = itertools.count() + generator = itertools.groupby(extractor, lambda x: next(counter) // db_data.chunk_size) + for chunk_idx, chunk_data in generator: + chunk_data = list(chunk_data) + original_chunk_path = db_data.get_original_chunk_path(chunk_idx) + original_chunk_writer.save_as_chunk(chunk_data, original_chunk_path) + + compressed_chunk_path = db_data.get_compressed_chunk_path(chunk_idx) + img_sizes = compressed_chunk_writer.save_as_chunk(chunk_data, compressed_chunk_path) + + if db_task.mode == 'annotation': + db_images.extend([ + models.Image( + data=db_data, + path=os.path.relpath(data[1], upload_dir), + frame=data[2], + width=size[0], + height=size[1]) + + for data, size in zip(chunk_data, img_sizes) + ]) + else: + video_size = img_sizes[0] + video_path = chunk_data[0][1] - db_data.size += len(chunk_data) - progress = extractor.get_progress(chunk_data[-1][2]) - update_progress(progress) + db_data.size += len(chunk_data) + progress = extractor.get_progress(chunk_data[-1][2]) + update_progress(progress) if db_task.mode == 'annotation': models.Image.objects.bulk_create(db_images) @@ -342,4 +380,4 @@ def _create_thread(tid, data): preview.save(db_data.get_preview_path()) slogger.glob.info("Founded frames {} for Data #{}".format(db_data.size, db_data.id)) - _save_task_to_db(db_task) + _save_task_to_db(db_task) \ No newline at end of file diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 120125ea..621f85e8 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -29,7 +29,7 @@ from rest_framework.exceptions import APIException from rest_framework.permissions import SAFE_METHODS, IsAuthenticated from rest_framework.renderers import JSONRenderer from rest_framework.response import Response -from sendfile import sendfile +from sendfile import sendfile#убрать import cvat.apps.dataset_manager as dm import cvat.apps.dataset_manager.views # pylint: disable=unused-import @@ -37,7 +37,7 @@ from cvat.apps.authentication import auth from cvat.apps.authentication.decorators import login_required from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import Job, Plugin, StatusChoice, Task +from cvat.apps.engine.models import Job, Plugin, StatusChoice, Task, DataChoice from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaSerializer, DataSerializer, ExceptionSerializer, @@ -49,6 +49,12 @@ from cvat.apps.engine.utils import av_scan_paths from . import models, task from .log import clogger, slogger +from .media_extractors import ( + Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, + ZipCompressedChunkWriter, ZipChunkWriter) +from .prepare import PrepareInfo +from diskcache import Cache +#from cvat.apps.engine.mime_types import mimetypes # drf-yasg component doesn't handle correctly URL_FORMAT_OVERRIDE and @@ -437,13 +443,63 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): try: db_task = self.get_object() + db_data = db_task.data frame_provider = FrameProvider(db_task.data) if data_type == 'chunk': data_id = int(data_id) + quality = data_quality + data_quality = FrameProvider.Quality.COMPRESSED \ if data_quality == 'compressed' else FrameProvider.Quality.ORIGINAL + path = os.path.realpath(frame_provider.get_chunk(data_id, data_quality)) + #TODO: av.FFmpegError processing + if settings.USE_CACHE: + with Cache(settings.CACHE_ROOT) as cache: + buff = None + chunk, tag = cache.get('{}_{}_{}'.format(db_task.id, data_id, quality), tag=True) + + if not chunk: + extractor_classes = { + 'compressed' : Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter, + 'original' : Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter, + } + + image_quality = 100 if extractor_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality + file_extension = 'mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'jpeg' + mime_type = 'video/mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip' + + extractor = extractor_classes[quality](image_quality) + + if 'interpolation' == db_task.mode: + + meta = PrepareInfo(source_path=os.path.join(db_data.get_upload_dirname(), db_data.video.path), + meta_path=db_data.get_meta_path()) + + frames = [] + for frame in meta.decode_needed_frames(data_id,db_data.chunk_size): + frames.append(frame) + + + buff = extractor.save_as_chunk_to_buff(frames, + format_=file_extension) + cache.set('{}_{}_{}'.format(db_task.id, data_id, quality), buff, tag=mime_type) + + else: + img_paths = cache.get('{}_{}'.format(db_task.id, data_id)) + buff = extractor.save_as_chunk_to_buff(img_paths, + format_=file_extension) + cache.set('{}_{}_{}'.format(db_task.id, data_id, quality), buff, tag=mime_type) + + + + elif 'process_creating' == tag: + pass + else: + buff, mime_type = cache.get('{}_{}_{}'.format(db_task.id, data_id, quality), tag=True) + + return HttpResponse(buff.getvalue(), content_type=mime_type) # Follow symbol links if the chunk is a link on a real image otherwise # mimetype detection inside sendfile will work incorrectly. diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index b2847cbd..cc650f3f 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -49,3 +49,4 @@ av==6.2.0 # The package is used by pyunpack as a command line tool to support multiple # archives. Don't use as a python module because it has GPL license. patool==1.12 +diskcache==4.1.0 \ No newline at end of file From e947a8252c37fba957500c9014b80592d5588b9d Mon Sep 17 00:00:00 2001 From: Maya Date: Sat, 15 Aug 2020 11:01:56 +0300 Subject: [PATCH 04/20] Added ability to select using cache or no in task creating form & some fixes --- cvat-core/src/session.js | 20 ++++++ cvat-ui/src/actions/tasks-actions.ts | 1 + .../advanced-configuration-form.tsx | 25 ++++++++ .../create-task-page/create-task-content.tsx | 1 + cvat/apps/engine/media_extractors.py | 61 +++++++++++++++++-- .../migrations/0028_data_storage_method.py | 19 ++++++ cvat/apps/engine/models.py | 12 ++++ cvat/apps/engine/prepare.py | 2 +- cvat/apps/engine/serializers.py | 5 +- cvat/apps/engine/task.py | 32 +++++----- cvat/apps/engine/views.py | 9 ++- 11 files changed, 162 insertions(+), 25 deletions(-) create mode 100644 cvat/apps/engine/migrations/0028_data_storage_method.py diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index 5d5bd7b2..ccaff82c 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -814,6 +814,7 @@ data_compressed_chunk_type: undefined, data_original_chunk_type: undefined, use_zip_chunks: undefined, + use_cache: undefined, }; for (const property in data) { @@ -1069,6 +1070,24 @@ data.use_zip_chunks = useZipChunks; }, }, + /** + * @name useCache + * @type {boolean} + * @memberof module:API.cvat.classes.Task + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + useCache: { + get: () => data.use_cache, + set: (useCache) => { + if (typeof (useCache) !== 'boolean') { + throw new ArgumentError( + 'Value must be a boolean', + ); + } + data.use_cache = useCache; + }, + }, /** * After task has been created value can be appended only. * @name labels @@ -1639,6 +1658,7 @@ remote_files: this.remoteFiles, image_quality: this.imageQuality, use_zip_chunks: this.useZipChunks, + use_cache: this.useCache, }; if (typeof (this.startFrame) !== 'undefined') { diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index fd1bfbc4..9fc4c1e8 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -390,6 +390,7 @@ ThunkAction, {}, {}, AnyAction> { z_order: data.advanced.zOrder, image_quality: 70, use_zip_chunks: data.advanced.useZipChunks, + use_cache: data.advanced.useCache, }; if (data.advanced.bugTracker) { diff --git a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx index b038408b..89a760e0 100644 --- a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx +++ b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx @@ -26,6 +26,7 @@ export interface AdvancedConfiguration { repository?: string; useZipChunks: boolean; dataChunkSize?: number; + useCache: boolean; } type Props = FormComponentProps & { @@ -380,6 +381,24 @@ class AdvancedConfigurationForm extends React.PureComponent { ); } + private renderCreateTaskMethod(): JSX.Element { + const { form } = this.props; + return ( + + {form.getFieldDecorator('useCache', { + initialValue: true, + valuePropName: 'checked', + })( + + + Use cache + + , + )} + + ); + } + private renderChunkSize(): JSX.Element { const { form } = this.props; @@ -433,6 +452,12 @@ class AdvancedConfigurationForm extends React.PureComponent { + + + {this.renderCreateTaskMethod()} + + + {this.renderImageQuality()} diff --git a/cvat-ui/src/components/create-task-page/create-task-content.tsx b/cvat-ui/src/components/create-task-page/create-task-content.tsx index 2bd32807..229015ef 100644 --- a/cvat-ui/src/components/create-task-page/create-task-content.tsx +++ b/cvat-ui/src/components/create-task-page/create-task-content.tsx @@ -39,6 +39,7 @@ const defaultState = { zOrder: false, lfs: false, useZipChunks: true, + useCache: true, }, labels: [], files: { diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index c2bc020f..0099b349 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -125,22 +125,21 @@ class DirectoryReader(ImageListReader): class ArchiveReader(DirectoryReader): def __init__(self, source_path, step=1, start=0, stop=None): - self._tmp_dir = create_tmp_dir() self._archive_source = source_path[0] - Archive(self._archive_source).extractall(self._tmp_dir) + Archive(self._archive_source).extractall(os.path.dirname(source_path[0])) super().__init__( - source_path=[self._tmp_dir], + source_path=[os.path.dirname(source_path[0])], step=step, start=start, stop=stop, ) def __del__(self): - delete_tmp_dir(self._tmp_dir) + os.remove(self._archive_source) def get_path(self, i): base_dir = os.path.dirname(self._archive_source) - return os.path.join(base_dir, os.path.relpath(self._source_path[i], self._tmp_dir)) + return os.path.join(base_dir, os.path.relpath(self._source_path[i], base_dir)) class PdfReader(DirectoryReader): def __init__(self, source_path, step=1, start=0, stop=None): @@ -193,6 +192,10 @@ class ZipReader(ImageListReader): def get_path(self, i): return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i]) + def extract(self): + self._zip_source.extractall(os.path.dirname(self._zip_source.filename)) + os.remove(self._zip_source.filename) + class VideoReader(IMediaReader): def __init__(self, source_path, step=1, start=0, stop=None): super().__init__( @@ -312,7 +315,7 @@ class ZipCompressedChunkWriter(IChunkWriter): def save_as_chunk_to_buff(self, images, format_='jpeg'): buff = io.BytesIO() - with zipfile.ZipFile(buff, 'w') as zip_file: + with zipfile.ZipFile(buff, 'x') as zip_file: for idx, image in enumerate(images): (_, _, image_buf) = self._compress_image(image, self._image_quality) arcname = '{:06d}.{}'.format(idx, format_) @@ -452,6 +455,52 @@ class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter): output_container.close() return [(input_w, input_h)] + def save_as_chunk_to_buff(self, frames, format_): + if not frames: + raise Exception('no images to save') + + buff = io.BytesIO() + input_w = frames[0].width + input_h = frames[0].height + + downscale_factor = 1 + while input_h / downscale_factor >= 1080: + downscale_factor *= 2 + + output_h = input_h // downscale_factor + output_w = input_w // downscale_factor + + + output_container, output_v_stream = self._create_av_container( + path=buff, + w=output_w, + h=output_h, + rate=self._output_fps, + options={ + 'profile': 'baseline', + 'coder': '0', + 'crf': str(self._image_quality), + 'wpredp': '0', + 'flags': '-loop' + }, + f=format_, + ) + + for frame in frames: + # let libav set the correct pts and time_base + frame.pts = None + frame.time_base = None + + for packet in output_v_stream.encode(frame): + output_container.mux(packet) + + # Flush streams + for packet in output_v_stream.encode(): + output_container.mux(packet) + output_container.close() + buff.seek(0) + return buff + def _is_archive(path): mime = mimetypes.guess_type(path) mime_type = mime[0] diff --git a/cvat/apps/engine/migrations/0028_data_storage_method.py b/cvat/apps/engine/migrations/0028_data_storage_method.py new file mode 100644 index 00000000..6490f5bf --- /dev/null +++ b/cvat/apps/engine/migrations/0028_data_storage_method.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.13 on 2020-08-13 05:49 + +import cvat.apps.engine.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0027_auto_20200719_1552'), + ] + + operations = [ + migrations.AddField( + model_name='data', + name='storage_method', + field=models.CharField(choices=[('cache', 'CACHE'), ('file_system', 'FILE_SYSTEM')], default=cvat.apps.engine.models.StorageMethodChoice('file_system'), max_length=15), + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index d2f8e26e..1abec5a8 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -43,6 +43,17 @@ class DataChoice(str, Enum): def __str__(self): return self.value +class StorageMethodChoice(str, Enum): + CACHE = 'cache' + FILE_SYSTEM = 'file_system' + + @classmethod + def choices(cls): + return tuple((x.value, x.name) for x in cls) + + def __str__(self): + return self.value + class Data(models.Model): chunk_size = models.PositiveIntegerField(null=True) size = models.PositiveIntegerField(default=0) @@ -54,6 +65,7 @@ class Data(models.Model): default=DataChoice.IMAGESET) original_chunk_type = models.CharField(max_length=32, choices=DataChoice.choices(), default=DataChoice.IMAGESET) + storage_method = models.CharField(max_length=15, choices=StorageMethodChoice.choices(), default=StorageMethodChoice.FILE_SYSTEM) class Meta: default_permissions = () diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 01d1c367..69d74b5a 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -4,7 +4,7 @@ import hashlib class WorkWithVideo: def __init__(self, **kwargs): if not kwargs.get('source_path'): - raise Exeption('No sourse path') + raise Exception('No sourse path') self.source_path = kwargs.get('source_path') def _open_video_container(self, sourse_path, mode, options=None): diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 90e24714..e789ae94 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -167,11 +167,13 @@ class DataSerializer(serializers.ModelSerializer): client_files = ClientFileSerializer(many=True, default=[]) server_files = ServerFileSerializer(many=True, default=[]) remote_files = RemoteFileSerializer(many=True, default=[]) + use_cache = serializers.BooleanField(default=False) class Meta: model = models.Data fields = ('chunk_size', 'size', 'image_quality', 'start_frame', 'stop_frame', 'frame_filter', - 'compressed_chunk_type', 'original_chunk_type', 'client_files', 'server_files', 'remote_files', 'use_zip_chunks') + 'compressed_chunk_type', 'original_chunk_type', 'client_files', 'server_files', 'remote_files', 'use_zip_chunks', + 'use_cache') # pylint: disable=no-self-use def validate_frame_filter(self, value): @@ -199,6 +201,7 @@ class DataSerializer(serializers.ModelSerializer): server_files = validated_data.pop('server_files') remote_files = validated_data.pop('remote_files') validated_data.pop('use_zip_chunks') + validated_data.pop('use_cache') db_data = models.Data.objects.create(**validated_data) data_path = db_data.get_data_dirname() diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index b468d4ef..fc2f3759 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -14,7 +14,7 @@ from urllib import parse as urlparse from urllib import request as urlrequest from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter -from cvat.apps.engine.models import DataChoice +from cvat.apps.engine.models import DataChoice, StorageMethodChoice from cvat.apps.engine.utils import av_scan_paths import django_rq @@ -245,6 +245,8 @@ def _create_thread(tid, data): start=db_data.start_frame, stop=data['stop_frame'], ) + if extractor.__class__ == MEDIA_TYPES['zip']['extractor']: + extractor.extract() db_task.mode = task_mode db_data.compressed_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' and not data['use_zip_chunks'] else models.DataChoice.IMAGESET db_data.original_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' else models.DataChoice.IMAGESET @@ -278,25 +280,22 @@ def _create_thread(tid, data): else: db_data.chunk_size = 36 - #it's better to add the field to the Task model - video_suitable_on_the_fly_processing = True video_path = "" video_size = (0, 0) - if settings.USE_CACHE: + if settings.USE_CACHE and db_data.storage_method == StorageMethodChoice.CACHE: for media_type, media_files in media.items(): if media_files: if task_mode == MEDIA_TYPES['video']['mode']: try: - analizer = AnalyzeVideo(source_path=os.path.join(upload_dir, media_files[0])) - analizer.check_type_first_frame() - analizer.check_video_timestamps_sequences() + analyzer = AnalyzeVideo(source_path=os.path.join(upload_dir, media_files[0])) + analyzer.check_type_first_frame() + analyzer.check_video_timestamps_sequences() meta_info = PrepareInfo(source_path=os.path.join(upload_dir, media_files[0]), meta_path=os.path.join(upload_dir, 'meta_info.txt')) meta_info.save_key_frames() - #meta_info.test_seek() meta_info.check_seek_key_frames() meta_info.save_meta_info() @@ -306,14 +305,17 @@ def _create_thread(tid, data): video_size = (frame.width, frame.height) except AssertionError as ex: - video_suitable_on_the_fly_processing = False + db_data.storage_method = StorageMethodChoice.FILE_SYSTEM except Exception as ex: - video_suitable_on_the_fly_processing = False + db_data.storage_method = StorageMethodChoice.FILE_SYSTEM - else:#images, TODO:archive + else:#images,archive with Cache(settings.CACHE_ROOT) as cache: counter_ = itertools.count() + if extractor.__class__ in [MEDIA_TYPES['archive']['extractor'], MEDIA_TYPES['zip']['extractor']]: + media_files = [os.path.join(upload_dir, f) for f in extractor._source_path] + for chunk_number, media_paths in itertools.groupby(media_files, lambda x: next(counter_) // db_data.chunk_size): media_paths = list(media_paths) cache.set('{}_{}'.format(tid, chunk_number), [os.path.join(upload_dir, file_name) for file_name in media_paths], tag='dummy') @@ -321,20 +323,20 @@ def _create_thread(tid, data): img_sizes = [] from PIL import Image for media_path in media_paths: - img_sizes += [Image.open(os.path.join(upload_dir, media_path)).size] + img_sizes += [Image.open(media_path).size] db_data.size += len(media_paths) db_images.extend([ models.Image( data=db_data, - path=data[1], + path=os.path.basename(data[1]), frame=data[0], width=size[0], height=size[1]) - for data, size in zip(enumerate(media_paths), img_sizes) + for data, size in zip(enumerate(media_paths, start=len(db_images)), img_sizes) ]) - if db_task.mode == 'interpolation' and not video_suitable_on_the_fly_processing or not settings.USE_CACHE: + if db_data.storage_method == StorageMethodChoice.FILE_SYSTEM or not settings.USE_CACHE: counter = itertools.count() generator = itertools.groupby(extractor, lambda x: next(counter) // db_data.chunk_size) for chunk_idx, chunk_data in generator: diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 621f85e8..f06f6d4b 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -37,7 +37,7 @@ from cvat.apps.authentication import auth from cvat.apps.authentication.decorators import login_required from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import Job, Plugin, StatusChoice, Task, DataChoice +from cvat.apps.engine.models import Job, Plugin, StatusChoice, Task, DataChoice, StorageMethodChoice from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaSerializer, DataSerializer, ExceptionSerializer, @@ -419,6 +419,11 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): db_task.save() data = {k:v for k, v in serializer.data.items()} data['use_zip_chunks'] = serializer.validated_data['use_zip_chunks'] + data['use_cache'] = serializer.validated_data['use_cache'] + if data['use_cache']: + db_task.data.storage_method = StorageMethodChoice.CACHE + db_task.data.save(update_fields=['storage_method']) + # if the value of stop_frame is 0, then inside the function we cannot know # the value specified by the user or it's default value from the database if 'stop_frame' not in serializer.validated_data: @@ -455,7 +460,7 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): path = os.path.realpath(frame_provider.get_chunk(data_id, data_quality)) #TODO: av.FFmpegError processing - if settings.USE_CACHE: + if settings.USE_CACHE and db_data.storage_method == StorageMethodChoice.CACHE: with Cache(settings.CACHE_ROOT) as cache: buff = None chunk, tag = cache.get('{}_{}_{}'.format(db_task.id, data_id, quality), tag=True) From 94bb3659c91630f3066d677ebafe953af32be6f6 Mon Sep 17 00:00:00 2001 From: Maya Date: Sat, 15 Aug 2020 11:10:25 +0300 Subject: [PATCH 05/20] Added data migration --- .../migrations/0029_auto_20200814_2153.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 cvat/apps/engine/migrations/0029_auto_20200814_2153.py diff --git a/cvat/apps/engine/migrations/0029_auto_20200814_2153.py b/cvat/apps/engine/migrations/0029_auto_20200814_2153.py new file mode 100644 index 00000000..f81c73c8 --- /dev/null +++ b/cvat/apps/engine/migrations/0029_auto_20200814_2153.py @@ -0,0 +1,53 @@ +from django.db import migrations +from django.conf import settings +import os +from cvat.apps.engine.mime_types import mimetypes +from pyunpack import Archive + +def unzip(apps, schema_editor): + Data = apps.get_model("engine", "Data") + data_q_set = Data.objects.all() + archive_paths = [] + archive_mimes = [ + 'application/gzip', + 'application/rar' + 'application/x-7z-compressed', + 'application/x-bzip', + 'application/x-bzip-compressed-tar', + 'application/x-compress', + 'application/x-compressed-tar', + 'application/x-cpio', + 'application/x-gtar-compressed', + 'application/x-lha', + 'application/x-lhz', + 'application/x-lrzip-compressed-tar', + 'application/x-lz4', + 'application/x-lzip', + 'application/x-lzip-compressed-tar', + 'application/x-lzma', + 'application/x-lzma-compressed-tar', + 'application/x-lzop', + 'application/x-tar', + 'application/x-tarz', + 'application/x-tzo', + 'application/x-xz-compressed-tar', + 'application/zip', + ] + + for data_instance in data_q_set: + for root, _, files in os.walk(os.path.join(settings.MEDIA_DATA_ROOT, '{}/raw/'.format(data_instance.id))): + archive_paths.extend([os.path.join(root, file) for file in files if mimetypes.guess_type(file)[0] in archive_mimes]) + + for path in archive_paths: + Archive(path).extractall(os.path.dirname(path)) + os.remove(path) + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0028_data_storage_method'), + ] + + operations = [ + migrations.RunPython(unzip) + ] \ No newline at end of file From f3630aa4b513c7a8bfcf48d7d13e66768d7ab7c0 Mon Sep 17 00:00:00 2001 From: Maya Date: Wed, 19 Aug 2020 08:09:34 +0300 Subject: [PATCH 06/20] Some additions: * moved cache implementation * fixed start, stop, step for tasks that created on the fly * changed frame provider --- cvat/apps/engine/cache.py | 61 ++++++++++++++++++++++++++++ cvat/apps/engine/frame_provider.py | 49 ++++++++++++++++++---- cvat/apps/engine/media_extractors.py | 5 ++- cvat/apps/engine/models.py | 3 ++ cvat/apps/engine/prepare.py | 12 +++--- cvat/apps/engine/task.py | 56 +++++++++++++------------ cvat/apps/engine/views.py | 57 ++------------------------ 7 files changed, 149 insertions(+), 94 deletions(-) create mode 100644 cvat/apps/engine/cache.py diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py new file mode 100644 index 00000000..c79bf4f2 --- /dev/null +++ b/cvat/apps/engine/cache.py @@ -0,0 +1,61 @@ +from diskcache import Cache +from django.conf import settings +from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, ZipChunkWriter, + Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter) +from cvat.apps.engine.models import DataChoice +from .prepare import PrepareInfo +import os + +class CacheInteraction: + def __init__(self): + self._cache = Cache(settings.CACHE_ROOT) + + def __del__(self): + self._cache.close() + + def get_buff_mime(self, chunk_number, quality, db_data): + chunk, tag = self._cache.get('{}_{}_{}'.format(db_data.id, chunk_number, quality), tag=True) + + if not chunk: + chunk, tag = self.prepare_chunk_buff(db_data, quality, chunk_number) + self.save_chunk(db_data.id, chunk_number, quality, chunk, tag) + return chunk, tag + + def get_buff(self, chunk_number, quality, db_data): + chunk, tag = self._cache.get('{}_{}_{}'.format(db_data.id, chunk_number, quality), tag=True) + + if not chunk: + chunk, tag = self.prepare_chunk_buff(db_data, quality, chunk_number) + self.save_chunk(db_data.id, chunk_number, quality, chunk, tag) + return chunk + + def prepare_chunk_buff(self, db_data, quality, chunk_number): + from cvat.apps.engine.frame_provider import FrameProvider + extractor_classes = { + FrameProvider.Quality.COMPRESSED : Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter, + FrameProvider.Quality.ORIGINAL : Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter, + } + + image_quality = 100 if extractor_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality + file_extension = 'mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'jpeg' + mime_type = 'video/mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip' + + extractor = extractor_classes[quality](image_quality) + + #if 'interpolation' == task_mode: + if os.path.exists(db_data.get_meta_path()): + meta = PrepareInfo(source_path=os.path.join(db_data.get_upload_dirname(), db_data.video.path), + meta_path=db_data.get_meta_path()) + frames = [] + for frame in meta.decode_needed_frames(chunk_number, db_data):#db_data.chunk_size + frames.append(frame) + buff = extractor.save_as_chunk_to_buff(frames, file_extension) + else: + img_paths = None + with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file: + img_paths = [line.strip() for line in dummy_file] + buff = extractor.save_as_chunk_to_buff(img_paths, file_extension) + return buff, mime_type + + def save_chunk(self, db_data_id, chunk_number, quality, buff, mime_type): + self._cache.set('{}_{}_{}'.format(db_data_id, chunk_number, quality), buff, tag=mime_type) \ No newline at end of file diff --git a/cvat/apps/engine/frame_provider.py b/cvat/apps/engine/frame_provider.py index 25575ea5..10e6b9e9 100644 --- a/cvat/apps/engine/frame_provider.py +++ b/cvat/apps/engine/frame_provider.py @@ -11,8 +11,8 @@ from PIL import Image from cvat.apps.engine.media_extractors import VideoReader, ZipReader from cvat.apps.engine.mime_types import mimetypes -from cvat.apps.engine.models import DataChoice - +from cvat.apps.engine.models import DataChoice, StorageMethodChoice +from .cache import CacheInteraction class RandomAccessIterator: def __init__(self, iterable): @@ -65,6 +65,20 @@ class FrameProvider: self.reader_class([self.get_chunk_path(chunk_id)])) return self.chunk_reader + class BuffChunkLoader(ChunkLoader): + def __init__(self, reader_class, path_getter, buff_mime_getter, quality, db_data): + super().__init__(reader_class, path_getter) + self.get_chunk = buff_mime_getter + self.quality = quality + self.db_data = db_data + + def load(self, chunk_id): + if self.chunk_id != chunk_id: + self.chunk_id = chunk_id + self.chunk_reader = RandomAccessIterator( + self.reader_class([self.get_chunk_path(chunk_id, self.quality, self.db_data)])) + return self.chunk_reader + def __init__(self, db_data): self._db_data = db_data self._loaders = {} @@ -73,12 +87,29 @@ class FrameProvider: DataChoice.IMAGESET: ZipReader, DataChoice.VIDEO: VideoReader, } - self._loaders[self.Quality.COMPRESSED] = self.ChunkLoader( - reader_class[db_data.compressed_chunk_type], - db_data.get_compressed_chunk_path) - self._loaders[self.Quality.ORIGINAL] = self.ChunkLoader( - reader_class[db_data.original_chunk_type], - db_data.get_original_chunk_path) + + if db_data.storage_method == StorageMethodChoice.CACHE: + cache = CacheInteraction() + + self._loaders[self.Quality.COMPRESSED] = self.BuffChunkLoader( + reader_class[db_data.compressed_chunk_type], + cache.get_buff, + cache.get_buff_mime, + self.Quality.COMPRESSED, + self._db_data) + self._loaders[self.Quality.ORIGINAL] = self.BuffChunkLoader( + reader_class[db_data.original_chunk_type], + cache.get_buff, + cache.get_buff_mime, + self.Quality.ORIGINAL, + self._db_data) + else: + self._loaders[self.Quality.COMPRESSED] = self.ChunkLoader( + reader_class[db_data.compressed_chunk_type], + db_data.get_compressed_chunk_path) + self._loaders[self.Quality.ORIGINAL] = self.ChunkLoader( + reader_class[db_data.original_chunk_type], + db_data.get_original_chunk_path) def __len__(self): return self._db_data.size @@ -129,6 +160,8 @@ class FrameProvider: def get_chunk(self, chunk_number, quality=Quality.ORIGINAL): chunk_number = self._validate_chunk_number(chunk_number) + if self._db_data.storage_method == StorageMethodChoice.CACHE: + return self._loaders[quality].get_chunk(chunk_number, quality, self._db_data) return self._loaders[quality].get_chunk_path(chunk_number) def get_frame(self, frame_number, quality=Quality.ORIGINAL, diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 0099b349..f2a59df4 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -190,7 +190,10 @@ class ZipReader(ImageListReader): return io.BytesIO(self._zip_source.read(self._source_path[i])) def get_path(self, i): - return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i]) + if self._zip_source.filename: + return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i]) + else: #для определения mime_type + return self._source_path[i] def extract(self): self._zip_source.extractall(os.path.dirname(self._zip_source.filename)) diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 1abec5a8..690ab216 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -117,6 +117,9 @@ class Data(models.Model): def get_meta_path(self): return os.path.join(self.get_upload_dirname(), 'meta_info.txt') + def get_dummy_chunk_path(self, chunk_number): + return os.path.join(self.get_upload_dirname(), 'dummy_{}.txt'.format(chunk_number)) + class Video(models.Model): data = models.OneToOneField(Data, on_delete=models.CASCADE, related_name="video", null=True) path = models.CharField(max_length=1024, default='') diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 69d74b5a..f8d1ab4d 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -144,11 +144,11 @@ class PrepareInfo(WorkWithVideo): return int(start_decode_frame_number), int(start_decode_timestamp) - def decode_needed_frames(self, chunk_number, chunk_size): - start_chunk_frame_number = chunk_number * chunk_size - end_chunk_frame_number = start_chunk_frame_number + chunk_size + def decode_needed_frames(self, chunk_number, db_data): + step = db_data.get_frame_step() + start_chunk_frame_number = db_data.start_frame + chunk_number * db_data.chunk_size * step + end_chunk_frame_number = min(start_chunk_frame_number + (db_data.chunk_size - 1) * step + 1, db_data.stop_frame + 1) start_decode_frame_number, start_decode_timestamp = self.get_nearest_left_key_frame(start_chunk_frame_number) - container = self._open_video_container(self.source_path, mode='r') video_stream = self._get_video_stream(container) container.seek(offset=start_decode_timestamp, stream=video_stream) @@ -159,8 +159,10 @@ class PrepareInfo(WorkWithVideo): frame_number += 1 if frame_number < start_chunk_frame_number: continue - elif frame_number < end_chunk_frame_number: + elif frame_number < end_chunk_frame_number and not (frame_number % step): yield frame + elif frame_number % step: + continue else: self._close_video_container(container) return diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index fc2f3759..546edb6e 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -25,7 +25,6 @@ from distutils.dir_util import copy_tree from . import models from .log import slogger from .prepare import PrepareInfo, AnalyzeVideo -from diskcache import Cache ############################# Low Level server API @@ -299,7 +298,8 @@ def _create_thread(tid, data): meta_info.check_seek_key_frames() meta_info.save_meta_info() - db_data.size = meta_info.get_task_size() + all_frames = meta_info.get_task_size() + db_data.size = len(range(db_data.start_frame, min(data['stop_frame'] + 1 if data['stop_frame'] else all_frames, all_frames), db_data.get_frame_step())) video_path = os.path.join(upload_dir, media_files[0]) frame = meta_info.key_frames.get(next(iter(meta_info.key_frames))) video_size = (frame.width, frame.height) @@ -310,31 +310,33 @@ def _create_thread(tid, data): db_data.storage_method = StorageMethodChoice.FILE_SYSTEM else:#images,archive - with Cache(settings.CACHE_ROOT) as cache: - counter_ = itertools.count() - - if extractor.__class__ in [MEDIA_TYPES['archive']['extractor'], MEDIA_TYPES['zip']['extractor']]: - media_files = [os.path.join(upload_dir, f) for f in extractor._source_path] - - for chunk_number, media_paths in itertools.groupby(media_files, lambda x: next(counter_) // db_data.chunk_size): - media_paths = list(media_paths) - cache.set('{}_{}'.format(tid, chunk_number), [os.path.join(upload_dir, file_name) for file_name in media_paths], tag='dummy') - - img_sizes = [] - from PIL import Image - for media_path in media_paths: - img_sizes += [Image.open(media_path).size] - db_data.size += len(media_paths) - db_images.extend([ - models.Image( - data=db_data, - path=os.path.basename(data[1]), - frame=data[0], - width=size[0], - height=size[1]) - for data, size in zip(enumerate(media_paths, start=len(db_images)), img_sizes) - ]) - + counter_ = itertools.count() + if extractor.__class__ in [MEDIA_TYPES['archive']['extractor'], MEDIA_TYPES['zip']['extractor']]: + media_files = [os.path.join(upload_dir, f) for f in extractor._source_path] + + numbers_sequence = range(db_data.start_frame, min(data['stop_frame'] if data['stop_frame'] else len(media_files), len(media_files)), db_data.get_frame_step()) + m_paths = [] + m_paths = [(path, numb) for numb, path in enumerate(media_files) if numb in numbers_sequence] + + for chunk_number, media_paths in itertools.groupby(m_paths, lambda x: next(counter_) // db_data.chunk_size): + media_paths = list(media_paths) + img_sizes = [] + from PIL import Image + with open(db_data.get_dummy_chunk_path(chunk_number), 'w') as dummy_chunk: + for path, _ in media_paths: + dummy_chunk.write(os.path.join(upload_dir, path)+'\n') + img_sizes += [Image.open(os.path.join(upload_dir, path)).size] + + db_data.size += len(media_paths) + db_images.extend([ + models.Image( + data=db_data, + path=os.path.basename(data[0]), + frame=data[1], + width=size[0], + height=size[1]) + for data, size in zip(media_paths, img_sizes) + ]) if db_data.storage_method == StorageMethodChoice.FILE_SYSTEM or not settings.USE_CACHE: counter = itertools.count() diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index f06f6d4b..542339b9 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -37,7 +37,7 @@ from cvat.apps.authentication import auth from cvat.apps.authentication.decorators import login_required from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import Job, Plugin, StatusChoice, Task, DataChoice, StorageMethodChoice +from cvat.apps.engine.models import Job, Plugin, StatusChoice, Task, StorageMethodChoice from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaSerializer, DataSerializer, ExceptionSerializer, @@ -49,13 +49,6 @@ from cvat.apps.engine.utils import av_scan_paths from . import models, task from .log import clogger, slogger -from .media_extractors import ( - Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, - ZipCompressedChunkWriter, ZipChunkWriter) -from .prepare import PrepareInfo -from diskcache import Cache -#from cvat.apps.engine.mime_types import mimetypes - # drf-yasg component doesn't handle correctly URL_FORMAT_OVERRIDE and # send requests with ?format=openapi suffix instead of ?scheme=openapi. @@ -458,56 +451,14 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): data_quality = FrameProvider.Quality.COMPRESSED \ if data_quality == 'compressed' else FrameProvider.Quality.ORIGINAL - path = os.path.realpath(frame_provider.get_chunk(data_id, data_quality)) #TODO: av.FFmpegError processing if settings.USE_CACHE and db_data.storage_method == StorageMethodChoice.CACHE: - with Cache(settings.CACHE_ROOT) as cache: - buff = None - chunk, tag = cache.get('{}_{}_{}'.format(db_task.id, data_id, quality), tag=True) - - if not chunk: - extractor_classes = { - 'compressed' : Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter, - 'original' : Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter, - } - - image_quality = 100 if extractor_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality - file_extension = 'mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'jpeg' - mime_type = 'video/mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip' - - extractor = extractor_classes[quality](image_quality) - - if 'interpolation' == db_task.mode: - - meta = PrepareInfo(source_path=os.path.join(db_data.get_upload_dirname(), db_data.video.path), - meta_path=db_data.get_meta_path()) - - frames = [] - for frame in meta.decode_needed_frames(data_id,db_data.chunk_size): - frames.append(frame) - - - buff = extractor.save_as_chunk_to_buff(frames, - format_=file_extension) - cache.set('{}_{}_{}'.format(db_task.id, data_id, quality), buff, tag=mime_type) - - else: - img_paths = cache.get('{}_{}'.format(db_task.id, data_id)) - buff = extractor.save_as_chunk_to_buff(img_paths, - format_=file_extension) - cache.set('{}_{}_{}'.format(db_task.id, data_id, quality), buff, tag=mime_type) - - - - elif 'process_creating' == tag: - pass - else: - buff, mime_type = cache.get('{}_{}_{}'.format(db_task.id, data_id, quality), tag=True) - - return HttpResponse(buff.getvalue(), content_type=mime_type) + buff, mime_type = frame_provider.get_chunk(data_id, data_quality) + return HttpResponse(buff.getvalue(), content_type=mime_type) # Follow symbol links if the chunk is a link on a real image otherwise # mimetype detection inside sendfile will work incorrectly. + path = os.path.realpath(frame_provider.get_chunk(data_id, data_quality)) return sendfile(request, path) elif data_type == 'frame': From ade215399c0dca208afcd8a5516ed30a8f8aaa2b Mon Sep 17 00:00:00 2001 From: Maya Date: Wed, 19 Aug 2020 10:25:33 +0300 Subject: [PATCH 07/20] Fix --- cvat/apps/engine/prepare.py | 4 ++-- cvat/apps/engine/task.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index f8d1ab4d..83a11f93 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -159,9 +159,9 @@ class PrepareInfo(WorkWithVideo): frame_number += 1 if frame_number < start_chunk_frame_number: continue - elif frame_number < end_chunk_frame_number and not (frame_number % step): + elif frame_number < end_chunk_frame_number and not ((frame_number - start_chunk_frame_number) % step): yield frame - elif frame_number % step: + elif (frame_number - start_chunk_frame_number) % step: continue else: self._close_video_container(container) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 546edb6e..08ecb275 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -316,7 +316,7 @@ def _create_thread(tid, data): numbers_sequence = range(db_data.start_frame, min(data['stop_frame'] if data['stop_frame'] else len(media_files), len(media_files)), db_data.get_frame_step()) m_paths = [] - m_paths = [(path, numb) for numb, path in enumerate(media_files) if numb in numbers_sequence] + m_paths = [(path, numb) for numb, path in enumerate(sorted(media_files)) if numb in numbers_sequence] for chunk_number, media_paths in itertools.groupby(m_paths, lambda x: next(counter_) // db_data.chunk_size): media_paths = list(media_paths) From 1520dcc605f608562d6f3115e04b1b42e306938a Mon Sep 17 00:00:00 2001 From: Maya Date: Thu, 20 Aug 2020 20:27:54 +0300 Subject: [PATCH 08/20] Deleted ArchiveReader/get_path --- cvat/apps/engine/media_extractors.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index f2a59df4..97bd0249 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -137,10 +137,6 @@ class ArchiveReader(DirectoryReader): def __del__(self): os.remove(self._archive_source) - def get_path(self, i): - base_dir = os.path.dirname(self._archive_source) - return os.path.join(base_dir, os.path.relpath(self._source_path[i], base_dir)) - class PdfReader(DirectoryReader): def __init__(self, source_path, step=1, start=0, stop=None): if not source_path: From ff62baab99e3f905b055621eb93ab07022f2f8c8 Mon Sep 17 00:00:00 2001 From: Maya Date: Thu, 20 Aug 2020 20:34:51 +0300 Subject: [PATCH 09/20] Changed paths --- cvat/apps/engine/cache.py | 2 +- cvat/apps/engine/task.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index c79bf4f2..c922f73e 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -53,7 +53,7 @@ class CacheInteraction: else: img_paths = None with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file: - img_paths = [line.strip() for line in dummy_file] + img_paths = [os.path.join(db_data.get_upload_dirname(), line.strip()) for line in dummy_file] buff = extractor.save_as_chunk_to_buff(img_paths, file_extension) return buff, mime_type diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 08ecb275..881bbed2 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -311,8 +311,10 @@ def _create_thread(tid, data): else:#images,archive counter_ = itertools.count() - if extractor.__class__ in [MEDIA_TYPES['archive']['extractor'], MEDIA_TYPES['zip']['extractor']]: - media_files = [os.path.join(upload_dir, f) for f in extractor._source_path] + if isinstance(extractor, MEDIA_TYPES['archive']['extractor']): + media_files = [os.path.relpath(path, upload_dir) for path in extractor._source_path] + elif isinstance(extractor, MEDIA_TYPES['zip']['extractor']): + media_files = extractor._source_path numbers_sequence = range(db_data.start_frame, min(data['stop_frame'] if data['stop_frame'] else len(media_files), len(media_files)), db_data.get_frame_step()) m_paths = [] @@ -324,14 +326,14 @@ def _create_thread(tid, data): from PIL import Image with open(db_data.get_dummy_chunk_path(chunk_number), 'w') as dummy_chunk: for path, _ in media_paths: - dummy_chunk.write(os.path.join(upload_dir, path)+'\n') + dummy_chunk.write(path+'\n') img_sizes += [Image.open(os.path.join(upload_dir, path)).size] db_data.size += len(media_paths) db_images.extend([ models.Image( data=db_data, - path=os.path.basename(data[0]), + path=data[0], frame=data[1], width=size[0], height=size[1]) From 3b9b1c686f735153f162479d73a011c1b5b98aea Mon Sep 17 00:00:00 2001 From: Maya Date: Mon, 24 Aug 2020 22:47:42 +0300 Subject: [PATCH 10/20] Disabled use cache default --- .../components/create-task-page/advanced-configuration-form.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx index c2dd2552..727eccd9 100644 --- a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx +++ b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx @@ -386,7 +386,7 @@ class AdvancedConfigurationForm extends React.PureComponent { return ( {form.getFieldDecorator('useCache', { - initialValue: true, + initialValue: false, valuePropName: 'checked', })( From 43c2c6ea8775883fa85b42efa79d4e2dcae74134 Mon Sep 17 00:00:00 2001 From: Maya Date: Mon, 24 Aug 2020 22:51:26 +0300 Subject: [PATCH 11/20] Merged migrations into one file --- .../migrations/0028_data_storage_method.py | 17 ++++++ .../migrations/0029_auto_20200814_2153.py | 53 ------------------- 2 files changed, 17 insertions(+), 53 deletions(-) delete mode 100644 cvat/apps/engine/migrations/0029_auto_20200814_2153.py diff --git a/cvat/apps/engine/migrations/0028_data_storage_method.py b/cvat/apps/engine/migrations/0028_data_storage_method.py index 6490f5bf..4bd8f757 100644 --- a/cvat/apps/engine/migrations/0028_data_storage_method.py +++ b/cvat/apps/engine/migrations/0028_data_storage_method.py @@ -1,8 +1,24 @@ # Generated by Django 2.2.13 on 2020-08-13 05:49 +from cvat.apps.engine.media_extractors import _is_archive, _is_zip import cvat.apps.engine.models +from django.conf import settings from django.db import migrations, models +import os +from pyunpack import Archive +def unzip(apps, schema_editor): + Data = apps.get_model("engine", "Data") + data_q_set = Data.objects.all() + archive_paths = [] + + for data_instance in data_q_set: + for root, _, files in os.walk(os.path.join(settings.MEDIA_DATA_ROOT, '{}/raw/'.format(data_instance.id))): + archive_paths.extend([os.path.join(root, file) for file in files if _is_archive(file) or _is_zip(file)]) + + for path in archive_paths: + Archive(path).extractall(os.path.dirname(path)) + os.remove(path) class Migration(migrations.Migration): @@ -16,4 +32,5 @@ class Migration(migrations.Migration): name='storage_method', field=models.CharField(choices=[('cache', 'CACHE'), ('file_system', 'FILE_SYSTEM')], default=cvat.apps.engine.models.StorageMethodChoice('file_system'), max_length=15), ), + migrations.RunPython(unzip), ] diff --git a/cvat/apps/engine/migrations/0029_auto_20200814_2153.py b/cvat/apps/engine/migrations/0029_auto_20200814_2153.py deleted file mode 100644 index f81c73c8..00000000 --- a/cvat/apps/engine/migrations/0029_auto_20200814_2153.py +++ /dev/null @@ -1,53 +0,0 @@ -from django.db import migrations -from django.conf import settings -import os -from cvat.apps.engine.mime_types import mimetypes -from pyunpack import Archive - -def unzip(apps, schema_editor): - Data = apps.get_model("engine", "Data") - data_q_set = Data.objects.all() - archive_paths = [] - archive_mimes = [ - 'application/gzip', - 'application/rar' - 'application/x-7z-compressed', - 'application/x-bzip', - 'application/x-bzip-compressed-tar', - 'application/x-compress', - 'application/x-compressed-tar', - 'application/x-cpio', - 'application/x-gtar-compressed', - 'application/x-lha', - 'application/x-lhz', - 'application/x-lrzip-compressed-tar', - 'application/x-lz4', - 'application/x-lzip', - 'application/x-lzip-compressed-tar', - 'application/x-lzma', - 'application/x-lzma-compressed-tar', - 'application/x-lzop', - 'application/x-tar', - 'application/x-tarz', - 'application/x-tzo', - 'application/x-xz-compressed-tar', - 'application/zip', - ] - - for data_instance in data_q_set: - for root, _, files in os.walk(os.path.join(settings.MEDIA_DATA_ROOT, '{}/raw/'.format(data_instance.id))): - archive_paths.extend([os.path.join(root, file) for file in files if mimetypes.guess_type(file)[0] in archive_mimes]) - - for path in archive_paths: - Archive(path).extractall(os.path.dirname(path)) - os.remove(path) - -class Migration(migrations.Migration): - - dependencies = [ - ('engine', '0028_data_storage_method'), - ] - - operations = [ - migrations.RunPython(unzip) - ] \ No newline at end of file From 2144c4aadd65d1072fdb5f67471576c3cb9a85bc Mon Sep 17 00:00:00 2001 From: Maya Date: Mon, 24 Aug 2020 23:07:49 +0300 Subject: [PATCH 12/20] Refactoring --- cvat/apps/engine/cache.py | 16 +++------------- cvat/apps/engine/frame_provider.py | 9 +++------ cvat/apps/engine/media_extractors.py | 16 ++++++++-------- cvat/apps/engine/prepare.py | 2 +- cvat/apps/engine/task.py | 4 +--- cvat/apps/engine/views.py | 1 - 6 files changed, 16 insertions(+), 32 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index c922f73e..e4d19f59 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -21,14 +21,6 @@ class CacheInteraction: self.save_chunk(db_data.id, chunk_number, quality, chunk, tag) return chunk, tag - def get_buff(self, chunk_number, quality, db_data): - chunk, tag = self._cache.get('{}_{}_{}'.format(db_data.id, chunk_number, quality), tag=True) - - if not chunk: - chunk, tag = self.prepare_chunk_buff(db_data, quality, chunk_number) - self.save_chunk(db_data.id, chunk_number, quality, chunk, tag) - return chunk - def prepare_chunk_buff(self, db_data, quality, chunk_number): from cvat.apps.engine.frame_provider import FrameProvider extractor_classes = { @@ -37,24 +29,22 @@ class CacheInteraction: } image_quality = 100 if extractor_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality - file_extension = 'mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'jpeg' mime_type = 'video/mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip' extractor = extractor_classes[quality](image_quality) - #if 'interpolation' == task_mode: if os.path.exists(db_data.get_meta_path()): meta = PrepareInfo(source_path=os.path.join(db_data.get_upload_dirname(), db_data.video.path), meta_path=db_data.get_meta_path()) frames = [] - for frame in meta.decode_needed_frames(chunk_number, db_data):#db_data.chunk_size + for frame in meta.decode_needed_frames(chunk_number, db_data): frames.append(frame) - buff = extractor.save_as_chunk_to_buff(frames, file_extension) + buff = extractor.save_as_chunk_to_buff(frames) else: img_paths = None with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file: img_paths = [os.path.join(db_data.get_upload_dirname(), line.strip()) for line in dummy_file] - buff = extractor.save_as_chunk_to_buff(img_paths, file_extension) + buff = extractor.save_as_chunk_to_buff(img_paths) return buff, mime_type def save_chunk(self, db_data_id, chunk_number, quality, buff, mime_type): diff --git a/cvat/apps/engine/frame_provider.py b/cvat/apps/engine/frame_provider.py index 10e6b9e9..8f56463a 100644 --- a/cvat/apps/engine/frame_provider.py +++ b/cvat/apps/engine/frame_provider.py @@ -66,9 +66,8 @@ class FrameProvider: return self.chunk_reader class BuffChunkLoader(ChunkLoader): - def __init__(self, reader_class, path_getter, buff_mime_getter, quality, db_data): + def __init__(self, reader_class, path_getter, quality, db_data): super().__init__(reader_class, path_getter) - self.get_chunk = buff_mime_getter self.quality = quality self.db_data = db_data @@ -76,7 +75,7 @@ class FrameProvider: if self.chunk_id != chunk_id: self.chunk_id = chunk_id self.chunk_reader = RandomAccessIterator( - self.reader_class([self.get_chunk_path(chunk_id, self.quality, self.db_data)])) + self.reader_class([self.get_chunk_path(chunk_id, self.quality, self.db_data)[0]])) return self.chunk_reader def __init__(self, db_data): @@ -93,13 +92,11 @@ class FrameProvider: self._loaders[self.Quality.COMPRESSED] = self.BuffChunkLoader( reader_class[db_data.compressed_chunk_type], - cache.get_buff, cache.get_buff_mime, self.Quality.COMPRESSED, self._db_data) self._loaders[self.Quality.ORIGINAL] = self.BuffChunkLoader( reader_class[db_data.original_chunk_type], - cache.get_buff, cache.get_buff_mime, self.Quality.ORIGINAL, self._db_data) @@ -161,7 +158,7 @@ class FrameProvider: def get_chunk(self, chunk_number, quality=Quality.ORIGINAL): chunk_number = self._validate_chunk_number(chunk_number) if self._db_data.storage_method == StorageMethodChoice.CACHE: - return self._loaders[quality].get_chunk(chunk_number, quality, self._db_data) + return self._loaders[quality].get_chunk_path(chunk_number, quality, self._db_data) return self._loaders[quality].get_chunk_path(chunk_number) def get_frame(self, frame_number, quality=Quality.ORIGINAL, diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 97bd0249..0ea98941 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -287,12 +287,12 @@ class ZipChunkWriter(IChunkWriter): # and does not decode it to know img size. return [] - def save_as_chunk_to_buff(self, images, format_='jpeg'): + def save_as_chunk_to_buff(self, images): buff = io.BytesIO() with zipfile.ZipFile(buff, 'w') as zip_file: for idx, image in enumerate(images): - arcname = '{:06d}.{}'.format(idx, format_) + arcname = '{:06d}.{}'.format(idx, os.path.splitext(image)[1]) if isinstance(image, av.VideoFrame): zip_file.writestr(arcname, image.to_image().tobytes().getvalue()) else: @@ -312,12 +312,12 @@ class ZipCompressedChunkWriter(IChunkWriter): return image_sizes - def save_as_chunk_to_buff(self, images, format_='jpeg'): + def save_as_chunk_to_buff(self, images): buff = io.BytesIO() with zipfile.ZipFile(buff, 'x') as zip_file: for idx, image in enumerate(images): (_, _, image_buf) = self._compress_image(image, self._image_quality) - arcname = '{:06d}.{}'.format(idx, format_) + arcname = '{:06d}.jpeg'.format(idx) zip_file.writestr(arcname, image_buf.getvalue()) buff.seek(0) return buff @@ -366,7 +366,7 @@ class Mpeg4ChunkWriter(IChunkWriter): output_container.close() return [(input_w, input_h)] - def save_as_chunk_to_buff(self, frames, format_): + def save_as_chunk_to_buff(self, frames): if not frames: raise Exception('no images to save') @@ -383,7 +383,7 @@ class Mpeg4ChunkWriter(IChunkWriter): "crf": str(self._image_quality), "preset": "ultrafast", }, - f=format_, + f='mp4', ) for frame in frames: @@ -454,7 +454,7 @@ class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter): output_container.close() return [(input_w, input_h)] - def save_as_chunk_to_buff(self, frames, format_): + def save_as_chunk_to_buff(self, frames): if not frames: raise Exception('no images to save') @@ -482,7 +482,7 @@ class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter): 'wpredp': '0', 'flags': '-loop' }, - f=format_, + f='mp4', ) for frame in frames: diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 83a11f93..7e0fe395 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -93,7 +93,7 @@ class PrepareInfo(WorkWithVideo): key_frames_copy = self.key_frames.copy() - for _, key_frame in key_frames_copy.items(): + for index, key_frame in key_frames_copy.items(): container.seek(offset=key_frame.pts, stream=video_stream) flag = True for packet in container.demux(video_stream): diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 881bbed2..1056e2f3 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -304,9 +304,7 @@ def _create_thread(tid, data): frame = meta_info.key_frames.get(next(iter(meta_info.key_frames))) video_size = (frame.width, frame.height) - except AssertionError as ex: - db_data.storage_method = StorageMethodChoice.FILE_SYSTEM - except Exception as ex: + except Exception: db_data.storage_method = StorageMethodChoice.FILE_SYSTEM else:#images,archive diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index e5800d15..94db4f39 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -408,7 +408,6 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): if data_type == 'chunk': data_id = int(data_id) - quality = data_quality data_quality = FrameProvider.Quality.COMPRESSED \ if data_quality == 'compressed' else FrameProvider.Quality.ORIGINAL From 7bb5e02ff1f8bfea6bcee9591fc594ef1194b2a4 Mon Sep 17 00:00:00 2001 From: Maya Date: Mon, 24 Aug 2020 23:10:08 +0300 Subject: [PATCH 13/20] Added license headers --- cvat/apps/engine/cache.py | 4 ++++ cvat/apps/engine/prepare.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index e4d19f59..34753557 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -1,3 +1,7 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + from diskcache import Cache from django.conf import settings from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, ZipChunkWriter, diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 7e0fe395..97c07a84 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -1,3 +1,7 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + import av import hashlib From e0461574ba6ffe62c3cd8d6d97ef4fc055849e8b Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 25 Aug 2020 08:36:20 +0300 Subject: [PATCH 14/20] Updated version --- cvat-core/package-lock.json | 2 +- cvat-core/package.json | 2 +- cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 0bc2c787..91ac8a0f 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.5.0", + "version": "3.6.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index ca65f045..4418f157 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.5.0", + "version": "3.6.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 4f97ca1e..4cfa9ddf 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.8.1", + "version": "1.9.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 64353225..e587666f 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.8.1", + "version": "1.9.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { From 761c83f3ab0d3f990010e95c09b187c82861b3ba Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 25 Aug 2020 09:20:38 +0300 Subject: [PATCH 15/20] Fix migration --- ...{0028_data_storage_method.py => 0029_data_storage_method.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename cvat/apps/engine/migrations/{0028_data_storage_method.py => 0029_data_storage_method.py} (96%) diff --git a/cvat/apps/engine/migrations/0028_data_storage_method.py b/cvat/apps/engine/migrations/0029_data_storage_method.py similarity index 96% rename from cvat/apps/engine/migrations/0028_data_storage_method.py rename to cvat/apps/engine/migrations/0029_data_storage_method.py index 4bd8f757..1c1aa814 100644 --- a/cvat/apps/engine/migrations/0028_data_storage_method.py +++ b/cvat/apps/engine/migrations/0029_data_storage_method.py @@ -23,7 +23,7 @@ def unzip(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('engine', '0027_auto_20200719_1552'), + ('engine', '0028_labelcolor'), ] operations = [ From b47d3d5683bc2a5b74f80794367a831674f038fd Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 25 Aug 2020 10:29:47 +0300 Subject: [PATCH 16/20] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca2dea56..3e5310b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Datumaro] Multi-dataset merge (https://github.com/opencv/cvat/pull/1695) - Link to django admin page from UI () - Notification message when users use wrong browser () +- Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007) ### Changed - Shape coordinates are rounded to 2 digits in dumped annotations () From d29f4cb6e0fa8b374602c869229a24be56a47c9e Mon Sep 17 00:00:00 2001 From: Maya Date: Thu, 27 Aug 2020 09:07:03 +0300 Subject: [PATCH 17/20] Added tests --- cvat/apps/engine/tests/_test_rest_api.py | 85 ++++++++++++++++++++++-- cvat/settings/testing.py | 2 + 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/cvat/apps/engine/tests/_test_rest_api.py b/cvat/apps/engine/tests/_test_rest_api.py index 73c81141..1e773b2f 100644 --- a/cvat/apps/engine/tests/_test_rest_api.py +++ b/cvat/apps/engine/tests/_test_rest_api.py @@ -72,13 +72,14 @@ import av import numpy as np from django.conf import settings from django.contrib.auth.models import Group, User +from django.http import HttpResponse from PIL import Image from pycocotools import coco as coco_loader from rest_framework import status from rest_framework.test import APIClient, APITestCase from cvat.apps.engine.models import (AttributeType, Data, Job, Project, - Segment, StatusChoice, Task) + Segment, StatusChoice, Task, StorageMethodChoice) _setUpModule() @@ -1670,7 +1671,8 @@ class TaskDataAPITestCase(APITestCase): stream = container.streams.video[0] return [f.to_image() for f in container.decode(stream)] - def _test_api_v1_tasks_id_data_spec(self, user, spec, data, expected_compressed_type, expected_original_type, image_sizes): + def _test_api_v1_tasks_id_data_spec(self, user, spec, data, expected_compressed_type, expected_original_type, image_sizes, + expected_storage_method=StorageMethodChoice.FILE_SYSTEM): # create task response = self._create_task(user, spec) self.assertEqual(response.status_code, status.HTTP_201_CREATED) @@ -1694,6 +1696,7 @@ class TaskDataAPITestCase(APITestCase): self.assertEqual(expected_compressed_type, task["data_compressed_chunk_type"]) self.assertEqual(expected_original_type, task["data_original_chunk_type"]) self.assertEqual(len(image_sizes), task["size"]) + self.assertEqual(expected_storage_method, Task.objects.get(pk=task_id).data.storage_method) # check preview response = self._get_preview(task_id, user) @@ -1706,7 +1709,10 @@ class TaskDataAPITestCase(APITestCase): response = self._get_compressed_chunk(task_id, user, 0) self.assertEqual(response.status_code, expected_status_code) if expected_status_code == status.HTTP_200_OK: - compressed_chunk = io.BytesIO(b"".join(response.streaming_content)) + if isinstance(response, HttpResponse): + compressed_chunk = io.BytesIO(response.content) + else: + compressed_chunk = io.BytesIO(b"".join(response.streaming_content)) if task["data_compressed_chunk_type"] == self.ChunkType.IMAGESET: images = self._extract_zip_chunk(compressed_chunk) else: @@ -1721,7 +1727,10 @@ class TaskDataAPITestCase(APITestCase): response = self._get_original_chunk(task_id, user, 0) self.assertEqual(response.status_code, expected_status_code) if expected_status_code == status.HTTP_200_OK: - original_chunk = io.BytesIO(b"".join(response.streaming_content)) + if isinstance(response, HttpResponse): + original_chunk = io.BytesIO(response.getvalue()) + else: + original_chunk = io.BytesIO(b"".join(response.streaming_content)) if task["data_original_chunk_type"] == self.ChunkType.IMAGESET: images = self._extract_zip_chunk(original_chunk) else: @@ -1909,6 +1918,74 @@ class TaskDataAPITestCase(APITestCase): self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes) + task_spec = { + "name": "use_cache video task #8", + "overlap": 0, + "segment_size": 0, + "labels": [ + {"name": "car"}, + {"name": "person"}, + ] + } + + task_data = { + "server_files[0]": 'test_video_1.mp4', + "image_quality": 70, + "use_cache": True, + } + + image_sizes = self._image_sizes[task_data["server_files[0]"]] + + self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, + self.ChunkType.VIDEO, image_sizes, StorageMethodChoice.CACHE) + + task_spec = { + "name": "use_cache images task #9", + "overlap": 0, + "segment_size": 0, + "labels": [ + {"name": "car"}, + {"name": "person"}, + ] + } + + task_data = { + "server_files[0]": "test_1.jpg", + "server_files[1]": "test_2.jpg", + "server_files[2]": "test_3.jpg", + "image_quality": 70, + "use_cache": True, + } + image_sizes = [ + self._image_sizes[task_data["server_files[0]"]], + self._image_sizes[task_data["server_files[1]"]], + self._image_sizes[task_data["server_files[2]"]], + ] + + self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, + self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE) + + task_spec = { + "name": "my zip archive task #10", + "overlap": 0, + "segment_size": 0, + "labels": [ + {"name": "car"}, + {"name": "person"}, + ] + } + + task_data = { + "server_files[0]": "test_archive_1.zip", + "image_quality": 70, + "use_cache": True + } + + image_sizes = self._image_sizes[task_data["server_files[0]"]] + + self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, + self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE) + def test_api_v1_tasks_id_data_admin(self): self._test_api_v1_tasks_id_data(self.admin) diff --git a/cvat/settings/testing.py b/cvat/settings/testing.py index 35bf5eaf..9825349f 100644 --- a/cvat/settings/testing.py +++ b/cvat/settings/testing.py @@ -22,6 +22,8 @@ os.makedirs(TASKS_ROOT, exist_ok=True) MODELS_ROOT = os.path.join(DATA_ROOT, 'models') os.makedirs(MODELS_ROOT, exist_ok=True) +CACHE_ROOT = os.path.join(DATA_ROOT, 'cache') +os.makedirs(CACHE_ROOT, exist_ok=True) # To avoid ERROR django.security.SuspiciousFileOperation: # The joined path (...) is located outside of the base path component From fb383c71e2a7e4e05bfd26e9c9c721e72cb01559 Mon Sep 17 00:00:00 2001 From: Maya Date: Thu, 27 Aug 2020 09:19:06 +0300 Subject: [PATCH 18/20] Deleted unnecessary --- cvat/apps/engine/media_extractors.py | 2 +- cvat/apps/engine/prepare.py | 23 ++--------------------- cvat/apps/engine/views.py | 2 +- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 0ea98941..49ee9c9f 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -188,7 +188,7 @@ class ZipReader(ImageListReader): def get_path(self, i): if self._zip_source.filename: return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i]) - else: #для определения mime_type + else: # necessary for mime_type definition return self._source_path[i] def extract(self): diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 97c07a84..3d4ca7da 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -54,25 +54,6 @@ class AnalyzeVideo(WorkWithVideo): frame_pts, frame_dts = frame.pts, frame.dts self._close_video_container(container) -# class Frame: -# def __init__(self, frame, frame_number=None): -# self.frame = frame -# if frame_number: -# self.frame_number = frame_number - -# def md5_hash(self): -# return hashlib.md5(self.frame.to_image().tobytes()).hexdigest() - -# def __eq__(self, image): -# return self.md5_hash(self) == image.md5_hash(image) and self.frame.pts == image.frame.pts - -# def __ne__(self, image): -# return md5_hash(self) != md5_hash(image) or self.frame.pts != image.frame.pts - -# def __len__(self): -# return (self.frame.width, self.frame.height) - - def md5_hash(frame): return hashlib.md5(frame.to_image().tobytes()).hexdigest() @@ -109,7 +90,8 @@ class PrepareInfo(WorkWithVideo): if not flag: break - if len(self.key_frames) == 0: #or self.frames // len(self.key_frames) > 300: + #TODO: correct ratio of number of frames to keyframes + if len(self.key_frames) == 0: raise Exception('Too few keyframes') def save_key_frames(self): @@ -139,7 +121,6 @@ class PrepareInfo(WorkWithVideo): for line in file: frame_number, timestamp = line.strip().split(' ') - #TODO: исправить если вдруг ключевой кадр окажется не первым if int(frame_number) <= start_chunk_frame_number: start_decode_frame_number = frame_number start_decode_timestamp = timestamp diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 94db4f39..7e46d836 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -27,7 +27,7 @@ from rest_framework.exceptions import APIException from rest_framework.permissions import SAFE_METHODS, IsAuthenticated from rest_framework.renderers import JSONRenderer from rest_framework.response import Response -from sendfile import sendfile#убрать +from sendfile import sendfile import cvat.apps.dataset_manager as dm import cvat.apps.dataset_manager.views # pylint: disable=unused-import From e7df942f6daa9a13b6b8e04aa0fea23ae738cba3 Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 28 Aug 2020 09:03:09 +0300 Subject: [PATCH 19/20] Refactoring --- cvat/apps/engine/cache.py | 18 +++-- cvat/apps/engine/media_extractors.py | 106 +-------------------------- 2 files changed, 11 insertions(+), 113 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 34753557..2f8b0a62 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -9,6 +9,7 @@ from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, ZipChunkWriter, from cvat.apps.engine.models import DataChoice from .prepare import PrepareInfo import os +from io import BytesIO class CacheInteraction: def __init__(self): @@ -37,18 +38,19 @@ class CacheInteraction: extractor = extractor_classes[quality](image_quality) + images = [] + buff = BytesIO() if os.path.exists(db_data.get_meta_path()): - meta = PrepareInfo(source_path=os.path.join(db_data.get_upload_dirname(), db_data.video.path), - meta_path=db_data.get_meta_path()) - frames = [] + source_path = os.path.join(db_data.get_upload_dirname(), db_data.video.path) + meta = PrepareInfo(source_path=source_path, meta_path=db_data.get_meta_path()) for frame in meta.decode_needed_frames(chunk_number, db_data): - frames.append(frame) - buff = extractor.save_as_chunk_to_buff(frames) + images.append(frame) + extractor.save_as_chunk([(image, source_path, None) for image in images], buff) else: - img_paths = None with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file: - img_paths = [os.path.join(db_data.get_upload_dirname(), line.strip()) for line in dummy_file] - buff = extractor.save_as_chunk_to_buff(img_paths) + images = [os.path.join(db_data.get_upload_dirname(), line.strip()) for line in dummy_file] + extractor.save_as_chunk([(image, image, None) for image in images], buff) + buff.seek(0) return buff, mime_type def save_chunk(self, db_data_id, chunk_number, quality, buff, mime_type): diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 49ee9c9f..08a660c5 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -287,19 +287,6 @@ class ZipChunkWriter(IChunkWriter): # and does not decode it to know img size. return [] - def save_as_chunk_to_buff(self, images): - buff = io.BytesIO() - - with zipfile.ZipFile(buff, 'w') as zip_file: - for idx, image in enumerate(images): - arcname = '{:06d}.{}'.format(idx, os.path.splitext(image)[1]) - if isinstance(image, av.VideoFrame): - zip_file.writestr(arcname, image.to_image().tobytes().getvalue()) - else: - zip_file.write(filename=image, arcname=arcname) - buff.seek(0) - return buff - class ZipCompressedChunkWriter(IChunkWriter): def save_as_chunk(self, images, chunk_path): image_sizes = [] @@ -312,23 +299,13 @@ class ZipCompressedChunkWriter(IChunkWriter): return image_sizes - def save_as_chunk_to_buff(self, images): - buff = io.BytesIO() - with zipfile.ZipFile(buff, 'x') as zip_file: - for idx, image in enumerate(images): - (_, _, image_buf) = self._compress_image(image, self._image_quality) - arcname = '{:06d}.jpeg'.format(idx) - zip_file.writestr(arcname, image_buf.getvalue()) - buff.seek(0) - return buff - class Mpeg4ChunkWriter(IChunkWriter): def __init__(self, _): super().__init__(17) self._output_fps = 25 @staticmethod - def _create_av_container(path, w, h, rate, options, f=None): + def _create_av_container(path, w, h, rate, options, f='mp4'): # x264 requires width and height must be divisible by 2 for yuv420p if h % 2: h += 1 @@ -366,41 +343,6 @@ class Mpeg4ChunkWriter(IChunkWriter): output_container.close() return [(input_w, input_h)] - def save_as_chunk_to_buff(self, frames): - if not frames: - raise Exception('no images to save') - - buff = io.BytesIO() - input_w = frames[0].width - input_h = frames[0].height - - output_container, output_v_stream = self._create_av_container( - path=buff, - w=input_w, - h=input_h, - rate=self._output_fps, - options={ - "crf": str(self._image_quality), - "preset": "ultrafast", - }, - f='mp4', - ) - - for frame in frames: - # let libav set the correct pts and time_base - frame.pts = None - frame.time_base = None - - for packet in output_v_stream.encode(frame): - output_container.mux(packet) - - # Flush streams - for packet in output_v_stream.encode(): - output_container.mux(packet) - output_container.close() - buff.seek(0) - return buff - @staticmethod def _encode_images(images, container, stream): for frame, _, _ in images: @@ -454,52 +396,6 @@ class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter): output_container.close() return [(input_w, input_h)] - def save_as_chunk_to_buff(self, frames): - if not frames: - raise Exception('no images to save') - - buff = io.BytesIO() - input_w = frames[0].width - input_h = frames[0].height - - downscale_factor = 1 - while input_h / downscale_factor >= 1080: - downscale_factor *= 2 - - output_h = input_h // downscale_factor - output_w = input_w // downscale_factor - - - output_container, output_v_stream = self._create_av_container( - path=buff, - w=output_w, - h=output_h, - rate=self._output_fps, - options={ - 'profile': 'baseline', - 'coder': '0', - 'crf': str(self._image_quality), - 'wpredp': '0', - 'flags': '-loop' - }, - f='mp4', - ) - - for frame in frames: - # let libav set the correct pts and time_base - frame.pts = None - frame.time_base = None - - for packet in output_v_stream.encode(frame): - output_container.mux(packet) - - # Flush streams - for packet in output_v_stream.encode(): - output_container.mux(packet) - output_container.close() - buff.seek(0) - return buff - def _is_archive(path): mime = mimetypes.guess_type(path) mime_type = mime[0] From f969e502dd5c8fb1b15f2a2424d8496965b44869 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Tue, 1 Sep 2020 12:47:42 +0300 Subject: [PATCH 20/20] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0029d86c..bdb6c11b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.2.0] - Unreleased ### Added - Added password reset functionality () +- Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007) ### Changed - UI models (like DEXTR) were redesigned to be more interactive () @@ -33,7 +34,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to configure email verification for new users () - Link to django admin page from UI () - Notification message when users use wrong browser () -- Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007) ### Changed - Shape coordinates are rounded to 2 digits in dumped annotations ()