diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index b72bf0cd..0383341b 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -313,7 +313,8 @@ class VideoReader(IMediaReader): class IChunkWriter(ABC): def __init__(self, quality, dimension=DimensionType.DIM_2D): - self._image_quality = quality + # translate inversed range [1:100] to [0:51] + self._image_quality = round(51 * (100 - quality) / 99) self._dimension = dimension @staticmethod @@ -373,12 +374,27 @@ class ZipCompressedChunkWriter(IChunkWriter): return image_sizes class Mpeg4ChunkWriter(IChunkWriter): - def __init__(self, _): - super().__init__(17) + def __init__(self, quality=67): + super().__init__(quality) self._output_fps = 25 - - @staticmethod - def _create_av_container(path, w, h, rate, options, f='mp4'): + try: + codec = av.codec.Codec('libopenh264', 'w') + self._codec_name = codec.name + self._codec_opts = { + 'profile': 'constrained_baseline', + 'qmin': str(self._image_quality), + 'qmax': str(self._image_quality), + 'rc_mode': 'buffer', + } + except av.codec.codec.UnknownCodecError: + codec = av.codec.Codec('libx264', 'w') + self._codec_name = codec.name + self._codec_opts = { + "crf": str(self._image_quality), + "preset": "ultrafast", + } + + def _create_av_container(self, path, w, h, rate, options, f='mp4'): # x264 requires width and height must be divisible by 2 for yuv420p if h % 2: h += 1 @@ -386,7 +402,7 @@ class Mpeg4ChunkWriter(IChunkWriter): w += 1 container = av.open(path, 'w',format=f) - video_stream = container.add_stream('libopenh264', rate=rate) + video_stream = container.add_stream(self._codec_name, rate=rate) video_stream.pix_fmt = "yuv420p" video_stream.width = w video_stream.height = h @@ -406,12 +422,7 @@ class Mpeg4ChunkWriter(IChunkWriter): w=input_w, h=input_h, rate=self._output_fps, - options={ - 'profile': 'constrained_baseline', - 'qmin': str(self._image_quality), - 'qmax': str(self._image_quality), - 'rc_mode': 'buffer', - }, + options=self._codec_opts, ) self._encode_images(images, output_container, output_v_stream) @@ -434,10 +445,15 @@ class Mpeg4ChunkWriter(IChunkWriter): class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter): def __init__(self, quality): - # translate inversed range [1:100] to [0:51] - self._image_quality = round(51 * (100 - quality) / 99) - self._output_fps = 25 - + super().__init__(quality) + if self._codec_name == 'libx264': + self._codec_opts = { + 'profile': 'baseline', + 'coder': '0', + 'crf': str(self._image_quality), + 'wpredp': '0', + 'flags': '-loop', + } def save_as_chunk(self, images, chunk_path): if not images: @@ -458,12 +474,7 @@ class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter): w=output_w, h=output_h, rate=self._output_fps, - options={ - 'profile': 'constrained_baseline', - 'qmin': str(self._image_quality), - 'qmax': str(self._image_quality), - 'rc_mode': 'buffer', - }, + options=self._codec_opts, ) self._encode_images(images, output_container, output_v_stream) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 46a4bf9a..b54b3af9 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -10,9 +10,9 @@ from re import findall import rq import shutil from traceback import print_exception -from urllib import error as urlerror from urllib import parse as urlparse from urllib import request as urlrequest +import requests from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter, ValidateDimension from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice, RelatedFile @@ -195,20 +195,16 @@ def _download_data(urls, upload_dir): job.meta['status'] = '{} is being downloaded..'.format(url) job.save_meta() - req = urlrequest.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) - try: - with urlrequest.urlopen(req) as fp, open(os.path.join(upload_dir, name), 'wb') as tfp: - while True: - block = fp.read(8192) - if not block: - break - tfp.write(block) - except urlerror.HTTPError as err: - raise Exception("Failed to download " + url + ". " + str(err.code) + ' - ' + err.reason) - except urlerror.URLError as err: - raise Exception("Invalid URL: " + url + ". " + err.reason) + response = requests.get(url, stream=True) + if response.status_code == 200: + response.raw.decode_content = True + with open(os.path.join(upload_dir, name), 'wb') as output_file: + shutil.copyfileobj(response.raw, output_file) + else: + raise Exception("Failed to download " + url) local_files[name] = True + return list(local_files.keys()) @transaction.atomic @@ -298,13 +294,20 @@ def _create_thread(tid, data): update_progress.call_counter = (update_progress.call_counter + 1) % len(progress_animation) compressed_chunk_writer_class = Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter - original_chunk_writer_class = Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter + if db_data.original_chunk_type == DataChoice.VIDEO: + original_chunk_writer_class = Mpeg4ChunkWriter + # Let's use QP=17 (that is 67 for 0-100 range) for the original chunks, which should be visually lossless or nearly so. + # A lower value will significantly increase the chunk size with a slight increase of quality. + original_quality = 67 + else: + original_chunk_writer_class = ZipChunkWriter + original_quality = 100 kwargs = {} if validate_dimension.dimension == DimensionType.DIM_3D: kwargs["dimension"] = validate_dimension.dimension compressed_chunk_writer = compressed_chunk_writer_class(db_data.image_quality, **kwargs) - original_chunk_writer = original_chunk_writer_class(100) + original_chunk_writer = original_chunk_writer_class(original_quality) # calculate chunk size if it isn't specified if db_data.chunk_size is None: