Fallback to libx264 encoder if libopenh264 isn't available (#2875)

* Fallback to libx264 encoder if libopenh264 isn't available

* Unified ChunkWriters interfaces: quality setting from 0..100 range for both writers (#2895)

* Fix bandit warning.

Co-authored-by: Andrey Zhavoronkov <andrey.zhavoronkov@intel.com>

cvat/apps/engine/media_extractors.py

@@ -313,7 +313,8 @@ class VideoReader(IMediaReader):
 
 class IChunkWriter(ABC):
     def __init__(self, quality, dimension=DimensionType.DIM_2D):
-        self._image_quality = quality
+        # translate inversed range [1:100] to [0:51]
+        self._image_quality = round(51 * (100 - quality) / 99)
         self._dimension = dimension
 
     @staticmethod
@@ -373,12 +374,27 @@ class ZipCompressedChunkWriter(IChunkWriter):
         return image_sizes
 
 class Mpeg4ChunkWriter(IChunkWriter):
-    def __init__(self, _):
+    def __init__(self, quality=67):
-        super().__init__(17)
+        super().__init__(quality)
         self._output_fps = 25
-
+        try:
-    @staticmethod
+            codec = av.codec.Codec('libopenh264', 'w')
-    def _create_av_container(path, w, h, rate, options, f='mp4'):
+            self._codec_name = codec.name
+            self._codec_opts = {
+                'profile': 'constrained_baseline',
+                'qmin': str(self._image_quality),
+                'qmax': str(self._image_quality),
+                'rc_mode': 'buffer',
+            }
+        except av.codec.codec.UnknownCodecError:
+            codec = av.codec.Codec('libx264', 'w')
+            self._codec_name = codec.name
+            self._codec_opts = {
+                "crf": str(self._image_quality),
+                "preset": "ultrafast",
+            }
+
+    def _create_av_container(self, path, w, h, rate, options, f='mp4'):
             # x264 requires width and height must be divisible by 2 for yuv420p
             if h % 2:
                 h += 1
@@ -386,7 +402,7 @@ class Mpeg4ChunkWriter(IChunkWriter):
                 w += 1
 
             container = av.open(path, 'w',format=f)
-            video_stream = container.add_stream('libopenh264', rate=rate)
+            video_stream = container.add_stream(self._codec_name, rate=rate)
             video_stream.pix_fmt = "yuv420p"
             video_stream.width = w
             video_stream.height = h
@@ -406,12 +422,7 @@ class Mpeg4ChunkWriter(IChunkWriter):
             w=input_w,
             h=input_h,
             rate=self._output_fps,
-            options={
+            options=self._codec_opts,
-                'profile': 'constrained_baseline',
-                'qmin': str(self._image_quality),
-                'qmax': str(self._image_quality),
-                'rc_mode': 'buffer',
-            },
         )
 
         self._encode_images(images, output_container, output_v_stream)
@@ -434,10 +445,15 @@ class Mpeg4ChunkWriter(IChunkWriter):
 
 class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter):
     def __init__(self, quality):
-        # translate inversed range [1:100] to [0:51]
+        super().__init__(quality)
-        self._image_quality = round(51 * (100 - quality) / 99)
+        if self._codec_name == 'libx264':
-        self._output_fps = 25
+            self._codec_opts = {
-
+                'profile': 'baseline',
+                'coder': '0',
+                'crf': str(self._image_quality),
+                'wpredp': '0',
+                'flags': '-loop',
+            }
 
     def save_as_chunk(self, images, chunk_path):
         if not images:
@@ -458,12 +474,7 @@ class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter):
             w=output_w,
             h=output_h,
             rate=self._output_fps,
-            options={
+            options=self._codec_opts,
-                'profile': 'constrained_baseline',
-                'qmin': str(self._image_quality),
-                'qmax': str(self._image_quality),
-                'rc_mode': 'buffer',
-            },
         )
 
         self._encode_images(images, output_container, output_v_stream)

cvat/apps/engine/task.py

@@ -10,9 +10,9 @@ from re import findall
 import rq
 import shutil
 from traceback import print_exception
-from urllib import error as urlerror
 from urllib import parse as urlparse
 from urllib import request as urlrequest
+import requests
 
 from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter, ValidateDimension
 from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice, RelatedFile
@@ -195,20 +195,16 @@ def _download_data(urls, upload_dir):
         job.meta['status'] = '{} is being downloaded..'.format(url)
         job.save_meta()
 
-        req = urlrequest.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        response = requests.get(url, stream=True)
-        try:
+        if response.status_code == 200:
-            with urlrequest.urlopen(req) as fp, open(os.path.join(upload_dir, name), 'wb') as tfp:
+            response.raw.decode_content = True
-                while True:
+            with open(os.path.join(upload_dir, name), 'wb') as output_file:
-                    block = fp.read(8192)
+                shutil.copyfileobj(response.raw, output_file)
-                    if not block:
+        else:
-                        break
+            raise Exception("Failed to download " + url)
-                    tfp.write(block)
-        except urlerror.HTTPError as err:
-            raise Exception("Failed to download " + url + ". " + str(err.code) + ' - ' + err.reason)
-        except urlerror.URLError as err:
-            raise Exception("Invalid URL: " + url + ". " + err.reason)
 
         local_files[name] = True
+
     return list(local_files.keys())
 
 @transaction.atomic
@@ -298,13 +294,20 @@ def _create_thread(tid, data):
         update_progress.call_counter = (update_progress.call_counter + 1) % len(progress_animation)
 
     compressed_chunk_writer_class = Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter
-    original_chunk_writer_class = Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter
+    if db_data.original_chunk_type == DataChoice.VIDEO:
+        original_chunk_writer_class = Mpeg4ChunkWriter
+        # Let's use QP=17 (that is 67 for 0-100 range) for the original chunks, which should be visually lossless or nearly so.
+        # A lower value will significantly increase the chunk size with a slight increase of quality.
+        original_quality = 67
+    else:
+        original_chunk_writer_class = ZipChunkWriter
+        original_quality = 100
 
     kwargs = {}
     if validate_dimension.dimension == DimensionType.DIM_3D:
         kwargs["dimension"] = validate_dimension.dimension
     compressed_chunk_writer = compressed_chunk_writer_class(db_data.image_quality, **kwargs)
-    original_chunk_writer = original_chunk_writer_class(100)
+    original_chunk_writer = original_chunk_writer_class(original_quality)
 
     # calculate chunk size if it isn't specified
     if db_data.chunk_size is None: