OpenH264 encoder (#2562)

* use Cisco openh264 encoder instead libx264 * installation of source code * multistage build * Reduced the size of docker image * updated travis config * set ubuntu 20.04 as build env
5 years ago · 04ac49a738
parent 8343dd776e
commit 04ac49a738
6 changed files with 122 additions and 68 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,7 +1,5 @@
-language: python
-
-python:
-  - '3.5'
+language: generic
+dist: focal

 cache:
  npm: true
--- a/137
+++ b/137
@ -1,3 +1,52 @@
+FROM ubuntu:20.04 as build-image
+
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+ARG socks_proxy
+ARG DJANGO_CONFIGURATION
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \
+        apache2-dev \
+        build-essential \
+        curl \
+        libldap2-dev \
+        libsasl2-dev \
+        nasm \
+        git \
+        pkg-config \
+        python3-dev \
+        python3-pip \
+        python3-venv && \
+    rm -rf /var/lib/apt/lists/*
+
+# Compile Openh264 and FFmpeg
+ARG PREFIX=/opt/ffmpeg
+ARG PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig
+
+ENV FFMPEG_VERSION=4.3.1 \
+    OPENH264_VERSION=2.1.1
+
+WORKDIR /tmp/openh264
+RUN curl -sL https://github.com/cisco/openh264/archive/v${OPENH264_VERSION}.tar.gz --output openh264-${OPENH264_VERSION}.tar.gz && \
+    tar -zx --strip-components=1 -f openh264-${OPENH264_VERSION}.tar.gz && \
+    make -j5 && make install PREFIX=${PREFIX} && make clean
+
+WORKDIR /tmp/ffmpeg
+RUN curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+    tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+    ./configure --disable-nonfree --disable-gpl --enable-libopenh264 --enable-shared --disable-static --prefix="${PREFIX}" && \
+    make -j5 && make install && make distclean
+
+# Install requirements
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:${PATH}"
+RUN python3 -m pip install --no-cache-dir -U pip==20.0.1 setuptools==49.6.0 wheel==0.35.1
+COPY cvat/requirements/ /tmp/requirements/
+RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
+
+
 FROM ubuntu:20.04

 ARG http_proxy
@ -21,45 +70,42 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION}

 # Install necessary apt packages
 RUN apt-get update && \
-    apt-get --no-install-recommends install -yq \
-        software-properties-common && \
-    apt-get update && \
    DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \
        apache2 \
-        apache2-dev \
-        apt-utils \
-        build-essential \
        libapache2-mod-xsendfile \
        supervisor \
-        libavcodec-dev=7:4.2.4-1ubuntu0.1 \
-        libavdevice-dev=7:4.2.4-1ubuntu0.1 \
-        libavfilter-dev=7:4.2.4-1ubuntu0.1 \
-        libavformat-dev=7:4.2.4-1ubuntu0.1 \
-        libavutil-dev=7:4.2.4-1ubuntu0.1 \
-        libswresample-dev=7:4.2.4-1ubuntu0.1 \
-        libswscale-dev=7:4.2.4-1ubuntu0.1 \
-        libldap2-dev \
-        libsasl2-dev \
-        pkg-config \
-        python3-dev \
-        python3-pip \
+        libldap-2.4-2 \
+        libsasl2-2 \
+        libpython3-dev \
        tzdata \
+        python3-distutils \
        p7zip-full \
        git \
        git-lfs \
-        ssh \
        poppler-utils \
+        ssh \
        curl && \
-    python3 -m pip install --no-cache-dir -U pip==20.0.1 setuptools==49.6.0 wheel==0.35.1 && \
    ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \
    dpkg-reconfigure -f noninteractive tzdata && \
    rm -rf /var/lib/apt/lists/* && \
    echo 'application/wasm wasm' >> /etc/mime.types

+ARG CLAM_AV
+ENV CLAM_AV=${CLAM_AV}
+RUN if [ "$CLAM_AV" = "yes" ]; then \
+        apt-get update && \
+        apt-get --no-install-recommends install -yq \
+            clamav \
+            libclamunrar9 && \
+        sed -i 's/ReceiveTimeout 30/ReceiveTimeout 300/g' /etc/clamav/freshclam.conf && \
+        freshclam && \
+        chown -R ${USER}:${USER} /var/lib/clamav && \
+        rm -rf /var/lib/apt/lists/*; \
+    fi
+
 # Add a non-root user
 ENV USER=${USER}
 ENV HOME /home/${USER}
-WORKDIR ${HOME}
 RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \
    if [ -z ${socks_proxy} ]; then \
        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
@ -67,40 +113,41 @@ RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \
        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
    fi

-COPY components /tmp/components
-
-# Install and initialize CVAT, copy all necessary files
-COPY cvat/requirements/ /tmp/requirements/
-COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
-RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
-
-ARG CLAM_AV
-ENV CLAM_AV=${CLAM_AV}
-RUN if [ "$CLAM_AV" = "yes" ]; then \
+ARG INSTALL_SOURCES='no'
+WORKDIR ${HOME}/sources
+RUN if [ "$INSTALL_SOURCES" = "yes" ]; then \
+        sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list && \
        apt-get update && \
-        apt-get --no-install-recommends install -yq \
-            clamav \
-            libclamunrar9 && \
-        sed -i 's/ReceiveTimeout 30/ReceiveTimeout 300/g' /etc/clamav/freshclam.conf && \
-        freshclam && \
-        chown -R ${USER}:${USER} /var/lib/clamav && \
+        dpkg --get-selections | while read -r line; do        \
+            package=$(echo "$line" | awk '{print $1}');       \
+            mkdir "$package";                                 \
+            (                                                 \
+                cd "$package";                                \
+                apt-get -q --download-only source "$package"; \
+            )                                                 \
+            done &&                                           \
        rm -rf /var/lib/apt/lists/*;                          \
    fi
+COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.bz2 ${HOME}/sources/

-COPY ssh ${HOME}/.ssh
-COPY utils ${HOME}/utils
-COPY cvat/ ${HOME}/cvat
-COPY cvat-core/ ${HOME}/cvat-core
-COPY cvat-data/ ${HOME}/cvat-data
-COPY tests ${HOME}/tests
+# Copy python virtual enviroment and FFmpeg binaries from build-image
+COPY --from=build-image /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:${PATH}"
+COPY --from=build-image /opt/ffmpeg /usr

-RUN chown -R ${USER}:${USER} .
+# Install and initialize CVAT, copy all necessary files
+COPY --chown=${USER} components /tmp/components
+COPY --chown=${USER} ssh ${HOME}/.ssh
+COPY --chown=${USER} supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
+COPY --chown=${USER} cvat/ ${HOME}/cvat
+COPY --chown=${USER} utils/ ${HOME}/utils

 # RUN all commands below as 'django' user
 USER ${USER}
+WORKDIR ${HOME}

 RUN mkdir data share media keys logs /tmp/supervisord
 RUN python3 manage.py collectstatic

-EXPOSE 8080 8443
+EXPOSE 8080
 ENTRYPOINT ["/usr/bin/supervisord"]
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@ -6,6 +6,7 @@ USER root
 RUN apt-get update && \
    DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \
        gpg-agent \
+        gnupg2 \
        apt-utils \
        build-essential \
        python3-dev \
@ -20,10 +21,17 @@ RUN apt-get update && \
        && \
    rm -rf /var/lib/apt/lists/*;

-RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt && \
+COPY cvat/requirements/ /tmp/requirements/
+
+RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt && \
    python3 -m pip install --no-cache-dir coveralls
 RUN gem install coveralls-lcov

+COPY utils ${HOME}/utils
+COPY cvat-core ${HOME}/cvat-core
+COPY cvat-data ${HOME}/cvat-data
+COPY tests ${HOME}/tests
+
 COPY .coveragerc .

 ENTRYPOINT []
--- a/cvat/apps/engine/media_extractors.py
+++ b/cvat/apps/engine/media_extractors.py
@ -348,7 +348,7 @@ class Mpeg4ChunkWriter(IChunkWriter):
                w += 1

            container = av.open(path, 'w',format=f)
-            video_stream = container.add_stream('libx264', rate=rate)
+            video_stream = container.add_stream('libopenh264', rate=rate)
            video_stream.pix_fmt = "yuv420p"
            video_stream.width = w
            video_stream.height = h
@ -369,8 +369,10 @@ class Mpeg4ChunkWriter(IChunkWriter):
            h=input_h,
            rate=self._output_fps,
            options={
-                "crf": str(self._image_quality),
-                "preset": "ultrafast",
+                'profile': 'constrained_baseline',
+                'qmin': str(self._image_quality),
+                'qmax': str(self._image_quality),
+                'rc_mode': 'buffer',
            },
        )

@ -419,11 +421,10 @@ class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter):
            h=output_h,
            rate=self._output_fps,
            options={
-                'profile': 'baseline',
-                'coder': '0',
-                'crf': str(self._image_quality),
-                'wpredp': '0',
-                'flags': '-loop'
+                'profile': 'constrained_baseline',
+                'qmin': str(self._image_quality),
+                'qmax': str(self._image_quality),
+                'rc_mode': 'buffer',
            },
        )

--- a/cvat/requirements/base.txt
+++ b/cvat/requirements/base.txt
@ -34,7 +34,7 @@ Shapely==1.7.1
 pdf2image==1.14.0
 django-rest-auth[with_social]==0.9.5
 cython==0.29.21
-opencv-python==4.4.0.42
+opencv-python-headless==4.4.0.42
 h5py==2.10.0
 django-cors-headers==3.5.0
 furl==2.1.0
@ -44,4 +44,4 @@ tensorflow==2.2.1 # Optional requirement of Datumaro
 # archives. Don't use as a python module because it has GPL license.
 patool==1.12
 diskcache==5.0.2
-git+https://github.com/openvinotoolkit/datumaro@v0.1.3
+git+https://github.com/openvinotoolkit/datumaro@v0.1.4
--- a/supervisord.conf
+++ b/supervisord.conf
@ -24,26 +24,26 @@ autorestart=true

 [program:rqworker_default]
 command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \
-    "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default"
+    "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 default"
 environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
 numprocs=2
 process_name=rqworker_default_%(process_num)s

 [program:rqworker_low]
 command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \
-    "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low"
+    "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 low"
 environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
 numprocs=1

 [program:git_status_updater]
 command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \
-    "/usr/bin/python3 ~/manage.py update_git_states"
+    "python3 ~/manage.py update_git_states"
 environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
 numprocs=1

 [program:rqscheduler]
 command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \
-    "/usr/bin/python3 /usr/local/bin/rqscheduler --host %(ENV_CVAT_REDIS_HOST)s -i 30"
+    "python3 /opt/venv/bin/rqscheduler --host %(ENV_CVAT_REDIS_HOST)s -i 30"
 environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
 numprocs=1

@ -59,9 +59,9 @@ numprocs=1
 ; with docker cache. Thus it is necessary to run collectstatic here for such
 ; apps.
 command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_POSTGRES_HOST)s:5432 -t 0 -- bash -ic \
-    "rm -f /tmp/cvat-server/httpd.pid && /usr/bin/python3 ~/manage.py migrate && \
-    /usr/bin/python3 ~/manage.py collectstatic --no-input && \
-    exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \
+    "rm -f /tmp/cvat-server/httpd.pid && python3 ~/manage.py migrate && \
+    python3 ~/manage.py collectstatic --no-input && \
+    exec python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \
    --limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \
    %(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s \
    --server-root /tmp/cvat-server"