Cleanup Dockerfiles for CVAT (#1060)

* Replaced wget by curl * Moved CI stuff into Dockerfile.ci * Use docker-compose to run commnands inside docker (need environment variables) * Added patool again (to support different archive formats) * Roll back tensorflow version: 1.15 -> 1.13.1 Fixed https://github.com/opencv/cvat/issues/982 Fixed https://github.com/opencv/cvat/issues/1017 * datumaro install tensorflow 2.x now. It breaks automatic annotation using TF. * Follow redirects in curl (auto_segmentation)
6 years ago · a5c3da1235
parent 7cf755fd38
commit a5c3da1235
15 changed files with 84 additions and 81 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -9,9 +9,9 @@ services:
  - docker

 before_script:
-  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up --build -d
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml build

 script:
-  - docker exec -it cvat /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
-  - docker exec -it cvat /bin/bash -c 'python3 manage.py test datumaro/'
-  - docker exec -it cvat /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls'
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test datumaro/'
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls'
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -91,8 +91,8 @@ to changes in ``.env/bin/activate`` file are active.
 - Perform all steps in the automatic annotation section
 - Download ReID model and save it somewhere:
 ```sh
-    wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -O reid.xml
-    wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid.bin
+    curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid.xml
+    curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid.bin
 ```
 - Add next line to ``.env/bin/activate``:
 ```sh
@ -103,7 +103,7 @@ to changes in ``.env/bin/activate`` file are active.
 - Perform all steps in the automatic annotation section
 - Download Deep Extreme Cut model, unpack it, and save somewhere:
 ```sh
-wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O dextr.zip
+curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o dextr.zip
 unzip dextr.zip
 ```
 - Add next lines to ``.env/bin/activate``:
@ -115,7 +115,7 @@ unzip dextr.zip
 ### Tensorflow RCNN
 - Download RCNN model, unpack it, and save it somewhere:
 ```sh
-wget -O model.tar.gz http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz && \
+curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \
 tar -xzf model.tar.gz
 ```
 - Add next lines to ``.env/bin/activate``:
@ -127,7 +127,7 @@ tar -xzf model.tar.gz
 ### Tensorflow Mask RCNN
 - Download Mask RCNN model, and save it somewhere:
 ```sh
-wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5
+curl https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 -o mask_rcnn_coco.h5
 ```
 - Add next lines to ``.env/bin/activate``:
 ```sh
--- a/71
+++ b/71
@ -22,9 +22,7 @@ ENV DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION}
 # Install necessary apt packages
 RUN apt-get update && \
    apt-get install -yq \
-        python-software-properties \
-        software-properties-common \
-        wget && \
+        software-properties-common && \
    add-apt-repository ppa:mc3man/xerus-media -y && \
    add-apt-repository ppa:mc3man/gstffmpeg-keep -y && \
    apt-get update && \
@ -40,14 +38,19 @@ RUN apt-get update && \
        python3-dev \
        python3-pip \
        tzdata \
-        unzip \
-        unrar \
        p7zip-full \
-        git-core \
-        libsm6 \
-        libxext6 && \
-    python3 -m pip install -U pip && \
-    python3 -m pip install -U setuptools && \
+        git \
+        ssh \
+        poppler-utils \
+        curl && \
+    curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
+    apt-get install -y git-lfs && git lfs install && \
+    if [ -z ${socks_proxy} ]; then \
+        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
+    else \
+        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
+    fi && \
+    python3 -m pip install --no-cache-dir -U pip setuptools && \
    ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \
    dpkg-reconfigure -f noninteractive tzdata && \
    add-apt-repository --remove ppa:mc3man/gstffmpeg-keep -y && \
@ -69,8 +72,8 @@ ENV REID_MODEL_DIR=${HOME}/reid
 RUN if [ "$OPENVINO_TOOLKIT" = "yes" ]; then \
        /tmp/components/openvino/install.sh && \
        mkdir ${REID_MODEL_DIR} && \
-        wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -O reid/reid.xml && \
-        wget https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -O reid/reid.bin; \
+        curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.xml -o reid/reid.xml && \
+        curl https://download.01.org/openvinotoolkit/2018_R5/open_model_zoo/person-reidentification-retail-0079/FP32/person-reidentification-retail-0079.bin -o reid/reid.bin; \
    fi

 # Tensorflow annotation support
@ -89,29 +92,6 @@ RUN if [ "$AUTO_SEGMENTATION" = "yes" ]; then \
    bash -i /tmp/components/auto_segmentation/install.sh; \
    fi

-ARG WITH_TESTS
-RUN if [ "$WITH_TESTS" = "yes" ]; then \
-        wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
-        echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \
-        wget -qO- https://deb.nodesource.com/setup_9.x | bash - && \
-        apt-get update && \
-        DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-            google-chrome-stable \
-            nodejs && \
-        rm -rf /var/lib/apt/lists/*; \
-        mkdir tests && cd tests && npm install \
-            eslint \
-            eslint-detailed-reporter \
-            karma \
-            karma-chrome-launcher \
-            karma-coveralls \
-            karma-coverage \
-            karma-junit-reporter \
-            karma-qunit \
-            qunit; \
-        echo "export PATH=~/tests/node_modules/.bin:${PATH}" >> ~/.bashrc; \
-    fi
-
 # Install and initialize CVAT, copy all necessary files
 COPY cvat/requirements/ /tmp/requirements/
 COPY supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/
@ -119,23 +99,6 @@ RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGUR
 # pycocotools package is impossible to install with its dependencies by one pip install command
 RUN python3 -m pip install --no-cache-dir pycocotools==2.0.0

-# Install git application dependencies
-RUN apt-get update && \
-    apt-get install -y ssh netcat-openbsd git curl zip  && \
-    wget -qO /dev/stdout https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
-    apt-get install -y git-lfs && \
-    git lfs install && \
-    rm -rf /var/lib/apt/lists/* && \
-    if [ -z ${socks_proxy} ]; then \
-        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
-    else \
-        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
-    fi
-
-# Install poppler for working with pdfs
-RUN apt-get update && \
-    apt install -y poppler-utils && \
-    rm -rf /var/lib/apt/lists/*

 # CUDA support
 ARG CUDA_SUPPORT
@ -150,8 +113,8 @@ ENV WITH_DEXTR=${WITH_DEXTR}
 ENV DEXTR_MODEL_DIR=${HOME}/dextr
 RUN if [ "$WITH_DEXTR" = "yes" ]; then \
        mkdir ${DEXTR_MODEL_DIR} -p && \
-        wget https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -O ${DEXTR_MODEL_DIR}/dextr.zip && \
-        unzip ${DEXTR_MODEL_DIR}/dextr.zip -d ${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \
+        curl https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip -o ${DEXTR_MODEL_DIR}/dextr.zip && \
+        7z e ${DEXTR_MODEL_DIR}/dextr.zip -o${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \
    fi

 COPY ssh ${HOME}/.ssh
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@ -0,0 +1,32 @@
+FROM cvat
+
+ENV DJANGO_CONFIGURATION=testing
+USER root
+
+RUN curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | tee /etc/apt/sources.list.d/google-chrome.list && \
+    curl https://deb.nodesource.com/setup_9.x | bash - && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+        google-chrome-stable \
+        nodejs && \
+    rm -rf /var/lib/apt/lists/*;
+
+RUN python3 -m pip install --no-cache-dir -r /tmp/requirements/${DJANGO_CONFIGURATION}.txt
+
+# RUN all commands below as 'django' user
+USER ${USER}
+
+RUN mkdir -p tests && cd tests && npm install \
+        eslint \
+        eslint-detailed-reporter \
+        karma \
+        karma-chrome-launcher \
+        karma-coveralls \
+        karma-coverage \
+        karma-junit-reporter \
+        karma-qunit \
+        qunit; \
+    echo "export PATH=~/tests/node_modules/.bin:${PATH}" >> ~/.bashrc;
+
+ENTRYPOINT []
--- a/components/auto_segmentation/install.sh
+++ b/components/auto_segmentation/install.sh
@ -3,10 +3,11 @@

 set -e

+MASK_RCNN_URL=https://github.com/matterport/Mask_RCNN
+
 cd ${HOME} && \
-git clone https://github.com/matterport/Mask_RCNN.git && \
-wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 && \
-mv mask_rcnn_coco.h5 Mask_RCNN/mask_rcnn_coco.h5
+git clone ${MASK_RCNN_URL}.git && \
+curl -L ${MASK_RCNN_URL}/releases/download/v2.0/mask_rcnn_coco.h5 -o Mask_RCNN/mask_rcnn_coco.h5

 # TODO remove useless files
 # tensorflow and Keras are installed globally
--- a/components/cuda/install.sh
+++ b/components/cuda/install.sh
@ -30,8 +30,8 @@ apt-get update && apt-get install -y --no-install-recommends --allow-unauthentic
    libnccl2=$NCCL_VERSION-1+cuda9.0 \
    libcudnn7=$CUDNN_VERSION-1+cuda9.0 && \
    ln -s cuda-9.0 /usr/local/cuda && \
-rm -rf /var/lib/apt/lists/* \
+    rm -rf /var/lib/apt/lists/* \
    /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list

 python3 -m pip uninstall -y tensorflow
-python3 -m pip install --no-cache-dir tensorflow-gpu==1.15.0
+python3 -m pip install --no-cache-dir tensorflow-gpu==1.13.1
--- a/components/tf_annotation/install.sh
+++ b/components/tf_annotation/install.sh
@ -7,7 +7,7 @@
 set -e

 cd ${HOME} && \
-wget -O model.tar.gz http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz && \
+curl http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz -o model.tar.gz && \
 tar -xzf model.tar.gz && rm model.tar.gz && \
 mv faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28 ${HOME}/rcnn && cd ${HOME} && \
 mv rcnn/frozen_inference_graph.pb rcnn/inference_graph.pb
--- a/cvat/apps/documentation/installation.md
+++ b/cvat/apps/documentation/installation.md
@ -100,7 +100,7 @@ server. Proxy is an advanced topic and it is not covered by the guide.
    install it as well. Type commands below in a terminal window:

    ```sh
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
+    curl https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
    sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
    sudo apt-get update
    sudo apt-get install -y google-chrome-stable
--- a/cvat/requirements/base.txt
+++ b/cvat/requirements/base.txt
@ -39,10 +39,13 @@ django-rest-auth[with_social]==0.9.5
 cython==0.29.13
 matplotlib==3.0.3
 scikit-image==0.15.0
-tensorflow==1.15.0
+tensorflow==1.13.1
 keras==2.2.5
 opencv-python==4.1.0.25
 h5py==2.9.0
 imgaug==0.2.9
 django-cors-headers==3.2.0
 furl==2.0.0
+# The package is used by pyunpack as a command line tool to support multiple
+# archives. Don't use as a python module because it has GPL license.
+patool==1.12
--- a/cvat/requirements/testing.txt
+++ b/cvat/requirements/testing.txt
@ -1,2 +1,5 @@
 -r development.txt
-fakeredis==1.0.3
+fakeredis==1.1.0
+# Fix dependencies for fakeredis 1.1.0
+# Pip will not reinstall six package if it is installed already
+six==1.12.0
--- a/datumaro/datumaro/util/image.py
+++ b/datumaro/datumaro/util/image.py
@ -14,7 +14,7 @@ _IMAGE_BACKEND = None
 try:
    import cv2
    _IMAGE_BACKEND = _IMAGE_BACKENDS.cv2
-except ModuleNotFoundError:
+except ImportError:
    import PIL
    _IMAGE_BACKEND = _IMAGE_BACKENDS.PIL

--- a/datumaro/requirements.txt
+++ b/datumaro/requirements.txt
@ -2,10 +2,10 @@ Cython>=0.27.3 # include before pycocotools
 GitPython>=2.1.11
 lxml>=4.4.1
 matplotlib<3.1 # 3.1+ requires python3.6, but we have 3.5 in cvat
-opencv-python>=4.1.0.25
+opencv-python-headless>=4.1.0.25
 Pillow>=6.1.0
 pycocotools>=2.0.0
 PyYAML>=5.1.1
 scikit-image>=0.15.0
 tensorboardX>=1.8
-tensorflow>=1.12.0
+tensorflow==1.13.1
--- a/docker-compose.ci.yml
+++ b/docker-compose.ci.yml
@ -1,12 +1,14 @@
 version: "2.3"

 services:
-  cvat:
+  cvat_ci:
+    image: cvat_ci
    build:
-      args:
-        DJANGO_CONFIGURATION: "testing"
-        WITH_TESTS: "yes"
      network: host
+      context: .
+      dockerfile: Dockerfile.ci
+    depends_on:
+      - cvat
    environment:
      COVERALLS_REPO_TOKEN:
      TRAVIS:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -49,7 +49,6 @@ services:
        AUTO_SEGMENTATION: "no"
        USER: "django"
        DJANGO_CONFIGURATION: "production"
-        WITH_TESTS: "no"
        TZ: "Etc/UTC"
        OPENVINO_TOOLKIT: "no"
    environment:
--- a/utils/tfrecords/requirements.txt
+++ b/utils/tfrecords/requirements.txt
@ -1,3 +1,3 @@
 argparse==1.1
-tensorflow==1.15.0
+tensorflow==1.13.1
 pathlib==1.0.1