Allow server debugging when deployed with Docker (#5445)

Closes https://github.com/opencv/cvat/issues/5327

- Added a way to debug the server in Docker-based deployments
- Added docs

.vscode/launch.json

@@ -4,6 +4,66 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "REST API tests: Attach to server",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "127.0.0.1",
+                "port": 9090
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/home/django/"
+                },
+                {
+                    "localRoot": "${workspaceFolder}/.env",
+                    "remoteRoot": "/opt/venv",
+                }
+            ],
+            "justMyCode": false,
+        },
+        {
+            "name": "REST API tests: Attach to RQ low",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "127.0.0.1",
+                "port": 9091
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/home/django/"
+                },
+                {
+                    "localRoot": "${workspaceFolder}/.env",
+                    "remoteRoot": "/opt/venv",
+                }
+            ],
+            "justMyCode": false,
+        },
+        {
+            "name": "REST API tests: Attach to RQ default",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "127.0.0.1",
+                "port": 9092
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/home/django/"
+                },
+                {
+                    "localRoot": "${workspaceFolder}/.env",
+                    "remoteRoot": "/opt/venv",
+                }
+            ],
+            "justMyCode": false,
+        },
         {
             "type": "pwa-chrome",
             "request": "launch",
@@ -48,7 +108,6 @@
                 "CVAT_SERVERLESS": "1",
                 "ALLOWED_HOSTS": "*",
                 "IAM_OPA_BUNDLE": "1"
-
             },
             "args": [
                 "runserver",

CHANGELOG.md

@@ -23,6 +23,7 @@ from online detectors & interactors) (<https://github.com/opencv/cvat/pull/4543>
 - Propagation backward on UI (<https://github.com/opencv/cvat/pull/5355>)
 - A PyTorch dataset adapter layer in the SDK
   (<https://github.com/opencv/cvat/pull/5417>)
+- A way to debug the server deployed with Docker (<https://github.com/opencv/cvat/issues/5327>)
 
 ### Changed
 - `api/docs`, `api/swagger`, `api/schema`, `server/about` endpoints now allow unauthorized access (<https://github.com/opencv/cvat/pull/4928>, <https://github.com/opencv/cvat/pull/4935>)

Dockerfile

@@ -148,6 +148,14 @@ ENV PATH="/opt/venv/bin:${PATH}"
 ENV NUMPROCS=1
 COPY --from=build-image /opt/ffmpeg /usr
 
+# These variables are required for supervisord substitutions in files
+# This library allows remote python debugging with VS Code
+ARG CVAT_DEBUG_ENABLED='no'
+ENV CVAT_DEBUG_PORT=''
+RUN if [ "${CVAT_DEBUG_ENABLED}" = 'yes' ]; then \
+        python3 -m pip install --no-cache-dir debugpy; \
+    fi
+
 # Install and initialize CVAT, copy all necessary files
 COPY --chown=${USER} components /tmp/components
 COPY --chown=${USER} supervisord/ ${HOME}/supervisord

cvat/wsgi.py

@@ -1,5 +1,6 @@
 
 # Copyright (C) 2018-2022 Intel Corporation
+# Copyright (C) 2022 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -20,3 +21,48 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cvat.settings.{}" \
     .format(os.environ.get("DJANGO_CONFIGURATION", "development")))
 
 application = get_wsgi_application()
+
+
+if os.environ.get('CVAT_DEBUG_ENABLED') == 'yes':
+    import debugpy
+
+    class Debugger:
+        """
+        Support for VS code debugger
+
+        Read docs: https://github.com/microsoft/debugpy
+        Read more: https://modwsgi.readthedocs.io/en/develop/user-guides/debugging-techniques.html
+        """
+
+        ENV_VAR_PORT = 'CVAT_DEBUG_PORT'
+        ENV_VAR_WAIT = 'CVAT_DEBUG_WAIT'
+
+        def __init__(self, obj):
+            self.__object = obj
+
+            port = int(os.environ[self.ENV_VAR_PORT])
+
+            # The only intended use is in Docker.
+            # Using 127.0.0.1 will not allow host connections
+            addr = ('0.0.0.0', port)  # nosec - B104:hardcoded_bind_all_interfaces
+
+            try:
+                # Debugpy is a singleton
+                # We put it in the main thread of the process and then report new threads
+                debugpy.listen(addr)
+
+                # In most cases it makes no sense to debug subprocesses
+                # Feel free to enable if needed.
+                debugpy.configure({"subProcess": False})
+
+                if os.environ.get(self.ENV_VAR_WAIT) == 'yes':
+                    debugpy.wait_for_client()
+            except Exception as ex:
+                print("failed to set debugger:", ex)
+
+        def __call__(self, *args, **kwargs):
+            debugpy.debug_this_thread()
+
+            return self.__object(*args, **kwargs)
+
+    application = Debugger(application)

docker-compose.dev.yml

@@ -1,5 +1,6 @@
 #
 # Copyright (C) 2021-2022 Intel Corporation
+# Copyright (C) 2022 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 #
@@ -15,6 +16,39 @@ services:
         socks_proxy:
         CLAM_AV:
         INSTALL_SOURCES:
+        CVAT_DEBUG_ENABLED:
+    command: '-c supervisord/server.conf'
+    environment:
+      CVAT_DEBUG_ENABLED: '${CVAT_DEBUG_ENABLED:-no}'
+      CVAT_DEBUG_PORT: '9090'
+      # If 'yes', wait for a debugger connection on startup
+      CVAT_DEBUG_WAIT: '${CVAT_DEBUG_WAIT_CLIENT:-no}'
+    ports:
+      - '9090:9090'
+
+  cvat_worker_default:
+    command: -c supervisord/worker.default.conf
+    environment:
+      # For debugging, make sure to set 1 process
+      # Due to the supervisord specifics, the extra processes will fail and
+      # after few attempts supervisord will give up restarting, leaving only 1 process
+      # NUMPROCS: 1
+      CVAT_DEBUG_ENABLED: '${CVAT_DEBUG_ENABLED:-no}'
+      CVAT_DEBUG_PORT: '9092'
+    ports:
+      - '9092:9092'
+
+  cvat_worker_low:
+    command: -c supervisord/worker.low.conf
+    environment:
+      # For debugging, make sure to set 1 process
+      # Due to the supervisord specifics, the extra processes will fail and
+      # after few attempts supervisord will give up restarting, leaving only 1 process
+      # NUMPROCS: 1
+      CVAT_DEBUG_ENABLED: '${CVAT_DEBUG_ENABLED:-no}'
+      CVAT_DEBUG_PORT: '9091'
+    ports:
+      - '9091:9091'
 
   cvat_ui:
     build:

site/content/en/docs/contributing/running-tests.md

@@ -39,6 +39,7 @@ yarn run cypress:run:chrome:canvas3d
 # REST API, SDK and CLI tests
 
 **Initial steps**
+
 1. Follow [this guide](/site/content/en/docs/api_sdk/sdk/developer-guide/) to prepare
    `cvat-sdk` and `cvat-cli` source code
 1. Install all necessary requirements before running REST API tests:
@@ -73,6 +74,51 @@ If you need to rebuild your CVAT images add `--rebuild` option:
 pytest ./tests/python --rebuild
 ```
 
+**Debugging**
+
+Currently, this is only supported in docker-compose deployments, which should be
+enough to fix errors arising in REST API tests.
+
+To debug a server deployed with Docker, you need to do the following:
+
+Rebuild the images and start the test containers:
+
+```bash
+CVAT_DEBUG_ENABLED=yes pytest --rebuild --start-services tests/python
+```
+
+Now, you can use VS Code tasks to attach to the running server containers.
+To attach to a container, run one of the following tasks:
+- `REST API tests: Attach to server` for the server container
+- `REST API tests: Attach to RQ low` for the low priority queue worker
+- `REST API tests: Attach to RQ default` for the default priority queue worker
+
+> If you have a custom development environment setup, you need to adjust
+host-remote path mappings in the `.vscode/launch.json`:
+```json
+...
+"pathMappings": [
+   {
+      "localRoot": "${workspaceFolder}/my_venv",
+      "remoteRoot": "/opt/venv",
+   },
+   {
+      "localRoot": "/some/other/path",
+      "remoteRoot": "/some/container/path",
+   }
+]
+```
+
+Extra options:
+- If you want the server to wait for a debugger on startup,
+  use the `CVAT_DEBUG_WAIT_CLIENT` environment variable:
+  ```bash
+  CVAT_DEBUG_WAIT_CLIENT=yes pytest ...
+  ```
+- If you want to change the default debugging ports, check the `*_DEBUG_PORT`
+  variables in the `docker-compose.dev.yml`
+
+
 # Unit tests
 
 **Initial steps**

supervisord/worker.default.conf

@@ -23,8 +23,15 @@ priority=1
 autorestart=true
 
 [program:rqworker_default]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
-    "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 default"
+    if [ \"%(ENV_CVAT_DEBUG_ENABLED)s\" = 'yes' ]; then \
+        _DEBUG_CMD=\"-m debugpy --listen 0.0.0.0:%(ENV_CVAT_DEBUG_PORT)s\";
+    else
+        _DEBUG_CMD=\"\";
+    fi && \
+    \
+    exec python3 ${_DEBUG_CMD} %(ENV_HOME)s/manage.py rqworker -v 3 default \
+    "
 environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
 numprocs=%(ENV_NUMPROCS)s
 process_name=rqworker_default_%(process_num)s

supervisord/worker.low.conf

@@ -23,8 +23,15 @@ priority=1
 autorestart=true
 
 [program:rqworker_low]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
-    "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 low"
+    if [ \"%(ENV_CVAT_DEBUG_ENABLED)s\" = 'yes' ]; then \
+        _DEBUG_CMD=\"-m debugpy --listen 0.0.0.0:%(ENV_CVAT_DEBUG_PORT)s\";
+    else
+        _DEBUG_CMD=\"\";
+    fi && \
+    \
+    exec python3 ${_DEBUG_CMD} %(ENV_HOME)s/manage.py rqworker -v 3 low \
+    "
 environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
 numprocs=%(ENV_NUMPROCS)s