PostgresSQL DB v15 and health check endpoint (#5312)

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "7.2.2",
+  "version": "7.3.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "src/api.ts",
   "scripts": {

cvat-core/src/api-implementation.ts

@@ -107,6 +107,16 @@ export default function implementAPI(cvat) {
         return result;
     };
 
+    cvat.server.healthCheck.implementation = async (
+        maxRetries = 1,
+        checkPeriod = 3000,
+        requestTimeout = 5000,
+        progressCallback = undefined,
+    ) => {
+        const result = await serverProxy.server.healthCheck(maxRetries, checkPeriod, requestTimeout, progressCallback);
+        return result;
+    };
+
     cvat.server.request.implementation = async (url, data) => {
         const result = await serverProxy.server.request(url, data);
         return result;

cvat-core/src/api.ts

@@ -257,6 +257,26 @@ function build() {
                 const result = await PluginRegistry.apiWrapper(cvat.server.authorized);
                 return result;
             },
+            /**
+             * Method allows to health check the server
+             * @method healthCheck
+             * @async
+             * @memberof module:API.cvat.server
+             * @param {number} requestTimeout
+             * @returns {Object | undefined} response data if exist
+             * @throws {module:API.cvat.exceptions.PluginError}
+             * @throws {module:API.cvat.exceptions.ServerError}
+             */
+            async healthCheck(maxRetries = 1, checkPeriod = 3000, requestTimeout = 5000, progressCallback = undefined) {
+                const result = await PluginRegistry.apiWrapper(
+                    cvat.server.healthCheck,
+                    maxRetries,
+                    checkPeriod,
+                    requestTimeout,
+                    progressCallback,
+                );
+                return result;
+            },
             /**
              * Method allows to do requests via cvat-core with authorization headers
              * @method request

cvat-core/src/server-proxy.ts

@@ -468,6 +468,29 @@ async function authorized() {
     return true;
 }
 
+async function healthCheck(maxRetries, checkPeriod, requestTimeout, progressCallback, attempt = 0) {
+    const { backendAPI } = config;
+    const url = `${backendAPI}/server/health/?format=json`;
+
+    if (progressCallback) {
+        progressCallback(`${attempt}/${attempt + maxRetries}`);
+    }
+
+    return Axios.get(url, {
+        proxy: config.proxy,
+        timeout: requestTimeout,
+    })
+        .then((response) => response.data)
+        .catch((errorData) => {
+            if (maxRetries > 0) {
+                return new Promise((resolve) => setTimeout(resolve, checkPeriod))
+                    .then(() => healthCheck(maxRetries - 1, checkPeriod,
+                        requestTimeout, progressCallback, attempt + 1));
+            }
+            throw generateError(errorData);
+        });
+}
+
 async function serverRequest(url, data) {
     try {
         return (
@@ -2227,6 +2250,7 @@ export default Object.freeze({
         requestPasswordReset,
         resetPassword,
         authorized,
+        healthCheck,
         register,
         request: serverRequest,
         userAgreements,

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.44.4",
+  "version": "1.45.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/components/cvat-app.tsx

@@ -11,6 +11,8 @@ import Layout from 'antd/lib/layout';
 import Modal from 'antd/lib/modal';
 import notification from 'antd/lib/notification';
 import Spin from 'antd/lib/spin';
+import { DisconnectOutlined } from '@ant-design/icons';
+import Space from 'antd/lib/space';
 import Text from 'antd/lib/typography/Text';
 import 'antd/dist/antd.css';
 
@@ -64,6 +66,7 @@ import showPlatformNotification, {
     showUnsupportedNotification,
 } from 'utils/platform-checker';
 import '../styles.scss';
+import consts from 'consts';
 import EmailConfirmationPage from './email-confirmation-pages/email-confirmed';
 import EmailVerificationSentPage from './email-confirmation-pages/email-verification-sent';
 import IncorrectEmailConfirmationPage from './email-confirmation-pages/incorrect-email-confirmation';
@@ -103,10 +106,23 @@ interface CVATAppProps {
     isModelPluginActive: boolean;
 }
 
-class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentProps> {
+interface CVATAppState {
+    healthIinitialized: boolean;
+    backendIsHealthy: boolean;
+}
+
+class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentProps, CVATAppState> {
+    constructor(props: CVATAppProps & RouteComponentProps) {
+        super(props);
+
+        this.state = {
+            healthIinitialized: false,
+            backendIsHealthy: false,
+        };
+    }
     public componentDidMount(): void {
         const core = getCore();
-        const { verifyAuthorized, history, location } = this.props;
+        const { history, location } = this.props;
         // configure({ ignoreRepeatedEventsWhenKeyHeldDown: false });
 
         // Logger configuration
@@ -121,7 +137,46 @@ class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentP
             customWaViewHit(_location.pathname, _location.search, _location.hash);
         });
 
-        verifyAuthorized();
+        const {
+            HEALH_CHECK_RETRIES, HEALTH_CHECK_PERIOD, HEALTH_CHECK_REQUEST_TIMEOUT, UPGRADE_GUIDE_URL,
+        } = consts;
+        core.server.healthCheck(
+            HEALH_CHECK_RETRIES,
+            HEALTH_CHECK_PERIOD,
+            HEALTH_CHECK_REQUEST_TIMEOUT,
+        ).then(() => {
+            this.setState({
+                healthIinitialized: true,
+                backendIsHealthy: true,
+            });
+        })
+            .catch(() => {
+                this.setState({
+                    healthIinitialized: true,
+                    backendIsHealthy: false,
+                });
+                Modal.error({
+                    title: 'Cannot connect to the server',
+                    className: 'cvat-modal-cannot-connect-server',
+                    closable: false,
+                    content: (
+                        <Text>
+                            Make sure the CVAT backend and all necessary services
+                            (Database, Redis and Open Policy Agent) are running and avaliable.
+                            If you upgraded from version 2.2.0 or earlier, manual actions may be needed, see the&nbsp;
+
+                            <a
+                                target='_blank'
+                                rel='noopener noreferrer'
+                                href={UPGRADE_GUIDE_URL}
+                            >
+                                Upgrade Guide
+                            </a>
+                            .
+                        </Text>
+                    ),
+                });
+            });
 
         const {
             name, version, engine, os,
@@ -198,6 +253,12 @@ class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentP
             isModelPluginActive,
         } = this.props;
 
+        const { backendIsHealthy } = this.state;
+
+        if (!backendIsHealthy) {
+            return;
+        }
+
         this.showErrors();
         this.showMessages();
 
@@ -332,6 +393,8 @@ class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentP
             isModelPluginActive,
         } = this.props;
 
+        const { healthIinitialized, backendIsHealthy } = this.state;
+
         const notRegisteredUserInitialized = (userInitialized && (user == null || !user.isVerified));
         let readyForRender = userAgreementsInitialized && authActionsInitialized;
         readyForRender = readyForRender && (notRegisteredUserInitialized ||
@@ -456,7 +519,15 @@ class CVATApplication extends React.PureComponent<CVATAppProps & RouteComponentP
             );
         }
 
-        return <Spin size='large' className='cvat-spinner' />;
+        if (healthIinitialized && !backendIsHealthy) {
+            return (
+                <Space align='center' direction='vertical' className='cvat-spinner'>
+                    <DisconnectOutlined className='cvat-disconnected' />
+                    Cannot connect to the server.
+                </Space>
+            );
+        }
+        return <Spin size='large' className='cvat-spinner' tip='Connecting...' />;
     }
 }
 

cvat-ui/src/consts.ts

@@ -12,6 +12,7 @@ const DISCORD_URL = 'https://discord.gg/fNR3eXfk6C';
 const GITHUB_URL = 'https://github.com/opencv/cvat';
 const GITHUB_IMAGE_URL = 'https://github.com/opencv/cvat/raw/develop/site/content/en/images/cvat.jpg';
 const GUIDE_URL = 'https://opencv.github.io/cvat/docs';
+const UPGRADE_GUIDE_URL = 'https://opencv.github.io/cvat/docs/administration/advanced/upgrade_guide';
 const SHARE_MOUNT_GUIDE_URL =
     'https://opencv.github.io/cvat/docs/administration/basics/installation/#share-path';
 const NUCLIO_GUIDE =
@@ -83,6 +84,10 @@ const DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS: string[][] = [
     ['NAM4', 'US-CENTRAL1 and US-EAST1'],
 ];
 
+const HEALH_CHECK_RETRIES = 10;
+const HEALTH_CHECK_PERIOD = 3000; // ms
+const HEALTH_CHECK_REQUEST_TIMEOUT = 5000; // ms
+
 export default {
     UNDEFINED_ATTRIBUTE_VALUE,
     NO_BREAK_SPACE,
@@ -93,6 +98,7 @@ export default {
     GITHUB_URL,
     GITHUB_IMAGE_URL,
     GUIDE_URL,
+    UPGRADE_GUIDE_URL,
     SHARE_MOUNT_GUIDE_URL,
     CANVAS_BACKGROUND_COLORS,
     NEW_LABEL_COLOR,
@@ -105,4 +111,7 @@ export default {
     DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS,
     OUTSIDE_PIC_URL,
     DATASET_MANIFEST_GUIDE_URL,
+    HEALH_CHECK_RETRIES,
+    HEALTH_CHECK_PERIOD,
+    HEALTH_CHECK_REQUEST_TIMEOUT,
 };

cvat-ui/src/styles.scss

@@ -17,6 +17,10 @@ hr {
     transform: translate(-50%, -50%);
 }
 
+.cvat-disconnected {
+    font-size: 36px;
+}
+
 .cvat-spinner-container {
     position: absolute;
     background: $background-color-1;

cvat/apps/health/__init__.py

@@ -0,0 +1,3 @@
+#  Copyright (C) 2022 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/health/apps.py

@@ -0,0 +1,14 @@
+# Copyright (C) 2022 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.apps import AppConfig
+
+from health_check.plugins import plugin_dir
+
+class HealthConfig(AppConfig):
+    name = 'cvat.apps.health'
+
+    def ready(self):
+        from .backends import OPAHealthCheck
+        plugin_dir.register(OPAHealthCheck)

cvat/apps/health/backends.py

@@ -0,0 +1,24 @@
+# Copyright (C) 2022 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import requests
+
+from health_check.backends import BaseHealthCheckBackend
+from health_check.exceptions import HealthCheckException
+
+from django.conf import settings
+
+class OPAHealthCheck(BaseHealthCheckBackend):
+    critical_service = True
+
+    def check_status(self):
+        opa_health_url = f'{settings.IAM_OPA_HOST}/health?bundles'
+        try:
+            response = requests.get(opa_health_url)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise HealthCheckException(str(e))
+
+    def identifier(self):
+        return self.__class__.__name__

cvat/requirements/base.txt

@@ -51,3 +51,5 @@ natsort==8.0.0
 mistune>=2.0.1 # not directly required, pinned by Snyk to avoid a vulnerability
 dnspython==2.2.0
 setuptools==65.5.1
+django-health-check==3.17.0
+psutil==5.9.4

cvat/settings/base.py

@@ -124,6 +124,11 @@ INSTALLED_APPS = [
     'allauth.socialaccount.providers.github',
     'allauth.socialaccount.providers.google',
     'dj_rest_auth.registration',
+    'health_check',
+    'health_check.db',
+    'health_check.cache',
+    'health_check.contrib.migrations',
+    'health_check.contrib.psutil',
     'cvat.apps.iam',
     'cvat.apps.dataset_manager',
     'cvat.apps.organizations',
@@ -132,6 +137,7 @@ INSTALLED_APPS = [
     'cvat.apps.lambda_manager',
     'cvat.apps.opencv',
     'cvat.apps.webhooks',
+    'cvat.apps.health',
 ]
 
 SITE_ID = 1
@@ -246,7 +252,8 @@ IAM_DEFAULT_ROLES = ['user']
 IAM_ADMIN_ROLE = 'admin'
 # Index in the list below corresponds to the priority (0 has highest priority)
 IAM_ROLES = [IAM_ADMIN_ROLE, 'business', 'user', 'worker']
-IAM_OPA_DATA_URL = 'http://opa:8181/v1/data'
+IAM_OPA_HOST = 'http://opa:8181'
+IAM_OPA_DATA_URL = f'{IAM_OPA_HOST}/v1/data'
 LOGIN_URL = 'rest_login'
 LOGIN_REDIRECT_URL = '/'
 

cvat/urls.py

@@ -45,3 +45,6 @@ if apps.is_installed('cvat.apps.webhooks'):
 
 if apps.is_installed('silk'):
     urlpatterns.append(path('profiler/', include('silk.urls')))
+
+if apps.is_installed('health_check'):
+    urlpatterns.append(path('api/server/health/', include('health_check.urls')))

docker-compose.yml

@@ -7,7 +7,7 @@ version: '3.3'
 services:
   cvat_db:
     container_name: cvat_db
-    image: postgres:10-alpine
+    image: postgres:15-alpine
     restart: always
     environment:
       POSTGRES_USER: root
@@ -161,7 +161,7 @@ services:
       - cvat
 
   traefik:
-    image: traefik:v2.4
+    image: traefik:v2.9
     container_name: traefik
     restart: always
     command:

site/content/en/docs/administration/advanced/upgrade_guide.md

@@ -9,6 +9,9 @@ description: 'Instructions for upgrading CVAT deployed with docker compose'
 
 ## Upgrade guide
 
+Note: updating CVAT from version 2.2.0 to version 2.3.0 requires additional manual actions with database data due to
+upgrading PostgreSQL base image major version. See details [here](#how-to-upgrade-postgresql-database-base-image)
+
 To upgrade CVAT, follow these steps:
 
 - It is highly recommended backup all CVAT data before updating, follow the
@@ -69,3 +72,43 @@ docker pull cvat/ui:v2.1.0
 docker tag cvat/ui:v2.1.0 openvino/cvat_ui:latest
 docker-compose up -d
 ```
+## How to upgrade PostgreSQL database base image
+
+1. It is highly recommended backup all CVAT data before updating, follow the
+   [backup guide](/docs/administration/advanced/backup_guide/) and backup CVAT database volume.
+
+1. Run previosly used CVAT version as usual
+
+1. Backup current database with `pg_dumpall` tool:
+   ```shell
+   docker exec -it cvat_db pg_dumpall > cvat.db.dump
+   ```
+
+1. Stop CVAT:
+   ```shell
+   docker-compose down
+   ```
+
+1. Delete current PostrgeSQL’s volume, that's why it's important to have a backup:
+   ```shell
+   docker volume rm cvat_cvat_db
+   ```
+
+1. Update CVAT source code by any preferable way: clone with git or download zip file from GitHub.
+   Check the
+   [installation guide](/docs/administration/basics/installation/#how-to-get-cvat-source-code) for details.
+
+1. Start database container only:
+   ```shell
+   docker-compose up -d cvat_db
+   ```
+
+1. Import PostgreSQL dump into new DB container:
+   ```shell
+   docker exec -i cvat_db psql -q -d postgres < cvat.db.dump
+   ```
+
+1. Start CVAT:
+   ```shell
+   docker-compose up -d
+   ```

site/content/en/docs/administration/basics/installation.md

@@ -345,6 +345,21 @@ unzip v1.7.0.zip && mv cvat-1.7.0 cvat
 cd cvat
 ```
 
+### CVAT healthcheck command
+The following command allows to test the CVAT container to make sure it works.
+```shell
+docker exec -t cvat_server python manage.py health_check
+```
+Expected output of a healthy CVAT container:
+```shell
+Cache backend: default   ... working
+DatabaseBackend          ... working
+DiskUsage                ... working
+MemoryUsage              ... working
+MigrationsHealthCheck    ... working
+OPAHealthCheck           ... working
+```
+
 ### Deploying CVAT behind a proxy
 
 If you deploy CVAT behind a proxy and do not plan to use any of [serverless functions](#semi-automatic-and-automatic-annotation)

supervisord/server.conf

@@ -34,4 +34,4 @@ command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_POSTGRES_HOST)s:5432 -t 0 -- bash
     --limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \
     %(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s \
     --server-root /tmp/cvat-server"
-numprocs=%(ENV_NUMPROCS)s
+numprocs=%(ENV_NUMPROCS)s

tests/python/shared/assets/cvat_db/restore.sql

@@ -1,8 +1,11 @@
-SELECT pg_terminate_backend(pg_stat_activity.pid)
-FROM pg_stat_activity
-WHERE pg_stat_activity.datname = 'cvat' AND pid <> pg_backend_pid();
+SELECT :'from' = 'cvat' AS fromcvat \gset
 
-DROP DATABASE IF EXISTS :to;
+\if :fromcvat
+    SELECT pg_terminate_backend(pg_stat_activity.pid)
+    FROM pg_stat_activity
+    WHERE pg_stat_activity.datname = 'cvat' AND pid <> pg_backend_pid();
+\endif
 
-CREATE DATABASE :to WITH TEMPLATE :from;
+DROP DATABASE IF EXISTS :to WITH (FORCE);
 
+CREATE DATABASE :to WITH TEMPLATE :from;