Init OPA rules with API instead of file binding (#5047)

Dockerfile

@@ -154,7 +154,9 @@ COPY --chown=${USER} cvat/ ${HOME}/cvat
 USER ${USER}
 WORKDIR ${HOME}
 
-RUN mkdir data share media keys logs /tmp/supervisord
+RUN mkdir -p data share keys logs /tmp/supervisord static/opa
+RUN find cvat/apps/iam/rules -name "*.rego" -and ! -name '*test*' -exec basename {} \; | \
+    tar -czf static/opa/bundle.tar.gz --transform 's,^,rules/,' -C cvat/apps/iam/rules/ -T -
 
 EXPOSE 8080
 ENTRYPOINT ["/usr/bin/supervisord"]

cvat/apps/iam/urls.py

@@ -12,12 +12,13 @@ from dj_rest_auth.views import (
 from allauth.account.views import ConfirmEmailView, EmailVerificationSentView
 from allauth.account import app_settings as allauth_settings
 
-from cvat.apps.iam.views import SigningView, RegisterViewEx
+from cvat.apps.iam.views import SigningView, RegisterViewEx, RulesView
 
 urlpatterns = [
     path('login', LoginView.as_view(), name='rest_login'),
     path('logout', LogoutView.as_view(), name='rest_logout'),
-    path('signing', SigningView.as_view(), name='signing')
+    path('signing', SigningView.as_view(), name='signing'),
+    path('rules', RulesView.as_view(), name='rules'),
 ]
 
 if settings.IAM_TYPE == 'BASIC':

cvat/apps/iam/views.py

@@ -3,11 +3,18 @@
 #
 # SPDX-License-Identifier: MIT
 
+import hashlib
+import os.path as osp
+from django_sendfile import sendfile
+
 from django.core.exceptions import BadRequest
 from django.utils.functional import SimpleLazyObject
 from rest_framework import views, serializers
 from rest_framework.exceptions import ValidationError
 from django.conf import settings
+from django.views import View
+from django.utils.decorators import method_decorator
+from django.views.decorators.http import etag
 from rest_framework.response import Response
 from dj_rest_auth.registration.views import RegisterView
 from allauth.account import app_settings as allauth_settings
@@ -16,7 +23,6 @@ from furl import furl
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import OpenApiResponse, extend_schema, inline_serializer, extend_schema_view
 
-
 from .authentication import Signer
 
 def get_context(request):
@@ -116,3 +122,20 @@ class RegisterViewEx(RegisterView):
             data['email_verification_required'] = False
             data['key'] = user.auth_token.key
         return data
+
+# Django Generic View is used here instead of DRF APIView due to native support of etag
+# that doesn't supported by DRF without extra dependencies
+class RulesView(View):
+    @staticmethod
+    def _get_bundle_path():
+        return osp.join(settings.STATIC_ROOT, 'opa', 'bundle.tar.gz')
+
+    @staticmethod
+    def _etag_func(file_path):
+        with open(file_path, 'rb') as f:
+            return hashlib.blake2b(f.read()).hexdigest()
+
+    @method_decorator(etag(lambda _: RulesView._etag_func(RulesView._get_bundle_path())))
+    def get(self, request):
+        file_path = self._get_bundle_path()
+        return sendfile(request, file_path)

docker-compose.dev.yml

@@ -25,3 +25,10 @@ services:
         no_proxy:
         socks_proxy:
       dockerfile: Dockerfile.ui
+
+  cvat_opa:
+    volumes:
+      - ./cvat/apps/iam/rules:/rules
+    ports:
+      - '8181:8181'
+    command: run --server --set=decision_logs.console=true /rules

docker-compose.yml

@@ -53,7 +53,9 @@ services:
       - cvat_keys:/home/django/keys
       - cvat_logs:/home/django/logs
     networks:
-      - cvat
+      cvat:
+        aliases:
+          - cvat-server
 
   cvat_utils:
     container_name: cvat_utils
@@ -185,11 +187,15 @@ services:
       cvat:
         aliases:
           - opa
-    volumes:
-      - ./cvat/apps/iam/rules:/rules
-    ports:
-      - '8181:8181'
-    command: run --server --addr :8181 --set=decision_logs.console=true /rules
+    command:
+      - run
+      - --server
+      - --set=decision_logs.console=true
+      - --set=services.cvat.url=http://cvat-server:8080
+      - --set=bundles.cvat.service=cvat
+      - --set=bundles.cvat.resource=/api/auth/rules
+      - --set=bundles.cvat.polling.min_delay_seconds=5
+      - --set=bundles.cvat.polling.max_delay_seconds=15
 
 volumes:
   cvat_db:

helm-chart/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.0
+version: 0.4.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm-chart/templates/cvat_opa/config.yml

@@ -1,17 +0,0 @@
-{{- if .Values.cvat.opa.defaultStorage.enabled }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ .Release.Name }}-opa-rules
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "cvat.labels" . | nindent 4 }}
-    app: cvat-app
-    tier: opa
-{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.GitVersion }}
-immutable: true
-{{- end }}
-binaryData:
-  rules.tar.gz: |-
-    {{ .Files.Get "rules.tar.gz" | b64enc }}
-{{- end}}

helm-chart/templates/cvat_opa/deployment.yml

@@ -36,10 +36,12 @@ spec:
           args:
             - run
             - --server
-            - --addr
-            - :8181
             - --set=decision_logs.console=true
-            - /rules/rules.tar.gz
+            - --set=services.cvat.url=http://{{ .Release.Name }}-backend-service:8080
+            - --set=bundles.cvat.service=cvat
+            - --set=bundles.cvat.resource=/api/auth/rules
+            - --set=bundles.cvat.polling.min_delay_seconds=10
+            - --set=bundles.cvat.polling.max_delay_seconds=15
           {{- with .Values.cvat.opa.resources }}
           resources:
           {{- toYaml . | nindent 12 }}
@@ -50,24 +52,14 @@ spec:
           env:
           {{- toYaml . | nindent 10 }}
           {{- end }}
-          volumeMounts:
-          - mountPath: /rules
-            name: cvat-opa-rules
           {{- with .Values.cvat.opa.additionalVolumeMounts }}
+          volumeMounts:
           {{- toYaml . | nindent 10 }}
           {{- end }}
+      {{- with .Values.cvat.opa.additionalVolumes }}
       volumes:
-        {{- if .Values.cvat.opa.defaultStorage.enabled }}
-        - name: cvat-opa-rules
-          configMap:
-            name: "{{ .Release.Name }}-opa-rules"
-            items:
-            - key: "rules.tar.gz"
-              path: "rules.tar.gz"
-        {{- end }}
-        {{- with .Values.cvat.opa.additionalVolumes }}
         {{- toYaml . | nindent 8 }}
-        {{- end }}
+      {{- end }}
       {{- with .Values.cvat.opa.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

helm-chart/values.yaml

@@ -166,8 +166,6 @@ cvat:
           targetPort: 8181
           protocol: TCP
           name: http
-    defaultStorage:
-        enabled: true
 
 postgresql:
   #See https://github.com/bitnami/charts/blob/master/bitnami/postgresql/ for more info

site/content/en/docs/administration/advanced/k8s_deployment_with_helm.md

@@ -69,10 +69,6 @@ helm dependency update
 1. Create `values.override.yaml` file inside `helm-chart` directory.
 1. Fill `values.override.yaml` with new parameters for chart.
 1. Override [postgresql password](#postgresql-password)
-1. Create a rules.tar.gz archive containing all OPA rules inside this `helm-chart` directory.
-   ```shell
-   find ../cvat/apps/iam/rules -name "*.rego" -and ! -name '*test*' -exec basename {} \; | tar -czf rules.tar.gz -C ../cvat/apps/iam/rules/ -T -
-   ```
 
 ### Postgresql password?
 Put below into your `values.override.yaml`

site/content/en/docs/contributing/development-environment.md

@@ -125,10 +125,10 @@ description: 'Installing a development environment for different operating syste
 
 - Install [Docker Engine](https://docs.docker.com/engine/install/ubuntu/) and [Docker-Compose](https://docs.docker.com/compose/install/)
 
-- Pull OpenPolicyAgent Docker-image (run from CVAT root dir):
+- Pull and run OpenPolicyAgent Docker image (run from CVAT root dir):
 
   ```bash
-  sudo docker-compose -f docker-compose.yml -f docker-compose.dev.yml up cvat_opa
+  sudo docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa
   ```
 
 ### Run CVAT

site/content/en/docs/manual/basics/workspace.md

@@ -31,7 +31,7 @@ In addition the workspace also has the following functions:
   ![](/images/image068_mapillary_vistas.jpg)
 
   - Adjust `Brightness`/`Contrast`/`Saturation` of too exposed or too
-  dark images using color settings (it affects only how a user sees the image, not the image itself).
+    dark images using color settings (it affects only how a user sees the image, not the image itself).
 
   ![](/images/image164_mapillary_vistas.jpg)