IAM OPA bundle for dev environment (#5190)

.dockerignore

@@ -7,4 +7,4 @@
 /db.sqlite3
 /keys
 **/node_modules
-
+/static

.github/workflows/full.yml

@@ -251,9 +251,9 @@ jobs:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
           CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
         run: |
-          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server
           max_tries=12
-          while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
+          while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
             -c 'python manage.py test cvat/apps -v 2'

.github/workflows/main.yml

@@ -215,10 +215,10 @@ jobs:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
           CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
         run: |
-          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server
 
           max_tries=12
-          while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
+          while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
           -c 'python manage.py test cvat/apps -v 2'

.github/workflows/schedule.yml

@@ -245,9 +245,9 @@ jobs:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
           CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
         run: |
-          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server
           max_tries=12
-          while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
+          while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
           -c 'coverage run -a manage.py test cvat/apps && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'

.vscode/launch.json

@@ -45,7 +45,10 @@
             "python": "${command:python.interpreterPath}",
             "program": "${workspaceRoot}/manage.py",
             "env": {
-                "CVAT_SERVERLESS": "1"
+                "CVAT_SERVERLESS": "1",
+                "ALLOWED_HOSTS": "*",
+                "IAM_OPA_BUNDLE": "1"
+
             },
             "args": [
                 "runserver",

Dockerfile

@@ -154,9 +154,7 @@ COPY --chown=${USER} cvat/ ${HOME}/cvat
 USER ${USER}
 WORKDIR ${HOME}
 
-RUN mkdir -p data share keys logs /tmp/supervisord static/opa
-RUN find cvat/apps/iam/rules -name "*.rego" -and ! -name '*test*' -exec basename {} \; | \
-    tar -czf static/opa/bundle.tar.gz --transform 's,^,rules/,' -C cvat/apps/iam/rules/ -T -
+RUN mkdir -p data share keys logs /tmp/supervisord static
 
 EXPOSE 8080
 ENTRYPOINT ["/usr/bin/supervisord"]

cvat/apps/iam/apps.py

@@ -1,8 +1,15 @@
+from distutils.util import strtobool
+import os
 from django.apps import AppConfig
 
+from .utils import create_opa_bundle
+
 class IAMConfig(AppConfig):
     name = 'cvat.apps.iam'
 
     def ready(self):
         from .signals import register_signals
         register_signals(self)
+
+        if strtobool(os.environ.get("IAM_OPA_BUNDLE", '0')):
+            create_opa_bundle()

cvat/apps/iam/utils.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+import tarfile
+
+from django.conf import settings
+
+def create_opa_bundle():
+    bundle_path = Path(settings.IAM_OPA_BUNDLE_PATH)
+    if bundle_path.is_file():
+        bundle_path.unlink()
+
+    rules_path = Path(settings.BASE_DIR) / 'cvat/apps/iam/rules'
+    with tarfile.open(bundle_path, 'w:gz') as tar:
+        for f in rules_path.glob('*[!.gen].rego'):
+            tar.add(name=f, arcname=f.relative_to(rules_path.parent))

cvat/apps/iam/views.py

@@ -3,18 +3,17 @@
 #
 # SPDX-License-Identifier: MIT
 
+import functools
 import hashlib
-import os.path as osp
-from django_sendfile import sendfile
 
 from django.core.exceptions import BadRequest
 from django.utils.functional import SimpleLazyObject
 from rest_framework import views, serializers
 from rest_framework.exceptions import ValidationError
+from rest_framework.permissions import AllowAny
 from django.conf import settings
-from django.views import View
-from django.utils.decorators import method_decorator
-from django.views.decorators.http import etag
+from django.http import HttpResponse
+from django.views.decorators.http import etag as django_etag
 from rest_framework.response import Response
 from dj_rest_auth.registration.views import RegisterView
 from allauth.account import app_settings as allauth_settings
@@ -74,6 +73,7 @@ def get_context(request):
     }
 
     return context
+
 class ContextMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
@@ -111,7 +111,6 @@ class SigningView(views.APIView):
         url = furl(url).add({Signer.QUERY_PARAM: sign}).url
         return Response(url)
 
-
 class RegisterViewEx(RegisterView):
     def get_response_data(self, user):
         data = self.get_serializer(user).data
@@ -123,19 +122,44 @@ class RegisterViewEx(RegisterView):
             data['key'] = user.auth_token.key
         return data
 
-# Django Generic View is used here instead of DRF APIView due to native support of etag
-# that doesn't supported by DRF without extra dependencies
-class RulesView(View):
+def _etag(etag_func):
+    """
+    Decorator to support conditional retrieval (or change)
+    for a Django Rest Framework's ViewSet.
+    It calls Django's original decorator but pass correct request object to it.
+    Django's original decorator doesn't work with DRF request object.
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(obj_self, request, *args, **kwargs):
+            drf_request = request
+            wsgi_request = request._request
+
+            @django_etag(etag_func=etag_func)
+            def patched_viewset_method(*_args, **_kwargs):
+                """Call original viewset method with correct type of request"""
+                return func(obj_self, drf_request, *args, **kwargs)
+
+            return patched_viewset_method(wsgi_request, *args, **kwargs)
+        return wrapper
+    return decorator
+
+class RulesView(views.APIView):
+    serializer_class = None
+    permission_classes = [AllowAny]
+    authentication_classes = []
+    iam_organization_field = None
+
     @staticmethod
     def _get_bundle_path():
-        return osp.join(settings.STATIC_ROOT, 'opa', 'bundle.tar.gz')
+        return settings.IAM_OPA_BUNDLE_PATH
 
     @staticmethod
     def _etag_func(file_path):
         with open(file_path, 'rb') as f:
             return hashlib.blake2b(f.read()).hexdigest()
 
-    @method_decorator(etag(lambda _: RulesView._etag_func(RulesView._get_bundle_path())))
+    @_etag(lambda _: RulesView._etag_func(RulesView._get_bundle_path()))
     def get(self, request):
-        file_path = self._get_bundle_path()
-        return sendfile(request, file_path)
+        file_obj = open(self._get_bundle_path() ,"rb")
+        return HttpResponse(file_obj, content_type='application/x-tar')

cvat/settings/base.py

@@ -390,6 +390,9 @@ os.makedirs(CLOUD_STORAGE_ROOT, exist_ok=True)
 TMP_FILES_ROOT = os.path.join(DATA_ROOT, 'tmp')
 os.makedirs(TMP_FILES_ROOT, exist_ok=True)
 
+IAM_OPA_BUNDLE_PATH = os.path.join(STATIC_ROOT, 'opa', 'bundle.tar.gz')
+os.makedirs(Path(IAM_OPA_BUNDLE_PATH).parent, exist_ok=True)
+
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,

docker-compose.dev.yml

@@ -27,8 +27,5 @@ services:
       dockerfile: Dockerfile.ui
 
   cvat_opa:
-    volumes:
-      - ./cvat/apps/iam/rules:/rules
     ports:
       - '8181:8181'
-    command: run --server --set=decision_logs.console=true /rules

docker-compose.yml

@@ -39,6 +39,7 @@ services:
       CVAT_REDIS_HOST: 'cvat_redis'
       CVAT_POSTGRES_HOST: 'cvat_db'
       ADAPTIVE_AUTO_ANNOTATION: 'false'
+      IAM_OPA_BUNDLE: '1'
       no_proxy: elasticsearch,kibana,logstash,nuclio,opa,${no_proxy}
       NUMPROCS: 1
     command: -c supervisord/server.conf

helm-chart/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.2
+version: 0.4.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm-chart/templates/cvat_backend/server/deployment.yml

@@ -56,6 +56,8 @@ spec:
             value: {{ .Values.cvat.backend.server.envs.ALLOWED_HOSTS | squote}}
           - name: DJANGO_MODWSGI_EXTRA_ARGS
             value: {{ .Values.cvat.backend.server.envs.DJANGO_MODWSGI_EXTRA_ARGS}}
+          - name: IAM_OPA_BUNDLE
+            value: "1"
           {{- if .Values.redis.enabled }}
           - name: CVAT_REDIS_HOST
             value: "{{ .Release.Name }}-redis-master"

site/content/en/docs/contributing/development-environment.md

@@ -167,10 +167,12 @@ description: 'Installing a development environment for different operating syste
 
 - Install [Docker Engine](https://docs.docker.com/engine/install/ubuntu/) and [Docker-Compose](https://docs.docker.com/compose/install/)
 
-- Pull and run OpenPolicyAgent Docker image (run from CVAT root dir):
+- Pull and run OpenPolicyAgent Docker image:
 
   ```bash
-  sudo docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa
+   docker run -d --rm --name cvat_opa_debug openpolicyagent/opa:0.34.2-rootless \
+   run --server --set=decision_logs.console=true --set=services.cvat.url=http://host.docker.internal:7000 \
+   --set=bundles.cvat.service=cvat --set=bundles.cvat.resource=/api/auth/rules
   ```
 
 ### Run CVAT