From 49bdef01f1bd13900c19451092a665527c3b9f67 Mon Sep 17 00:00:00 2001 From: Andrey Zhavoronkov Date: Wed, 2 Nov 2022 07:13:17 +0300 Subject: [PATCH] IAM OPA bundle for dev environment (#5190) --- .dockerignore | 2 +- .github/workflows/full.yml | 4 +- .github/workflows/main.yml | 4 +- .github/workflows/schedule.yml | 4 +- .vscode/launch.json | 5 +- Dockerfile | 4 +- cvat/apps/iam/apps.py | 7 +++ cvat/apps/iam/utils.py | 14 ++++++ cvat/apps/iam/views.py | 50 ++++++++++++++----- cvat/settings/base.py | 3 ++ docker-compose.dev.yml | 3 -- docker-compose.yml | 1 + helm-chart/Chart.yaml | 2 +- .../cvat_backend/server/deployment.yml | 2 + .../contributing/development-environment.md | 6 ++- 15 files changed, 81 insertions(+), 30 deletions(-) create mode 100644 cvat/apps/iam/utils.py diff --git a/.dockerignore b/.dockerignore index afe7b64b..21b715a6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,4 @@ /db.sqlite3 /keys **/node_modules - +/static diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index 335ec205..e0917eae 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -251,9 +251,9 @@ jobs: HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} CONTAINER_COVERAGE_DATA_DIR: "/coverage_data" run: | - docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa + docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server max_tries=12 - while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done + while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ -c 'python manage.py test cvat/apps -v 2' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d1a8f641..a53f41f3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -215,10 +215,10 @@ jobs: HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} CONTAINER_COVERAGE_DATA_DIR: "/coverage_data" run: | - docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa + docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server max_tries=12 - while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done + while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ -c 'python manage.py test cvat/apps -v 2' diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index d91f13c2..058827fe 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -245,9 +245,9 @@ jobs: HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} CONTAINER_COVERAGE_DATA_DIR: "/coverage_data" run: | - docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa + docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server max_tries=12 - while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done + while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ -c 'coverage run -a manage.py test cvat/apps && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}' diff --git a/.vscode/launch.json b/.vscode/launch.json index 6517d0db..710578cc 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -45,7 +45,10 @@ "python": "${command:python.interpreterPath}", "program": "${workspaceRoot}/manage.py", "env": { - "CVAT_SERVERLESS": "1" + "CVAT_SERVERLESS": "1", + "ALLOWED_HOSTS": "*", + "IAM_OPA_BUNDLE": "1" + }, "args": [ "runserver", diff --git a/Dockerfile b/Dockerfile index 00c925c7..ea9034f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -154,9 +154,7 @@ COPY --chown=${USER} cvat/ ${HOME}/cvat USER ${USER} WORKDIR ${HOME} -RUN mkdir -p data share keys logs /tmp/supervisord static/opa -RUN find cvat/apps/iam/rules -name "*.rego" -and ! -name '*test*' -exec basename {} \; | \ - tar -czf static/opa/bundle.tar.gz --transform 's,^,rules/,' -C cvat/apps/iam/rules/ -T - +RUN mkdir -p data share keys logs /tmp/supervisord static EXPOSE 8080 ENTRYPOINT ["/usr/bin/supervisord"] diff --git a/cvat/apps/iam/apps.py b/cvat/apps/iam/apps.py index d247b4d4..4f6979b7 100644 --- a/cvat/apps/iam/apps.py +++ b/cvat/apps/iam/apps.py @@ -1,8 +1,15 @@ +from distutils.util import strtobool +import os from django.apps import AppConfig +from .utils import create_opa_bundle + class IAMConfig(AppConfig): name = 'cvat.apps.iam' def ready(self): from .signals import register_signals register_signals(self) + + if strtobool(os.environ.get("IAM_OPA_BUNDLE", '0')): + create_opa_bundle() diff --git a/cvat/apps/iam/utils.py b/cvat/apps/iam/utils.py new file mode 100644 index 00000000..db552d7c --- /dev/null +++ b/cvat/apps/iam/utils.py @@ -0,0 +1,14 @@ +from pathlib import Path +import tarfile + +from django.conf import settings + +def create_opa_bundle(): + bundle_path = Path(settings.IAM_OPA_BUNDLE_PATH) + if bundle_path.is_file(): + bundle_path.unlink() + + rules_path = Path(settings.BASE_DIR) / 'cvat/apps/iam/rules' + with tarfile.open(bundle_path, 'w:gz') as tar: + for f in rules_path.glob('*[!.gen].rego'): + tar.add(name=f, arcname=f.relative_to(rules_path.parent)) diff --git a/cvat/apps/iam/views.py b/cvat/apps/iam/views.py index d651f0b3..42f6edb1 100644 --- a/cvat/apps/iam/views.py +++ b/cvat/apps/iam/views.py @@ -3,18 +3,17 @@ # # SPDX-License-Identifier: MIT +import functools import hashlib -import os.path as osp -from django_sendfile import sendfile from django.core.exceptions import BadRequest from django.utils.functional import SimpleLazyObject from rest_framework import views, serializers from rest_framework.exceptions import ValidationError +from rest_framework.permissions import AllowAny from django.conf import settings -from django.views import View -from django.utils.decorators import method_decorator -from django.views.decorators.http import etag +from django.http import HttpResponse +from django.views.decorators.http import etag as django_etag from rest_framework.response import Response from dj_rest_auth.registration.views import RegisterView from allauth.account import app_settings as allauth_settings @@ -74,6 +73,7 @@ def get_context(request): } return context + class ContextMiddleware: def __init__(self, get_response): self.get_response = get_response @@ -111,7 +111,6 @@ class SigningView(views.APIView): url = furl(url).add({Signer.QUERY_PARAM: sign}).url return Response(url) - class RegisterViewEx(RegisterView): def get_response_data(self, user): data = self.get_serializer(user).data @@ -123,19 +122,44 @@ class RegisterViewEx(RegisterView): data['key'] = user.auth_token.key return data -# Django Generic View is used here instead of DRF APIView due to native support of etag -# that doesn't supported by DRF without extra dependencies -class RulesView(View): +def _etag(etag_func): + """ + Decorator to support conditional retrieval (or change) + for a Django Rest Framework's ViewSet. + It calls Django's original decorator but pass correct request object to it. + Django's original decorator doesn't work with DRF request object. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(obj_self, request, *args, **kwargs): + drf_request = request + wsgi_request = request._request + + @django_etag(etag_func=etag_func) + def patched_viewset_method(*_args, **_kwargs): + """Call original viewset method with correct type of request""" + return func(obj_self, drf_request, *args, **kwargs) + + return patched_viewset_method(wsgi_request, *args, **kwargs) + return wrapper + return decorator + +class RulesView(views.APIView): + serializer_class = None + permission_classes = [AllowAny] + authentication_classes = [] + iam_organization_field = None + @staticmethod def _get_bundle_path(): - return osp.join(settings.STATIC_ROOT, 'opa', 'bundle.tar.gz') + return settings.IAM_OPA_BUNDLE_PATH @staticmethod def _etag_func(file_path): with open(file_path, 'rb') as f: return hashlib.blake2b(f.read()).hexdigest() - @method_decorator(etag(lambda _: RulesView._etag_func(RulesView._get_bundle_path()))) + @_etag(lambda _: RulesView._etag_func(RulesView._get_bundle_path())) def get(self, request): - file_path = self._get_bundle_path() - return sendfile(request, file_path) + file_obj = open(self._get_bundle_path() ,"rb") + return HttpResponse(file_obj, content_type='application/x-tar') diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 6769881c..1ce2842d 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -390,6 +390,9 @@ os.makedirs(CLOUD_STORAGE_ROOT, exist_ok=True) TMP_FILES_ROOT = os.path.join(DATA_ROOT, 'tmp') os.makedirs(TMP_FILES_ROOT, exist_ok=True) +IAM_OPA_BUNDLE_PATH = os.path.join(STATIC_ROOT, 'opa', 'bundle.tar.gz') +os.makedirs(Path(IAM_OPA_BUNDLE_PATH).parent, exist_ok=True) + LOGGING = { 'version': 1, 'disable_existing_loggers': False, diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index bdbc20fd..5c9449e4 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -27,8 +27,5 @@ services: dockerfile: Dockerfile.ui cvat_opa: - volumes: - - ./cvat/apps/iam/rules:/rules ports: - '8181:8181' - command: run --server --set=decision_logs.console=true /rules diff --git a/docker-compose.yml b/docker-compose.yml index a4680225..aceefd91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,7 @@ services: CVAT_REDIS_HOST: 'cvat_redis' CVAT_POSTGRES_HOST: 'cvat_db' ADAPTIVE_AUTO_ANNOTATION: 'false' + IAM_OPA_BUNDLE: '1' no_proxy: elasticsearch,kibana,logstash,nuclio,opa,${no_proxy} NUMPROCS: 1 command: -c supervisord/server.conf diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 0c31b8c8..50cb09a0 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.2 +version: 0.4.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm-chart/templates/cvat_backend/server/deployment.yml b/helm-chart/templates/cvat_backend/server/deployment.yml index ef5ca23b..38d1f581 100644 --- a/helm-chart/templates/cvat_backend/server/deployment.yml +++ b/helm-chart/templates/cvat_backend/server/deployment.yml @@ -56,6 +56,8 @@ spec: value: {{ .Values.cvat.backend.server.envs.ALLOWED_HOSTS | squote}} - name: DJANGO_MODWSGI_EXTRA_ARGS value: {{ .Values.cvat.backend.server.envs.DJANGO_MODWSGI_EXTRA_ARGS}} + - name: IAM_OPA_BUNDLE + value: "1" {{- if .Values.redis.enabled }} - name: CVAT_REDIS_HOST value: "{{ .Release.Name }}-redis-master" diff --git a/site/content/en/docs/contributing/development-environment.md b/site/content/en/docs/contributing/development-environment.md index 48576957..77c3544b 100644 --- a/site/content/en/docs/contributing/development-environment.md +++ b/site/content/en/docs/contributing/development-environment.md @@ -167,10 +167,12 @@ description: 'Installing a development environment for different operating syste - Install [Docker Engine](https://docs.docker.com/engine/install/ubuntu/) and [Docker-Compose](https://docs.docker.com/compose/install/) -- Pull and run OpenPolicyAgent Docker image (run from CVAT root dir): +- Pull and run OpenPolicyAgent Docker image: ```bash - sudo docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa + docker run -d --rm --name cvat_opa_debug openpolicyagent/opa:0.34.2-rootless \ + run --server --set=decision_logs.console=true --set=services.cvat.url=http://host.docker.internal:7000 \ + --set=bundles.cvat.service=cvat --set=bundles.cvat.resource=/api/auth/rules ``` ### Run CVAT