wangchunlin
/
cvat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove limit manager (#5642)

Browse Source
main
Kirill Sizov 3 years ago committed by GitHub
parent 654ff124fb
commit 2ab66fc32c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 68 additions and 1618 deletions
Show Stats Download Patch File Download Diff File

36
cvat/apps/iam/exceptions.py
Unescape Escape View File

@ -1,36 +0,0 @@
# Copyright (C) 2023 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
from django.conf import settings
from django.utils.module_loading import import_string
from rest_framework.exceptions import PermissionDenied
class DefaultLimitsReachedException(PermissionDenied):
default_personal_detail = "You've reached the maximum number of {}. Contact the administrator to extend the limits."
default_org_detail = "You've reached the maximum number of {}. Contact the administrator to extend the limits for `{}` organization."
def __init__(self, reasons, iam_context):
if not reasons or not isinstance(reasons, list):
super().__init__(reasons)
msg = self.default_personal_detail
if iam_context["organization"] is not None:
msg = self.default_org_detail
msg = msg.format(', '.join(reasons), iam_context["organization"].slug)
else:
msg = msg.format(', '.join(reasons))
super().__init__({"message": msg})
class ExceptionFactory:
def __call__(self, *args, **kwargs):
dotted_path = getattr(settings, "IAM_BASE_EXCEPTION", None)
print(dotted_path)
if dotted_path is None:
return DefaultLimitsReachedException(*args, **kwargs)
return import_string(dotted_path)(*args, **kwargs)
LimitsReachedException = ExceptionFactory()

268
cvat/apps/iam/permissions.py
Unescape Escape View File

@ -4,28 +4,22 @@
# SPDX-License-Identifier: MIT # SPDX-License-Identifier: MIT
from __future__ import annotations from __future__ import annotations
import operator
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
from collections import namedtuple from collections import namedtuple
from enum import Enum, auto from enum import Enum
import operator from typing import Any, List, Optional, Sequence, cast
from typing import Any, List, Optional, Sequence, Tuple, cast
from attrs import define, field
from rest_framework.exceptions import ValidationError, PermissionDenied
import requests import requests
from attrs import define, field
from django.conf import settings from django.conf import settings
from django.db.models import Q from django.db.models import Q
from rest_framework.exceptions import ValidationError
from rest_framework.permissions import BasePermission from rest_framework.permissions import BasePermission
from cvat.apps.engine.models import Issue, Job, Project, Task
from cvat.apps.organizations.models import Membership, Organization from cvat.apps.organizations.models import Membership, Organization
from cvat.apps.engine.models import Project, Task, Job, Issue
from cvat.apps.iam.exceptions import LimitsReachedException
from cvat.apps.limit_manager.core.limits import (CapabilityContext, LimitManager,
Limits, OrgCloudStoragesContext, OrgTasksContext, ProjectWebhooksContext,
OrgCommonWebhooksContext,
TasksInOrgProjectContext, TasksInUserSandboxProjectContext, UserOrgsContext,
UserSandboxCloudStoragesContext, UserSandboxTasksContext)
from cvat.apps.webhooks.models import WebhookTypeChoice from cvat.apps.webhooks.models import WebhookTypeChoice
@ -1347,233 +1341,10 @@ class IssuePermission(OpenPolicyAgentPermission):
return data return data
class LimitPermission(OpenPolicyAgentPermission):
@classmethod
def create(cls, request, view, obj):
return [] # There are no basic (unconditional) permissions
@classmethod
def create_from_scopes(cls, request, view, obj, scopes: List[OpenPolicyAgentPermission]):
scope_to_caps = [
(scope_handler, cls._prepare_capability_params(scope_handler))
for scope_handler in scopes
]
return [
cls.create_base_perm(request, view, str(scope_handler.scope), obj,
scope_handler=scope_handler, capabilities=capabilities,
)
for scope_handler, capabilities in scope_to_caps
if capabilities
]
def __init__(self, **kwargs):
self.url = settings.IAM_OPA_DATA_URL + '/limits/result'
self.scope_handler: OpenPolicyAgentPermission = kwargs.pop('scope_handler')
self.capabilities: Tuple[Limits, CapabilityContext] = kwargs.pop('capabilities')
super().__init__(**kwargs)
@classmethod
def get_scopes(cls, request, view, obj):
scopes = [
(scope_handler, cls._prepare_capability_params(scope_handler))
for ctx in OpenPolicyAgentPermission.__subclasses__()
if not issubclass(ctx, cls)
for scope_handler in ctx.create(request, view, obj)
]
return [
(scope, capabilities)
for scope, capabilities in scopes
if capabilities
]
def get_resource(self):
data = {}
limit_manager = LimitManager()
def _get_capability_status(
capability: Limits, context: Optional[CapabilityContext]
) -> dict:
status = limit_manager.get_status(limit=capability, context=context)
return { 'used': status.used, 'max': status.max }
for capability, context in self.capabilities:
data[self._get_capability_name(capability)] = _get_capability_status(
capability=capability, context=context,
)
return { 'limits': data }
@classmethod
def _get_capability_name(cls, capability: Limits) -> str:
return capability.name
@classmethod
def _prepare_capability_params(cls, scope: OpenPolicyAgentPermission
) -> List[Tuple[Limits, CapabilityContext]]:
scope_id = (type(scope), scope.scope)
results = []
if scope_id in [
(TaskPermission, TaskPermission.Scopes.CREATE),
(TaskPermission, TaskPermission.Scopes.IMPORT_BACKUP),
]:
if getattr(scope, 'org_id') is not None:
results.append((
Limits.ORG_TASKS,
OrgTasksContext(org_id=scope.org_id)
))
else:
results.append((
Limits.USER_SANDBOX_TASKS,
UserSandboxTasksContext(user_id=scope.user_id)
))
elif scope_id == (TaskPermission, TaskPermission.Scopes.CREATE_IN_PROJECT):
project = Project.objects.get(id=scope.project_id)
if getattr(project, 'organization') is not None:
results.append((
Limits.TASKS_IN_ORG_PROJECT,
TasksInOrgProjectContext(
org_id=project.organization.id,
project_id=project.id,
)
))
results.append((
Limits.ORG_TASKS,
OrgTasksContext(org_id=project.organization.id)
))
else:
results.append((
Limits.TASKS_IN_USER_SANDBOX_PROJECT,
TasksInUserSandboxProjectContext(
user_id=project.owner.id,
project_id=project.id
)
))
results.append((
Limits.USER_SANDBOX_TASKS,
UserSandboxTasksContext(user_id=project.owner.id)
))
elif scope_id == (TaskPermission, TaskPermission.Scopes.UPDATE_PROJECT):
task = cast(Task, scope.obj)
project = Project.objects.get(id=scope.project_id)
class OwnerType(Enum):
org = auto()
user = auto()
if getattr(task, 'organization', None):
old_owner = (OwnerType.org, task.organization.id)
else:
old_owner = (OwnerType.user, task.owner.id)
if getattr(project, 'organization', None) is not None:
results.append((
Limits.TASKS_IN_ORG_PROJECT,
TasksInOrgProjectContext(
org_id=project.organization.id,
project_id=project.id,
)
))
if old_owner != (OwnerType.org, project.organization.id):
results.append((
Limits.ORG_TASKS,
OrgTasksContext(org_id=project.organization.id)
))
else:
results.append((
Limits.TASKS_IN_USER_SANDBOX_PROJECT,
TasksInUserSandboxProjectContext(
user_id=project.owner.id,
project_id=project.id
)
))
if old_owner != (OwnerType.user, project.owner.id):
results.append((
Limits.USER_SANDBOX_TASKS,
UserSandboxTasksContext(user_id=project.owner.id)
))
elif scope_id == (TaskPermission, TaskPermission.Scopes.UPDATE_OWNER):
task = cast(Task, scope.obj)
class OwnerType(Enum):
org = auto()
user = auto()
if getattr(task, 'organization', None) is not None:
old_owner = (OwnerType.org, task.organization.id)
else:
old_owner = (OwnerType.user, task.owner.id)
new_owner = getattr(scope, 'owner_id', None)
if new_owner is not None and old_owner != (OwnerType.user, new_owner):
results.append((
Limits.USER_SANDBOX_TASKS,
UserSandboxTasksContext(user_id=new_owner)
))
elif scope_id in [
(ProjectPermission, ProjectPermission.Scopes.CREATE),
(ProjectPermission, ProjectPermission.Scopes.IMPORT_BACKUP),
]:
if getattr(scope, 'org_id') is not None:
results.append((
Limits.ORG_PROJECTS,
OrgTasksContext(org_id=scope.org_id)
))
else:
results.append((
Limits.USER_SANDBOX_PROJECTS,
UserSandboxTasksContext(user_id=scope.user_id)
))
elif scope_id == (CloudStoragePermission, CloudStoragePermission.Scopes.CREATE):
if getattr(scope, 'org_id') is not None:
results.append((
Limits.ORG_CLOUD_STORAGES,
OrgCloudStoragesContext(org_id=scope.org_id)
))
else:
results.append((
Limits.USER_SANDBOX_CLOUD_STORAGES,
UserSandboxCloudStoragesContext(user_id=scope.user_id)
))
elif scope_id == (OrganizationPermission, OrganizationPermission.Scopes.CREATE):
results.append((
Limits.USER_OWNED_ORGS,
UserOrgsContext(user_id=scope.user_id)
))
elif scope_id == (WebhookPermission, WebhookPermission.Scopes.CREATE_IN_ORG):
results.append((
Limits.ORG_COMMON_WEBHOOKS,
OrgCommonWebhooksContext(org_id=scope.org_id)
))
elif scope_id == (WebhookPermission, WebhookPermission.Scopes.CREATE_IN_PROJECT):
results.append((
Limits.PROJECT_WEBHOOKS,
ProjectWebhooksContext(project_id=scope.project_id)
))
return results
class PolicyEnforcer(BasePermission): class PolicyEnforcer(BasePermission):
# pylint: disable=no-self-use # pylint: disable=no-self-use
def check_permission(self, request, view, obj): def check_permission(self, request, view, obj):
# Some permissions are only needed to be checked if the action permissions: List[OpenPolicyAgentPermission] = []
# is permitted in general. To achieve this, we split checks
# into 2 groups, and check one after another.
basic_permissions: List[OpenPolicyAgentPermission] = []
conditional_permissions: List[OpenPolicyAgentPermission] = []
# DRF can send OPTIONS request. Internally it will try to get # DRF can send OPTIONS request. Internally it will try to get
# information about serializers for PUT and POST requests (clone # information about serializers for PUT and POST requests (clone
@ -1582,33 +1353,14 @@ class PolicyEnforcer(BasePermission):
# the condition below is enough. # the condition below is enough.
if not self.is_metadata_request(request, view): if not self.is_metadata_request(request, view):
for perm in OpenPolicyAgentPermission.__subclasses__(): for perm in OpenPolicyAgentPermission.__subclasses__():
basic_permissions.extend(perm.create(request, view, obj)) permissions.extend(perm.create(request, view, obj))
conditional_permissions.extend(LimitPermission.create_from_scopes(
request, view, obj, basic_permissions
))
self._iam_context = request.iam_context
allow = self._check_permissions(basic_permissions)
if allow and conditional_permissions:
allow = self._check_permissions(conditional_permissions)
return allow
def _check_permissions(self, permissions: List[OpenPolicyAgentPermission]) -> bool:
allow = True allow = True
reasons = []
for perm in permissions: for perm in permissions:
result = perm.check_access() result = perm.check_access()
allow &= result.allow allow &= result.allow
reasons.extend(result.reasons)
if allow: return allow
return True
elif reasons:
raise LimitsReachedException(reasons, self._iam_context)
else:
raise PermissionDenied("not authorized")
def has_permission(self, request, view): def has_permission(self, request, view):
if not view.detail: if not view.detail:

129
cvat/apps/iam/rules/limits.rego
Unescape Escape View File

@ -1,129 +0,0 @@
package limits
import future.keywords.if
import future.keywords.in
import future.keywords.contains
import data.utils
CAP_USER_SANDBOX_TASKS = "USER_SANDBOX_TASKS"
CAP_USER_SANDBOX_PROJECTS = "USER_SANDBOX_PROJECTS"
CAP_TASKS_IN_USER_SANDBOX_PROJECT = "TASKS_IN_USER_SANDBOX_PROJECT"
CAP_USER_OWNED_ORGS = "USER_OWNED_ORGS"
CAP_USER_SANDBOX_CLOUD_STORAGES = "USER_SANDBOX_CLOUD_STORAGES"
CAP_ORG_TASKS = "ORG_TASKS"
CAP_ORG_PROJECTS = "ORG_PROJECTS"
CAP_TASKS_IN_ORG_PROJECT = "TASKS_IN_ORG_PROJECT"
CAP_ORG_CLOUD_STORAGES = "ORG_CLOUD_STORAGES"
CAP_ORG_COMMON_WEBHOOKS = "ORG_COMMON_WEBHOOKS"
CAP_PROJECT_WEBHOOKS = "PROJECT_WEBHOOKS"
check_limit_exceeded(current, max) {
null != max
current >= max
}
problems contains "tasks per user" if {
check_limit_exceeded(
input.resource.limits[CAP_USER_SANDBOX_TASKS].used,
input.resource.limits[CAP_USER_SANDBOX_TASKS].max
)
}
problems contains "projects per user" if {
check_limit_exceeded(
input.resource.limits[CAP_USER_SANDBOX_PROJECTS].used,
input.resource.limits[CAP_USER_SANDBOX_PROJECTS].max
)
}
problems contains "tasks per project for the user" if {
check_limit_exceeded(
input.resource.limits[CAP_TASKS_IN_USER_SANDBOX_PROJECT].used,
input.resource.limits[CAP_TASKS_IN_USER_SANDBOX_PROJECT].max
)
}
problems contains "tasks per organization" if {
check_limit_exceeded(
input.resource.limits[CAP_ORG_TASKS].used,
input.resource.limits[CAP_ORG_TASKS].max
)
}
problems contains "projects per organization" if {
check_limit_exceeded(
input.resource.limits[CAP_ORG_PROJECTS].used,
input.resource.limits[CAP_ORG_PROJECTS].max
)
}
problems contains "tasks per project for the organization" if {
check_limit_exceeded(
input.resource.limits[CAP_TASKS_IN_ORG_PROJECT].used,
input.resource.limits[CAP_TASKS_IN_ORG_PROJECT].max
)
}
problems contains "webhooks per project" if {
check_limit_exceeded(
input.resource.limits[CAP_PROJECT_WEBHOOKS].used,
input.resource.limits[CAP_PROJECT_WEBHOOKS].max
)
}
problems contains "webhooks per organization" if {
check_limit_exceeded(
input.resource.limits[CAP_ORG_COMMON_WEBHOOKS].used,
input.resource.limits[CAP_ORG_COMMON_WEBHOOKS].max
)
}
problems contains "organizations per user" if {
check_limit_exceeded(
input.resource.limits[CAP_USER_OWNED_ORGS].used,
input.resource.limits[CAP_USER_OWNED_ORGS].max
)
}
problems contains "cloud storages per user" if {
check_limit_exceeded(
input.resource.limits[CAP_USER_SANDBOX_CLOUD_STORAGES].used,
input.resource.limits[CAP_USER_SANDBOX_CLOUD_STORAGES].max
)
}
problems contains "cloud storages per organization" if {
check_limit_exceeded(
input.resource.limits[CAP_ORG_CLOUD_STORAGES].used,
input.resource.limits[CAP_ORG_CLOUD_STORAGES].max
)
}
# In the case of invalid input or no applicable limits,
# we deny the request. We suppose that we always check at least 1
# limit, and this package is queried by IAM only when there are
# limits to check in the input scope.
default result = {
"allow": false,
"reasons": []
}
result := {
"allow": true,
"reasons": [],
} if {
utils.is_admin
} else := {
"allow": count(problems) == 0,
"reasons": problems
} if {
not utils.is_admin
count(input.resource.limits) != 0
}
allow := result.allow

14
cvat/apps/iam/rules/tests/configs/limits.csv
Unescape Escape View File

@ -1,14 +0,0 @@
TestKind,Capability,CapKind
single,USER_SANDBOX_TASKS,max
single,USER_SANDBOX_PROJECTS,max
single,TASKS_IN_USER_SANDBOX_PROJECT,max
single,USER_OWNED_ORGS,max
single,USER_SANDBOX_CLOUD_STORAGES,max
single,ORG_TASKS,max
single,ORG_PROJECTS,max
single,TASKS_IN_ORG_PROJECT,max
single,ORG_CLOUD_STORAGES,max
single,ORG_COMMON_WEBHOOKS,max
single,PROJECT_WEBHOOKS,max
multi,"USER_SANDBOX_TASKS,USER_SANDBOX_PROJECTS",N/A
multi,,N/A
1 TestKind Capability CapKind
2 single USER_SANDBOX_TASKS max
3 single USER_SANDBOX_PROJECTS max
4 single TASKS_IN_USER_SANDBOX_PROJECT max
5 single USER_OWNED_ORGS max
6 single USER_SANDBOX_CLOUD_STORAGES max
7 single ORG_TASKS max
8 single ORG_PROJECTS max
9 single TASKS_IN_ORG_PROJECT max
10 single ORG_CLOUD_STORAGES max
11 single ORG_COMMON_WEBHOOKS max
12 single PROJECT_WEBHOOKS max
13 multi USER_SANDBOX_TASKS,USER_SANDBOX_PROJECTS N/A
14 multi N/A

211
cvat/apps/iam/rules/tests/generators/limits_test.gen.rego.py
Unescape Escape View File

@ -1,211 +0,0 @@
# Copyright (C) 2022 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
import csv
import json
import os
import sys
import textwrap
from enum import Enum
from itertools import product
NAME = "limits"
class TestKinds(str, Enum):
single = "single"
multi = "multi"
def __str__(self) -> str:
return self.value.lower()
class CapKinds(str, Enum):
max = "max"
def __str__(self) -> str:
return self.value.lower()
def read_test_table(name):
# The table describes positive cases and test configurations
table = []
with open(os.path.join(sys.argv[1], f"{name}.csv")) as f:
for row in csv.DictReader(f):
table.append(row)
return table
test_table = read_test_table(NAME)
CAPABILITIES = {
entry["Capability"]: entry["CapKind"]
for entry in test_table
if entry["TestKind"] == TestKinds.single
}
ROLES = ["user", "admin"]
MAX_CAPABILITY_LIMIT_VALUES = [None, 5]
MAX_CAPABILITY_USED_VALUES = [2, 7]
def eval_rule(test_kind, role, capabilities, *, data):
if role == "admin":
return {
"allow": True,
"messages": 0,
}
allow = True
messages = 0
for capability in capabilities:
cap_name = capability["name"]
cap_kind = CAPABILITIES[cap_name]
cap_data = data["resource"]["limits"][cap_name]
if cap_kind == CapKinds.max:
cap_allow = (cap_data["max"] is None) or (cap_data["used"] < cap_data["max"])
messages += not cap_allow
allow &= cap_allow
else:
raise ValueError(f"Unknown capability kind {cap_kind}")
if not capabilities:
allow = False
messages = 0
return {
"allow": allow,
"messages": messages,
}
def _get_name(prefix, **kwargs):
name = prefix
for k, v in kwargs.items():
prefix = "_" + str(k)
if isinstance(v, dict):
if "id" in v:
v = v.copy()
v.pop("id")
if v:
name += _get_name(prefix, **v)
else:
name += "".join(
map(
lambda c: c if c.isalnum() else {"@": "_IN_"}.get(c, "_"),
f"{prefix}_{str(v).upper()}",
)
)
return name
def get_name(*args, **kwargs):
return _get_name("test", *args, **kwargs)
def generate_capability_cases(capability: str):
capability_kind = CAPABILITIES[capability]
if capability_kind == CapKinds.max:
for used, maximum in product(MAX_CAPABILITY_USED_VALUES, MAX_CAPABILITY_LIMIT_VALUES):
yield {"name": capability, "used": used, "max": maximum}
else:
raise ValueError(f"Unknown capability kind {capability_kind}")
def generate_test_data(test_kind, role, capabilities):
data = {
"auth": {"user": {"privilege": role}},
"resource": {
"limits": {},
},
}
for cap_case in capabilities:
cap_name = cap_case["name"]
cap_kind = CAPABILITIES[cap_case["name"]]
if cap_kind == CapKinds.max:
data["resource"]["limits"][cap_name] = {
"used": cap_case["used"],
"max": cap_case["max"],
}
else:
raise ValueError(f"Unknown capability type {cap_kind}")
return data
def generate_test_cases():
for config in test_table:
test_kind = config["TestKind"]
if test_kind == TestKinds.single:
capability = config["Capability"]
for role, cap_case in product(ROLES, generate_capability_cases(capability)):
yield dict(test_kind=test_kind, role=role, capabilities=[cap_case])
elif test_kind == TestKinds.multi:
if config["Capability"]:
capabilities = config["Capability"].split(",")
else:
capabilities = []
capability_cases = [
generate_capability_cases(capability) for capability in capabilities
]
for params in product(ROLES, *capability_cases):
role = params[0]
cap_case = params[1:]
yield dict(test_kind=test_kind, role=role, capabilities=cap_case)
else:
raise ValueError(f"Unknown test kind {test_kind}")
def gen_test_rego(name):
with open(f"{name}_test.gen.rego", "wt") as f:
f.write(f"package {name}\n\n")
for test_params in generate_test_cases():
test_data = generate_test_data(**test_params)
test_result = eval_rule(**test_params, data=test_data)
test_name = get_name(**test_params)
f.write(
textwrap.dedent(
"""
{test_name} {{
r := result with input as {data}
r.allow == {allow}
count(r.reasons) == {messages}
}}
"""
).format(
test_name=test_name,
allow=str(test_result["allow"]).lower(),
messages=test_result["messages"],
data=json.dumps(test_data),
)
)
# Write the script which is used to generate the file
with open(sys.argv[0]) as this_file:
f.write(f"\n\n# {os.path.split(sys.argv[0])[1]}\n")
for line in this_file:
if line.strip():
f.write(f"# {line}")
else:
f.write(f"#\n")
# Write rules which are used to generate the file
with open(os.path.join(sys.argv[1], f"{name}.csv")) as rego_file:
f.write(f"\n\n# {name}.csv\n")
for line in rego_file:
if line.strip():
f.write(f"# {line}")
else:
f.write(f"#\n")
gen_test_rego(NAME)

96
cvat/apps/iam/tests/test_rest_api.py
Unescape Escape View File

@ -3,11 +3,6 @@
# #
# SPDX-License-Identifier: MIT # SPDX-License-Identifier: MIT
from contextlib import contextmanager
from itertools import repeat
import itertools
from typing import Sequence, Type
from unittest import mock
from django.urls import reverse from django.urls import reverse
from rest_framework import status from rest_framework import status
from rest_framework.test import APITestCase, APIClient from rest_framework.test import APITestCase, APIClient
@ -16,10 +11,8 @@ from django.test import override_settings
from django.urls import path, re_path from django.urls import path, re_path
from allauth.account.views import EmailVerificationSentView from allauth.account.views import EmailVerificationSentView
from cvat.apps.iam.permissions import OpenPolicyAgentPermission, PermissionResult
from cvat.apps.iam.urls import urlpatterns as iam_url_patterns from cvat.apps.iam.urls import urlpatterns as iam_url_patterns
from cvat.apps.iam.views import ConfirmEmailViewEx from cvat.apps.iam.views import ConfirmEmailViewEx
from cvat.apps.engine.models import User
urlpatterns = iam_url_patterns + [ urlpatterns = iam_url_patterns + [
@ -98,92 +91,3 @@ class UserRegisterAPITestCase(APITestCase):
'username': 'test_username', 'email': 'test_email@test.com', 'username': 'test_username', 'email': 'test_email@test.com',
'email_verification_required': True, 'key': None}) 'email_verification_required': True, 'key': None})
class TestIamApi(APITestCase):
@classmethod
def _make_permission_class(cls, results) -> Type[OpenPolicyAgentPermission]:
class _TestPerm(OpenPolicyAgentPermission):
def get_resource(self):
return {}
@classmethod
def create(cls, request, view, obj) -> Sequence[OpenPolicyAgentPermission]:
return [
cls.create_base_perm(request, view, None, obj, result=result)
for result in results
]
def check_access(self) -> PermissionResult:
return PermissionResult(allow=self.result[0], reasons=self.result[1])
return _TestPerm
@classmethod
@contextmanager
def _mock_permissions(cls, *perm_results):
with mock.patch('cvat.apps.iam.permissions.OpenPolicyAgentPermission.__subclasses__',
lambda: [cls._make_permission_class(perm_results)]
):
yield
ENDPOINT_WITH_AUTH = '/api/users/self'
def setUp(self):
self.client = APIClient()
import sys
sys.modules.pop('cvat.apps.iam.permissions', None)
@classmethod
def _create_db_users(cls):
cls.user = User.objects.create_user(username="user", password="user")
@classmethod
def setUpTestData(cls) -> None:
super().setUpTestData()
cls._create_db_users()
def test_can_report_denial_reason(self):
expected_reasons = ["hello", "world"]
with self._mock_permissions((False, expected_reasons)), \
ForceLogin(user=self.user, client=self.client):
response = self.client.get(self.ENDPOINT_WITH_AUTH)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(all([reason in str(response.content) for reason in expected_reasons]))
def test_can_report_merged_denial_reasons(self):
expected_reasons = [["hello", "world"], ["hi", "there"]]
with self._mock_permissions(*zip(repeat(False), expected_reasons)), \
ForceLogin(user=self.user, client=self.client):
response = self.client.get(self.ENDPOINT_WITH_AUTH)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(
all(
[
reason in str(response.content)
for reason in itertools.chain(*expected_reasons)
]
)
)
def test_can_allow_if_no_permission_matches(self):
with self._mock_permissions(), ForceLogin(user=self.user, client=self.client):
response = self.client.get(self.ENDPOINT_WITH_AUTH)
self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_can_allow_if_permissions_allow(self):
with self._mock_permissions((True, [])), \
ForceLogin(user=self.user, client=self.client):
response = self.client.get(self.ENDPOINT_WITH_AUTH)
self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_can_deny_if_some_permissions_deny(self):
expected_reasons = ["hello"]
with self._mock_permissions((True, []), (False, expected_reasons)), \
ForceLogin(user=self.user, client=self.client):
response = self.client.get(self.ENDPOINT_WITH_AUTH)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(all([reason in str(response.content) for reason in expected_reasons]))

0
cvat/apps/limit_manager/__init__.py
Unescape Escape View File

6
cvat/apps/limit_manager/apps.py
Unescape Escape View File

@ -1,6 +0,0 @@
from django.apps import AppConfig
class LimitManagerConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'limit_manager'

230
cvat/apps/limit_manager/core/limits.py
Unescape Escape View File

@ -1,230 +0,0 @@
# Copyright (C) 2022 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
from enum import Enum, auto
from typing import Optional, cast
from attrs import define
from django.conf import settings
from cvat.apps.engine.models import CloudStorage, Project, Task
from cvat.apps.organizations.models import Organization
from cvat.apps.webhooks.models import Webhook
class Limits(Enum):
"""
Represents a capability which has an upper limit, and can be consumed.
Each capability is also supposed to have a separate CapabilityContext class,
representing its parameters. Different parameter combinations should each have
a different enum member, no member reuse is supposed for different limits.
"""
# TODO: for a capability with N params, there are O(k^N)
# possible limitation combinations. Not all are meaningful, but even though
# it is quite a large number. Example:
# A "task create" capability [user_id, org_id, project_id]
# yields the following possible limitations:
# - tasks from the user
# - tasks from the user outside orgs
# - tasks from the user inside orgs
# - tasks from the user in the org
# - tasks from the user in the project
# - tasks in the org
# - tasks in the org projects
# ...
#
# Currently, we will cover all of this with a single request to the limit manager.
# For each meaningful combination class a capability enum entry is supposed.
USER_SANDBOX_TASKS = auto()
USER_SANDBOX_PROJECTS = auto()
TASKS_IN_USER_SANDBOX_PROJECT = auto()
USER_OWNED_ORGS = auto()
USER_SANDBOX_CLOUD_STORAGES = auto()
ORG_TASKS = auto()
ORG_PROJECTS = auto()
TASKS_IN_ORG_PROJECT = auto()
ORG_CLOUD_STORAGES = auto()
ORG_COMMON_WEBHOOKS = auto()
PROJECT_WEBHOOKS = auto()
class CapabilityContext:
pass
@define(kw_only=True)
class UserCapabilityContext(CapabilityContext):
user_id: int
@define(kw_only=True)
class OrgCapabilityContext(CapabilityContext):
org_id: int
@define(kw_only=True)
class UserSandboxTasksContext(UserCapabilityContext):
pass
@define(kw_only=True)
class OrgTasksContext(OrgCapabilityContext):
pass
@define(kw_only=True)
class TasksInUserSandboxProjectContext(UserCapabilityContext):
project_id: int
@define(kw_only=True)
class TasksInOrgProjectContext(OrgCapabilityContext):
project_id: int
@define(kw_only=True)
class UserSandboxProjectsContext(UserCapabilityContext):
pass
@define(kw_only=True)
class OrgProjectsContext(OrgCapabilityContext):
pass
@define(kw_only=True)
class UserSandboxCloudStoragesContext(UserCapabilityContext):
pass
@define(kw_only=True)
class OrgCloudStoragesContext(OrgCapabilityContext):
pass
@define(kw_only=True)
class UserOrgsContext(UserCapabilityContext):
pass
@define(kw_only=True)
class ProjectWebhooksContext(CapabilityContext):
project_id: int
@define(kw_only=True)
class OrgCommonWebhooksContext(OrgCapabilityContext):
pass
@define(frozen=True)
class LimitStatus:
used: Optional[int]
max: Optional[int]
class LimitManager:
def get_status(self,
limit: Limits, *,
context: Optional[CapabilityContext] = None,
) -> LimitStatus:
if limit == Limits.USER_OWNED_ORGS:
assert context is not None
context = cast(UserOrgsContext, context)
return LimitStatus(
Organization.objects.filter(owner_id=context.user_id).count(),
settings.DEFAULT_LIMITS["USER_OWNED_ORGS"],
)
elif limit == Limits.USER_SANDBOX_PROJECTS:
assert context is not None
context = cast(UserSandboxProjectsContext, context)
return LimitStatus(
# TODO: check about active/removed projects
Project.objects.filter(owner=context.user_id, organization=None).count(),
settings.DEFAULT_LIMITS["USER_SANDBOX_PROJECTS"],
)
elif limit == Limits.ORG_PROJECTS:
assert context is not None
context = cast(OrgProjectsContext, context)
return LimitStatus(
# TODO: check about active/removed projects
Project.objects.filter(organization=context.org_id).count(),
settings.DEFAULT_LIMITS["ORG_PROJECTS"],
)
elif limit == Limits.USER_SANDBOX_TASKS:
assert context is not None
context = cast(UserSandboxTasksContext, context)
return LimitStatus(
# TODO: check about active/removed tasks
Task.objects.filter(owner=context.user_id, organization=None).count(),
settings.DEFAULT_LIMITS["USER_SANDBOX_TASKS"],
)
elif limit == Limits.ORG_TASKS:
assert context is not None
context = cast(OrgTasksContext, context)
return LimitStatus(
# TODO: check about active/removed tasks
Task.objects.filter(organization=context.org_id).count(),
settings.DEFAULT_LIMITS["ORG_TASKS"],
)
elif limit == Limits.TASKS_IN_USER_SANDBOX_PROJECT:
assert context is not None
context = cast(TasksInUserSandboxProjectContext, context)
return LimitStatus(
# TODO: check about active/removed tasks
Task.objects.filter(project=context.project_id).count(),
settings.DEFAULT_LIMITS["TASKS_IN_USER_SANDBOX_PROJECT"]
)
elif limit == Limits.TASKS_IN_ORG_PROJECT:
assert context is not None
context = cast(TasksInOrgProjectContext, context)
return LimitStatus(
# TODO: check about active/removed tasks
Task.objects.filter(project=context.project_id).count(),
settings.DEFAULT_LIMITS["TASKS_IN_ORG_PROJECT"]
)
elif limit == Limits.PROJECT_WEBHOOKS:
assert context is not None
context = cast(ProjectWebhooksContext, context)
return LimitStatus(
# We only limit webhooks per project, not per user
# TODO: think over this limit, maybe we should limit per user
Webhook.objects.filter(project=context.project_id).count(),
settings.DEFAULT_LIMITS["PROJECT_WEBHOOKS"]
)
elif limit == Limits.ORG_COMMON_WEBHOOKS:
assert context is not None
context = cast(OrgCommonWebhooksContext, context)
return LimitStatus(
Webhook.objects.filter(organization=context.org_id, project=None).count(),
settings.DEFAULT_LIMITS["ORG_COMMON_WEBHOOKS"]
)
elif limit == Limits.USER_SANDBOX_CLOUD_STORAGES:
assert context is not None
context = cast(UserSandboxCloudStoragesContext, context)
return LimitStatus(
CloudStorage.objects.filter(owner=context.user_id, organization=None).count(),
settings.DEFAULT_LIMITS["USER_SANDBOX_CLOUD_STORAGES"]
)
elif limit == Limits.ORG_CLOUD_STORAGES:
assert context is not None
context = cast(OrgCloudStoragesContext, context)
return LimitStatus(
CloudStorage.objects.filter(organization=context.org_id).count(),
settings.DEFAULT_LIMITS["ORG_CLOUD_STORAGES"]
)
raise NotImplementedError(f"Unknown capability {limit.name}")

0
cvat/apps/limit_manager/migrations/__init__.py
Unescape Escape View File

11
cvat/apps/limit_manager/models.py
Unescape Escape View File

@ -1,11 +0,0 @@
# Copyright (C) 2022 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
from django.db import models
import cvat.apps.limit_manager.core.limits as core
class Limits(core.Limits, models.TextChoices):
pass

16
cvat/settings/base.py
Unescape Escape View File

@ -264,22 +264,6 @@ IAM_OPA_DATA_URL = f'{IAM_OPA_HOST}/v1/data'
LOGIN_URL = 'rest_login' LOGIN_URL = 'rest_login'
LOGIN_REDIRECT_URL = '/' LOGIN_REDIRECT_URL = '/'
DEFAULT_LIMITS = {
"USER_SANDBOX_TASKS": 10,
"USER_SANDBOX_PROJECTS": 3,
"TASKS_IN_USER_SANDBOX_PROJECT": 5,
"USER_OWNED_ORGS": 1,
"USER_SANDBOX_CLOUD_STORAGES": 10,
"ORG_TASKS": 10,
"ORG_PROJECTS": 3,
"TASKS_IN_ORG_PROJECT": 5,
"ORG_CLOUD_STORAGES": 10,
"ORG_COMMON_WEBHOOKS": 20,
"PROJECT_WEBHOOKS": 10,
}
# ORG settings # ORG settings
ORG_INVITATION_CONFIRM = 'No' ORG_INVITATION_CONFIRM = 'No'

18
cvat/settings/testing.py
Unescape Escape View File

@ -64,24 +64,6 @@ PASSWORD_HASHERS = (
'django.contrib.auth.hashers.MD5PasswordHasher', 'django.contrib.auth.hashers.MD5PasswordHasher',
) )
# Most of the unit tests don't suppose user limitations
# Can be changed where the limits are supposed.
DEFAULT_LIMITS = {
"USER_SANDBOX_TASKS": None,
"USER_SANDBOX_PROJECTS": None,
"TASKS_IN_USER_SANDBOX_PROJECT": None,
"USER_OWNED_ORGS": None,
"USER_SANDBOX_CLOUD_STORAGES": None,
"ORG_TASKS": None,
"ORG_PROJECTS": None,
"TASKS_IN_ORG_PROJECT": None,
"ORG_CLOUD_STORAGES": None,
"ORG_COMMON_WEBHOOKS": None,
"PROJECT_WEBHOOKS": None,
}
# When you run ./manage.py test, Django looks at the TEST_RUNNER setting to # When you run ./manage.py test, Django looks at the TEST_RUNNER setting to
# determine what to do. By default, TEST_RUNNER points to # determine what to do. By default, TEST_RUNNER points to
# 'django.test.runner.DiscoverRunner'. This class defines the default Django # 'django.test.runner.DiscoverRunner'. This class defines the default Django

6
tests/python/rest_api/test_cloud_storages.py
Unescape Escape View File

@ -48,7 +48,7 @@ class TestGetCloudStorage:
("user", True, True), ("user", True, True),
], ],
) )
def test_sandbox_user_get_coud_storage( def test_sandbox_user_get_cloud_storage(
self, storage_id, group, is_owner, is_allow, users, cloud_storages self, storage_id, group, is_owner, is_allow, users, cloud_storages
): ):
org = "" org = ""
@ -80,7 +80,7 @@ class TestGetCloudStorage:
("worker", False, False), ("worker", False, False),
], ],
) )
def test_org_user_get_coud_storage( def test_org_user_get_cloud_storage(
self, org_id, storage_id, role, is_owner, is_allow, find_users, cloud_storages self, org_id, storage_id, role, is_owner, is_allow, find_users, cloud_storages
): ):
cloud_storage = cloud_storages[storage_id] cloud_storage = cloud_storages[storage_id]
@ -197,7 +197,7 @@ class TestPostCloudStorage:
("supervisor", False), ("supervisor", False),
], ],
) )
def test_org_user_create_coud_storage(self, org_id, role, is_allow, find_users): def test_org_user_create_cloud_storage(self, org_id, role, is_allow, find_users):
username = find_users(role=role, org=org_id)[0]["username"] username = find_users(role=role, org=org_id)[0]["username"]
if is_allow: if is_allow:

72
tests/python/rest_api/test_invitations.py
Unescape Escape View File

@ -13,19 +13,26 @@ from shared.utils.config import post_method
from .utils import CollectionSimpleFilterTestBase from .utils import CollectionSimpleFilterTestBase
@pytest.mark.usefixtures("restore_db_per_function")
class TestCreateInvitations: class TestCreateInvitations:
ROLES = ["worker", "supervisor", "maintainer", "owner"]
@pytest.fixture(autouse=True)
def setup(self, restore_db_per_function, organizations, memberships, admin_user):
self.org_id = 2
self.owner = self.get_member("owner", memberships, self.org_id)
def _test_post_invitation_201(self, user, data, invitee, **kwargs): def _test_post_invitation_201(self, user, data, invitee, **kwargs):
response = post_method(user, "invitations", data, **kwargs) response = post_method(user, "invitations", data, **kwargs)
assert response.status_code == HTTPStatus.CREATED assert response.status_code == HTTPStatus.CREATED, response.content
assert data["role"] == response.json()["role"] assert data["role"] == response.json()["role"]
assert invitee["id"] == response.json()["user"]["id"] assert invitee["id"] == response.json()["user"]["id"]
assert kwargs["org_id"] == response.json()["organization"] assert kwargs["org_id"] == response.json()["organization"]
def _test_post_invitation_403(self, user, data, **kwargs): def _test_post_invitation_403(self, user, data, **kwargs):
response = post_method(user, "invitations", data, **kwargs) response = post_method(user, "invitations", data, **kwargs)
assert response.status_code == HTTPStatus.FORBIDDEN assert response.status_code == HTTPStatus.FORBIDDEN, response.content
assert "You do not have permission" in str(response.content)
@staticmethod @staticmethod
def get_non_member_users(memberships, users): def get_non_member_users(memberships, users):
@ -44,50 +51,55 @@ class TestCreateInvitations:
return member return member
@pytest.mark.parametrize("org_id", [2]) @pytest.mark.parametrize("org_role", ROLES)
@pytest.mark.parametrize("org_role", ["worker", "supervisor", "maintainer", "owner"]) @pytest.mark.parametrize("invitee_role", ROLES)
def test_create_invitation(self, organizations, memberships, users, org_id, org_role): def test_create_invitation(self, organizations, memberships, users, org_role, invitee_role):
member = self.get_member(org_role, memberships, org_id) org_id = self.org_id
non_member_users = self.get_non_member_users(memberships, users) inviter_user = self.get_member(org_role, memberships, org_id)
invitee_user = self.get_non_member_users(memberships, users)[0]
if org_role in ["worker", "supervisor"]: if org_role in ["worker", "supervisor"]:
for invitee_role in ["worker", "supervisor", "maintainer", "owner"]: self._test_post_invitation_403(
self._test_post_invitation_403( inviter_user["username"],
member["username"], {"role": invitee_role, "email": invitee_user["email"]},
{"role": invitee_role, "email": non_member_users[0]["email"]}, org_id=org_id,
org_id=org_id, )
)
else:
for idx, invitee_role in enumerate(["worker", "supervisor"]):
self._test_post_invitation_201(
member["username"],
{"role": invitee_role, "email": non_member_users[idx]["email"]},
non_member_users[idx],
org_id=org_id,
)
# only the owner can invite a maintainer elif invitee_role in ["worker", "supervisor"]:
self._test_post_invitation_201(
inviter_user["username"],
{"role": invitee_role, "email": invitee_user["email"]},
invitee_user,
org_id=org_id,
)
elif invitee_role == "maintainer":
if org_role == "owner": if org_role == "owner":
# only the owner can invite a maintainer
self._test_post_invitation_201( self._test_post_invitation_201(
member["username"], inviter_user["username"],
{"role": "maintainer", "email": non_member_users[2]["email"]}, {"role": invitee_role, "email": invitee_user["email"]},
non_member_users[2], invitee_user,
org_id=org_id, org_id=org_id,
) )
else: else:
self._test_post_invitation_403( self._test_post_invitation_403(
member["username"], inviter_user["username"],
{"role": "maintainer", "email": non_member_users[3]["email"]}, {"role": invitee_role, "email": invitee_user["email"]},
org_id=org_id, org_id=org_id,
) )
elif invitee_role == "owner":
# nobody can invite an owner # nobody can invite an owner
self._test_post_invitation_403( self._test_post_invitation_403(
member["username"], inviter_user["username"],
{"role": "owner", "email": non_member_users[4]["email"]}, {"role": invitee_role, "email": invitee_user["email"]},
org_id=org_id, org_id=org_id,
) )
else:
assert False, "Unknown role"
class TestInvitationsListFilters(CollectionSimpleFilterTestBase): class TestInvitationsListFilters(CollectionSimpleFilterTestBase):
field_lookups = { field_lookups = {

554
tests/python/rest_api/test_limits.py
Unescape Escape View File

@ -1,554 +0,0 @@
# Copyright (C) 2022 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
import json
from contextlib import contextmanager
from functools import partial
from http import HTTPStatus
from pathlib import Path
from typing import Optional
from uuid import uuid4
import boto3
import pytest
from cvat_sdk import Client, exceptions
from cvat_sdk.api_client import ApiClient, models
from cvat_sdk.core.client import Config
from cvat_sdk.core.proxies.projects import Project
from cvat_sdk.core.proxies.tasks import ResourceType, Task
from shared.utils.config import (
BASE_URL,
MINIO_ENDPOINT_URL,
MINIO_KEY,
MINIO_SECRET_KEY,
USER_PASS,
post_method,
)
from shared.utils.helpers import generate_image_file
@pytest.fixture
def fxt_image_file(tmp_path: Path):
img_path = tmp_path / "img.png"
with img_path.open("wb") as f:
f.write(generate_image_file(filename=str(img_path), size=(5, 10)).getvalue())
return img_path
def get_common_storage_params():
return {
"provider_type": "AWS_S3_BUCKET",
"credentials_type": "KEY_SECRET_KEY_PAIR",
"key": "minio_access_key",
"secret_key": "minio_secret_key",
"specific_attributes": "endpoint_url=http://minio:9000",
}
def define_s3_client():
s3 = boto3.resource(
"s3",
aws_access_key_id=MINIO_KEY,
aws_secret_access_key=MINIO_SECRET_KEY,
endpoint_url=MINIO_ENDPOINT_URL,
)
return s3.meta.client
class TestUserLimits:
@classmethod
def _create_user(cls, api_client: ApiClient, email: str) -> str:
username = email.split("@", maxsplit=1)[0]
with api_client:
(user, _) = api_client.auth_api.create_register(
models.RegisterSerializerExRequest(
username=username, password1=USER_PASS, password2=USER_PASS, email=email
)
)
api_client.cookies.clear()
return user.username
def _make_client(self) -> Client:
return Client(BASE_URL, config=Config(status_check_period=0.01))
@pytest.fixture(autouse=True)
def setup(self, restore_db_per_function, tmp_path: Path, fxt_image_file: Path):
self.tmp_dir = tmp_path
self.image_file = fxt_image_file
self.client = self._make_client()
self.user = self._create_user(self.client.api_client, email="test_user_limits@localhost")
with self.client:
self.client.login((self.user, USER_PASS))
@pytest.fixture
def fxt_another_client(self) -> Client:
client = self._make_client()
user = self._create_user(self.client.api_client, email="test_user_limits2@localhost")
with client:
client.login((user, USER_PASS))
yield client
_DEFAULT_TASKS_LIMIT = 10
_DEFAULT_PROJECT_TASKS_LIMIT = 5
_DEFAULT_PROJECTS_LIMIT = 3
_DEFAULT_ORGS_LIMIT = 1
_DEFAULT_CLOUD_STORAGES_LIMIT = 10
_TASK_LIMIT_MESSAGE = "tasks per user"
_PROJECT_TASK_LIMIT_MESSAGE = "tasks per project for the user"
_PROJECTS_LIMIT_MESSAGE = "projects per user"
_ORGS_LIMIT_MESSAGE = "organizations per user"
_CLOUD_STORAGES_LIMIT_MESSAGE = "cloud storages per user"
def _create_task(
self, *, project: Optional[int] = None, client: Optional[Client] = None
) -> Task:
if client is None:
client = self.client
return client.tasks.create_from_data(
spec=models.TaskWriteRequest(
name="test_task",
labels=[models.PatchedLabelRequest(name="cat")] if not project else [],
project_id=project,
),
resource_type=ResourceType.LOCAL,
resources=[str(self.image_file)],
)
def _create_project(self, *, client: Optional[Client] = None) -> Project:
if client is None:
client = self.client
return client.projects.create(models.ProjectWriteRequest(name="test_project"))
def test_can_reach_tasks_limit(self):
for _ in range(self._DEFAULT_TASKS_LIMIT):
self._create_task()
with pytest.raises(exceptions.ApiException) as capture:
self._create_task()
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_tasks_limit_when_importing_backup(self):
for _ in range(self._DEFAULT_TASKS_LIMIT):
task = self._create_task()
backup_filename = self.tmp_dir / "task_backup.zip"
task.download_backup(backup_filename)
with pytest.raises(exceptions.ApiException) as capture:
self.client.tasks.create_from_backup(backup_filename)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_tasks_limit_when_creating_in_project(self):
project = self._create_project().id
for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project)
with pytest.raises(exceptions.ApiException) as capture:
self._create_task(project=project)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECT_TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_tasks_limit_when_creating_in_different_projects(self):
project1 = self._create_project().id
project2 = self._create_project().id
for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project1)
for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project2)
with pytest.raises(exceptions.ApiException) as capture:
self._create_task()
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_tasks_limit_when_creating_in_filled_project(self):
project = self._create_project().id
for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project)
for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task()
with pytest.raises(exceptions.ApiException) as capture:
self._create_task(project=project)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
assert self._PROJECT_TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_project_tasks_limit_when_moving_into_filled_project(self):
project = self._create_project().id
for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project)
task = self._create_task()
with pytest.raises(exceptions.ApiException) as capture:
task.update(models.PatchedTaskWriteRequest(project_id=project))
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECT_TASK_LIMIT_MESSAGE in str(capture.value.body)
@pytest.mark.xfail(
raises=AssertionError, reason="only admins can change ownership, but they ignore limits"
)
def test_can_reach_tasks_limit_when_giving_away_to_another_user(
self, fxt_another_client: Client
):
for _ in range(self._DEFAULT_TASKS_LIMIT):
self._create_task(client=fxt_another_client)
task = self._create_task()
with pytest.raises(exceptions.ApiException) as capture:
task.update(
models.PatchedTaskWriteRequest(
owner_id=fxt_another_client.users.retrieve_current_user().id
)
)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECT_TASK_LIMIT_MESSAGE in str(capture.value.body)
@pytest.mark.xfail(
raises=AssertionError, reason="only admins can change ownership, but they ignore limits"
)
def test_can_reach_project_tasks_limit_when_giving_away_to_another_users_filled_project(
self, fxt_another_client: Client
):
project = self._create_project(client=fxt_another_client).id
for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(client=fxt_another_client, project=project)
for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(client=fxt_another_client)
task = self._create_task()
with pytest.raises(exceptions.ApiException) as capture:
task.update(
models.PatchedTaskWriteRequest(
owner_id=fxt_another_client.users.retrieve_current_user().id
)
)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
assert self._PROJECT_TASK_LIMIT_MESSAGE in str(capture.value.body)
@pytest.mark.xfail(
raises=AssertionError, reason="only admins can change ownership, but they ignore limits"
)
def test_can_reach_projects_limit_when_giving_away_to_another_user(
self, fxt_another_client: Client
):
for _ in range(self._DEFAULT_PROJECTS_LIMIT):
self._create_project(client=fxt_another_client)
project = self._create_project()
with pytest.raises(exceptions.ApiException) as capture:
project.update(
models.PatchedProjectWriteRequest(
owner_id=fxt_another_client.users.retrieve_current_user().id
)
)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECT_TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_projects_limit(self):
for _ in range(self._DEFAULT_PROJECTS_LIMIT):
self._create_project()
with pytest.raises(exceptions.ApiException) as capture:
self._create_project()
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECTS_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_projects_limit_when_importing_backup(self):
for _ in range(self._DEFAULT_PROJECTS_LIMIT):
project = self._create_project()
backup_filename = self.tmp_dir / (project.name + "_backup.zip")
project.download_backup(backup_filename)
with pytest.raises(exceptions.ApiException) as capture:
self.client.projects.create_from_backup(backup_filename)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECTS_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_orgs_limit(self):
for i in range(self._DEFAULT_ORGS_LIMIT):
(_, response) = self.client.api_client.organizations_api.create(
models.OrganizationWriteRequest(slug=f"test_user_orgs_{i}"), _parse_response=False
)
assert response.status == HTTPStatus.CREATED
with pytest.raises(exceptions.ApiException) as capture:
self.client.api_client.organizations_api.create(
models.OrganizationWriteRequest(slug=f"test_user_orgs_{i}"), _parse_response=False
)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._ORGS_LIMIT_MESSAGE in str(capture.value.body)
@pytest.mark.with_external_services
def test_can_reach_cloud_storages_limit(self, request: pytest.FixtureRequest):
storage_params = get_common_storage_params()
# TODO: refactor after https://github.com/opencv/cvat/pull/4819
s3_client = define_s3_client()
def _create_bucket(name: str) -> str:
name = name + str(uuid4())
s3_client.create_bucket(Bucket=name)
request.addfinalizer(partial(s3_client.delete_bucket, Bucket=name))
return name
def _add_storage(idx: int):
response = post_method(
self.user,
"cloudstorages",
{
"display_name": f"test_storage{idx}",
"resource": _create_bucket(f"testbucket{idx}"),
**storage_params,
},
)
return response
for i in range(self._DEFAULT_CLOUD_STORAGES_LIMIT):
response = _add_storage(i)
assert response.status_code == HTTPStatus.CREATED
response = _add_storage(i)
assert response.status_code == HTTPStatus.FORBIDDEN
assert self._CLOUD_STORAGES_LIMIT_MESSAGE in str(response.content)
class TestOrgLimits:
@classmethod
def _create_org(cls, api_client: ApiClient) -> str:
with api_client:
(_, response) = api_client.organizations_api.create(
models.OrganizationWriteRequest(slug="test_org_limits"), _parse_response=False
)
return json.loads(response.data)
def _make_client(self) -> Client:
return Client(BASE_URL, config=Config(status_check_period=0.01))
@pytest.fixture(autouse=True)
def setup(
self, restore_db_per_function, tmp_path: Path, regular_user: str, fxt_image_file: Path
):
self.tmp_dir = tmp_path
self.image_file = fxt_image_file
self.client = self._make_client()
self.user = regular_user
with self.client:
self.client.login((self.user, USER_PASS))
org = self._create_org(self.client.api_client)
self.org = org["id"]
self.org_slug = org["slug"]
with self._patch_client_with_org(self.client):
yield
_DEFAULT_TASKS_LIMIT = 10
_DEFAULT_PROJECT_TASKS_LIMIT = 5
_DEFAULT_PROJECTS_LIMIT = 3
_DEFAULT_CLOUD_STORAGES_LIMIT = 10
_TASK_LIMIT_MESSAGE = "tasks per organization"
_PROJECT_TASK_LIMIT_MESSAGE = "tasks per project for the organization"
_PROJECTS_LIMIT_MESSAGE = "projects per organization"
_CLOUD_STORAGES_LIMIT_MESSAGE = "cloud storages per organization"
@contextmanager
def _patch_client_with_org(self, client: Optional[Client] = None):
if client is None:
client = self.client
new_headers = self.client.api_client.default_headers.copy()
new_headers["X-Organization"] = self.org_slug
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setattr(client.api_client, "default_headers", new_headers)
yield client
@pytest.fixture
def fxt_patch_client_with_org(self):
with self._patch_client_with_org(self.client):
yield
def _create_task(
self, *, project: Optional[int] = None, client: Optional[Client] = None
) -> Task:
if client is None:
client = self.client
return client.tasks.create_from_data(
spec=models.TaskWriteRequest(
name="test_task",
labels=[models.PatchedLabelRequest(name="cat")] if not project else [],
project_id=project,
),
resource_type=ResourceType.LOCAL,
resources=[str(self.image_file)],
)
def _create_project(self, *, client: Optional[Client] = None) -> Project:
if client is None:
client = self.client
return client.projects.create(models.ProjectWriteRequest(name="test_project"))
def test_can_reach_tasks_limit(self):
for _ in range(self._DEFAULT_TASKS_LIMIT):
self._create_task()
with pytest.raises(exceptions.ApiException) as capture:
self._create_task()
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_tasks_limit_when_importing_backup(self):
for _ in range(self._DEFAULT_TASKS_LIMIT):
task = self._create_task()
backup_filename = self.tmp_dir / "task_backup.zip"
task.download_backup(backup_filename)
with pytest.raises(exceptions.ApiException) as capture:
self.client.tasks.create_from_backup(backup_filename)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_tasks_limit_when_creating_in_project(self):
project = self._create_project().id
for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project)
with pytest.raises(exceptions.ApiException) as capture:
self._create_task(project=project)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECT_TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_tasks_limit_when_creating_in_different_projects(self):
project1 = self._create_project().id
project2 = self._create_project().id
for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project1)
for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project2)
with pytest.raises(exceptions.ApiException) as capture:
self._create_task()
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_tasks_limit_when_creating_in_filled_project(self):
project = self._create_project().id
for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task(project=project)
for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT):
self._create_task()
with pytest.raises(exceptions.ApiException) as capture:
self._create_task(project=project)
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._TASK_LIMIT_MESSAGE in str(capture.value.body)
assert self._PROJECT_TASK_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_projects_limit(self):
for _ in range(self._DEFAULT_PROJECTS_LIMIT):
self._create_project()
with pytest.raises(exceptions.ApiException) as capture:
self._create_project()
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECTS_LIMIT_MESSAGE in str(capture.value.body)
def test_can_reach_projects_limit_when_importing_backup(self):
for _ in range(self._DEFAULT_PROJECTS_LIMIT):
project = self._create_project()
backup_filename = self.tmp_dir / "test_project_backup.zip"
project.download_backup(str(backup_filename))
with pytest.raises(exceptions.ApiException) as capture:
self.client.projects.create_from_backup(str(backup_filename))
assert capture.value.status == HTTPStatus.FORBIDDEN
assert self._PROJECTS_LIMIT_MESSAGE in str(capture.value.body)
@pytest.mark.with_external_services
def test_can_reach_cloud_storages_limit(self, request: pytest.FixtureRequest):
storage_params = get_common_storage_params()
# TODO: refactor after https://github.com/opencv/cvat/pull/4819
s3_client = define_s3_client()
def _create_bucket(name: str) -> str:
name = name + str(uuid4())
s3_client.create_bucket(Bucket=name)
request.addfinalizer(partial(s3_client.delete_bucket, Bucket=name))
return name
def _add_storage(idx: int):
response = post_method(
self.user,
"cloudstorages",
{
"display_name": f"test_storage{idx}",
"resource": _create_bucket(f"testbucket{idx}"),
**storage_params,
},
org_id=self.org,
)
return response
for i in range(self._DEFAULT_CLOUD_STORAGES_LIMIT):
response = _add_storage(i)
assert response.status_code == HTTPStatus.CREATED
response = _add_storage(i)
assert response.status_code == HTTPStatus.FORBIDDEN
assert self._CLOUD_STORAGES_LIMIT_MESSAGE in str(response.content)

2
tests/python/shared/assets/users.json
Unescape Escape View File

@ -310,7 +310,7 @@
"is_active": true, "is_active": true,
"is_staff": true, "is_staff": true,
"is_superuser": true, "is_superuser": true,
"last_login": "2022-12-05T07:46:24.795659Z", "last_login": "2022-12-05T07:46:24.795000Z",
"last_name": "First", "last_name": "First",
"url": "http://localhost:8080/api/users/1", "url": "http://localhost:8080/api/users/1",
"username": "admin1" "username": "admin1"

17
tests/python/shared/fixtures/init.py
Unescape Escape View File

@ -89,6 +89,12 @@ def pytest_addoption(parser):
help="Platform identifier - 'kube' or 'local'. (default: %(default)s)", help="Platform identifier - 'kube' or 'local'. (default: %(default)s)",
) )
group._addoption(
"--no-run-services",
action="store_true",
help="Run tests without running containers. (default: %(default)s)",
)
def _run(command, capture_output=True): def _run(command, capture_output=True):
_command = command.split() if isinstance(command, str) else command _command = command.split() if isinstance(command, str) else command
@ -263,10 +269,6 @@ def start_services(rebuild=False):
capture_output=False, capture_output=False,
) )
docker_restore_data_volumes()
docker_cp(CVAT_DB_DIR / "restore.sql", f"{PREFIX}_cvat_db_1:/tmp/restore.sql")
docker_cp(CVAT_DB_DIR / "data.json", f"{PREFIX}_cvat_server_1:/tmp/data.json")
def pytest_sessionstart(session: pytest.Session) -> None: def pytest_sessionstart(session: pytest.Session) -> None:
stop = session.config.getoption("--stop-services") stop = session.config.getoption("--stop-services")
@ -274,6 +276,7 @@ def pytest_sessionstart(session: pytest.Session) -> None:
rebuild = session.config.getoption("--rebuild") rebuild = session.config.getoption("--rebuild")
cleanup = session.config.getoption("--cleanup") cleanup = session.config.getoption("--cleanup")
dumpdb = session.config.getoption("--dumpdb") dumpdb = session.config.getoption("--dumpdb")
no_init = session.config.getoption("--no-run-services")
if session.config.getoption("--collect-only"): if session.config.getoption("--collect-only"):
if any((stop, start, rebuild, cleanup, dumpdb)): if any((stop, start, rebuild, cleanup, dumpdb)):
@ -325,7 +328,11 @@ def pytest_sessionstart(session: pytest.Session) -> None:
) )
pytest.exit("All testing containers are stopped", returncode=0) pytest.exit("All testing containers are stopped", returncode=0)
start_services(rebuild) if not no_init:
start_services(rebuild)
docker_restore_data_volumes()
docker_cp(CVAT_DB_DIR / "restore.sql", f"{PREFIX}_cvat_db_1:/tmp/restore.sql")
docker_cp(CVAT_DB_DIR / "data.json", f"{PREFIX}_cvat_server_1:/tmp/data.json")
wait_for_services() wait_for_services()
docker_exec_cvat("python manage.py loaddata /tmp/data.json") docker_exec_cvat("python manage.py loaddata /tmp/data.json")

Write Preview
Loading…
Cancel
Save