REST API tests for IAM (#4090)

main
Kirill Sizov 4 years ago committed by GitHub
parent 90dcadd479
commit 6c96891fc4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -77,8 +77,6 @@ jobs:
run: |
docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml up -d
/bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done'
cat tests/rest_api/assets/cvat_db.sql | docker exec -i cvat_db psql -q -U root -d cvat
cat tests/rest_api/assets/cvat_data.tar.bz2 | docker run --rm -i --volumes-from cvat ubuntu tar -xj --strip 3 -C /home/django/data
pip3 install --user -r tests/rest_api/requirements.txt
pytest tests/rest_api/
docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml down -v

@ -35,7 +35,7 @@ available in the system like comments, users, issues, please use the following
procedure to add them:
1. Run a clean CVAT instance
1. Restore DB and data volume using commands below
1. Restore DB and data volume using commands below or running tests
1. Add new objects (e.g. issues, comments, tasks, projects)
1. Backup DB and data volume using commands below
1. Don't forget to dump new objects into corresponding json files inside
@ -58,35 +58,31 @@ for i, color in enumerate(colormap):
img.save(f'{i}.png')
```
## How to backup DB and data volume?
To backup DB and data volume, please use commands below.
```console
docker exec -i cvat_db pg_dump -c -U root -d cvat > assets/cvat_db.sql
docker exec cvat_db pg_dump -c -C -Fc -U root -d cvat > assets/cvat_db.dump
docker run --rm --volumes-from cvat ubuntu tar -cjv /home/django/data > assets/cvat_data.tar.bz2
```
To restore DB and data volume, please use commands below.
# How to update *.json files in the assets directory?
```console
cat assets/cvat_db.sql | docker exec -i cvat_db psql -q -U root -d cvat
cat assets/cvat_data.tar.bz2 | docker run --rm -i --volumes-from cvat ubuntu tar -xj --strip 3 -C /home/django/data
If you have updated the test database and want to update the assets/*.json
files as well, run the appropriate script:
```
python utils/dump_objects.py
```
To dump an object into JSON, please look at the sample code below. You also
can find similar code in `utils/dump_objects.py` script. All users in the
testing system has the same password `!Q@W#E$R`.
# How to restore DB and data volume?
```python
import requests
import json
with requests.Session() as session:
session.auth = ('admin1', '!Q@W#E$R')
To restore DB and data volume, please use commands below.
for obj in ['user', 'project', 'task', 'job']:
response = session.get(f'http://localhost:8080/api/v1/{obj}s')
with open(f'{obj}s.json', 'w') as f:
json.dump(response.json(), f, indent=2, sort_keys=True)
```console
cat assets/cvat_db/cvat_db.dump | docker exec -i cvat_db pg_restore -1 -c -U root -d cvat
cat assets/cvat_data.tar.bz2 | docker run --rm -i --volumes-from cvat ubuntu tar -xj --strip 3 -C /home/django/data
```
## FAQ
@ -97,7 +93,7 @@ with requests.Session() as session:
you have json description of all objects together with cvat_db.sql, it will
be possible to recreate them manually.
1. How to upgrade cvat_data.tar.bz2 and cvat_db.sql?
1. How to upgrade cvat_data.tar.bz2 and cvat_db.dump?
After every commit which changes the layout of DB and data directory it is
possible to break these files. But failed tests should be a clear indicator
@ -109,3 +105,9 @@ with requests.Session() as session:
Construction of some objects can be complex and takes time (backup
and restore should be much faster). Construction of objects in UI is more
intuitive.
1. How we solve the problem of dependent tests?
Since some tests change the database, these tests may be dependent on each
other, so in current implementation we avoid such problem by restoring
the database after each test function (see `conftest.py`)

File diff suppressed because one or more lines are too long

@ -0,0 +1,7 @@
SELECT pg_terminate_backend(pg_stat_activity.pid)
FROM pg_stat_activity
WHERE pg_stat_activity.datname = 'cvat' AND pid <> pg_backend_pid();
DROP DATABASE cvat;
CREATE DATABASE cvat WITH TEMPLATE test_db;

@ -1,8 +1,48 @@
{
"count": 7,
"count": 9,
"next": null,
"previous": null,
"results": [
{
"created_date": "2022-01-19T13:54:42.015131Z",
"key": "BrwoDmMNQQ1v9WXOukp9DwQVuqB3RDPjpUECCEq6QcAuG0Pi8k1IYtQ9uz9jg0Bv",
"organization": 2,
"owner": {
"first_name": "Business",
"id": 10,
"last_name": "First",
"url": "http://localhost:8080/api/v1/users/10",
"username": "business1"
},
"role": "maintainer",
"user": {
"first_name": "User",
"id": 5,
"last_name": "Fourth",
"url": "http://localhost:8080/api/v1/users/5",
"username": "user4"
}
},
{
"created_date": "2022-01-19T13:54:42.005381Z",
"key": "5FjIXya6fTGvlRpauFvi2QN1wDOqo1V9REB5rJinDR8FZO9gr0qmtWpghsCte8Y1",
"organization": 2,
"owner": {
"first_name": "Business",
"id": 10,
"last_name": "First",
"url": "http://localhost:8080/api/v1/users/10",
"username": "business1"
},
"role": "supervisor",
"user": {
"first_name": "User",
"id": 4,
"last_name": "Third",
"url": "http://localhost:8080/api/v1/users/4",
"username": "user3"
}
},
{
"created_date": "2021-12-14T19:55:13.745912Z",
"key": "h43G28di7vfs4Jv5VrKZ26xvGAfm6Yc2FFv14z9EKhiuIEDQ22pEnzmSCab8MnK1",

@ -1,8 +1,38 @@
{
"count": 9,
"count": 11,
"next": null,
"previous": null,
"results": [
{
"id": 11,
"invitation": "BrwoDmMNQQ1v9WXOukp9DwQVuqB3RDPjpUECCEq6QcAuG0Pi8k1IYtQ9uz9jg0Bv",
"is_active": true,
"joined_date": "2022-01-19T13:54:42.015131Z",
"organization": 2,
"role": "maintainer",
"user": {
"first_name": "User",
"id": 5,
"last_name": "Fourth",
"url": "http://localhost:8080/api/v1/users/5",
"username": "user4"
}
},
{
"id": 10,
"invitation": "5FjIXya6fTGvlRpauFvi2QN1wDOqo1V9REB5rJinDR8FZO9gr0qmtWpghsCte8Y1",
"is_active": true,
"joined_date": "2022-01-19T13:54:42.005381Z",
"organization": 2,
"role": "supervisor",
"user": {
"first_name": "User",
"id": 4,
"last_name": "Third",
"url": "http://localhost:8080/api/v1/users/4",
"username": "user3"
}
},
{
"id": 9,
"invitation": "h43G28di7vfs4Jv5VrKZ26xvGAfm6Yc2FFv14z9EKhiuIEDQ22pEnzmSCab8MnK1",

@ -14,7 +14,7 @@
"is_active": true,
"is_staff": true,
"is_superuser": true,
"last_login": "2021-12-22T07:32:58.602211Z",
"last_login": "2021-12-22T08:11:58.502575Z",
"last_name": "First",
"url": "http://localhost:8080/api/v1/users/1",
"username": "admin1"
@ -158,7 +158,7 @@
"is_active": true,
"is_staff": false,
"is_superuser": false,
"last_login": "2021-12-14T19:44:48.526708Z",
"last_login": "2022-01-19T13:52:59.477881Z",
"last_name": "First",
"url": "http://localhost:8080/api/v1/users/10",
"username": "business1"

@ -0,0 +1,119 @@
# Copyright (C) 2021 Intel Corporation
#
# SPDX-License-Identifier: MIT
from subprocess import run, CalledProcessError
import pytest
import json
import os.path as osp
from .utils.config import ASSETS_DIR
def cvat_db_container(command):
run(('docker exec cvat_db ' + command).split(), check=True) #nosec
def docker_cp(source, target):
run(' '.join(['docker container cp', source, target]).split(), check=True) #nosec
def restore_data_volume():
command = 'docker run --rm --volumes-from cvat --mount ' \
f'type=bind,source={ASSETS_DIR},target=/mnt/ ubuntu tar ' \
'--strip 3 -C /home/django/data -xjf /mnt/cvat_data.tar.bz2'
run(command.split(), check=True) #nosec
def drop_test_db():
cvat_db_container('pg_restore -c -U root -d cvat /cvat_db/cvat_db.dump')
cvat_db_container('rm -rf /cvat_db')
cvat_db_container('dropdb test_db')
def create_test_db():
docker_cp(source=osp.join(ASSETS_DIR, 'cvat_db'), target='cvat_db:/')
cvat_db_container('createdb test_db')
cvat_db_container('pg_restore -U root -d test_db /cvat_db/cvat_db.dump')
@pytest.fixture(scope='session', autouse=True)
def init_test_db():
try:
restore_data_volume()
create_test_db()
except CalledProcessError:
drop_test_db()
pytest.exit(f"Cannot to initialize test DB")
yield
drop_test_db()
@pytest.fixture(scope='function', autouse=True)
def restore_cvat_db():
cvat_db_container('psql -U root -d postgres -f /cvat_db/cvat_db.sql')
@pytest.fixture(scope='module')
def users():
with open(osp.join(ASSETS_DIR, 'users.json')) as f:
return json.load(f)['results']
@pytest.fixture(scope='module')
def organizations():
with open(osp.join(ASSETS_DIR, 'organizations.json')) as f:
data = json.load(f)
def _organizations(org_id=None):
if org_id:
return [org for org in data if org['id'] == org_id][0]
return data
return _organizations
@pytest.fixture(scope='module')
def memberships():
with open(osp.join(ASSETS_DIR, 'memberships.json')) as f:
return json.load(f)['results']
@pytest.fixture(scope='module')
def users_by_name(users):
return {user['username']: user for user in users}
@pytest.fixture(scope='module')
def find_users(test_db):
def find(**kwargs):
assert len(kwargs) > 0
assert any(kwargs)
data = test_db
kwargs = dict(filter(lambda a: a[1] is not None, kwargs.items()))
for field, value in kwargs.items():
if field.startswith('exclude_'):
field = field.split('_', maxsplit=1)[1]
exclude_rows = set(v['id'] for v in
filter(lambda a: a[field] == value, test_db))
data = list(filter(lambda a: a['id'] not in exclude_rows, data))
else:
data = list(filter(lambda a: a[field] == value, data))
return data
return find
@pytest.fixture(scope='module')
def test_db(users, users_by_name, memberships):
data = []
fields = ['username', 'id', 'privilege', 'role', 'org', 'membership_id']
def add_row(**kwargs):
data.append({field: kwargs.get(field) for field in fields})
for user in users:
for group in user['groups']:
add_row(username=user['username'], id=user['id'], privilege=group)
for membership in memberships:
username = membership['user']['username']
for group in users_by_name[username]['groups']:
add_row(username=username, role=membership['role'], privilege=group,
id=membership['user']['id'], org=membership['organization'],
membership_id=membership['id'])
return data

@ -2,23 +2,20 @@
#
# SPDX-License-Identifier: MIT
import os
import os.path as osp
import glob
import requests
import json
from deepdiff import DeepDiff
from .utils import config
import pytest
def test_check_objects_integrity():
with requests.Session() as session:
session.auth = ('admin1', config.USER_PASS)
@pytest.mark.parametrize('path', glob.glob(osp.join(config.ASSETS_DIR, '*.json')))
def test_check_objects_integrity(path):
with open(path) as f:
endpoint = osp.basename(path).rsplit('.')[0]
response = config.get_method('admin1', endpoint, page_size='all')
json_objs = json.load(f)
resp_objs = response.json()
for filename in glob.glob(os.path.join(config.ASSETS_DIR, '*.json')):
with open(filename) as f:
endpoint = os.path.basename(filename).rsplit('.')[0]
response = session.get(config.get_api_url(endpoint, page_size='all'))
json_objs = json.load(f)
resp_objs = response.json()
assert DeepDiff(json_objs, resp_objs, ignore_order=True,
exclude_regex_paths="root\['results'\]\[\d+\]\['last_login'\]") == {}
assert DeepDiff(json_objs, resp_objs, ignore_order=True,
exclude_regex_paths="root\['results'\]\[\d+\]\['last_login'\]") == {}

@ -0,0 +1,52 @@
# Copyright (C) 2021 Intel Corporation
#
# SPDX-License-Identifier: MIT
from http import HTTPStatus
from deepdiff import DeepDiff
from .utils.config import get_method
class TestGetUsers:
def _test_can_see(self, user, data, endpoint='users', exclude_paths='', **kwargs):
response = get_method(user, endpoint, **kwargs)
response_data = response.json()
response_data = response_data.get('results', response_data)
assert response.status_code == HTTPStatus.OK
assert DeepDiff(data, response_data, ignore_order=True,
exclude_paths=exclude_paths) == {}
def _test_cannot_see(self, user, endpoint='users', **kwargs):
response = get_method(user, endpoint, **kwargs)
assert response.status_code == HTTPStatus.FORBIDDEN
def test_admin_can_see_all_others(self, users):
exclude_paths = [f"root[{i}]['last_login']" for i in range(len(users))]
self._test_can_see('admin2', users, exclude_paths=exclude_paths,
page_size="all")
def test_everybody_can_see_self(self, users_by_name):
for user, data in users_by_name.items():
self._test_can_see(user, data, "users/self", "root['last_login']")
def test_non_members_cannot_see_list_of_members(self):
self._test_cannot_see('user2', org='org1')
def test_non_admin_cannot_see_others(self, users):
non_admins = (v for v in users if not v['is_superuser'])
user = next(non_admins)['username']
user_id = next(non_admins)['id']
self._test_cannot_see(user, f"users/{user_id}")
def test_all_members_can_see_list_of_members(self, find_users, users):
org_members = [user['username'] for user in find_users(org=1)]
available_fields = ['url', 'id', 'username', 'first_name', 'last_name']
data = [dict(filter(lambda row: row[0] in available_fields, user.items()))
for user in users if user['username'] in org_members]
for member in org_members:
self._test_can_see(member, data, org='org1')

@ -1,35 +0,0 @@
# Copyright (C) 2021 Intel Corporation
#
# SPDX-License-Identifier: MIT
import os
from http import HTTPStatus
import requests
import json
from .utils import config
from deepdiff import DeepDiff
def test_non_admin_cannot_see_others():
for username in ['dummy1', 'worker1', 'user1', 'business1']:
response = requests.get(config.get_api_url('users'), auth=(username, config.USER_PASS))
assert response.status_code == HTTPStatus.OK
assert response.json()['count'] == 1
def test_admin_can_see_all_others():
response = requests.get(config.get_api_url('users'), auth=('admin2', config.USER_PASS))
assert response.status_code == HTTPStatus.OK
with open(os.path.join(config.ASSETS_DIR, 'users.json')) as f:
data = json.load(f)
assert response.json()['count'] == data['count']
def test_everybody_can_see_self():
with open(os.path.join(config.ASSETS_DIR, 'users.json')) as f:
data = json.load(f)['results']
users = {user['username']:user for user in data}
for username in ['dummy1', 'worker1', 'user1', 'business1', 'admin1']:
response = requests.get(config.get_api_url('users/self'), auth=(username, config.USER_PASS))
assert response.status_code == HTTPStatus.OK
assert DeepDiff(users[username], response.json(), ignore_order=True,
exclude_paths="root['last_login']") == {}

@ -2,24 +2,101 @@
#
# SPDX-License-Identifier: MIT
import os
from http import HTTPStatus
import requests
import json
from .utils import config
import pytest
from .utils.config import get_method, patch_method, delete_method
from deepdiff import DeepDiff
def compare_organizations(org_id, response):
assert response.status_code == HTTPStatus.OK
with open(os.path.join(config.ASSETS_DIR, 'organizations.json')) as f:
org = next(filter(lambda org: org['id'] == org_id, json.load(f)))
DeepDiff(org, response.json())
class TestGetOrganizations:
_ORG = 2
def test_admin1_get_organization_id_1():
response = requests.get(config.get_api_url('organizations/1'), auth=('admin1', config.USER_PASS))
compare_organizations(1, response)
@pytest.mark.parametrize('privilege, role, is_member, is_allow', [
('admin', None, None, True),
('user', None, False, False),
('business', None, False, False),
('worker', None, False, False),
(None, 'owner', True, True),
(None, 'maintainer', True, True),
(None, 'worker', True, True),
(None, 'supervisor', True, True),
])
def test_can_see_specific_organization(self, privilege, role, is_member,
is_allow, find_users, organizations):
exclude_org = None if is_member else self._ORG
org = self._ORG if is_member else None
user = find_users(privilege=privilege, role=role, org=org,
exclude_org=exclude_org)[0]['username']
def test_user1_get_organization_id_1():
response = requests.get(config.get_api_url('organizations/1'), auth=('user1', config.USER_PASS))
compare_organizations(1, response)
response = get_method(user, f'organizations/{self._ORG}')
if is_allow:
assert response.status_code == HTTPStatus.OK
assert DeepDiff(organizations(self._ORG), response.json()) == {}
else:
assert response.status_code == HTTPStatus.NOT_FOUND
class TestPatchOrganizations:
_ORG = 2
@pytest.fixture(scope='class')
def request_data(self):
return {'slug': 'new', 'name': 'new', 'description': 'new',
'contact': {'email': 'new@cvat.org'}}
@pytest.fixture(scope='class')
def expected_data(self, organizations, request_data):
data = organizations(self._ORG).copy()
data.update(request_data)
return data
@pytest.mark.parametrize('privilege, role, is_member, is_allow', [
('admin', None, None, True),
('user', None, False, False),
('business', None, False, False),
('worker', None, False, False),
(None, 'owner', True, True),
(None, 'maintainer', True, True),
(None, 'worker', True, False),
(None, 'supervisor', True, False),
])
def test_can_update_specific_organization(self, privilege, role, is_member,
is_allow, find_users, request_data, expected_data):
exclude_org = None if is_member else self._ORG
org = self._ORG if is_member else None
user = find_users(privilege=privilege, role=role, org=org,
exclude_org=exclude_org)[0]['username']
response = patch_method(user, f'organizations/{self._ORG}', request_data)
if is_allow:
assert response.status_code == HTTPStatus.OK
assert DeepDiff(expected_data, response.json(),
exclude_paths="root['updated_date']") == {}
else:
assert response.status_code != HTTPStatus.OK
class TestDeleteOrganizations:
_ORG = 2
@pytest.mark.parametrize('privilege, role, is_member, is_allow', [
('admin', None, None, True),
(None, 'owner', True, True),
(None, 'maintainer', True, False),
(None, 'worker', True, False),
(None, 'supervisor', True, False),
('user', None, False, False),
('business', None, False, False),
('worker', None, False, False),
])
def test_can_delete(self, privilege, role, is_member,
is_allow, find_users):
exclude_org = None if is_member else self._ORG
org = self._ORG if is_member else None
user = find_users(privilege=privilege, role=role, org=org,
exclude_org=exclude_org)[0]['username']
response = delete_method(user, f'organizations/{self._ORG}')
if is_allow:
assert response.status_code == HTTPStatus.NO_CONTENT
else:
assert response.status_code != HTTPStatus.OK

@ -0,0 +1,79 @@
# Copyright (C) 2021 Intel Corporation
#
# SPDX-License-Identifier: MIT
import pytest
from http import HTTPStatus
from deepdiff import DeepDiff
from .utils.config import get_method, patch_method
class TestGetMemberships:
def _test_can_see_memberships(self, user, data, **kwargs):
response = get_method(user, 'memberships', **kwargs)
assert response.status_code == HTTPStatus.OK
assert DeepDiff(data, response.json()['results']) == {}
def _test_cannot_see_memberships(self, user, **kwargs):
response = get_method(user, 'memberships', **kwargs)
assert response.status_code == HTTPStatus.FORBIDDEN
def test_admin_can_see_all_memberships(self, memberships):
self._test_can_see_memberships('admin2', memberships, page_size='all')
def test_non_admin_can_see_only_self_memberships(self, memberships):
non_admins= ['business1', 'user1', 'dummy1','worker2']
for user in non_admins:
data = [m for m in memberships if m['user']['username'] == user]
self._test_can_see_memberships(user, data)
def test_all_members_can_see_other_members_membership(self, memberships):
data = [m for m in memberships if m['organization'] == 1]
for membership in data:
self._test_can_see_memberships(membership['user']['username'],
data, org_id=1)
def test_non_members_cannot_see_members_membership(self):
non_org1_users = ['user2', 'worker3']
for user in non_org1_users:
self._test_cannot_see_memberships(user, org_id=1)
class TestPatchMemberships:
_ORG = 2
def _test_can_change_membership(self, user, membership_id, new_role):
response = patch_method(user, f"memberships/{membership_id}",
{'role': new_role}, org_id=self._ORG)
assert response.status_code == HTTPStatus.OK
assert response.json()['role'] == new_role
def _test_cannot_change_membership(self, user, membership_id, new_role):
response = patch_method(user, f"memberships/{membership_id}",
{'role': new_role}, org_id=self._ORG)
assert response.status_code == HTTPStatus.FORBIDDEN
@pytest.mark.parametrize('who, whom, new_role, is_allow', [
('supervisor', 'worker', 'supervisor', False),
('supervisor', 'maintainer', 'supervisor', False),
('worker', 'supervisor', 'worker', False),
('worker', 'maintainer', 'worker', False),
('maintainer', 'maintainer', 'worker', False),
('maintainer', 'supervisor', 'worker', True),
('maintainer', 'worker', 'supervisor', True),
('owner', 'maintainer', 'worker', True),
('owner', 'supervisor', 'worker', True),
('owner', 'worker', 'supervisor', True),
])
def test_user_can_change_role_of_member(self, who, whom, new_role, is_allow, find_users):
user = find_users(org=self._ORG, role=who)[0]['username']
membership_id = find_users(org=self._ORG, role=whom)[1]['membership_id']
if is_allow:
self._test_can_change_membership(user, membership_id, new_role)
else:
self._test_cannot_change_membership(user, membership_id, new_role)

@ -2,13 +2,23 @@
#
# SPDX-License-Identifier: MIT
import os
import os.path as osp
import requests
ROOT_DIR = os.path.dirname(__file__)
ASSETS_DIR = os.path.join(ROOT_DIR, '..', 'assets')
ROOT_DIR = osp.dirname(__file__)
ASSETS_DIR = osp.abspath(osp.join(ROOT_DIR, '..', 'assets'))
# Suppress the warning from Bandit about hardcoded passwords
USER_PASS = '!Q@W#E$R' # nosec
BASE_URL = 'http://localhost:8080/api/v1/'
def get_api_url(endpoint, **kwargs):
return BASE_URL + endpoint + '?' + '&'.join([f'{k}={v}' for k,v in kwargs.items()])
return BASE_URL + endpoint + '?' + '&'.join([f'{k}={v}' for k,v in kwargs.items()])
def get_method(username, endpoint, **kwargs):
return requests.get(get_api_url(endpoint, **kwargs), auth=(username, USER_PASS))
def delete_method(username, endpoint, **kwargs):
return requests.delete(get_api_url(endpoint, **kwargs), auth=(username, USER_PASS))
def patch_method(username, endpoint, data, **kwargs):
return requests.patch(get_api_url(endpoint, **kwargs), json=data, auth=(username, USER_PASS))

@ -1,14 +1,10 @@
import os
import requests
import os.path as osp
from config import get_method, ASSETS_DIR
import json
import config
with requests.Session() as session:
session.auth = ('admin1', config.USER_PASS)
for obj in ['user', 'project', 'task', 'job', 'organization', 'membership',
'invitation']:
response = session.get(f'http://localhost:8080/api/v1/{obj}s?page_size=all')
with open(os.path.join(config.ASSETS_DIR, f'{obj}s.json'), 'w') as f:
json.dump(response.json(), f, indent=2, sort_keys=True)
for obj in ['user', 'project', 'task', 'job', 'organization', 'membership',
'invitation']:
response = get_method('admin1', obj, page_size='all')
with open(osp.join(ASSETS_DIR, f'{obj}s.json'), 'w') as f:
json.dump(response.json(), f, indent=2, sort_keys=True)

Loading…
Cancel
Save