diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 188b4572..5b6023df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,12 +74,15 @@ jobs: - name: Running REST API tests env: API_ABOUT_PAGE: "localhost:8080/api/server/about" + # Access key length should be at least 3, and secret key length at least 8 characters + MINIO_ACCESS_KEY: "minio_access_key" + MINIO_SECRET_KEY: "minio_secret_key" run: | - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml up -d + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml -f tests/rest_api/docker-compose.minio.yml up -d /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done' pip3 install --user -r tests/rest_api/requirements.txt pytest tests/rest_api/ - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml down -v + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml -f components/analytics/docker-compose.analytics.yml -f tests/rest_api/docker-compose.minio.yml down -v - name: Running unit tests env: HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} diff --git a/cvat-ui/src/components/create-cloud-storage-page/cloud-storage-form.tsx b/cvat-ui/src/components/create-cloud-storage-page/cloud-storage-form.tsx index bd34dd95..d4724a23 100644 --- a/cvat-ui/src/components/create-cloud-storage-page/cloud-storage-form.tsx +++ b/cvat-ui/src/components/create-cloud-storage-page/cloud-storage-form.tsx @@ -48,6 +48,7 @@ interface CloudStorageForm { prefix?: string; project_id?: string; manifests: string[]; + endpoint_url?: string; } const { Dragger } = Upload; @@ -117,16 +118,20 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { const location = parsedOptions.get('region') || parsedOptions.get('location'); const prefix = parsedOptions.get('prefix'); const projectId = parsedOptions.get('project_id'); + const endpointUrl = parsedOptions.get('endpoint_url'); + if (location) { setSelectedRegion(location); } if (prefix) { fieldsValue.prefix = prefix; } - if (projectId) { fieldsValue.project_id = projectId; } + if (endpointUrl) { + fieldsValue.endpoint_url = endpointUrl; + } } form.setFieldsValue(fieldsValue); @@ -222,6 +227,10 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { delete cloudStorageData.project_id; specificAttributes.append('project_id', formValues.project_id); } + if (formValues.endpoint_url) { + delete cloudStorageData.endpoint_url; + specificAttributes.append('endpoint_url', formValues.endpoint_url); + } cloudStorageData.specific_attributes = specificAttributes.toString(); @@ -489,6 +498,14 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { {credentialsBlok()} + + + + /bin/sh -c " + $${MC_PATH} config host add --quiet --api s3v4 $${MINIO_ALIAS} $${MINIO_HOST} $${MINIO_ACCESS_KEY} $${MINIO_SECRET_KEY}; + $${MC_PATH} mb $${MINIO_ALIAS}/$${PRIVATE_BUCKET} $${MINIO_ALIAS}/$${PUBLIC_BUCKET} $${MINIO_ALIAS}/$${TEST_BUCKET}; + for BUCKET in $${MINIO_ALIAS}/$${PRIVATE_BUCKET} $${MINIO_ALIAS}/$${PUBLIC_BUCKET} $${MINIO_ALIAS}/$${TEST_BUCKET}; + do + $${MC_PATH} cp --recursive /storage/ $${BUCKET}; + for i in 1 2; + do + $${MC_PATH} cp /storage/manifest.jsonl $${BUCKET}/manifest_$${i}.jsonl; + done; + done; + $${MC_PATH} policy set public $${MINIO_ALIAS}/$${PUBLIC_BUCKET}; + exit 0; + " diff --git a/tests/rest_api/test_cloud_storages.py b/tests/rest_api/test_cloud_storages.py new file mode 100644 index 00000000..ed1b5371 --- /dev/null +++ b/tests/rest_api/test_cloud_storages.py @@ -0,0 +1,188 @@ +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import pytest +from http import HTTPStatus +from deepdiff import DeepDiff + +from .utils.config import get_method, patch_method, post_method + +class TestGetCloudStorage: + + def _test_can_see(self, user, storage_id, data, **kwargs): + response = get_method(user, f'cloudstorages/{storage_id}', **kwargs) + response_data = response.json() + response_data = response_data.get('results', response_data) + + assert response.status_code == HTTPStatus.OK + assert DeepDiff(data, response_data, ignore_order=True) == {} + + def _test_cannot_see(self, user, storage_id, **kwargs): + response = get_method(user, f'cloudstorages/{storage_id}', **kwargs) + + assert response.status_code == HTTPStatus.FORBIDDEN + + @pytest.mark.parametrize('storage_id', [1]) + @pytest.mark.parametrize('group, is_owner, is_allow', [ + ('admin', False, True), + ('business', False, False), + ('user', True, True), + ]) + def test_sandbox_user_get_coud_storage(self, storage_id, group, is_owner, is_allow, users, cloud_storages): + org = '' + cloud_storage = cloud_storages[storage_id] + username = cloud_storage['owner']['username'] if is_owner else \ + next((u for u in users if group in u['groups'] and u['id'] != cloud_storage['owner']['id']))['username'] + + if is_allow: + self._test_can_see(username, storage_id, cloud_storage, org=org) + else: + self._test_cannot_see(username, storage_id, org=org) + + + @pytest.mark.parametrize('org_id', [2]) + @pytest.mark.parametrize('storage_id', [2]) + @pytest.mark.parametrize('role, is_owner, is_allow', [ + ('worker', True, True), + ('supervisor', False, True), + ('worker', False, False), + ]) + def test_org_user_get_coud_storage(self, org_id, storage_id, role, is_owner, is_allow, find_users, cloud_storages): + cloud_storage = cloud_storages[storage_id] + username = cloud_storage['owner']['username'] if is_owner else \ + next((u for u in find_users(role=role, org=org_id) if u['id'] != cloud_storage['owner']['id']))['username'] + + if is_allow: + self._test_can_see(username, storage_id, cloud_storage, org_id=org_id) + else: + self._test_cannot_see(username, storage_id, org_id=org_id) + + +class TestPostCloudStorage: + _SPEC = { + 'provider_type': 'AWS_S3_BUCKET', + 'resource': 'test', + 'display_name': 'Bucket', + 'credentials_type': 'KEY_SECRET_KEY_PAIR', + 'key': 'minio_access_key', 'secret_key': 'minio_secret_key', + 'specific_attributes': 'endpoint_url=http://minio:9000', + 'description': 'Some description', + 'manifests': [ + 'manifest.jsonl' + ], + } + _EXCLUDE_PATHS = [ + f"root['{extra_field}']" for extra_field in { + # unchanged fields + 'created_date', 'id', 'organization', 'owner', 'updated_date', + # credentials that server doesn't return + 'key', 'secret_key', + }] + + def _test_can_create(self, user, spec, **kwargs): + response = post_method(user, 'cloudstorages', spec, **kwargs) + response_data = response.json() + response_data = response_data.get('results', response_data) + + assert response.status_code == HTTPStatus.CREATED + assert DeepDiff(self._SPEC, response_data, ignore_order=True, + exclude_paths=self._EXCLUDE_PATHS) == {} + + def _test_cannot_create(self, user, spec, **kwargs): + response = post_method(user, 'cloudstorages', spec, **kwargs) + + assert response.status_code == HTTPStatus.FORBIDDEN + + + @pytest.mark.parametrize('group, is_allow', [ + ('user', True), ('worker', False) + ]) + def test_sandbox_user_create_cloud_storage(self, group, is_allow, users): + org = '' + username = [u for u in users if group in u['groups']][0]['username'] + + if is_allow: + self._test_can_create(username, self._SPEC, org=org) + else: + self._test_cannot_create(username, self._SPEC, org=org) + + @pytest.mark.parametrize('org_id', [2]) + @pytest.mark.parametrize('role, is_allow', [ + ('owner', True), ('maintainer', True), + ('worker', False), ('supervisor', False), + ]) + def test_org_user_create_coud_storage(self, org_id, role, is_allow, find_users): + username = find_users(role=role, org=org_id)[0]['username'] + + if is_allow: + self._test_can_create(username, self._SPEC, org_id=org_id) + else: + self._test_cannot_create(username, self._SPEC, org_id=org_id) + +class TestPatchCloudStorage: + _SPEC = { + 'display_name': 'New display name', + 'description': 'New description', + 'manifests': [ + 'manifest_1.jsonl', + 'manifest_2.jsonl', + ], + } + _EXCLUDE_PATHS = [ + f"root['{extra_field}']" for extra_field in { + # unchanged fields + 'created_date', 'credentials_type', 'id', 'organization', 'owner', + 'provider_type', 'resource', 'specific_attributes', 'updated_date', + }] + + def _test_can_update(self, user, storage_id, spec, **kwargs): + response = patch_method(user, f'cloudstorages/{storage_id}', spec, **kwargs) + response_data = response.json() + response_data = response_data.get('results', response_data) + + assert response.status_code == HTTPStatus.OK + assert DeepDiff(self._SPEC, response_data, ignore_order=True, + exclude_paths=self._EXCLUDE_PATHS) == {} + + assert response.status_code == HTTPStatus.OK + + def _test_cannot_update(self, user, storage_id, spec, **kwargs): + response = patch_method(user, f'cloudstorages/{storage_id}', spec, **kwargs) + + assert response.status_code == HTTPStatus.FORBIDDEN + + + @pytest.mark.parametrize('storage_id', [1]) + @pytest.mark.parametrize('group, is_owner, is_allow', [ + ('admin', False, True), + ('business', False, False), + ('worker', True, True), + ]) + def test_sandbox_user_update_cloud_storage(self, storage_id, group, is_owner, is_allow, users, cloud_storages): + org = '' + cloud_storage = cloud_storages[storage_id] + username = cloud_storage['owner']['username'] if is_owner else \ + next((u for u in users if group in u['groups'] and u['id'] != cloud_storage['owner']['id']))['username'] + + if is_allow: + self._test_can_update(username, storage_id, self._SPEC, org=org) + else: + self._test_cannot_update(username, storage_id, self._SPEC, org=org) + + @pytest.mark.parametrize('org_id', [2]) + @pytest.mark.parametrize('storage_id', [2]) + @pytest.mark.parametrize('role, is_owner, is_allow', [ + ('worker', True, True), + ('maintainer', False, True), + ('supervisor', False, False), + ]) + def test_org_user_update_coud_storage(self, org_id, storage_id, role, is_owner, is_allow, find_users, cloud_storages): + cloud_storage = cloud_storages[storage_id] + username = cloud_storage['owner']['username'] if is_owner else \ + next((u for u in find_users(role=role, org=org_id) if u['id'] != cloud_storage['owner']['id']))['username'] + + if is_allow: + self._test_can_update(username, storage_id, self._SPEC, org_id=org_id) + else: + self._test_cannot_update(username, storage_id, self._SPEC, org_id=org_id) diff --git a/tests/rest_api/utils/dump_objects.py b/tests/rest_api/utils/dump_objects.py index 45fad7de..9ff0826c 100644 --- a/tests/rest_api/utils/dump_objects.py +++ b/tests/rest_api/utils/dump_objects.py @@ -4,7 +4,7 @@ import json annotations = {} for obj in ['user', 'project', 'task', 'job', 'organization', 'membership', - 'invitation', 'issue']: + 'invitation', 'cloudstorage', 'issue']: response = get_method('admin1', f'{obj}s', page_size='all') with open(osp.join(ASSETS_DIR, f'{obj}s.json'), 'w') as f: json.dump(response.json(), f, indent=2, sort_keys=True)