From af65707eee191ecd610f179e820ad4c6f74ca4ee Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 23 Dec 2022 11:52:40 +0300 Subject: [PATCH] Mark tests that require external services (#5474) Related: #5225 External services are not available when we execute in Helm. - Added a way to mark REST API tests that require external services to run - Changed the way of filtering tests in Helm tests Currently, we can't execute external service mocks in tests, and we ignore related tests in the Helm execution command. But this command may be outdated, because Helm tests are not executed in each PR. This PR allows to indicate related tests and filter them out without the need to synchronize the CI command. --- .github/workflows/helm.yml | 8 +------- tests/python/pytest.ini | 5 +++++ tests/python/rest_api/test_analytics.py | 3 +++ tests/python/rest_api/test_cloud_storages.py | 3 +++ tests/python/rest_api/test_resource_import_export.py | 4 ++++ tests/python/rest_api/test_tasks.py | 2 ++ tests/python/rest_api/test_webhooks.py | 2 +- tests/python/rest_api/test_webhooks_sender.py | 3 +++ tests/python/sdk/test_tasks.py | 1 + 9 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index c541bfac..e0045d24 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -100,10 +100,4 @@ jobs: - name: REST API and SDK tests run: | kubectl cp tests/mounted_file_share/images $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}'):/home/django/share - pytest --platform=kube \ - --ignore=tests/python/rest_api/test_cloud_storages.py \ - --ignore=tests/python/rest_api/test_analytics.py \ - --ignore=tests/python/rest_api/test_resource_import_export.py \ - --ignore=tests/python/rest_api/test_webhooks_sender.py \ - -k 'not create_task_with_cloud_storage_files' \ - tests/python + pytest --platform=kube -m "not with_external_services" tests/python diff --git a/tests/python/pytest.ini b/tests/python/pytest.ini index 461f36c6..982760d8 100644 --- a/tests/python/pytest.ini +++ b/tests/python/pytest.ini @@ -1,6 +1,11 @@ [pytest] required_plugins = pytest-timeout addopts = --verbose --capture=tee-sys + # We expect no regular individual test to run too long # can be overridden for specific tests with a test decorator timeout = 15 + +markers = + with_external_services: The test requires services extrernal to the default CVAT deployment, e.g. a Git server etc. + diff --git a/tests/python/rest_api/test_analytics.py b/tests/python/rest_api/test_analytics.py index 2e7287c7..e06be66b 100644 --- a/tests/python/rest_api/test_analytics.py +++ b/tests/python/rest_api/test_analytics.py @@ -9,6 +9,9 @@ import pytest from shared.utils.config import server_get +# https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules +pytestmark = [pytest.mark.with_external_services] + @pytest.mark.usefixtures("restore_db_per_class") class TestGetAnalytics: diff --git a/tests/python/rest_api/test_cloud_storages.py b/tests/python/rest_api/test_cloud_storages.py index 4c9c0e94..5f304326 100644 --- a/tests/python/rest_api/test_cloud_storages.py +++ b/tests/python/rest_api/test_cloud_storages.py @@ -10,6 +10,9 @@ from deepdiff import DeepDiff from shared.utils.config import get_method, patch_method, post_method +# https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules +pytestmark = [pytest.mark.with_external_services] + @pytest.mark.usefixtures("restore_db_per_class") class TestGetCloudStorage: diff --git a/tests/python/rest_api/test_resource_import_export.py b/tests/python/rest_api/test_resource_import_export.py index 6e29a106..6ebbf803 100644 --- a/tests/python/rest_api/test_resource_import_export.py +++ b/tests/python/rest_api/test_resource_import_export.py @@ -20,6 +20,10 @@ FILENAME_TEMPLATE = "cvat/{}/{}.zip" FORMAT = "COCO 1.0" +# https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules +pytestmark = [pytest.mark.with_external_services] + + def _make_custom_resource_params(obj: str, resource: str, cloud_storage_id: int) -> Dict[str, Any]: params = { "filename": FILENAME_TEMPLATE.format(obj, resource), diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index b29f27f4..9df6d10c 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -636,6 +636,7 @@ class TestPostTaskData: response = get_method(self._USERNAME, f"jobs/{job_id}/annotations") assert response.status_code == HTTPStatus.OK + @pytest.mark.with_external_services @pytest.mark.parametrize( "cloud_storage_id, manifest, use_bucket_content, org", [ @@ -674,6 +675,7 @@ class TestPostTaskData: self._USERNAME, task_spec, data_spec, content_type="application/json", org=org ) + @pytest.mark.with_external_services @pytest.mark.parametrize( "cloud_storage_id, manifest, org", [(1, "manifest.jsonl", "")], # public bucket diff --git a/tests/python/rest_api/test_webhooks.py b/tests/python/rest_api/test_webhooks.py index e87a67f6..8c931005 100644 --- a/tests/python/rest_api/test_webhooks.py +++ b/tests/python/rest_api/test_webhooks.py @@ -304,8 +304,8 @@ class TestPostWebhooks: assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + @pytest.mark.skip("Not implemented yet") def test_cannot_create_non_unique_webhook(self): - pytest.skip("Not implemented yet") response = post_method("admin2", "webhooks", self.proj_webhook) response = post_method("admin2", "webhooks", self.proj_webhook) diff --git a/tests/python/rest_api/test_webhooks_sender.py b/tests/python/rest_api/test_webhooks_sender.py index b532d2a0..8da59b17 100644 --- a/tests/python/rest_api/test_webhooks_sender.py +++ b/tests/python/rest_api/test_webhooks_sender.py @@ -20,6 +20,9 @@ from shared.utils.config import delete_method, get_method, patch_method, post_me # 1) trigger some webhook # 2) check that webhook is sent by checking value of `response` field for the last delivery of this webhook +# https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules +pytestmark = [pytest.mark.with_external_services] + def target_url(): env_data = {} diff --git a/tests/python/sdk/test_tasks.py b/tests/python/sdk/test_tasks.py index 291ed38c..4ca82a5b 100644 --- a/tests/python/sdk/test_tasks.py +++ b/tests/python/sdk/test_tasks.py @@ -168,6 +168,7 @@ class TestTaskUsecases: assert capture.match("No media data found") assert self.stdout.getvalue() == "" + @pytest.mark.with_external_services def test_can_create_task_with_git_repo(self, fxt_image_file: Path): pbar_out = io.StringIO() pbar = make_pbar(file=pbar_out)