Mark tests that require external services (#5474)

Related: #5225 

External services are not available when we execute in Helm.

- Added a way to mark REST API tests that require external services to
run
- Changed the way of filtering tests in Helm tests

Currently, we can't execute external service mocks in tests, and we
ignore related tests in the Helm
execution command. But this command may be outdated, because Helm tests
are not executed in each PR.
This PR allows to indicate related tests and filter them out without the
need to synchronize the CI command.

.github/workflows/helm.yml

@@ -100,10 +100,4 @@ jobs:
     - name: REST API and SDK tests
       run: |
         kubectl cp tests/mounted_file_share/images $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}'):/home/django/share
-        pytest --platform=kube \
-          --ignore=tests/python/rest_api/test_cloud_storages.py \
-          --ignore=tests/python/rest_api/test_analytics.py \
-          --ignore=tests/python/rest_api/test_resource_import_export.py \
-          --ignore=tests/python/rest_api/test_webhooks_sender.py \
-          -k 'not create_task_with_cloud_storage_files' \
-        tests/python
+        pytest --platform=kube -m "not with_external_services" tests/python

tests/python/pytest.ini

@@ -1,6 +1,11 @@
 [pytest]
 required_plugins = pytest-timeout
 addopts = --verbose --capture=tee-sys
+
 # We expect no regular individual test to run too long
 # can be overridden for specific tests with a test decorator
 timeout = 15
+
+markers =
+    with_external_services: The test requires services extrernal to the default CVAT deployment, e.g. a Git server etc.
+

tests/python/rest_api/test_analytics.py

@@ -9,6 +9,9 @@ import pytest
 
 from shared.utils.config import server_get
 
+# https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules
+pytestmark = [pytest.mark.with_external_services]
+
 
 @pytest.mark.usefixtures("restore_db_per_class")
 class TestGetAnalytics:

tests/python/rest_api/test_cloud_storages.py

@@ -10,6 +10,9 @@ from deepdiff import DeepDiff
 
 from shared.utils.config import get_method, patch_method, post_method
 
+# https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules
+pytestmark = [pytest.mark.with_external_services]
+
 
 @pytest.mark.usefixtures("restore_db_per_class")
 class TestGetCloudStorage:

tests/python/rest_api/test_resource_import_export.py

@@ -20,6 +20,10 @@ FILENAME_TEMPLATE = "cvat/{}/{}.zip"
 FORMAT = "COCO 1.0"
 
 
+# https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules
+pytestmark = [pytest.mark.with_external_services]
+
+
 def _make_custom_resource_params(obj: str, resource: str, cloud_storage_id: int) -> Dict[str, Any]:
     params = {
         "filename": FILENAME_TEMPLATE.format(obj, resource),

tests/python/rest_api/test_tasks.py

@@ -636,6 +636,7 @@ class TestPostTaskData:
         response = get_method(self._USERNAME, f"jobs/{job_id}/annotations")
         assert response.status_code == HTTPStatus.OK
 
+    @pytest.mark.with_external_services
     @pytest.mark.parametrize(
         "cloud_storage_id, manifest, use_bucket_content, org",
         [
@@ -674,6 +675,7 @@ class TestPostTaskData:
             self._USERNAME, task_spec, data_spec, content_type="application/json", org=org
         )
 
+    @pytest.mark.with_external_services
     @pytest.mark.parametrize(
         "cloud_storage_id, manifest, org",
         [(1, "manifest.jsonl", "")],  # public bucket

tests/python/rest_api/test_webhooks.py

@@ -304,8 +304,8 @@ class TestPostWebhooks:
 
         assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
 
+    @pytest.mark.skip("Not implemented yet")
     def test_cannot_create_non_unique_webhook(self):
-        pytest.skip("Not implemented yet")
         response = post_method("admin2", "webhooks", self.proj_webhook)
 
         response = post_method("admin2", "webhooks", self.proj_webhook)

tests/python/rest_api/test_webhooks_sender.py

@@ -20,6 +20,9 @@ from shared.utils.config import delete_method, get_method, patch_method, post_me
 #  1) trigger some webhook
 #  2) check that webhook is sent by checking value of `response` field for the last delivery of this webhook
 
+# https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules
+pytestmark = [pytest.mark.with_external_services]
+
 
 def target_url():
     env_data = {}

tests/python/sdk/test_tasks.py

@@ -168,6 +168,7 @@ class TestTaskUsecases:
         assert capture.match("No media data found")
         assert self.stdout.getvalue() == ""
 
+    @pytest.mark.with_external_services
     def test_can_create_task_with_git_repo(self, fxt_image_file: Path):
         pbar_out = io.StringIO()
         pbar = make_pbar(file=pbar_out)