REST API tests: update data.json (#27)

cvat/apps/engine/models.py

@@ -144,7 +144,7 @@ class AbstractArrayField(models.TextField):
             return []
         if value.startswith('[') and value.endswith(']'):
             value = value[1:-1]
-        return [self.converter(v) for v in value.split(self.separator)]
+        return [self.converter(v) for v in value.split(self.separator) if v]
 
     def to_python(self, value):
         if isinstance(value, list):

tests/rest_api/README.md

@@ -67,7 +67,7 @@ for i, color in enumerate(colormap):
 To backup DB and data volume, please use commands below.
 
 ```console
-docker exec test_cvat_1 python manage.py dumpdata --indent 2 > assets/cvat_db/data.json
+docker exec test_cvat_1 python manage.py dumpdata --indent 2 --natural-foreign --exclude=auth.permission --exclude=contenttypes > assets/cvat_db/data.json
 docker exec test_cvat_1 tar -cjv /home/django/data > assets/cvat_db/cvat_data.tar.bz2
 ```
 

tests/rest_api/fixtures/init.py

@@ -53,16 +53,28 @@ def pytest_addoption(parser):
         help="Delete files that was create by tests without running tests. (default: %(default)s)",
     )
 
+    group._addoption(
+        "--dumpdb",
+        action="store_true",
+        help="Update data.json without running tests. (default: %(default)s)",
+    )
 
-def _run(command):
+
+def _run(command, capture_output=True):
     try:
-        proc = run(command.split(), check=True, stdout=PIPE, stderr=PIPE)  # nosec
-        return proc.stdout.decode(), proc.stderr.decode()
+        stdout, stderr = "", ""
+        if capture_output:
+            proc = run(command.split(), check=True, stdout=PIPE, stderr=PIPE)  # nosec
+            stdout, stderr = proc.stdout.decode(), proc.stderr.decode()
+        else:
+            proc = run(command.split(), check=True)  # nosec
+        return stdout, stderr
     except CalledProcessError as exc:
+        stderr = exc.stderr.decode() if capture_output else "see above"
         pytest.exit(
             f"Command failed: {command}.\n"
-            f"Error message: {exc.stderr.decode()}.\n"
-            f"Add `-s` option to see more details"
+            f"Error message: {stderr}.\n"
+            "Add `-s` option to see more details"
         )
 
 
@@ -79,14 +91,41 @@ def exec_cvat_db(command):
 
 
 def restore_db():
-    exec_cvat_db("psql -U root -d postgres -v from=test_db -v to=cvat -f /tmp/restore.sql")
+    exec_cvat_db(
+        "psql -U root -d postgres -v from=test_db -v to=cvat -f /tmp/restore.sql"
+    )
+
+def running_containers():
+    return [cn for cn in _run("docker ps --format {{.Names}}")[0].split("\n") if cn]
+
+
+def dump_db():
+    if 'test_cvat_1' not in running_containers():
+        pytest.exit("CVAT is not running")
+    with open(osp.join(CVAT_DB_DIR, "data.json"), "w") as f:
+        try:
+            run( # nosec
+                "docker exec test_cvat_1 \
+                    python manage.py dumpdata \
+                    --indent 2 --natural-foreign \
+                    --exclude=auth.permission --exclude=contenttypes".split(),
+                stdout=f, check=True
+            )
+        except CalledProcessError:
+            pytest.exit("Database dump failed.\n")
 
 
 def create_compose_files():
     for filename in CONTAINER_NAME_FILES:
-        with open(filename.replace(".tests.yml", ".yml"), "r") as dcf, open(filename, "w") as ndcf:
+        with open(filename.replace(".tests.yml", ".yml"), "r") as dcf, open(
+            filename, "w"
+        ) as ndcf:
             ndcf.writelines(
-                [line for line in dcf.readlines() if not re.match("^.+container_name.+$", line)]
+                [
+                    line
+                    for line in dcf.readlines()
+                    if not re.match("^.+container_name.+$", line)
+                ]
             )
 
 
@@ -97,33 +136,40 @@ def delete_compose_files():
 
 
 def wait_for_server():
-    for _ in range(60):
+    for _ in range(30):
         response = requests.get(get_api_url("users/self"))
         if response.status_code == HTTPStatus.UNAUTHORIZED:
             break
         sleep(5)
 
+
 def restore_data_volumes():
-    docker_cp(osp.join(CVAT_DB_DIR, "cvat_data.tar.bz2"), f"{PREFIX}_cvat_1:/tmp/cvat_data.tar.bz2")
+    docker_cp(
+        osp.join(CVAT_DB_DIR, "cvat_data.tar.bz2"),
+        f"{PREFIX}_cvat_1:/tmp/cvat_data.tar.bz2",
+    )
     exec_cvat("tar --strip 3 -xjf /tmp/cvat_data.tar.bz2 -C /home/django/data/")
 
-def start_services(rebuild=False):
-    running_containers = [cn for cn in _run("docker ps --format {{.Names}}")[0].split("\n") if cn]
 
-    if any([cn in ["cvat", "cvat_db"] for cn in running_containers]):
+def start_services(rebuild=False):
+    if any([cn in ["cvat", "cvat_db"] for cn in running_containers()]):
         pytest.exit(
             "It's looks like you already have running cvat containers. Stop them and try again. "
-            f"List of running containers: {', '.join(running_containers)}"
+            f"List of running containers: {', '.join(running_containers())}"
         )
 
-    out = _run(f"docker-compose -p {PREFIX} -f {' -f '.join(DC_FILES)} up -d " + "--build" * rebuild)[1]
+    _run(
+        f"docker-compose -p {PREFIX} -f {' -f '.join(DC_FILES)} up -d "
+        + "--build" * rebuild,
+        capture_output=False,
+    )
 
     restore_data_volumes()
-    docker_cp(osp.join(CVAT_DB_DIR, "restore.sql"), f"{PREFIX}_cvat_db_1:/tmp/restore.sql")
+    docker_cp(
+        osp.join(CVAT_DB_DIR, "restore.sql"), f"{PREFIX}_cvat_db_1:/tmp/restore.sql"
+    )
     docker_cp(osp.join(CVAT_DB_DIR, "data.json"), f"{PREFIX}_cvat_1:/tmp/data.json")
 
-    return out
-
 
 @pytest.fixture(autouse=True, scope="session")
 def services(request):
@@ -131,32 +177,41 @@ def services(request):
     start = request.config.getoption("--start-services")
     rebuild = request.config.getoption("--rebuild")
     cleanup = request.config.getoption("--cleanup")
+    dumpdb = request.config.getoption("--dumpdb")
 
     if start and stop:
         raise Exception("--start-services and --stop-services are incompatible")
 
+    if dumpdb:
+        dump_db()
+        pytest.exit("data.json has been updated", returncode=0)
+
     if cleanup:
         delete_compose_files()
-        pytest.exit(f"All generated test files have been deleted", returncode=0)
+        pytest.exit("All generated test files have been deleted", returncode=0)
 
-    if not all([osp.exists(f) for f in CONTAINER_NAME_FILES]):
+    if not all([osp.exists(f) for f in CONTAINER_NAME_FILES]) or rebuild:
+        delete_compose_files()
         create_compose_files()
 
     if stop:
-        out = _run(f"docker-compose -p {PREFIX} -f {' -f '.join(DC_FILES)} down -v")[1]
-        out = set(l.split()[1] for l in out.split("\n") if "done" in l.split())
-        pytest.exit(f"All testing  containers are stopped: {', '.join(out)}", returncode=0)
+        _run(
+            f"docker-compose -p {PREFIX} -f {' -f '.join(DC_FILES)} down -v",
+            capture_output=False,
+        )
+        pytest.exit("All testing containers are stopped", returncode=0)
 
-    started_services = start_services(rebuild)
+    start_services(rebuild)
     wait_for_server()
 
     exec_cvat("python manage.py loaddata /tmp/data.json")
-    exec_cvat_db("psql -U root -d postgres -v from=cvat -v to=test_db -f /tmp/restore.sql")
+    exec_cvat_db(
+        "psql -U root -d postgres -v from=cvat -v to=test_db -f /tmp/restore.sql"
+    )
 
     if start:
         pytest.exit(
-            f"All necessary containers have been created and started: {started_services}",
-            returncode=0,
+            "All necessary containers have been created and started.", returncode=0
         )
 
     yield
@@ -174,6 +229,7 @@ def changedb():
 def dontchangedb():
     restore_db()
 
+
 @pytest.fixture(scope="function")
 def restore_cvat_data():
     restore_data_volumes()

tests/rest_api/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = -s -v

tests/rest_api/test_jobs.py

@@ -44,7 +44,8 @@ class TestGetJobs:
         response = get_method(user, f'jobs/{jid}', **kwargs)
 
         assert response.status_code == HTTPStatus.OK
-        assert DeepDiff(data, response.json(), exclude_paths="root['updated_date']") == {}
+        assert DeepDiff(data, response.json(), exclude_paths="root['updated_date']",
+            ignore_order=True) == {}
 
     def _test_get_job_403(self, user, jid, **kwargs):
         response = get_method(user, f'jobs/{jid}', **kwargs)
@@ -83,7 +84,8 @@ class TestListJobs:
         response = get_method(user, 'jobs', **kwargs, page_size='all')
 
         assert response.status_code == HTTPStatus.OK
-        assert DeepDiff(data, response.json()['results'], exclude_paths="root['updated_date']") == {}
+        assert DeepDiff(data, response.json()['results'], exclude_paths="root['updated_date']",
+            ignore_order=True) == {}
 
     def _test_list_jobs_403(self, user, **kwargs):
         response = get_method(user, 'jobs', **kwargs)
@@ -295,6 +297,7 @@ class TestPatchJob:
             members -= {assignee_id(jobs[jid]), user_id}
             return members.pop()
         return find_new_assignee
+
     @pytest.mark.parametrize('org', [2])
     @pytest.mark.parametrize('role, task_staff, is_allow', [
         ('maintainer', False, True),  ('owner',  False, True),
@@ -314,6 +317,6 @@ class TestPatchJob:
         if is_allow:
             assert response.status_code == HTTPStatus.OK
             assert DeepDiff(expected_data(jid, assignee), response.json(),
-                exclude_paths="root['updated_date']") == {}
+                exclude_paths="root['updated_date']", ignore_order=True) == {}
         else:
             assert response.status_code == HTTPStatus.FORBIDDEN

tests/rest_api/test_remote_url.py

@@ -21,12 +21,13 @@ def _post_task_remote_data(username, task_id, resources):
 def _wait_until_task_is_created(username, task_id):
     url = f'tasks/{task_id}/status'
 
-    while True:
+    for _ in range(100):
         response = get_method(username, url)
         response_json = response.json()
         if response_json['state'] == 'Finished' or response_json['state'] == 'Failed':
             return response
         sleep(1)
+    raise Exception('Cannot create task')
 
 
 @pytest.mark.usefixtures('changedb')

tests/rest_api/test_tasks.py

@@ -275,11 +275,12 @@ class TestGetTaskDataset:
 class TestPostTaskData:
     @staticmethod
     def _wait_until_task_is_created(api: TasksApi, task_id: int) -> RqStatus:
-        while True:
+        for _ in range(100):
             (status, _) = api.retrieve_status(task_id)
             if status.state.value in ['Finished', 'Failed']:
                 return status
             sleep(1)
+        raise Exception('Cannot create task')
 
     def _test_create_task(self, username, spec, data, files):
         with make_api_client(username) as api_client: