diff --git a/CHANGELOG.md b/CHANGELOG.md index 622318f7..afd6f709 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 from online detectors & interactors) () - Added Webhooks () - Authentication with social accounts google & github () +- REST API tests to export job datasets & annotations and validate their structure () ### Changed - `api/docs`, `api/swagger`, `api/schema`, `server/about` endpoints now allow unauthorized access (, ) diff --git a/cvat/apps/dataset_manager/task.py b/cvat/apps/dataset_manager/task.py index d1d6af67..fdedb494 100644 --- a/cvat/apps/dataset_manager/task.py +++ b/cvat/apps/dataset_manager/task.py @@ -70,9 +70,18 @@ def _merge_table_rows(rows, keys_for_merge, field_id): return list(merged_rows.values()) class JobAnnotation: - def __init__(self, pk): - self.db_job = models.Job.objects.select_related('segment__task') \ - .select_for_update().get(id=pk) + def __init__(self, pk, is_prefetched=False): + if is_prefetched: + self.db_job = models.Job.objects.select_related('segment__task') \ + .select_for_update().get(id=pk) + else: + self.db_job = models.Job.objects.prefetch_related( + 'segment', + 'segment__task', + Prefetch('segment__task__data', queryset=models.Data.objects.select_related('video').prefetch_related( + Prefetch('images', queryset=models.Image.objects.order_by('frame')) + )) + ).get(pk=pk) db_segment = self.db_job.segment self.start_frame = db_segment.start_frame @@ -630,7 +639,7 @@ class TaskAnnotation: self.reset() for db_job in self.db_jobs: - annotation = JobAnnotation(db_job.id) + annotation = JobAnnotation(db_job.id, is_prefetched=True) annotation.init_from_db() if annotation.ir_data.version > self.ir_data.version: self.ir_data.version = annotation.ir_data.version diff --git a/tests/python/README.md b/tests/python/README.md index 3cc30e17..eff40c5e 100644 --- a/tests/python/README.md +++ b/tests/python/README.md @@ -68,8 +68,8 @@ for i, color in enumerate(colormap): To backup DB and data volume, please use commands below. ```console -docker exec test_cvat_server_1 python manage.py dumpdata --indent 2 --natural-foreign --exclude=auth.permission --exclude=contenttypes > assets/cvat_db/data.json -docker exec test_cvat_server_1 tar -cjv /home/django/data > assets/cvat_db/cvat_data.tar.bz2 +docker exec test_cvat_server_1 python manage.py dumpdata --indent 2 --natural-foreign --exclude=auth.permission --exclude=contenttypes > shared/assets/cvat_db/data.json +docker exec test_cvat_server_1 tar -cjv /home/django/data > shared/assets/cvat_db/cvat_data.tar.bz2 ``` > Note: if you won't be use --indent options or will be use with other value @@ -166,7 +166,7 @@ Assets directory has two parts: ``` Just dump JSON assets with: ``` - python3 tests/python/shared/utils/dump_objests.py + python3 tests/python/shared/utils/dump_objects.py ``` 1. If your test infrastructure has been corrupted and you have errors during db restoring. diff --git a/tests/python/rest_api/test_jobs.py b/tests/python/rest_api/test_jobs.py index 2cd03b02..ee6b9f5a 100644 --- a/tests/python/rest_api/test_jobs.py +++ b/tests/python/rest_api/test_jobs.py @@ -4,8 +4,11 @@ # SPDX-License-Identifier: MIT import json +import xml.etree.ElementTree as ET +import zipfile from copy import deepcopy from http import HTTPStatus +from io import BytesIO from typing import List import pytest @@ -489,6 +492,36 @@ class TestPatchJob: assert response.status == HTTPStatus.FORBIDDEN +def _check_coco_job_annotations(content, values_to_be_checked): + exported_annotations = json.loads(content) + assert values_to_be_checked["shapes_length"] == len(exported_annotations["annotations"]) + assert values_to_be_checked["job_size"] == len(exported_annotations["images"]) + assert values_to_be_checked["task_size"] > len(exported_annotations["images"]) + + +def _check_cvat_job_annotations(content, values_to_be_checked): + document = ET.fromstring(content) + # check meta information + meta = document.find("meta") + instance = list(meta)[0] + assert instance.tag == "job" + assert instance.find("id").text == values_to_be_checked["job_id"] + assert instance.find("size").text == str(values_to_be_checked["job_size"]) + assert instance.find("start_frame").text == str(values_to_be_checked["start_frame"]) + assert instance.find("stop_frame").text == str(values_to_be_checked["stop_frame"]) + assert instance.find("mode").text == values_to_be_checked["mode"] + assert len(instance.find("segments")) == 1 + + # check number of images, their sorting, number of annotations + images = document.findall("image") + assert len(images) == values_to_be_checked["job_size"] + assert len(list(document.iter("box"))) == values_to_be_checked["shapes_length"] + current_id = values_to_be_checked["start_frame"] + for image_elem in images: + assert image_elem.attrib["id"] == str(current_id) + current_id += 1 + + @pytest.mark.usefixtures("restore_db_per_class") class TestJobDataset: def _export_dataset(self, username, jid, **kwargs): @@ -510,3 +543,45 @@ class TestJobDataset: job = jobs_with_shapes[0] response = self._export_annotations(admin_user, job["id"], format="CVAT for images 1.1") assert response.data + + @pytest.mark.parametrize("username, jid", [("admin1", 14)]) + @pytest.mark.parametrize( + "anno_format, anno_file_name, check_func", + [ + ("COCO 1.0", "annotations/instances_default.json", _check_coco_job_annotations), + ("CVAT for images 1.1", "annotations.xml", _check_cvat_job_annotations), + ], + ) + def test_exported_job_dataset_structure( + self, + username, + jid, + anno_format, + anno_file_name, + check_func, + tasks, + jobs, + annotations, + ): + job_data = jobs[jid] + annotations_before = annotations["job"][str(jid)] + + values_to_be_checked = { + "task_size": tasks[job_data["task_id"]]["size"], + # NOTE: data step is not stored in assets, default = 1 + "job_size": job_data["stop_frame"] - job_data["start_frame"] + 1, + "start_frame": job_data["start_frame"], + "stop_frame": job_data["stop_frame"], + "shapes_length": len(annotations_before["shapes"]), + "job_id": str(jid), + "mode": job_data["mode"], + } + + response = self._export_dataset(username, jid, format=anno_format) + assert response.data + with zipfile.ZipFile(BytesIO(response.data)) as zip_file: + assert ( + len(zip_file.namelist()) == values_to_be_checked["job_size"] + 1 + ) # images + annotation file + content = zip_file.read(anno_file_name) + check_func(content, values_to_be_checked) diff --git a/tests/python/shared/assets/annotations.json b/tests/python/shared/assets/annotations.json index c93ee597..80fe0133 100644 --- a/tests/python/shared/assets/annotations.json +++ b/tests/python/shared/assets/annotations.json @@ -280,7 +280,53 @@ "version": 0 }, "14": { - "shapes": [], + "shapes": [ + { + "attributes": [], + "elements": [], + "frame": 15, + "group": 0, + "id": 41, + "label_id": 6, + "occluded": false, + "outside": false, + "points": [ + 53.062929061787145, + 301.6390160183091, + 197.94851258581548, + 763.3266590389048 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 1, + "value": "mazda" + } + ], + "elements": [], + "frame": 16, + "group": 0, + "id": 42, + "label_id": 5, + "occluded": false, + "outside": false, + "points": [ + 172.0810546875, + 105.990234375, + 285.97262095255974, + 138.40000000000146 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + } + ], "tags": [], "tracks": [], "version": 0 @@ -932,6 +978,51 @@ "source": "manual", "type": "rectangle", "z_order": 0 + }, + { + "attributes": [], + "elements": [], + "frame": 15, + "group": 0, + "id": 41, + "label_id": 6, + "occluded": false, + "outside": false, + "points": [ + 53.062929061787145, + 301.6390160183091, + 197.94851258581548, + 763.3266590389048 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 + }, + { + "attributes": [ + { + "spec_id": 1, + "value": "mazda" + } + ], + "elements": [], + "frame": 16, + "group": 0, + "id": 42, + "label_id": 5, + "occluded": false, + "outside": false, + "points": [ + 172.0810546875, + 105.990234375, + 285.97262095255974, + 138.40000000000146 + ], + "rotation": 0.0, + "source": "manual", + "type": "rectangle", + "z_order": 0 } ], "tags": [], diff --git a/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2 b/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2 index cec26a03..d05fa40c 100644 Binary files a/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2 and b/tests/python/shared/assets/cvat_db/cvat_data.tar.bz2 differ diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index 1b4a382b..3c371804 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -36,7 +36,7 @@ "pk": 1, "fields": { "password": "pbkdf2_sha256$260000$DevmxlmLwciP1P6sZs2Qag$U9DFtjTWx96Sk95qY6UXVcvpdQEP2LcoFBftk5D2RKY=", - "last_login": "2022-10-17T17:09:16.903Z", + "last_login": "2022-11-03T13:56:42.744Z", "is_superuser": true, "username": "admin1", "first_name": "Admin", @@ -2192,7 +2192,7 @@ ], "bug_tracker": "", "created_date": "2021-12-14T19:46:37.969Z", - "updated_date": "2022-03-05T09:47:49.679Z", + "updated_date": "2022-11-03T13:57:25.895Z", "status": "annotation", "organization": null, "source_storage": null, @@ -2463,7 +2463,7 @@ ], "bug_tracker": "", "created_date": "2022-03-05T09:33:10.420Z", - "updated_date": "2022-03-05T09:47:49.667Z", + "updated_date": "2022-11-03T13:57:26.007Z", "overlap": 0, "segment_size": 5, "status": "annotation", @@ -3556,10 +3556,10 @@ "fields": { "segment": 14, "assignee": null, - "updated_date": "2022-06-22T09:18:45.296Z", + "updated_date": "2022-11-03T13:57:26.346Z", "status": "annotation", "stage": "annotation", - "state": "new" + "state": "in progress" } }, { @@ -4751,6 +4751,55 @@ "job": 17 } }, +{ + "model": "engine.jobcommit", + "pk": 74, + "fields": { + "scope": "create", + "owner": [ + "business1" + ], + "timestamp": "2022-11-03T13:57:26.015Z", + "data": { + "stage": "annotation", + "state": "new", + "assignee": null + }, + "job": 14 + } +}, +{ + "model": "engine.jobcommit", + "pk": 75, + "fields": { + "scope": "create", + "owner": [ + "business1" + ], + "timestamp": "2022-11-03T13:57:26.351Z", + "data": { + "stage": "annotation", + "state": "in progress", + "assignee": null + }, + "job": 14 + } +}, +{ + "model": "engine.jobcommit", + "pk": 76, + "fields": { + "scope": "update", + "owner": [ + "admin1" + ], + "timestamp": "2022-11-03T13:57:26.356Z", + "data": { + "state": "in progress" + }, + "job": 14 + } +}, { "model": "engine.labeledimage", "pk": 1, @@ -5079,6 +5128,42 @@ "parent": 36 } }, +{ + "model": "engine.labeledshape", + "pk": 41, + "fields": { + "job": 14, + "label": 6, + "frame": 15, + "group": 0, + "source": "manual", + "type": "rectangle", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "[53.062929061787145, 301.6390160183091, 197.94851258581548, 763.3266590389048]", + "rotation": 0.0, + "parent": null + } +}, +{ + "model": "engine.labeledshape", + "pk": 42, + "fields": { + "job": 14, + "label": 5, + "frame": 16, + "group": 0, + "source": "manual", + "type": "rectangle", + "occluded": false, + "outside": false, + "z_order": 0, + "points": "[172.0810546875, 105.990234375, 285.97262095255974, 138.40000000000146]", + "rotation": 0.0, + "parent": null + } +}, { "model": "engine.labeledshapeattributeval", "pk": 1, @@ -5097,6 +5182,15 @@ "shape": 39 } }, +{ + "model": "engine.labeledshapeattributeval", + "pk": 3, + "fields": { + "spec": 1, + "value": "mazda", + "shape": 42 + } +}, { "model": "engine.labeledtrack", "pk": 1, @@ -5920,6 +6014,85 @@ "organization": 1 } }, +{ + "model": "webhooks.webhookdelivery", + "pk": 1, + "fields": { + "webhook": 2, + "event": "update:job", + "status_code": 200, + "redelivery": false, + "created_date": "2022-11-03T13:57:26.380Z", + "updated_date": "2022-11-03T13:57:26.908Z", + "changed_fields": "state", + "request": { + "job": { + "id": 14, + "url": "http://localhost:8080/api/jobs/14", + "mode": "annotation", + "stage": "annotation", + "state": "in progress", + "labels": [ + { + "id": 6, + "name": "person", + "type": "any", + "color": "#c06060", + "sublabels": [], + "attributes": [], + "has_parent": false + }, + { + "id": 5, + "name": "car", + "type": "any", + "color": "#2080c0", + "sublabels": [], + "attributes": [ + { + "id": 1, + "name": "model", + "values": [ + "mazda", + "volvo", + "bmw" + ], + "mutable": false, + "input_type": "select", + "default_value": "mazda" + } + ], + "has_parent": false + } + ], + "status": "annotation", + "task_id": 9, + "assignee": null, + "dimension": "2d", + "project_id": 1, + "stop_frame": 19, + "bug_tracker": "", + "start_frame": 15, + "updated_date": "2022-11-03T13:57:26.346824Z", + "data_chunk_size": 72, + "data_compressed_chunk_type": "imageset" + }, + "event": "update:job", + "sender": { + "id": 1, + "url": "http://localhost:8080/api/users/1", + "username": "admin1", + "last_name": "First", + "first_name": "Admin" + }, + "webhook_id": 2, + "before_update": { + "state": "new" + } + }, + "response": "\n\n\n Example Domain\n\n \n \n \n \n\n\n\n
\n

Example Domain

\n

This domain is for use in illustrative examples in documents. You may use this\n domain in literature without prior coordination or asking for permission.

\n

More information...

\n
\n\n\n" + } +}, { "model": "admin.logentry", "pk": 1, diff --git a/tests/python/shared/assets/jobs.json b/tests/python/shared/assets/jobs.json index c1c10d25..2c65db5f 100644 --- a/tests/python/shared/assets/jobs.json +++ b/tests/python/shared/assets/jobs.json @@ -326,11 +326,11 @@ "project_id": 1, "stage": "annotation", "start_frame": 15, - "state": "new", + "state": "in progress", "status": "annotation", "stop_frame": 19, "task_id": 9, - "updated_date": "2022-06-22T09:18:45.296000Z", + "updated_date": "2022-11-03T13:57:26.346000Z", "url": "http://localhost:8080/api/jobs/14" }, { diff --git a/tests/python/shared/assets/projects.json b/tests/python/shared/assets/projects.json index 8dd2a11e..51e9ee02 100644 --- a/tests/python/shared/assets/projects.json +++ b/tests/python/shared/assets/projects.json @@ -508,7 +508,7 @@ "tasks": [ 9 ], - "updated_date": "2022-03-05T09:47:49.679000Z", + "updated_date": "2022-11-03T13:57:25.895000Z", "url": "http://localhost:8080/api/projects/1" } ] diff --git a/tests/python/shared/assets/tasks.json b/tests/python/shared/assets/tasks.json index 91308d46..9642092f 100644 --- a/tests/python/shared/assets/tasks.json +++ b/tests/python/shared/assets/tasks.json @@ -550,7 +550,7 @@ "assignee": null, "id": 14, "stage": "annotation", - "state": "new", + "state": "in progress", "status": "annotation", "url": "http://localhost:8080/api/jobs/14" } @@ -564,7 +564,7 @@ "status": "annotation", "subset": "", "target_storage": null, - "updated_date": "2022-03-05T09:47:49.667000Z", + "updated_date": "2022-11-03T13:57:26.007000Z", "url": "http://localhost:8080/api/tasks/9" }, { diff --git a/tests/python/shared/assets/users.json b/tests/python/shared/assets/users.json index 4fe99c39..83e6ad10 100644 --- a/tests/python/shared/assets/users.json +++ b/tests/python/shared/assets/users.json @@ -310,7 +310,7 @@ "is_active": true, "is_staff": true, "is_superuser": true, - "last_login": "2022-10-17T17:09:16.903140Z", + "last_login": "2022-11-03T13:56:42.744000Z", "last_name": "First", "url": "http://localhost:8080/api/users/1", "username": "admin1" diff --git a/tests/python/shared/assets/webhooks.json b/tests/python/shared/assets/webhooks.json index ec2a71bd..f2cbe54c 100644 --- a/tests/python/shared/assets/webhooks.json +++ b/tests/python/shared/assets/webhooks.json @@ -91,6 +91,8 @@ ], "id": 2, "is_active": true, + "last_delivery_date": "2022-11-03T13:57:26.908000Z", + "last_status": 200, "organization": null, "owner": { "first_name": "Business",