Fix pagination in some endpoints (#5557)

- Added missing pagination or page parameters in `/projects/{id}/tasks`,
`/tasks/{id}/jobs`, `/jobs/{id}/issues`, `/jobs/{id}/commits`,
`/issues/{id}/comments`, `/organizations`
- Updated SDK, tests and UI
main
Maxim Zhiltsov 3 years ago committed by GitHub
parent 330f123785
commit 58b05536f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- \[SDK\] The `resource_type` args now have the default value of `local` in task creation functions.
The corresponding arguments are keyword-only now.
(<https://github.com/opencv/cvat/pull/5502>)
- \[Server API\] Added missing pagination or pagination parameters in
`/project/{id}/tasks`, `/tasks/{id}/jobs`, `/jobs/{id}/issues`,
`/jobs/{id}/commits`, `/issues/{id}/comments`, `/organizations`
(<https://github.com/opencv/cvat/pull/5557>)
- Windows Installation Instructions adjusted to work around <https://github.com/nuclio/nuclio/issues/1821>
- The contour detection function for semantic segmentation (<https://github.com/opencv/cvat/pull/4665>)

@ -5,7 +5,7 @@
import FormData from 'form-data';
import store from 'store';
import Axios from 'axios';
import Axios, { AxiosResponse } from 'axios';
import * as tus from 'tus-js-client';
import { Storage } from './storage';
import { StorageLocation, WebhookSourceType } from './enums';
@ -1258,19 +1258,69 @@ async function getJobs(filter = {}) {
return response.data;
}
function fetchAll(url): Promise<any[]> {
const pageSize = 500;
let collection = [];
return new Promise((resolve, reject) => {
Axios.get(url, {
params: {
page_size: pageSize,
page: 1,
},
proxy: config.proxy,
}).then((initialData) => {
const { count, results } = initialData.data;
collection = collection.concat(results);
if (count <= pageSize) {
resolve(collection);
return;
}
const pages = Math.ceil(count / pageSize);
const promises = Array(pages).fill(0).map((_: number, i: number) => {
if (i) {
return Axios.get(url, {
params: {
page_size: pageSize,
page: i + 1,
},
proxy: config.proxy,
});
}
return Promise.resolve(null);
});
Promise.all(promises).then((responses: AxiosResponse<any, any>[]) => {
responses.forEach((resp) => {
if (resp) {
collection = collection.concat(resp.data.results);
}
});
// removing possible dublicates
const obj = collection.reduce((acc: Record<string, any>, item: any) => {
acc[item.id] = item;
return acc;
}, {});
resolve(Object.values(obj));
}).catch((error) => reject(error));
}).catch((error) => reject(error));
});
}
async function getJobIssues(jobID) {
const { backendAPI } = config;
let response = null;
try {
response = await Axios.get(`${backendAPI}/jobs/${jobID}/issues`, {
proxy: config.proxy,
});
response = await fetchAll(`${backendAPI}/jobs/${jobID}/issues`);
} catch (errorData) {
throw generateError(errorData);
}
return response.data;
return response;
}
async function createComment(data) {
@ -1989,14 +2039,12 @@ async function getOrganizations() {
let response = null;
try {
response = await Axios.get(`${backendAPI}/organizations`, {
proxy: config.proxy,
});
response = await fetchAll(`${backendAPI}/organizations?page_size`);
} catch (errorData) {
throw generateError(errorData);
}
return response.data;
return response;
}
async function createOrganization(data) {

@ -1,10 +1,13 @@
# Copyright (C) 2022 CVAT.ai Corporation
# Copyright (C) 2022-2023 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
from __future__ import annotations
from typing import List
from cvat_sdk.api_client import apis, models
from cvat_sdk.core.helpers import get_paginated_collection
from cvat_sdk.core.proxies.model_proxy import (
ModelCreateMixin,
ModelDeleteMixin,
@ -50,6 +53,12 @@ class Issue(
):
_model_partial_update_arg = "patched_issue_write_request"
def get_comments(self) -> List[Comment]:
return [
Comment(self._client, m)
for m in get_paginated_collection(self.api.list_comments_endpoint, id=self.id)
]
class IssuesRepo(
_IssueRepoBase,

@ -1,4 +1,4 @@
# Copyright (C) 2022 CVAT.ai Corporation
# Copyright (C) 2022-2023 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
@ -161,7 +161,10 @@ class Job(
)
def get_issues(self) -> List[Issue]:
return [Issue(self._client, m) for m in self.api.list_issues(id=self.id)[0]]
return [
Issue(self._client, m)
for m in get_paginated_collection(self.api.list_issues_endpoint, id=self.id)
]
def get_commits(self) -> List[models.IJobCommit]:
return get_paginated_collection(self.api.list_commits_endpoint, id=self.id)

@ -1,4 +1,4 @@
# Copyright (C) 2022 CVAT.ai Corporation
# Copyright (C) 2022-2023 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, List, Optional
from cvat_sdk.api_client import apis, models
from cvat_sdk.core.downloading import Downloader
from cvat_sdk.core.helpers import get_paginated_collection
from cvat_sdk.core.progress import ProgressReporter
from cvat_sdk.core.proxies.model_proxy import (
ModelCreateMixin,
@ -124,7 +125,10 @@ class Project(
return annotations
def get_tasks(self) -> List[Task]:
return [Task(self._client, m) for m in self.api.list_tasks(id=self.id)[0].results]
return [
Task(self._client, m)
for m in get_paginated_collection(self.api.list_tasks_endpoint, id=self.id)
]
def get_preview(
self,

@ -1,4 +1,4 @@
# Copyright (C) 2022 CVAT.ai Corporation
# Copyright (C) 2022-2023 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
@ -18,6 +18,7 @@ from PIL import Image
from cvat_sdk.api_client import apis, exceptions, models
from cvat_sdk.core import git
from cvat_sdk.core.downloading import Downloader
from cvat_sdk.core.helpers import get_paginated_collection
from cvat_sdk.core.progress import ProgressReporter
from cvat_sdk.core.proxies.annotations import AnnotationCrudMixin
from cvat_sdk.core.proxies.jobs import Job
@ -302,7 +303,10 @@ class Task(
self._client.logger.info(f"Backup for task {self.id} has been downloaded to {filename}")
def get_jobs(self) -> List[Job]:
return [Job(self._client, m) for m in self.api.list_jobs(id=self.id)[0]]
return [
Job(self._client, model=m)
for m in get_paginated_collection(self.api.list_jobs_endpoint, id=self.id)
]
def get_meta(self) -> models.IDataMetaRead:
(meta, _) = self.api.retrieve_data_meta(self.id)

@ -26,6 +26,7 @@ import cvat.apps.dataset_manager as dm
from cvat.apps.dataset_manager.bindings import CvatTaskOrJobDataExtractor, TaskData
from cvat.apps.dataset_manager.task import TaskAnnotation
from cvat.apps.engine.models import Task
from cvat.apps.engine.tests.utils import get_paginated_collection
projects_path = osp.join(osp.dirname(__file__), 'assets', 'projects.json')
with open(projects_path) as file:
@ -174,8 +175,10 @@ class _DbTestBase(APITestCase):
def _get_jobs(self, task_id):
with ForceLogin(self.admin, self.client):
response = self.client.get("/api/tasks/{}/jobs".format(task_id))
return response.data
values = get_paginated_collection(lambda page:
self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page))
)
return values
def _get_request(self, path, user):
with ForceLogin(user, self.client):

@ -19,6 +19,7 @@ from django.contrib.auth.models import Group, User
from cvat.apps.engine.models import Task
from cvat.apps.dataset_repo.dataset_repo import (Git, initial_create, push, get)
from cvat.apps.dataset_repo.models import GitData, GitStatusChoice
from cvat.apps.engine.tests.utils import get_paginated_collection
orig_execute = git.cmd.Git.execute
GIT_URL = "https://1.2.3.4/repo/exist.git"
@ -198,8 +199,10 @@ class GitDatasetRepoTest(APITestCase):
def _get_jobs(self, task_id):
with ForceLogin(self.admin, self.client):
response = self.client.get("/api/tasks/{}/jobs".format(task_id))
return response.data
values = get_paginated_collection(lambda page:
self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page))
)
return values
def _create_task(self, init_repos=False):
data = {

@ -185,14 +185,14 @@ class JobReadSerializer(serializers.ModelSerializer):
project_id = serializers.ReadOnlyField(source="get_project_id", allow_null=True)
start_frame = serializers.ReadOnlyField(source="segment.start_frame")
stop_frame = serializers.ReadOnlyField(source="segment.stop_frame")
assignee = BasicUserSerializer(allow_null=True)
dimension = serializers.CharField(max_length=2, source='segment.task.dimension')
labels = LabelSerializer(many=True, source='get_labels')
assignee = BasicUserSerializer(allow_null=True, read_only=True)
dimension = serializers.CharField(max_length=2, source='segment.task.dimension', read_only=True)
labels = LabelSerializer(many=True, source='get_labels', read_only=True)
data_chunk_size = serializers.ReadOnlyField(source='segment.task.data.chunk_size')
data_compressed_chunk_type = serializers.ReadOnlyField(source='segment.task.data.compressed_chunk_type')
mode = serializers.ReadOnlyField(source='segment.task.mode')
bug_tracker = serializers.CharField(max_length=2000, source='get_bug_tracker',
allow_null=True)
allow_null=True, read_only=True)
class Meta:
model = models.Job

@ -35,6 +35,7 @@ from cvat.apps.engine.models import (AttributeSpec, AttributeType, Data, Job,
Project, Segment, StageChoice, StatusChoice, Task, Label, StorageMethodChoice,
StorageChoice, DimensionType, SortingMethod)
from cvat.apps.engine.media_extractors import ValidateDimension, sort
from cvat.apps.engine.tests.utils import get_paginated_collection
from utils.dataset_manifest import ImageManifestManager, VideoManifestManager
#supress av warnings
@ -4225,8 +4226,9 @@ class JobAnnotationAPITestCase(APITestCase):
response = self.client.get("/api/tasks/{}".format(tid))
task = response.data
response = self.client.get("/api/tasks/{}/jobs".format(tid))
jobs = response.data
jobs = get_paginated_collection(lambda page:
self.client.get("/api/tasks/{}/jobs?page={}".format(tid, page))
)
return (task, jobs)

@ -24,6 +24,8 @@ from cvat.apps.engine.media_extractors import ValidateDimension
from cvat.apps.dataset_manager.task import TaskAnnotation
from datumaro.util.test_utils import TestDir
from cvat.apps.engine.tests.utils import get_paginated_collection
CREATE_ACTION = "create"
UPDATE_ACTION = "update"
DELETE_ACTION = "delete"
@ -140,8 +142,10 @@ class _DbTestBase(APITestCase):
def _get_jobs(self, task_id):
with ForceLogin(self.admin, self.client):
response = self.client.get("/api/tasks/{}/jobs".format(task_id))
return response.data
values = get_paginated_collection(lambda page:
self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page))
)
return values
def _get_request(self, path, user):
with ForceLogin(user, self.client):

@ -0,0 +1,25 @@
# Copyright (C) 2023 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
import itertools
from typing import Callable, Iterator, TypeVar
from django.http.response import HttpResponse
T = TypeVar('T')
def get_paginated_collection(
request_chunk_callback: Callable[[int], HttpResponse]
) -> Iterator[T]:
values = []
for page in itertools.count(start=1):
response = request_chunk_callback(page)
data = response.json()
values.extend(data["results"])
if not data.get('next'):
break
return values

@ -0,0 +1,47 @@
# Copyright (C) 2023 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
# NOTE: importing in the header leads to circular importing
from typing import Optional, Type
from django.db.models.query import QuerySet
from django.http.request import HttpRequest
from django.http.response import HttpResponse
from rest_framework.serializers import Serializer
from rest_framework.viewsets import GenericViewSet
def make_paginated_response(
queryset: QuerySet,
*,
viewset: GenericViewSet,
response_type: Optional[Type[HttpResponse]] = None,
serializer_type: Optional[Type[Serializer]] = None,
request: Optional[Type[HttpRequest]] = None,
**serializer_params
):
# Adapted from the mixins.ListModelMixin.list()
serializer_params.setdefault('many', True)
if response_type is None:
from rest_framework.response import Response
response_type = Response
if request is None:
request = getattr(viewset, 'request', None)
if request is not None:
context = serializer_params.setdefault('context', {})
context.setdefault('request', request)
if serializer_type is None:
serializer_type = viewset.get_serializer
page = viewset.paginate_queryset(queryset)
if page is not None:
serializer = serializer_type(page, **serializer_params)
return viewset.get_paginated_response(serializer.data)
serializer = serializer_type(queryset, **serializer_params)
return response_type(serializer.data)

@ -46,7 +46,7 @@ from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer
from cvat.apps.engine.frame_provider import FrameProvider
from cvat.apps.engine.media_extractors import get_mime
from cvat.apps.engine.models import (
Job, Task, Project, Issue, Data,
Job, JobCommit, Task, Project, Issue, Data,
Comment, StorageMethodChoice, StorageChoice,
CloudProviderChoice, Location
)
@ -62,6 +62,7 @@ from cvat.apps.engine.serializers import (
ProjectFileSerializer, TaskFileSerializer)
from utils.dataset_manifest import ImageManifestManager
from cvat.apps.engine.view_utils import make_paginated_response
from cvat.apps.engine.utils import (
av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message
)
@ -309,23 +310,17 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
@extend_schema(
summary='Method returns information of the tasks of the project with the selected id',
responses={
'200': TaskReadSerializer(many=True),
})
@action(detail=True, methods=['GET'], serializer_class=TaskReadSerializer)
responses=TaskReadSerializer(many=True)) # Duplicate to still get 'list' op. nam
@action(detail=True, methods=['GET'], serializer_class=TaskReadSerializer,
pagination_class=viewsets.GenericViewSet.pagination_class,
# Remove regular list() parameters from the swagger schema.
# Unset, they would be taken from the enclosing class, which is wrong.
# https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want
filter_fields=None, search_fields=None, ordering_fields=None)
def tasks(self, request, pk):
self.get_object() # force to call check_object_permissions
queryset = Task.objects.filter(project_id=pk).order_by('-id')
page = self.paginate_queryset(queryset)
if page is not None:
serializer = self.get_serializer(page, many=True,
context={"request": request})
return self.get_paginated_response(serializer.data)
serializer = self.get_serializer(queryset, many=True,
context={"request": request})
return Response(serializer.data)
return make_paginated_response(Task.objects.filter(project_id=pk).order_by('-id'),
viewset=self, serializer_type=self.serializer_class) # from @action
@extend_schema(methods=['GET'], summary='Export project as a dataset in a specific format',
@ -866,17 +861,16 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
@extend_schema(summary='Method returns a list of jobs for a specific task',
responses=JobReadSerializer(many=True)) # Duplicate to still get 'list' op. name
@action(detail=True, methods=['GET'], serializer_class=JobReadSerializer(many=True),
# Remove regular list() parameters from swagger schema
@action(detail=True, methods=['GET'], serializer_class=JobReadSerializer,
pagination_class=viewsets.GenericViewSet.pagination_class,
# Remove regular list() parameters from the swagger schema.
# Unset, they would be taken from the enclosing class, which is wrong.
# https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want
pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None)
filter_fields=None, search_fields=None, ordering_fields=None)
def jobs(self, request, pk):
self.get_object() # force to call check_object_permissions
queryset = Job.objects.filter(segment__task_id=pk)
serializer = JobReadSerializer(queryset, many=True,
context={"request": request})
return Response(serializer.data)
return make_paginated_response(Job.objects.filter(segment__task_id=pk).order_by('id'),
viewset=self, serializer_type=self.serializer_class) # from @action
# UploadMixin method
def get_upload_dir(self):
@ -1635,18 +1629,16 @@ class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
@extend_schema(summary='Method returns list of issues for the job',
responses=IssueReadSerializer(many=True)) # Duplicate to still get 'list' op. name
@action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer(many=True),
# Remove regular list() parameters from swagger schema
@action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer,
pagination_class=viewsets.GenericViewSet.pagination_class,
# Remove regular list() parameters from the swagger schema.
# Unset, they would be taken from the enclosing class, which is wrong.
# https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want
pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None)
filter_fields=None, search_fields=None, ordering_fields=None)
def issues(self, request, pk):
db_job = self.get_object()
queryset = db_job.issues
serializer = IssueReadSerializer(queryset,
context={'request': request}, many=True)
return Response(serializer.data)
self.get_object() # force to call check_object_permissions
return make_paginated_response(Issue.objects.filter(job_id=pk).order_by('id'),
viewset=self, serializer_type=self.serializer_class) # from @action
@extend_schema(summary='Method returns data for a specific job',
parameters=[
@ -1748,21 +1740,17 @@ class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
return Response(serializer.data)
@extend_schema(summary='The action returns the list of tracked changes for the job',
responses={
'200': JobCommitSerializer(many=True),
})
@action(detail=True, methods=['GET'], serializer_class=None)
responses=JobCommitSerializer(many=True)) # Duplicate to still get 'list' op. name
@action(detail=True, methods=['GET'], serializer_class=JobCommitSerializer,
pagination_class=viewsets.GenericViewSet.pagination_class,
# Remove regular list() parameters from the swagger schema.
# Unset, they would be taken from the enclosing class, which is wrong.
# https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want
filter_fields=None, search_fields=None, ordering_fields=None)
def commits(self, request, pk):
db_job = self.get_object()
queryset = db_job.commits.order_by('-id')
page = self.paginate_queryset(queryset)
if page is not None:
serializer = JobCommitSerializer(page, context={'request': request}, many=True)
return self.get_paginated_response(serializer.data)
serializer = JobCommitSerializer(queryset, context={'request': request}, many=True)
return Response(serializer.data)
self.get_object() # force to call check_object_permissions
return make_paginated_response(JobCommit.objects.filter(job_id=pk).order_by('-id'),
viewset=self, serializer_type=self.serializer_class) # from @action
@extend_schema(summary='Method returns a preview image for the job',
responses={
@ -1848,19 +1836,16 @@ class IssueViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
@extend_schema(summary='The action returns all comments of a specific issue',
responses=CommentReadSerializer(many=True)) # Duplicate to still get 'list' op. name
@action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer(many=True),
# Remove regular list() parameters from swagger schema
@action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer,
pagination_class=viewsets.GenericViewSet.pagination_class,
# Remove regular list() parameters from the swagger schema.
# Unset, they would be taken from the enclosing class, which is wrong.
# https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want
pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None)
filter_fields=None, search_fields=None, ordering_fields=None)
def comments(self, request, pk):
# TODO: remove this endpoint? It is totally covered by issue body.
db_issue = self.get_object()
queryset = db_issue.comments
serializer = CommentReadSerializer(queryset,
context={'request': request}, many=True)
return Response(serializer.data)
self.get_object() # force to call check_object_permissions
return make_paginated_response(Comment.objects.filter(issue_id=pk).order_by('-id'),
viewset=self, serializer_type=self.serializer_class) # from @action
@extend_schema(tags=['comments'])
@extend_schema_view(

@ -17,6 +17,8 @@ from PIL import Image
from rest_framework import status
from rest_framework.test import APIClient, APITestCase
from cvat.apps.engine.tests.utils import get_paginated_collection
LAMBDA_ROOT_PATH = '/api/lambda'
LAMBDA_FUNCTIONS_PATH = f'{LAMBDA_ROOT_PATH}/functions'
LAMBDA_REQUESTS_PATH = f'{LAMBDA_ROOT_PATH}/requests'
@ -1044,10 +1046,13 @@ class Issue4996_Cases(_LambdaTestCaseBase):
)
self.task = task
jobs = self._get_request(f"/api/tasks/{self.task['id']}/jobs", self.admin,
org_id=self.org['id'])
assert jobs.status_code == status.HTTP_200_OK
self.job = jobs.json()[1]
jobs = get_paginated_collection(lambda page:
self._get_request(
f"/api/tasks/{self.task['id']}/jobs?page={page}",
self.admin, org_id=self.org['id']
)
)
self.job = jobs[1]
self.common_data = {
"task": self.task['id'],

@ -61,7 +61,6 @@ class OrganizationViewSet(viewsets.GenericViewSet,
ordering_fields = filter_fields
ordering = '-id'
http_method_names = ['get', 'post', 'patch', 'delete', 'head', 'options']
pagination_class = None
iam_organization_field = None
def get_queryset(self):

@ -70,7 +70,7 @@ describe('Delete users, tasks, projects, organizations created during the tests
Authorization: `Token ${authKey}`,
},
}).then((response) => {
const responseResult = response.body;
const responseResult = response.body.results;
for (const org of responseResult) {
const { id } = org;
cy.request({

@ -36,7 +36,7 @@ Cypress.Commands.add('deleteOrganizations', (authResponse, otrganizationsToDelet
Authorization: `Token ${authKey}`,
},
}).then((_response) => {
const responceResult = _response.body;
const responceResult = _response.body.results;
for (const organization of responceResult) {
const { id, slug } = organization;
for (const organizationToDelete of otrganizationsToDelete) {

@ -1,4 +1,8 @@
[
{
"count": 2,
"next": null,
"previous": null,
"results": [
{
"contact": {
"email": "org2@cvat.org"
@ -36,3 +40,4 @@
"updated_date": "2021-12-14T18:45:40.172000Z"
}
]
}

@ -43,7 +43,7 @@ def users():
@pytest.fixture(scope="session")
def organizations():
with open(ASSETS_DIR / "organizations.json") as f:
return Container(json.load(f))
return Container(json.load(f)["results"])
@pytest.fixture(scope="session")

Loading…
Cancel
Save