From 23257a2a26f49fe31f24549a7a94c603e80b0902 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Wed, 16 Nov 2022 19:27:30 +0200 Subject: [PATCH] Significantly reduced the number of queries to DB from api/jobs, api/tasks, and api/projects endpoints. (#5304) --- CHANGELOG.md | 1 + cvat/apps/engine/models.py | 4 ++++ cvat/apps/engine/serializers.py | 7 +------ cvat/apps/engine/views.py | 25 +++++++++++++++++-------- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3a22517..8c1c1599 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ from online detectors & interactors) ( - The `--https` option of CLI () ### Fixed +- Significantly optimized access to DB for api/jobs, api/tasks, and api/projects. - Removed a possibly duplicated encodeURI() calls in `server-proxy.ts` to prevent doubly encoding non-ascii paths while adding files from "Connected file share" (issue #4428) - Removed unnecessary volumes defined in docker-compose.serverless.yml diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index d12f5200..44ca08ee 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -364,6 +364,10 @@ class Task(models.Model): class Meta: default_permissions = () + def get_labels(self): + project = self.project + return project.label_set if project else self.label_set + def get_dirname(self): return os.path.join(settings.TASKS_ROOT, str(self.id)) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 163c90d2..fd2448b5 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -454,7 +454,7 @@ class StorageSerializer(serializers.ModelSerializer): fields = ('id', 'location', 'cloud_storage_id') class TaskReadSerializer(serializers.ModelSerializer): - labels = LabelSerializer(many=True, source='label_set', partial=True, required=False) + labels = LabelSerializer(many=True, source='get_labels') segments = SegmentSerializer(many=True, source='segment_set', read_only=True) data_chunk_size = serializers.ReadOnlyField(source='data.chunk_size', required=False) data_compressed_chunk_type = serializers.ReadOnlyField(source='data.compressed_chunk_type', required=False) @@ -483,11 +483,6 @@ class TaskReadSerializer(serializers.ModelSerializer): 'overlap': { 'allow_null': True }, } - def to_representation(self, instance): - response = super().to_representation(instance) - if instance.project_id: - response["labels"] = LabelSerializer(many=True).to_representation(instance.project.label_set) - return response class TaskWriteSerializer(WriteOnceMixin, serializers.ModelSerializer): labels = LabelSerializer(many=True, source='label_set', partial=True, required=False) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index a4dba89b..ba3059c4 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -308,9 +308,10 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin ): - queryset = models.Project.objects.prefetch_related(Prefetch('label_set', - queryset=models.Label.objects.order_by('id') - )) + queryset = models.Project.objects.select_related('assignee', 'owner', + 'target_storage', 'source_storage').prefetch_related( + 'tasks', 'label_set__sublabels__attributespec_set', + 'label_set__attributespec_set') # NOTE: The search_fields attribute should be a list of names of text # type fields on the model,such as CharField or TextField @@ -752,10 +753,12 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin ): - queryset = Task.objects.prefetch_related( - Prefetch('label_set', queryset=models.Label.objects.order_by('id')), - "label_set__attributespec_set", - "segment_set__job_set") + queryset = Task.objects.all().select_related('data', 'assignee', 'owner', + 'target_storage', 'source_storage').prefetch_related( + 'segment_set__job_set__assignee', 'label_set__attributespec_set', + 'project__label_set__attributespec_set', + 'label_set__sublabels__attributespec_set', + 'project__label_set__sublabels__attributespec_set') lookup_fields = {'project_name': 'project__name', 'owner': 'owner__username', 'assignee': 'assignee__username'} search_fields = ('project_name', 'name', 'owner', 'status', 'assignee', 'subset', 'mode', 'dimension') filter_fields = list(search_fields) + ['id', 'project_id', 'updated_date'] @@ -1318,10 +1321,16 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, '200': JobReadSerializer, # check JobWriteSerializer.to_representation }) ) + class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin ): - queryset = Job.objects.all() + queryset = Job.objects.all().select_related('segment__task__data').prefetch_related( + 'segment__task__label_set', 'segment__task__project__label_set', + 'segment__task__label_set__sublabels__attributespec_set', + 'segment__task__project__label_set__sublabels__attributespec_set', + 'segment__task__label_set__attributespec_set', + 'segment__task__project__label_set__attributespec_set') iam_organization_field = 'segment__task__organization' search_fields = ('task_name', 'project_name', 'assignee', 'state', 'stage') filter_fields = list(search_fields) + ['id', 'task_id', 'project_id', 'updated_date']