From d99125a8ea0edb1ff77c457e562ae7e5ac8cfb98 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Fri, 27 Jan 2023 17:55:38 +0200 Subject: [PATCH] Clean up data when a project is deleted (#5632) Fix https://github.com/opencv/cvat/issues/5595 --- CHANGELOG.md | 1 + cvat/apps/engine/signals.py | 57 ++++++++++++++++++++++++++----------- cvat/apps/engine/views.py | 18 ------------ 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2149ff86..da4fb87a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Export annotations to Azure container () - Fix the type of the credentials parameter of make_client from the Python SDK - Reduced number of noisy information on ortho views for 3D canvas () +- Clean up disk space after a project is removed () ### Security - Fixed vulnerability with social authentication () diff --git a/cvat/apps/engine/signals.py b/cvat/apps/engine/signals.py index a8f861be..59573855 100644 --- a/cvat/apps/engine/signals.py +++ b/cvat/apps/engine/signals.py @@ -3,21 +3,20 @@ # SPDX-License-Identifier: MIT import shutil +from django.contrib.auth.models import User from django.db.models.signals import post_delete, post_save from django.dispatch import receiver -from django.contrib.auth.models import User -from .models import ( - Data, - Job, - StatusChoice, - Task, - Profile, -) +from .models import (CloudStorage, Data, Job, Profile, Project, + StatusChoice, Task) +# TODO: need to log any problems reported by shutil.rmtree when the new +# analytics feature is available. Now the log system can write information +# into a file inside removed directory. -@receiver(post_save, sender=Job, dispatch_uid="update_task_status") -def update_task_status(instance, **kwargs): +@receiver(post_save, sender=Job, + dispatch_uid=__name__ + ".save_job_handler") +def __save_job_handler(instance, **kwargs): db_task = instance.segment.task db_jobs = list(Job.objects.filter(segment__task_id=db_task.id)) status = StatusChoice.COMPLETED @@ -30,18 +29,44 @@ def update_task_status(instance, **kwargs): db_task.status = status db_task.save() -@receiver(post_save, sender=User, dispatch_uid="create_a_profile_on_create_a_user") -def create_profile(instance, **kwargs): +@receiver(post_save, sender=User, + dispatch_uid=__name__ + ".save_user_handler") +def __save_user_handler(instance, **kwargs): if not hasattr(instance, 'profile'): profile = Profile() profile.user = instance profile.save() -@receiver(post_delete, sender=Task, dispatch_uid="delete_task_files_on_delete_task") -def delete_task_files_on_delete_task(instance, **kwargs): +@receiver(post_delete, sender=Project, + dispatch_uid=__name__ + ".delete_project_handler") +def __delete_project_handler(instance, **kwargs): + shutil.rmtree(instance.get_dirname(), ignore_errors=True) + +@receiver(post_delete, sender=Task, + dispatch_uid=__name__ + ".delete_task_handler") +def __delete_task_handler(instance, **kwargs): shutil.rmtree(instance.get_dirname(), ignore_errors=True) + if instance.data and not instance.data.tasks.exists(): + instance.data.delete() + try: + if instance.project: # update project + db_project = instance.project + db_project.save() + except Project.DoesNotExist: + pass # probably the project has been deleted -@receiver(post_delete, sender=Data, dispatch_uid="delete_data_files_on_delete_data") -def delete_data_files_on_delete_data(instance, **kwargs): +@receiver(post_delete, sender=Job, + dispatch_uid=__name__ + ".delete_job_handler") +def __delete_job_handler(instance, **kwargs): + shutil.rmtree(instance.get_dirname(), ignore_errors=True) + +@receiver(post_delete, sender=Data, + dispatch_uid=__name__ + ".delete_data_handler") +def __delete_data_handler(instance, **kwargs): shutil.rmtree(instance.get_data_dirname(), ignore_errors=True) + +@receiver(post_delete, sender=CloudStorage, + dispatch_uid=__name__ + ".delete_cloudstorage_handler") +def __delete_cloudstorage_handler(instance, **kwargs): + shutil.rmtree(instance.get_storage_dirname(), ignore_errors=True) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 54f0dec7..90a772e9 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -7,7 +7,6 @@ import io import os import os.path as osp import pytz -import shutil import traceback from datetime import datetime from distutils.util import strtobool @@ -847,18 +846,6 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, db_project.save() assert serializer.instance.organization == db_project.organization - def perform_destroy(self, instance): - task_dirname = instance.get_dirname() - super().perform_destroy(instance) - shutil.rmtree(task_dirname, ignore_errors=True) - if instance.data and not instance.data.tasks.all(): - shutil.rmtree(instance.data.get_data_dirname(), ignore_errors=True) - instance.data.delete() - if instance.project: - db_project = instance.project - db_project.save() - - @extend_schema(summary='Method returns a list of jobs for a specific task', responses=JobReadSerializer(many=True)) # Duplicate to still get 'list' op. name @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer, @@ -2056,11 +2043,6 @@ class CloudStorageViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, owner=self.request.user, organization=self.request.iam_context['organization']) - def perform_destroy(self, instance): - cloud_storage_dirname = instance.get_storage_dirname() - super().perform_destroy(instance) - shutil.rmtree(cloud_storage_dirname, ignore_errors=True) - def create(self, request, *args, **kwargs): try: response = super().create(request, *args, **kwargs)