Clean up data when a project is deleted (#5632)

Fix https://github.com/opencv/cvat/issues/5595

CHANGELOG.md

@@ -51,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Export annotations to Azure container (<https://github.com/opencv/cvat/pull/5596>)
 - Fix the type of the credentials parameter of make_client from the Python SDK
 - Reduced number of noisy information on ortho views for 3D canvas (<https://github.com/opencv/cvat/pull/5608>)
+- Clean up disk space after a project is removed (<https://github.com/opencv/cvat/pull/5632>)
 
 ### Security
 - Fixed vulnerability with social authentication (<https://github.com/opencv/cvat/pull/5521>)

cvat/apps/engine/signals.py

@@ -3,21 +3,20 @@
 # SPDX-License-Identifier: MIT
 import shutil
 
+from django.contrib.auth.models import User
 from django.db.models.signals import post_delete, post_save
 from django.dispatch import receiver
-from django.contrib.auth.models import User
 
-from .models import (
+from .models import (CloudStorage, Data, Job, Profile, Project,
-    Data,
+    StatusChoice, Task)
-    Job,
-    StatusChoice,
-    Task,
-    Profile,
-)
 
+# TODO: need to log any problems reported by shutil.rmtree when the new
+# analytics feature is available. Now the log system can write information
+# into a file inside removed directory.
 
-@receiver(post_save, sender=Job, dispatch_uid="update_task_status")
+@receiver(post_save, sender=Job,
-def update_task_status(instance, **kwargs):
+    dispatch_uid=__name__ + ".save_job_handler")
+def __save_job_handler(instance, **kwargs):
     db_task = instance.segment.task
     db_jobs = list(Job.objects.filter(segment__task_id=db_task.id))
     status = StatusChoice.COMPLETED
@@ -30,18 +29,44 @@ def update_task_status(instance, **kwargs):
         db_task.status = status
         db_task.save()
 
-@receiver(post_save, sender=User, dispatch_uid="create_a_profile_on_create_a_user")
+@receiver(post_save, sender=User,
-def create_profile(instance, **kwargs):
+    dispatch_uid=__name__ + ".save_user_handler")
+def __save_user_handler(instance, **kwargs):
     if not hasattr(instance, 'profile'):
         profile = Profile()
         profile.user = instance
         profile.save()
 
-@receiver(post_delete, sender=Task, dispatch_uid="delete_task_files_on_delete_task")
+@receiver(post_delete, sender=Project,
-def delete_task_files_on_delete_task(instance, **kwargs):
+    dispatch_uid=__name__ + ".delete_project_handler")
+def __delete_project_handler(instance, **kwargs):
+    shutil.rmtree(instance.get_dirname(), ignore_errors=True)
+
+@receiver(post_delete, sender=Task,
+    dispatch_uid=__name__ + ".delete_task_handler")
+def __delete_task_handler(instance, **kwargs):
     shutil.rmtree(instance.get_dirname(), ignore_errors=True)
+    if instance.data and not instance.data.tasks.exists():
+        instance.data.delete()
 
+    try:
+        if instance.project: # update project
+            db_project = instance.project
+            db_project.save()
+    except Project.DoesNotExist:
+        pass # probably the project has been deleted
 
-@receiver(post_delete, sender=Data, dispatch_uid="delete_data_files_on_delete_data")
+@receiver(post_delete, sender=Job,
-def delete_data_files_on_delete_data(instance, **kwargs):
+    dispatch_uid=__name__ + ".delete_job_handler")
+def __delete_job_handler(instance, **kwargs):
+    shutil.rmtree(instance.get_dirname(), ignore_errors=True)
+
+@receiver(post_delete, sender=Data,
+    dispatch_uid=__name__ + ".delete_data_handler")
+def __delete_data_handler(instance, **kwargs):
     shutil.rmtree(instance.get_data_dirname(), ignore_errors=True)
+
+@receiver(post_delete, sender=CloudStorage,
+    dispatch_uid=__name__ + ".delete_cloudstorage_handler")
+def __delete_cloudstorage_handler(instance, **kwargs):
+    shutil.rmtree(instance.get_storage_dirname(), ignore_errors=True)

cvat/apps/engine/views.py

@@ -7,7 +7,6 @@ import io
 import os
 import os.path as osp
 import pytz
-import shutil
 import traceback
 from datetime import datetime
 from distutils.util import strtobool
@@ -847,18 +846,6 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
             db_project.save()
             assert serializer.instance.organization == db_project.organization
 
-    def perform_destroy(self, instance):
-        task_dirname = instance.get_dirname()
-        super().perform_destroy(instance)
-        shutil.rmtree(task_dirname, ignore_errors=True)
-        if instance.data and not instance.data.tasks.all():
-            shutil.rmtree(instance.data.get_data_dirname(), ignore_errors=True)
-            instance.data.delete()
-        if instance.project:
-            db_project = instance.project
-            db_project.save()
-
-
     @extend_schema(summary='Method returns a list of jobs for a specific task',
         responses=JobReadSerializer(many=True)) # Duplicate to still get 'list' op. name
     @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer,
@@ -2056,11 +2043,6 @@ class CloudStorageViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
             owner=self.request.user,
             organization=self.request.iam_context['organization'])
 
-    def perform_destroy(self, instance):
-        cloud_storage_dirname = instance.get_storage_dirname()
-        super().perform_destroy(instance)
-        shutil.rmtree(cloud_storage_dirname, ignore_errors=True)
-
     def create(self, request, *args, **kwargs):
         try:
             response = super().create(request, *args, **kwargs)