Fixed chunk and preview cache usage (#5569)

CHANGELOG.md

@@ -35,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Helm: Empty password for Redis (<https://github.com/opencv/cvat/pull/5520>)
+- Preview & chunk cache settings are ignored (<https://github.com/opencv/cvat/pull/5569>)
 
 ### Security
 - Fixed vulnerability with social authentication (<https://github.com/opencv/cvat/pull/5521>)

cvat/apps/engine/cache.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from tempfile import NamedTemporaryFile
 import pytz
 
-from diskcache import Cache
+from django.core.cache import cache
 from django.conf import settings
 from rest_framework.exceptions import ValidationError, NotFound
 
@@ -25,42 +25,42 @@ from cvat.apps.engine.mime_types import mimetypes
 from utils.dataset_manifest import ImageManifestManager
 
 
-class CacheInteraction:
+class MediaCache:
     def __init__(self, dimension=DimensionType.DIM_2D):
-        self._cache = Cache(settings.CACHE_ROOT)
         self._dimension = dimension
 
-    def __del__(self):
-        self._cache.close()
+    @staticmethod
+    def _get_or_set_cache_item(key, create_function):
+        item = cache.get(key)
+        if not item:
+            item = create_function()
+            cache.set(key, item)
 
-    def get_buf_chunk_with_mime(self, chunk_number, quality, db_data):
-        cache_key = f'{db_data.id}_{chunk_number}_{quality}'
-        chunk, tag = self._cache.get(cache_key, tag=True)
+        return item
 
-        if not chunk:
-            chunk, tag = self._prepare_chunk_buff(db_data, quality, chunk_number)
-            self._cache.set(cache_key, chunk, tag=tag)
+    def get_buf_chunk_with_mime(self, chunk_number, quality, db_data):
+        item = self._get_or_set_cache_item(
+            key=f'{db_data.id}_{chunk_number}_{quality}',
+            create_function=lambda: self._prepare_chunk_buff(db_data, quality, chunk_number),
+        )
 
-        return chunk, tag
+        return item
 
     def get_local_preview_with_mime(self, frame_number, db_data):
-        key = f'data_{db_data.id}_{frame_number}_preview'
-        buf, mime = self._cache.get(key, tag=True)
-        if not buf:
-            buf, mime = self._prepare_local_preview(frame_number, db_data)
-            self._cache.set(key, buf, tag=mime)
+        item = self._get_or_set_cache_item(
+            key=f'data_{db_data.id}_{frame_number}_preview',
+            create_function=lambda: self._prepare_local_preview(frame_number, db_data),
+        )
 
-        return buf, mime
+        return item
 
     def get_cloud_preview_with_mime(self, db_storage):
-        key = f'cloudstorage_{db_storage.id}_preview'
-        preview, mime = self._cache.get(key, tag=True)
-
-        if not preview:
-            preview, mime = self._prepare_cloud_preview(db_storage)
-            self._cache.set(key, preview, tag=mime)
+        item = self._get_or_set_cache_item(
+            key=f'cloudstorage_{db_storage.id}_preview',
+            create_function=lambda: self._prepare_cloud_preview(db_storage)
+        )
 
-        return preview, mime
+        return item
 
     @staticmethod
     def _get_frame_provider():
@@ -147,9 +147,9 @@ class CacheInteraction:
     def _prepare_local_preview(self, frame_number, db_data):
         FrameProvider = self._get_frame_provider()
         frame_provider = FrameProvider(db_data, self._dimension)
-        buf, mime = frame_provider.get_preview(frame_number)
+        buff, mime_type = frame_provider.get_preview(frame_number)
 
-        return buf, mime
+        return buff, mime_type
 
     def _prepare_cloud_preview(self, db_storage):
         storage = db_storage_to_storage_instance(db_storage)
@@ -179,7 +179,7 @@ class CacheInteraction:
             slogger.cloud_storage[db_storage.pk].info(msg)
             raise NotFound(msg)
 
-        preview = storage.download_fileobj(preview_path)
-        mime = mimetypes.guess_type(preview_path)[0]
+        buff = storage.download_fileobj(preview_path)
+        mime_type = mimetypes.guess_type(preview_path)[0]
 
-        return preview, mime
+        return buff, mime_type

cvat/apps/engine/frame_provider.py

@@ -12,7 +12,7 @@ import cv2
 import numpy as np
 from PIL import Image
 
-from cvat.apps.engine.cache import CacheInteraction
+from cvat.apps.engine.cache import MediaCache
 from cvat.apps.engine.media_extractors import VideoReader, ZipReader
 from cvat.apps.engine.mime_types import mimetypes
 from cvat.apps.engine.models import DataChoice, StorageMethodChoice, DimensionType
@@ -97,7 +97,7 @@ class FrameProvider:
         }
 
         if db_data.storage_method == StorageMethodChoice.CACHE:
-            cache = CacheInteraction(dimension=dimension)
+            cache = MediaCache(dimension=dimension)
 
             self._loaders[self.Quality.COMPRESSED] = self.BuffChunkLoader(
                 reader_class[db_data.compressed_chunk_type],

cvat/apps/engine/views.py

@@ -75,7 +75,7 @@ from .log import clogger, slogger
 from cvat.apps.iam.permissions import (CloudStoragePermission,
     CommentPermission, IssuePermission, JobPermission, ProjectPermission,
     TaskPermission, UserPermission)
-from cvat.apps.engine.cache import CacheInteraction
+from cvat.apps.engine.cache import MediaCache
 
 
 @extend_schema(tags=['server'])
@@ -721,7 +721,7 @@ class DataChunkGetter:
                         f'[{start}, {stop}] range')
 
                 if self.type == 'preview':
-                    cache = CacheInteraction(self.dimension)
+                    cache = MediaCache(self.dimension)
                     buf, mime = cache.get_local_preview_with_mime(self.number, db_data)
                 else:
                     buf, mime = frame_provider.get_frame(self.number, self.quality)
@@ -2185,7 +2185,7 @@ class CloudStorageViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
     def preview(self, request, pk):
         try:
             db_storage = self.get_object()
-            cache = CacheInteraction()
+            cache = MediaCache()
             preview, mime = cache.get_cloud_preview_with_mime(db_storage)
             return HttpResponse(preview, mime)
         except CloudStorageModel.DoesNotExist:

cvat/settings/base.py

@@ -514,6 +514,7 @@ CACHES = {
        'BACKEND' : 'diskcache.DjangoCache',
        'LOCATION' : CACHE_ROOT,
        'TIMEOUT' : None,
+       'SHARDS': 32,
        'OPTIONS' : {
             'size_limit' : 2 ** 40, # 1 Tb
        }