Add documentation/tests for tasks large files uploads (#4036)

* added api documentation

* added test with 108mb file

* fixed linter issues

* added upload chunk size param

* fixed initialization

* udpated doc for uploadChunkSize

* reworked setting as global

* small fix

* moved uploadChunkSize setting setup to hooks

* fix comments

* change this to globalThis

cvat-core/src/api.js

@@ -699,6 +699,7 @@ function build() {
              * @memberof module:API.cvat.config
              * @property {string} origin ui URL origin
              * @memberof module:API.cvat.config
+             * @property {number} uploadChunkSize max size of one data request in mb
              * @memberof module:API.cvat.config
              */
             get backendAPI() {
@@ -719,6 +720,12 @@ function build() {
             set origin(value) {
                 config.origin = value;
             },
+            get uploadChunkSize() {
+                return config.uploadChunkSize;
+            },
+            set uploadChunkSize(value) {
+                config.uploadChunkSize = value;
+            },
         },
         /**
          * Namespace contains some library information e.g. api version

cvat-core/src/config.js

@@ -7,4 +7,5 @@ module.exports = {
     proxy: false,
     organizationID: null,
     origin: '',
+    uploadChunkSize: 100,
 };

cvat-core/src/server-proxy.js

@@ -772,7 +772,7 @@
                     });
                 }
 
-                const chunkSize = 1024 * 1024 * 100; // 100 mb
+                const chunkSize = config.uploadChunkSize * 1024 * 1024;
                 const clientFiles = taskDataSpec.client_files;
                 const chunkFiles = [];
                 const bulkFiles = [];
@@ -819,7 +819,7 @@
                 async function chunkUpload(taskId, file) {
                     return new Promise((resolve, reject) => {
                         const upload = new tus.Upload(file, {
-                            endpoint: `${origin}/${backendAPI}/tasks/${taskId}/data/`,
+                            endpoint: `${origin}${backendAPI}/tasks/${taskId}/data/`,
                             metadata: {
                                 filename: file.name,
                                 filetype: file.type,

cvat-ui/src/cvat-core-wrapper.ts

@@ -8,6 +8,8 @@ const cvat: any = _cvat;
 
 cvat.config.backendAPI = '/api/v1';
 cvat.config.origin = window.location.origin;
+cvat.config.uploadChunkSize = 100;
+(globalThis as any).cvat = cvat;
 
 export default function getCore(): any {
     return cvat;

cvat/apps/engine/views.py

@@ -648,8 +648,16 @@ class TaskViewSet(UploadMixin, viewsets.ModelViewSet):
         task.create(db_task.id, data)
         return Response(serializer.data, status=status.HTTP_202_ACCEPTED)
 
-    @swagger_auto_schema(method='post', operation_summary='Method permanently attaches images or video to a task',
+    @swagger_auto_schema(method='post', operation_summary='Method permanently attaches images or video to a task. Supports tus uploads, see more https://tus.io/',
         request_body=DataSerializer,
+        manual_parameters=[
+                openapi.Parameter('Upload-Start', in_=openapi.IN_HEADER, type=openapi.TYPE_BOOLEAN,
+                    description="Initializes data upload. No data should be sent with this header"),
+                openapi.Parameter('Upload-Multiple', in_=openapi.IN_HEADER, type=openapi.TYPE_BOOLEAN,
+                    description="Indicates that data with this request are single or multiple files that should be attached to a task"),
+                openapi.Parameter('Upload-Finish', in_=openapi.IN_HEADER, type=openapi.TYPE_BOOLEAN,
+                    description="Finishes data upload. Can be combined with Upload-Start header to create task data with one request"),
+         ]
     )
     @swagger_auto_schema(method='get', operation_summary='Method returns data for a specific task',
         manual_parameters=[

tests/cypress/integration/actions_tasks3/case_112_tus_upload.js

@@ -0,0 +1,50 @@
+// Copyright (C) 2020-2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+/// <reference types="cypress" />
+
+context('Create task with tus file', () => {
+    const caseId = '112';
+    const labelName = `Case ${caseId}`;
+    const taskName = `New annotation task for ${labelName}`;
+    const attrName = `Attr for ${labelName}`;
+    const textDefaultValue = 'Some default value for type Text';
+    const imagesCount = 1;
+    const imageFileName = `image_${labelName.replace(' ', '_').toLowerCase()}`;
+    const width = 1920;
+    const height = 1080;
+    const posX = 10;
+    const posY = 10;
+    const color = 'gray';
+    const archiveName = `${imageFileName}.zip`;
+    const archivePath = `cypress/fixtures/${archiveName}`;
+    const imagesFolder = `cypress/fixtures/${imageFileName}`;
+    const directoryToArchive = imagesFolder;
+    const zipLevel = 0;
+    const extension = 'bmp';
+
+    before(() => {
+        cy.visit('auth/login');
+        cy.login();
+        cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX,
+            posY, labelName, imagesCount, extension);
+        cy.createZipArchive(directoryToArchive, archivePath, zipLevel);
+        cy.window().then((win) => { win.cvat.config.uploadChunkSize = 5; });
+    });
+
+    describe(`Testing "${labelName}"`, () => {
+        it('Create a task with 5mb upload chunk size', () => {
+            cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, archiveName);
+        });
+
+        it('Check if task exist', () => {
+            cy.goToTaskList();
+            cy.contains('.cvat-item-task-name', taskName).should('exist');
+        });
+    });
+
+    after(() => {
+        cy.window().then((win) => { win.cvat.config.uploadChunkSize = 100; });
+    });
+});

tests/cypress/plugins/createZipArchive/addPlugin.js

@@ -1,22 +1,24 @@
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
-// eslint-disable-next-line no-undef
+// eslint-disable-next-line no-use-before-define
 exports.createZipArchive = createZipArchive;
 
 const archiver = require('archiver');
+// eslint-disable-next-line import/no-extraneous-dependencies
 const fs = require('fs-extra');
 
 function createZipArchive(args) {
-    const directoryToArchive = args.directoryToArchive;
+    const { directoryToArchive } = args;
+    const { level } = args;
     const output = fs.createWriteStream(args.arhivePath);
     const archive = archiver('zip', {
         gzip: true,
-        zlib: { level: 9 },
+        zlib: { level },
     });
 
-    archive.on('error', function (err) {
+    archive.on('error', (err) => {
         throw err;
     });
 

tests/cypress/plugins/createZipArchive/createZipArchiveCommand.js

@@ -1,10 +1,9 @@
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
-Cypress.Commands.add('createZipArchive', function (directoryToArchive, arhivePath) {
+Cypress.Commands.add('createZipArchive', (directoryToArchive, arhivePath, level = 9) => cy.task('createZipArchive', {
-    return cy.task('createZipArchive', {
+    directoryToArchive,
-        directoryToArchive: directoryToArchive,
+    arhivePath,
-        arhivePath: arhivePath,
+    level,
-    });
+}));
-});

tests/cypress/plugins/imageGenerator/addPlugin.js

@@ -1,33 +1,45 @@
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
+// eslint-disable-next-line no-use-before-define
 exports.imageGenerator = imageGenerator;
 
-const jimp = require('jimp');
 const path = require('path');
+const jimp = require('jimp');
 
-function imageGenerator(args) {
+function createImage(width, height, color) {
-    const directory = args.directory;
+    return new Promise((resolve, reject) => {
-    const fileName = args.fileName;
+        // eslint-disable-next-line new-cap, no-new
-    const width = args.width;
+        new jimp(width, height, color, ((err, img) => {
-    const height = args.height;
+            if (err) reject(err);
-    const color = args.color;
+            resolve(img);
-    const posX = args.posX;
+        }));
-    const posY = args.posY;
+    });
-    const message = args.message;
+}
-    const file = path.join(directory, fileName);
+
-    const count = args.count;
+function appendText(image, posX, posY, message, index) {
     return new Promise((resolve, reject) => {
+        jimp.loadFont(jimp.FONT_SANS_64_BLACK, (err, font) => {
+            if (err) reject(err);
+            image.print(font, Number(posX), Number(posY), `${message}. Num ${index}`);
+            resolve(image);
+        });
+    });
+}
+
+async function imageGenerator(args) {
+    const {
+        directory, fileName, width, height, color, posX, posY, message, count, extension,
+    } = args;
+    const file = path.join(directory, fileName);
+    try {
         for (let i = 1; i <= count; i++) {
-            new jimp(width, height, color, function (err, image) {
+            let image = await createImage(width, height, color);
-                if (err) reject(err);
+            image = await appendText(image, posX, posY, message, i);
-                jimp.loadFont(jimp.FONT_SANS_64_BLACK, function (err, font) {
+            image.write(`${file}_${i}.${extension}`);
-                    if (err) reject(err);
-                    image.print(font, Number(posX), Number(posY), `${message}. Num ${i}`).write(`${file}_${i}.png`);
-                });
-            });
         }
-        setTimeout(() => resolve(null), '1000');
+    // eslint-disable-next-line no-empty
-    });
+    } catch (e) {}
+    return null;
 }

tests/cypress/plugins/imageGenerator/imageGeneratorCommand.js

@@ -1,17 +1,16 @@
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
-Cypress.Commands.add('imageGenerator', (directory, fileName, width, height, color, posX, posY, message, count) => {
+Cypress.Commands.add('imageGenerator', (directory, fileName, width, height, color, posX, posY, message, count, extension = 'png') => cy.task('imageGenerator', {
-    return cy.task('imageGenerator', {
+    directory,
-        directory: directory,
+    fileName,
-        fileName: fileName,
+    width,
-        width: width,
+    height,
-        height: height,
+    color,
-        color: color,
+    posX,
-        posX: posX,
+    posY,
-        posY: posY,
+    message,
-        message: message,
+    count,
-        count: count,
+    extension,
-    });
+}));
-});