Add documentation/tests for tasks large files uploads (#4036)

* added api documentation

* added test with 108mb file

* fixed linter issues

* added upload chunk size param

* fixed initialization

* udpated doc for uploadChunkSize

* reworked setting as global

* small fix

* moved uploadChunkSize setting setup to hooks

* fix comments

* change this to globalThis
main
Kirill Lakhov 4 years ago committed by GitHub
parent 2ebe7176cf
commit 69d3ad79f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -699,6 +699,7 @@ function build() {
* @memberof module:API.cvat.config
* @property {string} origin ui URL origin
* @memberof module:API.cvat.config
* @property {number} uploadChunkSize max size of one data request in mb
* @memberof module:API.cvat.config
*/
get backendAPI() {
@ -719,6 +720,12 @@ function build() {
set origin(value) {
config.origin = value;
},
get uploadChunkSize() {
return config.uploadChunkSize;
},
set uploadChunkSize(value) {
config.uploadChunkSize = value;
},
},
/**
* Namespace contains some library information e.g. api version

@ -7,4 +7,5 @@ module.exports = {
proxy: false,
organizationID: null,
origin: '',
uploadChunkSize: 100,
};

@ -772,7 +772,7 @@
});
}
const chunkSize = 1024 * 1024 * 100; // 100 mb
const chunkSize = config.uploadChunkSize * 1024 * 1024;
const clientFiles = taskDataSpec.client_files;
const chunkFiles = [];
const bulkFiles = [];
@ -819,7 +819,7 @@
async function chunkUpload(taskId, file) {
return new Promise((resolve, reject) => {
const upload = new tus.Upload(file, {
endpoint: `${origin}/${backendAPI}/tasks/${taskId}/data/`,
endpoint: `${origin}${backendAPI}/tasks/${taskId}/data/`,
metadata: {
filename: file.name,
filetype: file.type,

@ -8,6 +8,8 @@ const cvat: any = _cvat;
cvat.config.backendAPI = '/api/v1';
cvat.config.origin = window.location.origin;
cvat.config.uploadChunkSize = 100;
(globalThis as any).cvat = cvat;
export default function getCore(): any {
return cvat;

@ -648,8 +648,16 @@ class TaskViewSet(UploadMixin, viewsets.ModelViewSet):
task.create(db_task.id, data)
return Response(serializer.data, status=status.HTTP_202_ACCEPTED)
@swagger_auto_schema(method='post', operation_summary='Method permanently attaches images or video to a task',
@swagger_auto_schema(method='post', operation_summary='Method permanently attaches images or video to a task. Supports tus uploads, see more https://tus.io/',
request_body=DataSerializer,
manual_parameters=[
openapi.Parameter('Upload-Start', in_=openapi.IN_HEADER, type=openapi.TYPE_BOOLEAN,
description="Initializes data upload. No data should be sent with this header"),
openapi.Parameter('Upload-Multiple', in_=openapi.IN_HEADER, type=openapi.TYPE_BOOLEAN,
description="Indicates that data with this request are single or multiple files that should be attached to a task"),
openapi.Parameter('Upload-Finish', in_=openapi.IN_HEADER, type=openapi.TYPE_BOOLEAN,
description="Finishes data upload. Can be combined with Upload-Start header to create task data with one request"),
]
)
@swagger_auto_schema(method='get', operation_summary='Method returns data for a specific task',
manual_parameters=[

@ -0,0 +1,50 @@
// Copyright (C) 2020-2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
/// <reference types="cypress" />
context('Create task with tus file', () => {
const caseId = '112';
const labelName = `Case ${caseId}`;
const taskName = `New annotation task for ${labelName}`;
const attrName = `Attr for ${labelName}`;
const textDefaultValue = 'Some default value for type Text';
const imagesCount = 1;
const imageFileName = `image_${labelName.replace(' ', '_').toLowerCase()}`;
const width = 1920;
const height = 1080;
const posX = 10;
const posY = 10;
const color = 'gray';
const archiveName = `${imageFileName}.zip`;
const archivePath = `cypress/fixtures/${archiveName}`;
const imagesFolder = `cypress/fixtures/${imageFileName}`;
const directoryToArchive = imagesFolder;
const zipLevel = 0;
const extension = 'bmp';
before(() => {
cy.visit('auth/login');
cy.login();
cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX,
posY, labelName, imagesCount, extension);
cy.createZipArchive(directoryToArchive, archivePath, zipLevel);
cy.window().then((win) => { win.cvat.config.uploadChunkSize = 5; });
});
describe(`Testing "${labelName}"`, () => {
it('Create a task with 5mb upload chunk size', () => {
cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, archiveName);
});
it('Check if task exist', () => {
cy.goToTaskList();
cy.contains('.cvat-item-task-name', taskName).should('exist');
});
});
after(() => {
cy.window().then((win) => { win.cvat.config.uploadChunkSize = 100; });
});
});

@ -1,22 +1,24 @@
// Copyright (C) 2020 Intel Corporation
// Copyright (C) 2020-2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
// eslint-disable-next-line no-undef
// eslint-disable-next-line no-use-before-define
exports.createZipArchive = createZipArchive;
const archiver = require('archiver');
// eslint-disable-next-line import/no-extraneous-dependencies
const fs = require('fs-extra');
function createZipArchive(args) {
const directoryToArchive = args.directoryToArchive;
const { directoryToArchive } = args;
const { level } = args;
const output = fs.createWriteStream(args.arhivePath);
const archive = archiver('zip', {
gzip: true,
zlib: { level: 9 },
zlib: { level },
});
archive.on('error', function (err) {
archive.on('error', (err) => {
throw err;
});

@ -1,10 +1,9 @@
// Copyright (C) 2020 Intel Corporation
// Copyright (C) 2020-2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
Cypress.Commands.add('createZipArchive', function (directoryToArchive, arhivePath) {
return cy.task('createZipArchive', {
directoryToArchive: directoryToArchive,
arhivePath: arhivePath,
});
});
Cypress.Commands.add('createZipArchive', (directoryToArchive, arhivePath, level = 9) => cy.task('createZipArchive', {
directoryToArchive,
arhivePath,
level,
}));

@ -1,33 +1,45 @@
// Copyright (C) 2020 Intel Corporation
// Copyright (C) 2020-2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
// eslint-disable-next-line no-use-before-define
exports.imageGenerator = imageGenerator;
const jimp = require('jimp');
const path = require('path');
const jimp = require('jimp');
function imageGenerator(args) {
const directory = args.directory;
const fileName = args.fileName;
const width = args.width;
const height = args.height;
const color = args.color;
const posX = args.posX;
const posY = args.posY;
const message = args.message;
const file = path.join(directory, fileName);
const count = args.count;
function createImage(width, height, color) {
return new Promise((resolve, reject) => {
// eslint-disable-next-line new-cap, no-new
new jimp(width, height, color, ((err, img) => {
if (err) reject(err);
resolve(img);
}));
});
}
function appendText(image, posX, posY, message, index) {
return new Promise((resolve, reject) => {
jimp.loadFont(jimp.FONT_SANS_64_BLACK, (err, font) => {
if (err) reject(err);
image.print(font, Number(posX), Number(posY), `${message}. Num ${index}`);
resolve(image);
});
});
}
async function imageGenerator(args) {
const {
directory, fileName, width, height, color, posX, posY, message, count, extension,
} = args;
const file = path.join(directory, fileName);
try {
for (let i = 1; i <= count; i++) {
new jimp(width, height, color, function (err, image) {
if (err) reject(err);
jimp.loadFont(jimp.FONT_SANS_64_BLACK, function (err, font) {
if (err) reject(err);
image.print(font, Number(posX), Number(posY), `${message}. Num ${i}`).write(`${file}_${i}.png`);
});
});
let image = await createImage(width, height, color);
image = await appendText(image, posX, posY, message, i);
image.write(`${file}_${i}.${extension}`);
}
setTimeout(() => resolve(null), '1000');
});
// eslint-disable-next-line no-empty
} catch (e) {}
return null;
}

@ -1,17 +1,16 @@
// Copyright (C) 2020 Intel Corporation
// Copyright (C) 2020-2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
Cypress.Commands.add('imageGenerator', (directory, fileName, width, height, color, posX, posY, message, count) => {
return cy.task('imageGenerator', {
directory: directory,
fileName: fileName,
width: width,
height: height,
color: color,
posX: posX,
posY: posY,
message: message,
count: count,
});
});
Cypress.Commands.add('imageGenerator', (directory, fileName, width, height, color, posX, posY, message, count, extension = 'png') => cy.task('imageGenerator', {
directory,
fileName,
width,
height,
color,
posX,
posY,
message,
count,
extension,
}));

Loading…
Cancel
Save