[cvat-core] support cloud storage (#3313)

* Update server-proxy

* Add cloud storage implementation && update enums

* Add api && fix server-proxy

* Add fixes

* small update

* Move from ui_support_cloud_storage

* Remove temporary credentials && fix typos

* Remove trailing spaces

* Apply lost changes

* manifest_set -> manifests

* [cvat-core] Add status && updated getpreview for getting the desired error message from the server

* Remove excess code

* Fix missing

* Add tests

* move from cycle

* Update CHANGELOG

* Increase version
main
Maria Khrustaleva 5 years ago committed by GitHub
parent 9f31fb3882
commit 720d79845b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added intelligent scissors blocking feature (<https://github.com/openvinotoolkit/cvat/pull/3510>)
- Support cloud storage status (<https://github.com/openvinotoolkit/cvat/pull/3386>)
- Support cloud storage preview (<https://github.com/openvinotoolkit/cvat/pull/3386>)
- cvat-core: support cloud storages (<https://github.com/openvinotoolkit/cvat/pull/3313>)
### Changed

@ -1,6 +1,6 @@
{
"name": "cvat-core",
"version": "3.15.0",
"version": "3.16.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

@ -1,6 +1,6 @@
{
"name": "cvat-core",
"version": "3.15.0",
"version": "3.16.0",
"description": "Part of Computer Vision Tool which presents an interface for client-side integration",
"main": "babel.config.js",
"scripts": {

@ -16,13 +16,20 @@
camelToSnake,
} = require('./common');
const { TaskStatus, TaskMode, DimensionType } = require('./enums');
const {
TaskStatus,
TaskMode,
DimensionType,
CloudStorageProviderType,
CloudStorageCredentialsType,
} = require('./enums');
const User = require('./user');
const { AnnotationFormats } = require('./annotation-formats');
const { ArgumentError } = require('./exceptions');
const { Task } = require('./session');
const { Project } = require('./project');
const { CloudStorage } = require('./cloud-storage');
function implementAPI(cvat) {
cvat.plugins.list.implementation = PluginRegistry.list;
@ -262,6 +269,49 @@
cvat.projects.searchNames.implementation = async (search, limit) => serverProxy.projects.searchNames(search, limit);
cvat.cloudStorages.get.implementation = async (filter) => {
checkFilter(filter, {
page: isInteger,
displayName: isString,
resourceName: isString,
description: isString,
id: isInteger,
owner: isString,
search: isString,
providerType: isEnum.bind(CloudStorageProviderType),
credentialsType: isEnum.bind(CloudStorageCredentialsType),
});
checkExclusiveFields(filter, ['id', 'search'], ['page']);
const searchParams = new URLSearchParams();
for (const field of [
'displayName',
'credentialsType',
'providerType',
'owner',
'search',
'id',
'page',
'description',
]) {
if (Object.prototype.hasOwnProperty.call(filter, field)) {
searchParams.set(camelToSnake(field), filter[field]);
}
}
if (Object.prototype.hasOwnProperty.call(filter, 'resourceName')) {
searchParams.set('resource', filter.resourceName);
}
const cloudStoragesData = await serverProxy.cloudStorages.get(searchParams.toString());
const cloudStorages = cloudStoragesData.map((cloudStorage) => new CloudStorage(cloudStorage));
cloudStorages.count = cloudStoragesData.count;
return cloudStorages;
};
return cvat;
}

@ -22,6 +22,8 @@ function build() {
const { Attribute, Label } = require('./labels');
const MLModel = require('./ml-model');
const { FrameData } = require('./frames');
const { CloudStorage } = require('./cloud-storage');
const enums = require('./enums');
@ -748,6 +750,41 @@ function build() {
PluginError,
ServerError,
},
/**
* Namespace is used for getting cloud storages
* @namespace cloudStorages
* @memberof module:API.cvat
*/
cloudStorages: {
/**
* @typedef {Object} CloudStorageFilter
* @property {string} displayName Check if displayName contains this value
* @property {string} resourceName Check if resourceName contains this value
* @property {module:API.cvat.enums.ProviderType} providerType Check if providerType equal this value
* @property {integer} id Check if id equals this value
* @property {integer} page Get specific page
* (default REST API returns 20 clouds storages per request.
* In order to get more, it is need to specify next page)
* @property {string} owner Check if an owner name contains this value
* @property {string} search Combined search of contains among all the fields
* @global
*/
/**
* Method returns a list of cloud storages corresponding to a filter
* @method get
* @async
* @memberof module:API.cvat.cloudStorages
* @param {CloudStorageFilter} [filter={}] cloud storage filter
* @returns {module:API.cvat.classes.CloudStorage[]}
* @throws {module:API.cvat.exceptions.PluginError}
* @throws {module:API.cvat.exceptions.ServerError}
*/
async get(filter = {}) {
const result = await PluginRegistry.apiWrapper(cvat.cloudStorages.get, filter);
return result;
},
},
/**
* Namespace is used for access to classes
* @namespace classes
@ -768,6 +805,7 @@ function build() {
Issue,
Review,
FrameData,
CloudStorage,
},
};
@ -780,6 +818,7 @@ function build() {
cvat.lambda = Object.freeze(cvat.lambda);
cvat.client = Object.freeze(cvat.client);
cvat.enums = Object.freeze(cvat.enums);
cvat.cloudStorages = Object.freeze(cvat.cloudStorages);
const implementAPI = require('./api-implementation');

@ -0,0 +1,520 @@
// Copyright (C) 2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
(() => {
const PluginRegistry = require('./plugins');
const serverProxy = require('./server-proxy');
const { isBrowser, isNode } = require('browser-or-node');
const { ArgumentError } = require('./exceptions');
const { CloudStorageCredentialsType, CloudStorageProviderType } = require('./enums');
/**
* Class representing a cloud storage
* @memberof module:API.cvat.classes
*/
class CloudStorage {
// TODO: add storage availability status (avaliable/unavaliable)
constructor(initialData) {
const data = {
id: undefined,
display_name: undefined,
description: undefined,
credentials_type: undefined,
provider_type: undefined,
resource: undefined,
account_name: undefined,
key: undefined,
secret_key: undefined,
session_token: undefined,
specific_attributes: undefined,
owner: undefined,
created_date: undefined,
updated_date: undefined,
manifest_path: undefined,
manifests: undefined,
};
for (const property in data) {
if (Object.prototype.hasOwnProperty.call(data, property) && property in initialData) {
data[property] = initialData[property];
}
}
Object.defineProperties(
this,
Object.freeze({
/**
* @name id
* @type {integer}
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
*/
id: {
get: () => data.id,
},
/**
* Storage name
* @name displayName
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
displayName: {
get: () => data.display_name,
set: (value) => {
if (typeof value !== 'string') {
throw new ArgumentError(`Value must be string. ${typeof value} was found`);
} else if (!value.trim().length) {
throw new ArgumentError('Value must not be empty string');
}
data.display_name = value;
},
},
/**
* Storage description
* @name description
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
description: {
get: () => data.description,
set: (value) => {
if (typeof value !== 'string') {
throw new ArgumentError('Value must be string');
}
data.description = value;
},
},
/**
* Azure account name
* @name accountName
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
accountName: {
get: () => data.account_name,
set: (value) => {
if (typeof value === 'string') {
if (value.trim().length) {
data.account_name = value;
} else {
throw new ArgumentError('Value must not be empty');
}
} else {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
}
},
},
/**
* AWS access key id
* @name accessKey
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
accessKey: {
get: () => data.key,
set: (value) => {
if (typeof value === 'string') {
if (value.trim().length) {
data.key = value;
} else {
throw new ArgumentError('Value must not be empty');
}
} else {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
}
},
},
/**
* AWS secret key
* @name secretKey
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
secretKey: {
get: () => data.secret_key,
set: (value) => {
if (typeof value === 'string') {
if (value.trim().length) {
data.secret_key = value;
} else {
throw new ArgumentError('Value must not be empty');
}
} else {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
}
},
},
/**
* Session token
* @name token
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
token: {
get: () => data.session_token,
set: (value) => {
if (typeof value === 'string') {
if (value.trim().length) {
data.session_token = value;
} else {
throw new ArgumentError('Value must not be empty');
}
} else {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
}
},
},
/**
* Unique resource name
* @name resourceName
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
resourceName: {
get: () => data.resource,
set: (value) => {
if (typeof value !== 'string') {
throw new ArgumentError(`Value must be string. ${typeof value} was found`);
} else if (!value.trim().length) {
throw new ArgumentError('Value must not be empty');
}
data.resource = value;
},
},
/**
* @name manifestPath
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
manifestPath: {
get: () => data.manifest_path,
set: (value) => {
if (typeof value === 'string') {
data.manifest_path = value;
} else {
throw new ArgumentError('Value must be a string');
}
},
},
/**
* @name providerType
* @type {module:API.cvat.enums.ProviderType}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
providerType: {
get: () => data.provider_type,
set: (key) => {
if (key !== undefined && !!CloudStorageProviderType[key]) {
data.provider_type = CloudStorageProviderType[key];
} else {
throw new ArgumentError('Value must be one CloudStorageProviderType keys');
}
},
},
/**
* @name credentialsType
* @type {module:API.cvat.enums.CloudStorageCredentialsType}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
credentialsType: {
get: () => data.credentials_type,
set: (key) => {
if (key !== undefined && !!CloudStorageCredentialsType[key]) {
data.credentials_type = CloudStorageCredentialsType[key];
} else {
throw new ArgumentError('Value must be one CloudStorageCredentialsType keys');
}
},
},
/**
* @name specificAttributes
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
specificAttributes: {
get: () => data.specific_attributes,
set: (attributesValue) => {
if (typeof attributesValue === 'string') {
const attrValues = new URLSearchParams(
Array.from(new URLSearchParams(attributesValue).entries()).filter(
([key, value]) => !!key && !!value,
),
).toString();
if (!attrValues) {
throw new ArgumentError('Value must match the key1=value1&key2=value2');
}
data.specific_attributes = attributesValue;
} else {
throw new ArgumentError('Value must be a string');
}
},
},
/**
* Instance of a user who has created the cloud storage
* @name owner
* @type {module:API.cvat.classes.User}
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
*/
owner: {
get: () => data.owner,
},
/**
* @name createdDate
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
*/
createdDate: {
get: () => data.created_date,
},
/**
* @name updatedDate
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
*/
updatedDate: {
get: () => data.updated_date,
},
/**
* @name manifests
* @type {string[]}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
manifests: {
get: () => data.manifests,
set: (manifests) => {
if (Array.isArray(manifests)) {
for (const elem of manifests) {
if (typeof elem !== 'string') {
throw new ArgumentError('Each element of the manifests array must be a string');
}
}
data.manifests = manifests;
} else {
throw new ArgumentError('Value must be an array');
}
},
},
}),
);
}
/**
* Method updates data of a created cloud storage or creates new cloud storage
* @method save
* @returns {module:API.cvat.classes.CloudStorage}
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
* @async
* @throws {module:API.cvat.exceptions.ServerError}
* @throws {module:API.cvat.exceptions.PluginError}
*/
async save() {
const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.save);
return result;
}
/**
* Method deletes a cloud storage from a server
* @method delete
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
* @async
* @throws {module:API.cvat.exceptions.ServerError}
* @throws {module:API.cvat.exceptions.PluginError}
*/
async delete() {
const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.delete);
return result;
}
/**
* Method returns cloud storage content
* @method getContent
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
* @async
* @throws {module:API.cvat.exceptions.ServerError}
* @throws {module:API.cvat.exceptions.PluginError}
*/
async getContent() {
const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getContent);
return result;
}
/**
* Method returns the cloud storage preview
* @method getPreview
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
* @async
* @throws {module:API.cvat.exceptions.ServerError}
* @throws {module:API.cvat.exceptions.PluginError}
*/
async getPreview() {
const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getPreview);
return result;
}
/**
* Method returns cloud storage status
* @method getStatus
* @memberof module:API.cvat.classes.CloudStorage
* @readonly
* @instance
* @async
* @throws {module:API.cvat.exceptions.ServerError}
* @throws {module:API.cvat.exceptions.PluginError}
*/
async getStatus() {
const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getStatus);
return result;
}
}
CloudStorage.prototype.save.implementation = async function () {
function prepareOptionalFields(cloudStorageInstance) {
const data = {};
if (cloudStorageInstance.description) {
data.description = cloudStorageInstance.description;
}
if (cloudStorageInstance.accountName) {
data.account_name = cloudStorageInstance.accountName;
}
if (cloudStorageInstance.accessKey) {
data.key = cloudStorageInstance.accessKey;
}
if (cloudStorageInstance.secretKey) {
data.secret_key = cloudStorageInstance.secretKey;
}
if (cloudStorageInstance.token) {
data.session_token = cloudStorageInstance.token;
}
if (cloudStorageInstance.specificAttributes) {
data.specific_attributes = cloudStorageInstance.specificAttributes;
}
return data;
}
// update
if (typeof this.id !== 'undefined') {
// providr_type and recource should not change;
// send to the server only the values that have changed
const initialData = {};
if (this.displayName) {
initialData.display_name = this.displayName;
}
if (this.credentialsType) {
initialData.credentials_type = this.credentialsType;
}
if (this.manifests) {
initialData.manifests = this.manifests;
}
const cloudStorageData = {
...initialData,
...prepareOptionalFields(this),
};
await serverProxy.cloudStorages.update(this.id, cloudStorageData);
return this;
}
// create
const initialData = {
display_name: this.displayName,
credentials_type: this.credentialsType,
provider_type: this.providerType,
resource: this.resourceName,
manifests: this.manifests,
};
const cloudStorageData = {
...initialData,
...prepareOptionalFields(this),
};
const cloudStorage = await serverProxy.cloudStorages.create(cloudStorageData);
return new CloudStorage(cloudStorage);
};
CloudStorage.prototype.delete.implementation = async function () {
const result = await serverProxy.cloudStorages.delete(this.id);
return result;
};
CloudStorage.prototype.getContent.implementation = async function () {
const result = await serverProxy.cloudStorages.getContent(this.id, this.manifestPath);
return result;
};
CloudStorage.prototype.getPreview.implementation = async function getPreview() {
return new Promise((resolve, reject) => {
serverProxy.cloudStorages
.getPreview(this.id)
.then((result) => {
if (isNode) {
resolve(global.Buffer.from(result, 'binary').toString('base64'));
} else if (isBrowser) {
const reader = new FileReader();
reader.onload = () => {
resolve(reader.result);
};
reader.readAsDataURL(result);
}
})
.catch((error) => {
reject(error);
});
});
};
CloudStorage.prototype.getStatus.implementation = async function () {
const result = await serverProxy.cloudStorages.getStatus(this.id);
return result;
};
module.exports = {
CloudStorage,
};
})();

@ -1,4 +1,4 @@
// Copyright (C) 2019-2020 Intel Corporation
// Copyright (C) 2019-2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
@ -333,6 +333,36 @@
'#733380',
];
/**
* Types of cloud storage providers
* @enum {string}
* @name CloudStorageProviderType
* @memberof module:API.cvat.enums
* @property {string} AWS_S3 'AWS_S3_BUCKET'
* @property {string} AZURE 'AZURE_CONTAINER'
* @readonly
*/
const CloudStorageProviderType = Object.freeze({
AWS_S3_BUCKET: 'AWS_S3_BUCKET',
AZURE_CONTAINER: 'AZURE_CONTAINER',
});
/**
* Types of cloud storage credentials
* @enum {string}
* @name CloudStorageCredentialsType
* @memberof module:API.cvat.enums
* @property {string} KEY_SECRET_KEY_PAIR 'KEY_SECRET_KEY_PAIR'
* @property {string} ACCOUNT_NAME_TOKEN_PAIR 'ACCOUNT_NAME_TOKEN_PAIR'
* @property {string} ANONYMOUS_ACCESS 'ANONYMOUS_ACCESS'
* @readonly
*/
const CloudStorageCredentialsType = Object.freeze({
KEY_SECRET_KEY_PAIR: 'KEY_SECRET_KEY_PAIR',
ACCOUNT_NAME_TOKEN_PAIR: 'ACCOUNT_NAME_TOKEN_PAIR',
ANONYMOUS_ACCESS: 'ANONYMOUS_ACCESS',
});
module.exports = {
ShareFileType,
TaskStatus,
@ -348,5 +378,7 @@
colors,
Source,
DimensionType,
CloudStorageProviderType,
CloudStorageCredentialsType,
};
})();

@ -1145,9 +1145,7 @@
const closureId = Date.now();
predictAnnotations.latestRequest.id = closureId;
const predicate = () => (
!predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId
);
const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId;
if (predictAnnotations.latestRequest.fetching) {
waitFor(5, predicate).then(() => {
if (predictAnnotations.latestRequest.id !== closureId) {
@ -1181,6 +1179,121 @@
}
}
async function createCloudStorage(storageDetail) {
const { backendAPI } = config;
try {
const response = await Axios.post(`${backendAPI}/cloudstorages`, JSON.stringify(storageDetail), {
proxy: config.proxy,
headers: {
'Content-Type': 'application/json',
},
});
return response.data;
} catch (errorData) {
throw generateError(errorData);
}
}
async function updateCloudStorage(id, cloudStorageData) {
const { backendAPI } = config;
try {
await Axios.patch(`${backendAPI}/cloudstorages/${id}`, JSON.stringify(cloudStorageData), {
proxy: config.proxy,
headers: {
'Content-Type': 'application/json',
},
});
} catch (errorData) {
throw generateError(errorData);
}
}
async function getCloudStorages(filter = '') {
const { backendAPI } = config;
let response = null;
try {
response = await Axios.get(`${backendAPI}/cloudstorages?page_size=12&${filter}`, {
proxy: config.proxy,
});
} catch (errorData) {
throw generateError(errorData);
}
response.data.results.count = response.data.count;
return response.data.results;
}
async function getCloudStorageContent(id, manifestPath) {
const { backendAPI } = config;
let response = null;
try {
const url = `${backendAPI}/cloudstorages/${id}/content${
manifestPath ? `?manifest_path=${manifestPath}` : ''
}`;
response = await Axios.get(url, {
proxy: config.proxy,
});
} catch (errorData) {
throw generateError(errorData);
}
return response.data;
}
async function getCloudStoragePreview(id) {
const { backendAPI } = config;
let response = null;
try {
const url = `${backendAPI}/cloudstorages/${id}/preview`;
response = await workerAxios.get(url, {
proxy: config.proxy,
responseType: 'arraybuffer',
});
} catch (errorData) {
throw generateError({
...errorData,
message: '',
response: {
...errorData.response,
data: String.fromCharCode.apply(null, new Uint8Array(errorData.response.data)),
},
});
}
return new Blob([new Uint8Array(response)]);
}
async function getCloudStorageStatus(id) {
const { backendAPI } = config;
let response = null;
try {
const url = `${backendAPI}/cloudstorages/${id}/status`;
response = await Axios.get(url, {
proxy: config.proxy,
});
} catch (errorData) {
throw generateError(errorData);
}
return response.data;
}
async function deleteCloudStorage(id) {
const { backendAPI } = config;
try {
await Axios.delete(`${backendAPI}/cloudstorages/${id}`);
} catch (errorData) {
throw generateError(errorData);
}
}
Object.defineProperties(
this,
Object.freeze({
@ -1310,6 +1423,19 @@
}),
writable: false,
},
cloudStorages: {
value: Object.freeze({
get: getCloudStorages,
getContent: getCloudStorageContent,
getPreview: getCloudStoragePreview,
getStatus: getCloudStorageStatus,
create: createCloudStorage,
delete: deleteCloudStorage,
update: updateCloudStorage,
}),
writable: false,
},
}),
);
}

@ -1012,6 +1012,7 @@
use_cache: undefined,
copy_data: undefined,
dimension: undefined,
cloud_storage_id: undefined,
};
const updatedFields = new FieldUpdateTrigger({
@ -1373,7 +1374,7 @@
get: () => [...data.jobs],
},
/**
* List of files from shared resource
* List of files from shared resource or list of cloud storage files
* @name serverFiles
* @type {string[]}
* @memberof module:API.cvat.classes.Task
@ -1535,6 +1536,15 @@
*/
get: () => data.dimension,
},
/**
* @name cloudStorageId
* @type {integer|null}
* @memberof module:API.cvat.classes.Task
* @instance
*/
cloudStorageId: {
get: () => data.cloud_storage_id,
},
_internalData: {
get: () => data,
},
@ -2062,6 +2072,9 @@
if (typeof this.copyData !== 'undefined') {
taskDataSpec.copy_data = this.copyData;
}
if (typeof this.cloudStorageId !== 'undefined') {
taskDataSpec.cloud_storage_id = this.cloudStorageId;
}
const task = await serverProxy.tasks.createTask(taskSpec, taskDataSpec, onUpdate);
return new Task(task);

@ -0,0 +1,178 @@
// Copyright (C) 2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
// Setup mock for a server
jest.mock('../../src/server-proxy', () => {
const mock = require('../mocks/server-proxy.mock');
return mock;
});
// Initialize api
window.cvat = require('../../src/api');
const { CloudStorage } = require('../../src/cloud-storage');
const { cloudStoragesDummyData } = require('../mocks/dummy-data.mock');
describe('Feature: get cloud storages', () => {
test('get all cloud storages', async () => {
const result = await window.cvat.cloudStorages.get();
expect(Array.isArray(result)).toBeTruthy();
expect(result).toHaveLength(cloudStoragesDummyData.count);
for (const item of result) {
expect(item).toBeInstanceOf(CloudStorage);
}
});
test('get cloud storage by id', async () => {
const result = await window.cvat.cloudStorages.get({
id: 1,
});
const cloudStorage = result[0];
expect(Array.isArray(result)).toBeTruthy();
expect(result).toHaveLength(1);
expect(cloudStorage).toBeInstanceOf(CloudStorage);
expect(cloudStorage.id).toBe(1);
expect(cloudStorage.providerType).toBe('AWS_S3_BUCKET');
expect(cloudStorage.credentialsType).toBe('KEY_SECRET_KEY_PAIR');
expect(cloudStorage.resourceName).toBe('bucket');
expect(cloudStorage.displayName).toBe('Demonstration bucket');
expect(cloudStorage.manifests).toHaveLength(1);
expect(cloudStorage.manifests[0]).toBe('manifest.jsonl');
expect(cloudStorage.specificAttributes).toBe('');
expect(cloudStorage.description).toBe('It is first bucket');
});
test('get a cloud storage by an unknown id', async () => {
const result = await window.cvat.cloudStorages.get({
id: 10,
});
expect(Array.isArray(result)).toBeTruthy();
expect(result).toHaveLength(0);
});
test('get a cloud storage by an invalid id', async () => {
expect(
window.cvat.cloudStorages.get({
id: '1',
}),
).rejects.toThrow(window.cvat.exceptions.ArgumentError);
});
test('get cloud storages by filters', async () => {
const filters = new Map([
['providerType', 'AWS_S3_BUCKET'],
['resourceName', 'bucket'],
['displayName', 'Demonstration bucket'],
['credentialsType', 'KEY_SECRET_KEY_PAIR'],
['description', 'It is first bucket'],
]);
const result = await window.cvat.cloudStorages.get(Object.fromEntries(filters));
const [cloudStorage] = result;
expect(Array.isArray(result)).toBeTruthy();
expect(result).toHaveLength(1);
expect(cloudStorage).toBeInstanceOf(CloudStorage);
expect(cloudStorage.id).toBe(1);
filters.forEach((value, key) => {
expect(cloudStorage[key]).toBe(value);
});
});
test('get cloud storage by invalid filters', async () => {
expect(
window.cvat.cloudStorages.get({
unknown: '5',
}),
).rejects.toThrow(window.cvat.exceptions.ArgumentError);
});
});
describe('Feature: create a cloud storage', () => {
test('create new cloud storage without an id', async () => {
const cloudStorage = new window.cvat.classes.CloudStorage({
display_name: 'new cloud storage',
provider_type: 'AZURE_CONTAINER',
resource: 'newcontainer',
credentials_type: 'ACCOUNT_NAME_TOKEN_PAIR',
account_name: 'accountname',
session_token: 'x'.repeat(135),
manifests: ['manifest.jsonl'],
});
const result = await cloudStorage.save();
expect(typeof result.id).toBe('number');
});
});
describe('Feature: update a cloud storage', () => {
test('update cloud storage with some new field values', async () => {
const newValues = new Map([
['displayName', 'new display name'],
['credentialsType', 'ANONYMOUS_ACCESS'],
['description', 'new description'],
['specificAttributes', 'region=eu-west-1'],
]);
let result = await window.cvat.cloudStorages.get({
id: 1,
});
let [cloudStorage] = result;
for (const [key, value] of newValues) {
cloudStorage[key] = value;
}
cloudStorage.save();
result = await window.cvat.cloudStorages.get({
id: 1,
});
[cloudStorage] = result;
newValues.forEach((value, key) => {
expect(cloudStorage[key]).toBe(value);
});
});
test('Update manifests in a cloud storage', async () => {
const newManifests = [
'sub1/manifest.jsonl',
'sub2/manifest.jsonl',
];
let result = await window.cvat.cloudStorages.get({
id: 1,
});
let [cloudStorage] = result;
cloudStorage.manifests = newManifests;
cloudStorage.save();
result = await window.cvat.cloudStorages.get({
id: 1,
});
[cloudStorage] = result;
expect(cloudStorage.manifests).toEqual(newManifests);
});
});
describe('Feature: delete a cloud storage', () => {
test('delete a cloud storage', async () => {
let result = await window.cvat.cloudStorages.get({
id: 2,
});
await result[0].delete();
result = await window.cvat.cloudStorages.get({
id: 2,
});
expect(Array.isArray(result)).toBeTruthy();
expect(result).toHaveLength(0);
});
});

@ -2547,6 +2547,56 @@ const frameMetaDummyData = {
},
};
const cloudStoragesDummyData = {
count: 2,
next: null,
previous: null,
results: [
{
id: 2,
owner: {
url: 'http://localhost:7000/api/v1/users/1',
id: 1,
username: 'maya',
first_name: '',
last_name: ''
},
manifests: [
'manifest.jsonl'
],
provider_type: 'AZURE_CONTAINER',
resource: 'container',
display_name: 'Demonstration container',
created_date: '2021-09-01T09:29:47.094244Z',
updated_date: '2021-09-01T09:29:47.103264Z',
credentials_type: 'ACCOUNT_NAME_TOKEN_PAIR',
specific_attributes: '',
description: 'It is first container'
},
{
id: 1,
owner: {
url: 'http://localhost:7000/api/v1/users/1',
id: 1,
username: 'maya',
first_name: '',
last_name: ''
},
manifests: [
'manifest.jsonl'
],
provider_type: 'AWS_S3_BUCKET',
resource: 'bucket',
display_name: 'Demonstration bucket',
created_date: '2021-08-31T09:03:09.350817Z',
updated_date: '2021-08-31T15:16:21.394773Z',
credentials_type: 'KEY_SECRET_KEY_PAIR',
specific_attributes: '',
description: 'It is first bucket'
}
]
};
module.exports = {
tasksDummyData,
projectsDummyData,
@ -2557,4 +2607,5 @@ module.exports = {
jobAnnotationsDummyData,
frameMetaDummyData,
formatsDummyData,
cloudStoragesDummyData,
};

@ -12,6 +12,7 @@ const {
taskAnnotationsDummyData,
jobAnnotationsDummyData,
frameMetaDummyData,
cloudStoragesDummyData,
} = require('./dummy-data.mock');
function QueryStringToJSON(query, ignoreList = []) {
@ -318,6 +319,63 @@ class ServerProxy {
return null;
}
async function getCloudStorages(filter = '') {
const queries = QueryStringToJSON(filter);
const result = cloudStoragesDummyData.results.filter((item) => {
for (const key in queries) {
if (Object.prototype.hasOwnProperty.call(queries, key)) {
if (queries[key] !== item[key]) {
return false;
}
}
}
return true;
});
return result;
}
async function updateCloudStorage(id, cloudStorageData) {
const cloudStorage = cloudStoragesDummyData.results.find((item) => item.id === id);
if (cloudStorage) {
for (const prop in cloudStorageData) {
if (
Object.prototype.hasOwnProperty.call(cloudStorageData, prop)
&& Object.prototype.hasOwnProperty.call(cloudStorage, prop)
) {
cloudStorage[prop] = cloudStorageData[prop];
}
}
}
}
async function createCloudStorage(cloudStorageData) {
const id = Math.max(...cloudStoragesDummyData.results.map((item) => item.id)) + 1;
cloudStoragesDummyData.results.push({
id,
provider_type: cloudStorageData.provider_type,
resource: cloudStorageData.resource,
display_name: cloudStorageData.display_name,
credentials_type: cloudStorageData.credentials_type,
specific_attributes: cloudStorageData.specific_attributes,
description: cloudStorageData.description,
owner: 1,
created_date: '2021-09-01T09:29:47.094244+03:00',
updated_date: '2021-09-01T09:29:47.103264+03:00',
});
const result = await getCloudStorages(`?id=${id}`);
return result[0];
}
async function deleteCloudStorage(id) {
const cloudStorages = cloudStoragesDummyData.results;
const cloudStorageId = cloudStorages.findIndex((item) => item.id === id);
if (cloudStorageId !== -1) {
cloudStorages.splice(cloudStorageId);
}
}
Object.defineProperties(
this,
Object.freeze({
@ -384,6 +442,16 @@ class ServerProxy {
getAnnotations,
},
},
cloudStorages: {
value: Object.freeze({
get: getCloudStorages,
update: updateCloudStorage,
create: createCloudStorage,
delete: deleteCloudStorage,
}),
writable: false,
},
}),
);
}

Loading…
Cancel
Save