diff --git a/CHANGELOG.md b/CHANGELOG.md index e39e25b3..25a2c93c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Open Images V6 format () - Rotated bounding boxes () - Player option: Smooth image when zoom-in, enabled by default () +- Google Cloud Storage support in UI () - Add project tasks paginations () ### Changed diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 52b293a4..4389ca73 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,12 +1,12 @@ { "name": "cvat-core", - "version": "3.20.0", + "version": "3.20.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cvat-core", - "version": "3.20.0", + "version": "3.20.1", "license": "MIT", "dependencies": { "axios": "^0.21.4", diff --git a/cvat-core/package.json b/cvat-core/package.json index 7d6c09d6..3552b5be 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.20.0", + "version": "3.20.1", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-core/src/cloud-storage.js b/cvat-core/src/cloud-storage.js index 4fd8bd3a..c27d7729 100644 --- a/cvat-core/src/cloud-storage.js +++ b/cvat-core/src/cloud-storage.js @@ -9,12 +9,19 @@ const { ArgumentError } = require('./exceptions'); const { CloudStorageCredentialsType, CloudStorageProviderType } = require('./enums'); + function validateNotEmptyString(value) { + if (typeof value !== 'string') { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } else if (!value.trim().length) { + throw new ArgumentError('Value mustn\'t be empty string'); + } + } + /** * Class representing a cloud storage * @memberof module:API.cvat.classes */ class CloudStorage { - // TODO: add storage availability status (avaliable/unavaliable) constructor(initialData) { const data = { id: undefined, @@ -27,6 +34,8 @@ key: undefined, secret_key: undefined, session_token: undefined, + key_file_path: undefined, + key_file: undefined, specific_attributes: undefined, owner: undefined, created_date: undefined, @@ -65,11 +74,7 @@ displayName: { get: () => data.display_name, set: (value) => { - if (typeof value !== 'string') { - throw new ArgumentError(`Value must be string. ${typeof value} was found`); - } else if (!value.trim().length) { - throw new ArgumentError('Value must not be empty string'); - } + validateNotEmptyString(value); data.display_name = value; }, }, @@ -101,15 +106,8 @@ accountName: { get: () => data.account_name, set: (value) => { - if (typeof value === 'string') { - if (value.trim().length) { - data.account_name = value; - } else { - throw new ArgumentError('Value must not be empty'); - } - } else { - throw new ArgumentError(`Value must be a string. ${typeof value} was found`); - } + validateNotEmptyString(value); + data.account_name = value; }, }, /** @@ -123,15 +121,8 @@ accessKey: { get: () => data.key, set: (value) => { - if (typeof value === 'string') { - if (value.trim().length) { - data.key = value; - } else { - throw new ArgumentError('Value must not be empty'); - } - } else { - throw new ArgumentError(`Value must be a string. ${typeof value} was found`); - } + validateNotEmptyString(value); + data.key = value; }, }, /** @@ -145,15 +136,8 @@ secretKey: { get: () => data.secret_key, set: (value) => { - if (typeof value === 'string') { - if (value.trim().length) { - data.secret_key = value; - } else { - throw new ArgumentError('Value must not be empty'); - } - } else { - throw new ArgumentError(`Value must be a string. ${typeof value} was found`); - } + validateNotEmptyString(value); + data.secret_key = value; }, }, /** @@ -167,14 +151,40 @@ token: { get: () => data.session_token, set: (value) => { - if (typeof value === 'string') { - if (value.trim().length) { - data.session_token = value; - } else { - throw new ArgumentError('Value must not be empty'); - } + validateNotEmptyString(value); + data.session_token = value; + }, + }, + /** + * Key file path + * @name keyFilePath + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + keyFilePath: { + get: () => data.key_file_path, + set: (value) => { + validateNotEmptyString(value); + data.key_file_path = value; + }, + }, + /** + * Key file + * @name keyFile + * @type {File} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + keyFile: { + get: () => data.key_file, + set: (file) => { + if (file instanceof File) { + data.key_file = file; } else { - throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + throw new ArgumentError(`Should be a file. ${typeof file} was found`); } }, }, @@ -189,11 +199,7 @@ resourceName: { get: () => data.resource, set: (value) => { - if (typeof value !== 'string') { - throw new ArgumentError(`Value must be string. ${typeof value} was found`); - } else if (!value.trim().length) { - throw new ArgumentError('Value must not be empty'); - } + validateNotEmptyString(value); data.resource = value; }, }, @@ -207,11 +213,8 @@ manifestPath: { get: () => data.manifest_path, set: (value) => { - if (typeof value === 'string') { - data.manifest_path = value; - } else { - throw new ArgumentError('Value must be a string'); - } + validateNotEmptyString(value); + data.manifest_path = value; }, }, /** @@ -410,7 +413,7 @@ CloudStorage.prototype.save.implementation = async function () { function prepareOptionalFields(cloudStorageInstance) { const data = {}; - if (cloudStorageInstance.description) { + if (cloudStorageInstance.description !== undefined) { data.description = cloudStorageInstance.description; } @@ -430,14 +433,22 @@ data.session_token = cloudStorageInstance.token; } - if (cloudStorageInstance.specificAttributes) { + if (cloudStorageInstance.keyFilePath) { + data.key_file_path = cloudStorageInstance.keyFilePath; + } + + if (cloudStorageInstance.keyFile) { + data.key_file = cloudStorageInstance.keyFile; + } + + if (cloudStorageInstance.specificAttributes !== undefined) { data.specific_attributes = cloudStorageInstance.specificAttributes; } return data; } // update if (typeof this.id !== 'undefined') { - // providr_type and recource should not change; + // provider_type and recource should not change; // send to the server only the values that have changed const initialData = {}; if (this.displayName) { diff --git a/cvat-core/src/enums.js b/cvat-core/src/enums.js index c8ecab55..03fcb9e5 100644 --- a/cvat-core/src/enums.js +++ b/cvat-core/src/enums.js @@ -340,11 +340,13 @@ * @memberof module:API.cvat.enums * @property {string} AWS_S3 'AWS_S3_BUCKET' * @property {string} AZURE 'AZURE_CONTAINER' + * @property {string} GOOGLE_CLOUD_STORAGE 'GOOGLE_CLOUD_STORAGE' * @readonly */ const CloudStorageProviderType = Object.freeze({ AWS_S3_BUCKET: 'AWS_S3_BUCKET', AZURE_CONTAINER: 'AZURE_CONTAINER', + GOOGLE_CLOUD_STORAGE: 'GOOGLE_CLOUD_STORAGE', }); /** @@ -355,12 +357,14 @@ * @property {string} KEY_SECRET_KEY_PAIR 'KEY_SECRET_KEY_PAIR' * @property {string} ACCOUNT_NAME_TOKEN_PAIR 'ACCOUNT_NAME_TOKEN_PAIR' * @property {string} ANONYMOUS_ACCESS 'ANONYMOUS_ACCESS' + * @property {string} KEY_FILE_PATH 'KEY_FILE_PATH' * @readonly */ const CloudStorageCredentialsType = Object.freeze({ KEY_SECRET_KEY_PAIR: 'KEY_SECRET_KEY_PAIR', ACCOUNT_NAME_TOKEN_PAIR: 'ACCOUNT_NAME_TOKEN_PAIR', ANONYMOUS_ACCESS: 'ANONYMOUS_ACCESS', + KEY_FILE_PATH: 'KEY_FILE_PATH', }); module.exports = { diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 9e6a846d..b9c59d29 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -45,6 +45,20 @@ return new ServerError(message, 0); } + function prepareData(details) { + const data = new FormData(); + for (const [key, value] of Object.entries(details)) { + if (Array.isArray(value)) { + value.forEach((element, idx) => { + data.append(`${key}[${idx}]`, element); + }); + } else { + data.set(key, value); + } + } + return data; + } + class WorkerWrappedAxios { constructor() { const worker = new DownloadWorker(); @@ -1181,12 +1195,10 @@ async function createCloudStorage(storageDetail) { const { backendAPI } = config; + const storageDetailData = prepareData(storageDetail); try { - const response = await Axios.post(`${backendAPI}/cloudstorages`, JSON.stringify(storageDetail), { + const response = await Axios.post(`${backendAPI}/cloudstorages`, storageDetailData, { proxy: config.proxy, - headers: { - 'Content-Type': 'application/json', - }, }); return response.data; } catch (errorData) { @@ -1194,15 +1206,13 @@ } } - async function updateCloudStorage(id, cloudStorageData) { + async function updateCloudStorage(id, storageDetail) { const { backendAPI } = config; + const storageDetailData = prepareData(storageDetail); try { - await Axios.patch(`${backendAPI}/cloudstorages/${id}`, JSON.stringify(cloudStorageData), { + await Axios.patch(`${backendAPI}/cloudstorages/${id}`, storageDetailData, { proxy: config.proxy, - headers: { - 'Content-Type': 'application/json', - }, }); } catch (errorData) { throw generateError(errorData); diff --git a/cvat-core/tests/api/cloud-storages.js b/cvat-core/tests/api/cloud-storages.js index 7b5d1bf4..0af69d62 100644 --- a/cvat-core/tests/api/cloud-storages.js +++ b/cvat-core/tests/api/cloud-storages.js @@ -61,24 +61,41 @@ describe('Feature: get cloud storages', () => { }); test('get cloud storages by filters', async () => { - const filters = new Map([ - ['providerType', 'AWS_S3_BUCKET'], - ['resourceName', 'bucket'], - ['displayName', 'Demonstration bucket'], - ['credentialsType', 'KEY_SECRET_KEY_PAIR'], - ['description', 'It is first bucket'], - ]); - - const result = await window.cvat.cloudStorages.get(Object.fromEntries(filters)); + const filters = [ + new Map([ + ['providerType', 'AWS_S3_BUCKET'], + ['resourceName', 'bucket'], + ['displayName', 'Demonstration bucket'], + ['credentialsType', 'KEY_SECRET_KEY_PAIR'], + ['description', 'It is first bucket'], + ]), + new Map([ + ['providerType', 'AZURE_CONTAINER'], + ['resourceName', 'container'], + ['displayName', 'Demonstration container'], + ['credentialsType', 'ACCOUNT_NAME_TOKEN_PAIR'], + ]), + new Map([ + ['providerType', 'GOOGLE_CLOUD_STORAGE'], + ['resourceName', 'gcsbucket'], + ['displayName', 'Demo GCS'], + ['credentialsType', 'KEY_FILE_PATH'], + ]), + ]; - const [cloudStorage] = result; - expect(Array.isArray(result)).toBeTruthy(); - expect(result).toHaveLength(1); - expect(cloudStorage).toBeInstanceOf(CloudStorage); - expect(cloudStorage.id).toBe(1); - filters.forEach((value, key) => { - expect(cloudStorage[key]).toBe(value); - }); + const ids = [1, 2, 3]; + + await Promise.all(filters.map(async (_, idx) => { + const result = await window.cvat.cloudStorages.get(Object.fromEntries(filters[idx])); + const [cloudStorage] = result; + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(1); + expect(cloudStorage).toBeInstanceOf(CloudStorage); + expect(cloudStorage.id).toBe(ids[idx]); + filters[idx].forEach((value, key) => { + expect(cloudStorage[key]).toBe(value); + }); + })); }); test('get cloud storage by invalid filters', async () => { diff --git a/cvat-core/tests/mocks/dummy-data.mock.js b/cvat-core/tests/mocks/dummy-data.mock.js index 66809f04..683fe64f 100644 --- a/cvat-core/tests/mocks/dummy-data.mock.js +++ b/cvat-core/tests/mocks/dummy-data.mock.js @@ -2551,10 +2551,31 @@ const frameMetaDummyData = { }; const cloudStoragesDummyData = { - count: 2, + count: 3, next: null, previous: null, results: [ + { + id: 3, + owner: { + url: 'http://localhost:7000/api/v1/users/1', + id: 1, + username: 'maya', + first_name: '', + last_name: '' + }, + manifests: [ + 'manifest.jsonl' + ], + provider_type: 'GOOGLE_CLOUD_STORAGE', + resource: 'gcsbucket', + display_name: 'Demo GCS', + created_date: '2021-09-01T09:29:47.094244Z', + updated_date: '2021-09-01T09:29:47.103264Z', + credentials_type: 'KEY_FILE_PATH', + specific_attributes: '', + description: 'It is first google cloud storage' + }, { id: 2, owner: { diff --git a/cvat-core/tests/mocks/server-proxy.mock.js b/cvat-core/tests/mocks/server-proxy.mock.js index 7c9e2e15..5854f760 100644 --- a/cvat-core/tests/mocks/server-proxy.mock.js +++ b/cvat-core/tests/mocks/server-proxy.mock.js @@ -97,8 +97,8 @@ class ServerProxy { const object = projectsDummyData.results.filter((project) => project.id === id)[0]; for (const prop in projectData) { if ( - Object.prototype.hasOwnProperty.call(projectData, prop) - && Object.prototype.hasOwnProperty.call(object, prop) + Object.prototype.hasOwnProperty.call(projectData, prop) && + Object.prototype.hasOwnProperty.call(object, prop) ) { if (prop === 'labels') { object[prop] = projectData[prop].filter((label) => !label.deleted); @@ -160,8 +160,8 @@ class ServerProxy { const object = tasksDummyData.results.filter((task) => task.id === id)[0]; for (const prop in taskData) { if ( - Object.prototype.hasOwnProperty.call(taskData, prop) - && Object.prototype.hasOwnProperty.call(object, prop) + Object.prototype.hasOwnProperty.call(taskData, prop) && + Object.prototype.hasOwnProperty.call(object, prop) ) { if (prop === 'labels') { object[prop] = taskData[prop].filter((label) => !label.deleted); @@ -249,8 +249,8 @@ class ServerProxy { for (const prop in jobData) { if ( - Object.prototype.hasOwnProperty.call(jobData, prop) - && Object.prototype.hasOwnProperty.call(object, prop) + Object.prototype.hasOwnProperty.call(jobData, prop) && + Object.prototype.hasOwnProperty.call(object, prop) ) { object[prop] = jobData[prop]; } @@ -339,8 +339,8 @@ class ServerProxy { if (cloudStorage) { for (const prop in cloudStorageData) { if ( - Object.prototype.hasOwnProperty.call(cloudStorageData, prop) - && Object.prototype.hasOwnProperty.call(cloudStorage, prop) + Object.prototype.hasOwnProperty.call(cloudStorageData, prop) && + Object.prototype.hasOwnProperty.call(cloudStorage, prop) ) { cloudStorage[prop] = cloudStorageData[prop]; } @@ -375,7 +375,6 @@ class ServerProxy { } } - Object.defineProperties( this, Object.freeze({ diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 8e1d2cb1..8744ae45 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "cvat-ui", - "version": "1.27.1", + "version": "1.28.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cvat-ui", - "version": "1.27.1", + "version": "1.28.0", "license": "MIT", "dependencies": { "@ant-design/icons": "^4.6.3", diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 20fe291e..93bc5923 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.27.1", + "version": "1.28.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/assets/google-cloud.svg b/cvat-ui/src/assets/google-cloud.svg new file mode 100644 index 00000000..08ca1710 --- /dev/null +++ b/cvat-ui/src/assets/google-cloud.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/cvat-ui/src/components/cloud-storages-page/cloud-storage-item.tsx b/cvat-ui/src/components/cloud-storages-page/cloud-storage-item.tsx index ee415b3e..a3933d15 100644 --- a/cvat-ui/src/components/cloud-storages-page/cloud-storage-item.tsx +++ b/cvat-ui/src/components/cloud-storages-page/cloud-storage-item.tsx @@ -85,6 +85,7 @@ export default function CloudStorageItemComponent(props: Props): JSX.Element { size='small' style={style} className='cvat-cloud-storage-item' + hoverable > ([]); + const [keyFilePathIsDisabled, setKeyFilePathIsDisabled] = useState(false); + const [keyFileIsDisabled, setKeyFileIsDisabled] = useState(false); + + const [uploadedKeyFile, setUploadedKeyFile] = useState(null); + function initializeFields(): void { setManifestNames(cloudStorage.manifests); const fieldsValue: CloudStorageForm = { @@ -95,12 +112,24 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { } else if (cloudStorage.credentialsType === CredentialsType.KEY_SECRET_KEY_PAIR) { fieldsValue.key = fakeCredentialsData.key; fieldsValue.secret_key = fakeCredentialsData.secretKey; + } else if (cloudStorage.credentialsType === CredentialsType.KEY_FILE_PATH) { + fieldsValue.key_file_path = fakeCredentialsData.keyFilePath; } - if (cloudStorage.providerType === ProviderType.AWS_S3_BUCKET && cloudStorage.specificAttributes) { - const region = new URLSearchParams(cloudStorage.specificAttributes).get('region'); - if (region) { - setSelectedRegion(region); + if (cloudStorage.specificAttributes) { + const parsedOptions = new URLSearchParams(cloudStorage.specificAttributes); + const location = parsedOptions.get('region') || parsedOptions.get('location'); + const prefix = parsedOptions.get('prefix'); + const projectId = parsedOptions.get('project_id'); + if (location) { + setSelectedRegion(location); + } + if (prefix) { + fieldsValue.prefix = prefix; + } + + if (projectId) { + fieldsValue.project_id = projectId; } } @@ -161,7 +190,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { }, [updatedCloudStorageId]); useEffect(() => { - if (cloudStorage && cloudStorage.credentialsType !== CredentialsType.ANONYMOUS_ACCESS) { + if (cloudStorageId && cloudStorage.credentialsType !== CredentialsType.ANONYMOUS_ACCESS) { notification.info({ message: `For security reasons, your credentials are hidden and represented by fake values that will not be taken into account when updating the cloud storage. @@ -170,15 +199,37 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { duration: 15, }); } - }, [cloudStorage]); + }, []); const onSubmit = async (): Promise => { let cloudStorageData: Record = {}; const formValues = await form.validateFields(); cloudStorageData = { ...formValues }; - if (formValues.region !== undefined) { - delete cloudStorageData.region; - cloudStorageData.specific_attributes = `region=${selectedRegion}`; + // specific attributes + const specificAttributes = new URLSearchParams(); + + if (selectedRegion) { + if (cloudStorageData.provider_type === ProviderType.AWS_S3_BUCKET) { + delete cloudStorageData.region; + specificAttributes.append('region', selectedRegion as string); + } else if (cloudStorageData.provider_type === ProviderType.GOOGLE_CLOUD_STORAGE) { + delete cloudStorageData.location; + specificAttributes.append('location', selectedRegion as string); + } + } + if (formValues.prefix) { + delete cloudStorageData.prefix; + specificAttributes.append('prefix', formValues.prefix); + } + if (formValues.project_id) { + delete cloudStorageData.project_id; + specificAttributes.append('project_id', formValues.project_id); + } + + cloudStorageData.specific_attributes = specificAttributes.toString(); + + if (uploadedKeyFile) { + cloudStorageData.key_file = uploadedKeyFile; } if (cloudStorageData.credentials_type === CredentialsType.ACCOUNT_NAME_TOKEN_PAIR) { @@ -195,6 +246,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { if (cloudStorage) { cloudStorageData.id = cloudStorage.id; + if (cloudStorageData.account_name === fakeCredentialsData.accountName) { delete cloudStorageData.account_name; } @@ -207,6 +259,9 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { if (cloudStorageData.session_token === fakeCredentialsData.sessionToken) { delete cloudStorageData.session_token; } + if (cloudStorageData.key_file_path === fakeCredentialsData.keyFilePath) { + delete cloudStorageData.key_file_path; + } dispatch(updateCloudStorageAsync(cloudStorageData)); } else { dispatch(createCloudStorageAsync(cloudStorageData)); @@ -219,6 +274,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { secret_key: undefined, session_token: undefined, account_name: undefined, + key_file_path: undefined, }); }; @@ -264,7 +320,6 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { const internalCommonProps = { ...commonProps, labelCol: { span: 8, offset: 2 }, - wrapperCol: { offset: 1 }, }; if (providerType === ProviderType.AWS_S3_BUCKET && credentialsType === CredentialsType.KEY_SECRET_KEY_PAIR) { @@ -320,8 +375,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { visibilityToggle={accountNameVisibility} onChange={() => setAccountNameVisibility(true)} onFocus={() => onFocusCredentialsItem('accountName', 'account_name')} - onBlur={() => - onBlurCredentialsItem('accountName', 'account_name', setAccountNameVisibility)} + onBlur={() => onBlurCredentialsItem('accountName', 'account_name', setAccountNameVisibility)} /> setSessionTokenVisibility(true)} onFocus={() => onFocusCredentialsItem('sessionToken', 'session_token')} - onBlur={() => - onBlurCredentialsItem('sessionToken', 'session_token', setSessionTokenVisibility)} + onBlur={() => onBlurCredentialsItem('sessionToken', 'session_token', setSessionTokenVisibility)} /> @@ -363,6 +416,72 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element { ); } + if (providerType === ProviderType.GOOGLE_CLOUD_STORAGE && credentialsType === CredentialsType.KEY_FILE_PATH) { + return ( + + Key file + + + )} + > + + + { + setKeyFilePathVisibility(true); + const isDisabled = !!(e.target.value); + setKeyFileIsDisabled(isDisabled); + }} + onFocus={() => onFocusCredentialsItem('keyFilePath', 'key_file_path')} + onBlur={() => onBlurCredentialsItem('keyFilePath', 'key_file_path', setKeyFilePathVisibility)} + disabled={keyFilePathIsDisabled} + /> + + + + { + if (form.getFieldValue('key_file_path')) { + form.setFieldsValue({ + key_file_path: undefined, + }); + } + setKeyFilePathIsDisabled(true); + setUploadedKeyFile(file); + return false; + }} + > + + + + )} + name={name} + {...internalCommonProps} + > + setNewRegionKey(event.target.value)} + maxLength={14} + placeholder='key' + /> + setNewRegionName(event.target.value)} + placeholder='name' + /> + + + + )} + onSelect={(_, instance) => onSelectRegion(instance.key)} + > + { + Array.from(Object.entries(locations)).map( + ([key, value]): JSX.Element => ( + + ), + ) + } + + + ); +} diff --git a/cvat-ui/src/components/create-cloud-storage-page/s3-region.tsx b/cvat-ui/src/components/create-cloud-storage-page/s3-region.tsx index 2b5413aa..a8484a3b 100644 --- a/cvat-ui/src/components/create-cloud-storage-page/s3-region.tsx +++ b/cvat-ui/src/components/create-cloud-storage-page/s3-region.tsx @@ -1,119 +1,31 @@ // Copyright (C) 2021 Intel Corporation // // SPDX-License-Identifier: MIT - -import React, { useState } from 'react'; -import Divider from 'antd/lib/divider'; -import Select from 'antd/lib/select'; -import { PlusCircleOutlined, QuestionCircleOutlined } from '@ant-design/icons'; -import Input from 'antd/lib/input'; -import Button from 'antd/lib/button'; -import Form from 'antd/lib/form'; -import notification from 'antd/lib/notification'; -import Tooltip from 'antd/lib/tooltip'; +import React from 'react'; +import Location from './location'; import consts from '../../consts'; -const { Option } = Select; interface Props { - selectedRegion: undefined | string; + selectedRegion: any; onSelectRegion: any; internalCommonProps: any; } -function prepareDefaultRegions(): Map { - const temp = new Map(); - for (const [key, value] of consts.DEFAULT_AWS_S3_REGIONS) { - temp.set(key, value); - } - return temp; -} - export default function S3Region(props: Props): JSX.Element { - const { selectedRegion, onSelectRegion, internalCommonProps } = props; - const [regions, setRegions] = useState>(() => prepareDefaultRegions()); - const [newRegionKey, setNewRegionKey] = useState(''); - const [newRegionName, setNewRegionName] = useState(''); - - const handleAddingRegion = (): void => { - if (!newRegionKey || !newRegionName) { - notification.warning({ - message: 'Incorrect region', - className: 'cvat-incorrect-add-region-notification', - }); - } else if (regions.has(newRegionKey)) { - notification.warning({ - message: 'This region already exists', - className: 'cvat-incorrect-add-region-notification', - }); - } else { - const regionsCopy = regions; - setRegions(regionsCopy.set(newRegionKey, newRegionName)); - setNewRegionKey(''); - setNewRegionName(''); - } - }; - + const { + selectedRegion, + onSelectRegion, + internalCommonProps, + } = props; return ( - - Region - - - - - )} + - setNewRegionKey(event.target.value)} - maxLength={14} - placeholder='key' - /> - setNewRegionName(event.target.value)} - placeholder='name' - /> - - - - )} - onSelect={(_, instance) => onSelectRegion(instance.key)} - > - { - Array.from(regions.entries()).map( - ([key, value]): JSX.Element => ( - - ), - ) - } - - + label='Region' + href='https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions' + /> ); } diff --git a/cvat-ui/src/components/create-cloud-storage-page/styles.scss b/cvat-ui/src/components/create-cloud-storage-page/styles.scss index 3a021bd1..31ba6ad0 100644 --- a/cvat-ui/src/components/create-cloud-storage-page/styles.scss +++ b/cvat-ui/src/components/create-cloud-storage-page/styles.scss @@ -35,6 +35,14 @@ } } + .cvat-cloud-storage-form-item-key-file { + width: 100%; + + :nth-child(1) { + flex-grow: 1; + } + } + > div:not(first-child) { margin-top: $grid-unit-size; } diff --git a/cvat-ui/src/components/file-manager/cloud-storages-tab.tsx b/cvat-ui/src/components/file-manager/cloud-storages-tab.tsx index 8c2cb2be..0aabdd49 100644 --- a/cvat-ui/src/components/file-manager/cloud-storages-tab.tsx +++ b/cvat-ui/src/components/file-manager/cloud-storages-tab.tsx @@ -13,7 +13,7 @@ import { debounce } from 'lodash'; import Select from 'antd/lib/select'; import getCore from 'cvat-core-wrapper'; import { CloudStorage } from 'reducers/interfaces'; -import { AzureProvider, S3Provider } from 'icons'; +import { AzureProvider, GoogleCloudProvider, S3Provider } from 'icons'; import { ProviderType } from 'utils/enums'; import CloudStorageFiles from './cloud-storages-files'; @@ -125,11 +125,12 @@ export default function CloudStorageTab(props: Props): JSX.Element { - {_cloudStorage.providerType === ProviderType.AWS_S3_BUCKET ? ( - - ) : ( - - )} + {_cloudStorage.providerType === ProviderType.AWS_S3_BUCKET && } + {_cloudStorage.providerType === ProviderType.AZURE_CONTAINER && } + { + _cloudStorage.providerType === ProviderType.GOOGLE_CLOUD_STORAGE && + + } {_cloudStorage.displayName} ), diff --git a/cvat-ui/src/components/search-tooltip/search-tooltip.tsx b/cvat-ui/src/components/search-tooltip/search-tooltip.tsx index 46a20aab..3fc5dcb7 100644 --- a/cvat-ui/src/components/search-tooltip/search-tooltip.tsx +++ b/cvat-ui/src/components/search-tooltip/search-tooltip.tsx @@ -67,6 +67,8 @@ export default function SearchTooltip(props: Props): JSX.Element { AWS_S3_BUCKET or AZURE_CONTAINER + or + GOOGLE_CLOUD_STORAGE ) : null} @@ -78,6 +80,8 @@ export default function SearchTooltip(props: Props): JSX.Element { or ACCOUNT_NAME_TOKEN_PAIR or + KEY_FILE_PATH + or ANONYMOUS_ACCESS diff --git a/cvat-ui/src/consts.ts b/cvat-ui/src/consts.ts index fcdd5cb9..27da2c90 100644 --- a/cvat-ui/src/consts.ts +++ b/cvat-ui/src/consts.ts @@ -45,6 +45,45 @@ const DEFAULT_AWS_S3_REGIONS: string[][] = [ ['sa-east-1', 'South America (São Paulo)'], ]; +const DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS: string[][] = [ + ['NORTHAMERICA-NORTHEAST1', 'Montréal'], + ['NORTHAMERICA-NORTHEAST2', 'Toronto'], + ['US-CENTRAL1', 'Iowa'], + ['US-EAST1', 'South Carolina'], + ['US-EAST4', 'Northern Virginia'], + ['US-WEST1', 'Oregon'], + ['US-WEST2', 'Los Angeles'], + ['US-WEST3', 'Salt Lake City'], + ['US-WEST4', 'Las Vegas'], + ['SOUTHAMERICA-EAST1', 'São Paulo'], + ['EUROPE-CENTRAL2', 'Warsaw'], + ['EUROPE-NORTH1', 'Finland'], + ['EUROPE-WEST1', 'Belgium'], + ['EUROPE-WEST2', 'London'], + ['EUROPE-WEST3', 'Frankfurt'], + ['EUROPE-WEST4', 'Netherlands'], + ['EUROPE-WEST6', 'Zürich'], + ['ASIA-EAST1', 'Taiwan'], + ['ASIA-EAST2', 'Hong Kong'], + ['ASIA-NORTHEAST1', 'Tokyo'], + ['ASIA-NORTHEAST2', 'Osaka'], + ['ASIA-NORTHEAST3', 'Seoul'], + ['ASIA-SOUTH1', 'Mumbai'], + ['ASIA-SOUTH2', 'Delhi'], + ['ASIA-SOUTHEAST1', 'Singapore'], + ['ASIA-SOUTHEAST2', 'Jakarta'], + ['AUSTRALIA-SOUTHEAST1', 'Sydney'], + ['AUSTRALIA-SOUTHEAST2', 'Melbourne'], + // Multi-regions + ['ASIA', 'Data centers in Asia'], + ['EU', 'Data centers within member states of the European Union'], + ['US', 'Data centers in the United States'], + // Dual-regions + ['ASIA1', 'ASIA-NORTHEAST1 and ASIA-NORTHEAST2'], + ['EUR4', 'EUROPE-NORTH1 and EUROPE-WEST4'], + ['NAM4', 'US-CENTRAL1 and US-EAST1'], +]; + export default { UNDEFINED_ATTRIBUTE_VALUE, NO_BREAK_SPACE, @@ -67,4 +106,5 @@ export default { INTEL_COOKIES_URL, INTEL_PRIVACY_URL, DEFAULT_AWS_S3_REGIONS, + DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS, }; diff --git a/cvat-ui/src/icons.tsx b/cvat-ui/src/icons.tsx index d90ba180..e43f2d69 100644 --- a/cvat-ui/src/icons.tsx +++ b/cvat-ui/src/icons.tsx @@ -52,6 +52,7 @@ import SVGOpenCV from './assets/opencv.svg'; import SVGFilterIcon from './assets/object-filter-icon.svg'; import SVGCVATAzureProvider from './assets/vscode-icons_file-type-azure.svg'; import SVGCVATS3Provider from './assets/S3.svg'; +import SVGCVATGoogleCloudProvider from './assets/google-cloud.svg'; export const CVATLogo = React.memo((): JSX.Element => ); export const AccountIcon = React.memo((): JSX.Element => ); @@ -101,3 +102,4 @@ export const OpenCVIcon = React.memo((): JSX.Element => ); export const FilterIcon = React.memo((): JSX.Element => ); export const AzureProvider = React.memo((): JSX.Element => ); export const S3Provider = React.memo((): JSX.Element => ); +export const GoogleCloudProvider = React.memo((): JSX.Element => ); diff --git a/cvat-ui/src/utils/enums.tsx b/cvat-ui/src/utils/enums.tsx index 3cb8529f..68672486 100644 --- a/cvat-ui/src/utils/enums.tsx +++ b/cvat-ui/src/utils/enums.tsx @@ -5,12 +5,14 @@ export enum ProviderType { AWS_S3_BUCKET = 'AWS_S3_BUCKET', AZURE_CONTAINER = 'AZURE_CONTAINER', + GOOGLE_CLOUD_STORAGE = 'GOOGLE_CLOUD_STORAGE', } export enum CredentialsType { KEY_SECRET_KEY_PAIR = 'KEY_SECRET_KEY_PAIR', ACCOUNT_NAME_TOKEN_PAIR = 'ACCOUNT_NAME_TOKEN_PAIR', ANONYMOUS_ACCESS = 'ANONYMOUS_ACCESS', + KEY_FILE_PATH = 'KEY_FILE_PATH', } export enum StorageStatuses { diff --git a/cvat/apps/engine/cloud_provider.py b/cvat/apps/engine/cloud_provider.py index 869b2ad1..3af72df2 100644 --- a/cvat/apps/engine/cloud_provider.py +++ b/cvat/apps/engine/cloud_provider.py @@ -119,6 +119,7 @@ def get_cloud_storage_instance(cloud_provider, resource, credentials, specific_a instance = GoogleCloudStorage( bucket_name=resource, service_account_json=credentials.key_file_path, + anonymous_access = credentials.credentials_type == CredentialsTypeChoice.ANONYMOUS_ACCESS, prefix=specific_attributes.get('prefix'), location=specific_attributes.get('location'), project=specific_attributes.get('project') @@ -356,18 +357,18 @@ def _define_gcs_status(func): class GoogleCloudStorage(_CloudStorage): - def __init__(self, bucket_name, prefix=None, service_account_json=None, project=None, location=None): + def __init__(self, bucket_name, prefix=None, service_account_json=None, anonymous_access=False, project=None, location=None): super().__init__() if service_account_json: self._storage_client = storage.Client.from_service_account_json(service_account_json) + elif anonymous_access: + self._storage_client = storage.Client.create_anonymous_client() else: + # If no credentials were provided when constructing the client, the + # client library will look for credentials in the environment. self._storage_client = storage.Client() - bucket = self._storage_client.lookup_bucket(bucket_name) - if bucket is None: - bucket = self._storage_client.bucket(bucket_name, user_project=project) - - self._bucket = bucket + self._bucket = self._storage_client.bucket(bucket_name, user_project=project) self._bucket_location = location self._prefix = prefix @@ -464,7 +465,6 @@ class Credentials: elif self.credentials_type == CredentialsTypeChoice.ACCOUNT_NAME_TOKEN_PAIR: self.account_name, self.session_token = credentials.get('value').split() elif self.credentials_type == CredentialsTypeChoice.ANONYMOUS_ACCESS: - self.session_token, self.key, self.secret_key = ('', '', '') # account_name will be in [some_value, ''] self.account_name = credentials.get('value') elif self.credentials_type == CredentialsTypeChoice.KEY_FILE_PATH: @@ -472,31 +472,25 @@ class Credentials: else: raise NotImplementedError('Found {} not supported credentials type'.format(self.credentials_type)) + def reset(self, exclusion): + for i in set(self.__slots__) - exclusion - {'credentials_type'}: + self.__setattr__(i, '') + def mapping_with_new_values(self, credentials): self.credentials_type = credentials.get('credentials_type', self.credentials_type) if self.credentials_type == CredentialsTypeChoice.ANONYMOUS_ACCESS: - self.key = '' - self.secret_key = '' - self.session_token = '' - self.key_file_path = '' + self.reset(exclusion={'account_name'}) self.account_name = credentials.get('account_name', self.account_name) elif self.credentials_type == CredentialsTypeChoice.KEY_SECRET_KEY_PAIR: + self.reset(exclusion={'key', 'secret_key'}) self.key = credentials.get('key', self.key) self.secret_key = credentials.get('secret_key', self.secret_key) - self.session_token = '' - self.account_name = '' - self.key_file_path = '' elif self.credentials_type == CredentialsTypeChoice.ACCOUNT_NAME_TOKEN_PAIR: + self.reset(exclusion={'session_token', 'account_name'}) self.session_token = credentials.get('session_token', self.session_token) self.account_name = credentials.get('account_name', self.account_name) - self.key = '' - self.secret_key = '' - self.key_file_path = '' elif self.credentials_type == CredentialsTypeChoice.KEY_FILE_PATH: - self.key = '' - self.secret_key = '' - self.session_token = '' - self.account_name = '' + self.reset(exclusion={'key_file_path'}) self.key_file_path = credentials.get('key_file_path', self.key_file_path) else: raise NotImplementedError('Mapping credentials: unsupported credentials type') diff --git a/cvat/apps/engine/migrations/0044_auto_20211123_0824.py b/cvat/apps/engine/migrations/0044_auto_20211123_0824.py new file mode 100644 index 00000000..41541f69 --- /dev/null +++ b/cvat/apps/engine/migrations/0044_auto_20211123_0824.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.13 on 2021-11-23 08:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0044_auto_20211115_0858'), + ] + + operations = [ + migrations.AlterField( + model_name='cloudstorage', + name='resource', + field=models.CharField(max_length=222), + ), + migrations.AlterField( + model_name='cloudstorage', + name='specific_attributes', + field=models.CharField(blank=True, max_length=128), + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index f2edd6d1..31107e48 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -585,15 +585,19 @@ class Manifest(models.Model): class CloudStorage(models.Model): # restrictions: - # AWS bucket name, Azure container name - 63 + # AWS bucket name, Azure container name - 63, Google bucket name - 63 without dots and 222 with dots + # https://cloud.google.com/storage/docs/naming-buckets#requirements # AWS access key id - 20 # AWS secret access key - 40 # AWS temporary session tocken - None # The size of the security token that AWS STS API operations return is not fixed. # We strongly recommend that you make no assumptions about the maximum size. # The typical token size is less than 4096 bytes, but that can vary. + # specific attributes: + # location - max 23 + # project ID: 6 - 30 (https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin) provider_type = models.CharField(max_length=20, choices=CloudProviderChoice.choices()) - resource = models.CharField(max_length=63) + resource = models.CharField(max_length=222) display_name = models.CharField(max_length=63) owner = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL, related_name="cloud_storages") @@ -601,7 +605,7 @@ class CloudStorage(models.Model): updated_date = models.DateTimeField(auto_now=True) credentials = models.CharField(max_length=500) credentials_type = models.CharField(max_length=29, choices=CredentialsTypeChoice.choices())#auth_type - specific_attributes = models.CharField(max_length=50, blank=True) + specific_attributes = models.CharField(max_length=128, blank=True) description = models.TextField(blank=True) class Meta: @@ -625,3 +629,6 @@ class CloudStorage(models.Model): def get_specific_attributes(self): return parse_specific_attributes(self.specific_attributes) + + def get_key_file_path(self): + return os.path.join(self.get_storage_dirname(), 'key.json') diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 0021a2ae..8e595c71 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -6,6 +6,8 @@ import os import re import shutil +from tempfile import NamedTemporaryFile + from rest_framework import serializers, exceptions from django.contrib.auth.models import User, Group @@ -798,6 +800,7 @@ class CloudStorageSerializer(serializers.ModelSerializer): key = serializers.CharField(max_length=20, allow_blank=True, required=False) secret_key = serializers.CharField(max_length=40, allow_blank=True, required=False) key_file_path = serializers.CharField(max_length=64, allow_blank=True, required=False) + key_file = serializers.FileField(required=False) account_name = serializers.CharField(max_length=24, allow_blank=True, required=False) manifests = ManifestSerializer(many=True, default=[]) @@ -806,7 +809,8 @@ class CloudStorageSerializer(serializers.ModelSerializer): fields = ( 'provider_type', 'resource', 'display_name', 'owner', 'credentials_type', 'created_date', 'updated_date', 'session_token', 'account_name', 'key', - 'secret_key', 'key_file_path', 'specific_attributes', 'description', 'id', 'manifests', + 'secret_key', 'key_file_path', 'key_file', 'specific_attributes', + 'description', 'id', 'manifests', ) read_only_fields = ('created_date', 'updated_date', 'owner') @@ -820,20 +824,33 @@ class CloudStorageSerializer(serializers.ModelSerializer): return value def validate(self, attrs): - if attrs.get('provider_type') == models.CloudProviderChoice.AZURE_CONTAINER: + provider_type = attrs.get('provider_type') + if provider_type == models.CloudProviderChoice.AZURE_CONTAINER: if not attrs.get('account_name', ''): raise serializers.ValidationError('Account name for Azure container was not specified') + if attrs.get('key_file', '') and attrs.get('key_file_path', ''): + raise serializers.ValidationError('Should be specified key file or key file path') return attrs def create(self, validated_data): provider_type = validated_data.get('provider_type') should_be_created = validated_data.pop('should_be_created', None) + + key_file = validated_data.pop('key_file', None) + # we need to save it to temporary file to check the granted permissions + temporary_file = '' + if key_file: + with NamedTemporaryFile(mode='wb', prefix='cvat', delete=False) as temp_key: + temp_key.write(key_file.read()) + temporary_file = temp_key.name + key_file.close() + del key_file credentials = Credentials( account_name=validated_data.pop('account_name', ''), key=validated_data.pop('key', ''), secret_key=validated_data.pop('secret_key', ''), session_token=validated_data.pop('session_token', ''), - key_file_path=validated_data.pop('key_file_path', ''), + key_file_path=validated_data.pop('key_file_path', '') or temporary_file, credentials_type = validated_data.get('credentials_type') ) details = { @@ -880,6 +897,15 @@ class CloudStorageSerializer(serializers.ModelSerializer): shutil.rmtree(cloud_storage_path) os.makedirs(db_storage.get_storage_logs_dirname(), exist_ok=True) + if temporary_file: + # so, gcs key file is valid and we need to set correct path to the file + real_path_to_key_file = db_storage.get_key_file_path() + shutil.copyfile(temporary_file, real_path_to_key_file) + os.remove(temporary_file) + + credentials.key_file_path = real_path_to_key_file + db_storage.credentials = credentials.convert_to_db() + db_storage.save() return db_storage elif storage_status == Status.FORBIDDEN: field = 'credentials' @@ -887,6 +913,8 @@ class CloudStorageSerializer(serializers.ModelSerializer): else: field = 'recource' message = 'The resource {} not found. It may have been deleted.'.format(storage.name) + if temporary_file: + os.remove(temporary_file) slogger.glob.error(message) raise serializers.ValidationError({field: message}) @@ -897,8 +925,23 @@ class CloudStorageSerializer(serializers.ModelSerializer): 'type': instance.credentials_type, 'value': instance.credentials, }) - tmp = {k:v for k,v in validated_data.items() if k in {'key', 'secret_key', 'account_name', 'session_token', 'key_file_path', 'credentials_type'}} - credentials.mapping_with_new_values(tmp) + credentials_dict = {k:v for k,v in validated_data.items() if k in { + 'key','secret_key', 'account_name', 'session_token', 'key_file_path', + 'credentials_type' + }} + + key_file = validated_data.pop('key_file', None) + temporary_file = '' + if key_file: + with NamedTemporaryFile(mode='wb', prefix='cvat', delete=False) as temp_key: + temp_key.write(key_file.read()) + temporary_file = temp_key.name + # pair (key_file, key_file_path) isn't supported by server, so only one value may be specified + credentials_dict['key_file_path'] = temporary_file + key_file.close() + del key_file + + credentials.mapping_with_new_values(credentials_dict) instance.credentials = credentials.convert_to_db() instance.credentials_type = validated_data.get('credentials_type', instance.credentials_type) instance.resource = validated_data.get('resource', instance.resource) @@ -937,6 +980,13 @@ class CloudStorageSerializer(serializers.ModelSerializer): }) manifest_instances = [models.Manifest(filename=f, cloud_storage=instance) for f in delta_to_create] models.Manifest.objects.bulk_create(manifest_instances) + if temporary_file: + # so, gcs key file is valid and we need to set correct path to the file + real_path_to_key_file = instance.get_key_file_path() + shutil.copyfile(temporary_file, real_path_to_key_file) + os.remove(temporary_file) + + instance.credentials = real_path_to_key_file instance.save() return instance elif storage_status == Status.FORBIDDEN: @@ -945,6 +995,8 @@ class CloudStorageSerializer(serializers.ModelSerializer): else: field = 'recource' message = 'The resource {} not found. It may have been deleted.'.format(storage.name) + if temporary_file: + os.remove(temporary_file) slogger.glob.error(message) raise serializers.ValidationError({field: message}) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 1b058935..935898ff 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1334,6 +1334,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS ) @action(detail=True, methods=['GET'], url_path='content') def content(self, request, pk): + storage = None try: db_storage = CloudStorageModel.objects.get(pk=pk) credentials = Credentials() @@ -1378,7 +1379,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS return Response(data=msg, status=status.HTTP_404_NOT_FOUND) except Exception as ex: # check that cloud storage was not deleted - storage_status = storage.get_status() + storage_status = storage.get_status() if storage else None if storage_status == Status.FORBIDDEN: msg = 'The resource {} is no longer available. Access forbidden.'.format(storage.name) elif storage_status == Status.NOT_FOUND: @@ -1397,6 +1398,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS ) @action(detail=True, methods=['GET'], url_path='preview') def preview(self, request, pk): + storage = None try: db_storage = CloudStorageModel.objects.get(pk=pk) if not os.path.exists(db_storage.get_preview_path()): @@ -1455,7 +1457,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS return HttpResponseNotFound(message) except Exception as ex: # check that cloud storage was not deleted - storage_status = storage.get_status() + storage_status = storage.get_status() if storage else None if storage_status == Status.FORBIDDEN: msg = 'The resource {} is no longer available. Access forbidden.'.format(storage.name) elif storage_status == Status.NOT_FOUND: