Support GCS (#3919)

Co-authored-by: Boris Sekachev <boris.sekachev@intel.com>
main
Maria Khrustaleva 4 years ago committed by GitHub
parent 130b815f61
commit f59d1f57f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Add Open Images V6 format (<https://github.com/openvinotoolkit/cvat/pull/3679>)
- Rotated bounding boxes (<https://github.com/openvinotoolkit/cvat/pull/3832>)
- Player option: Smooth image when zoom-in, enabled by default (<https://github.com/openvinotoolkit/cvat/pull/3933>)
- Google Cloud Storage support in UI (<https://github.com/openvinotoolkit/cvat/pull/3919>)
- Add project tasks paginations (<https://github.com/openvinotoolkit/cvat/pull/3910>)
### Changed

@ -1,12 +1,12 @@
{
"name": "cvat-core",
"version": "3.20.0",
"version": "3.20.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "cvat-core",
"version": "3.20.0",
"version": "3.20.1",
"license": "MIT",
"dependencies": {
"axios": "^0.21.4",

@ -1,6 +1,6 @@
{
"name": "cvat-core",
"version": "3.20.0",
"version": "3.20.1",
"description": "Part of Computer Vision Tool which presents an interface for client-side integration",
"main": "babel.config.js",
"scripts": {

@ -9,12 +9,19 @@
const { ArgumentError } = require('./exceptions');
const { CloudStorageCredentialsType, CloudStorageProviderType } = require('./enums');
function validateNotEmptyString(value) {
if (typeof value !== 'string') {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
} else if (!value.trim().length) {
throw new ArgumentError('Value mustn\'t be empty string');
}
}
/**
* Class representing a cloud storage
* @memberof module:API.cvat.classes
*/
class CloudStorage {
// TODO: add storage availability status (avaliable/unavaliable)
constructor(initialData) {
const data = {
id: undefined,
@ -27,6 +34,8 @@
key: undefined,
secret_key: undefined,
session_token: undefined,
key_file_path: undefined,
key_file: undefined,
specific_attributes: undefined,
owner: undefined,
created_date: undefined,
@ -65,11 +74,7 @@
displayName: {
get: () => data.display_name,
set: (value) => {
if (typeof value !== 'string') {
throw new ArgumentError(`Value must be string. ${typeof value} was found`);
} else if (!value.trim().length) {
throw new ArgumentError('Value must not be empty string');
}
validateNotEmptyString(value);
data.display_name = value;
},
},
@ -101,15 +106,8 @@
accountName: {
get: () => data.account_name,
set: (value) => {
if (typeof value === 'string') {
if (value.trim().length) {
data.account_name = value;
} else {
throw new ArgumentError('Value must not be empty');
}
} else {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
}
validateNotEmptyString(value);
data.account_name = value;
},
},
/**
@ -123,15 +121,8 @@
accessKey: {
get: () => data.key,
set: (value) => {
if (typeof value === 'string') {
if (value.trim().length) {
data.key = value;
} else {
throw new ArgumentError('Value must not be empty');
}
} else {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
}
validateNotEmptyString(value);
data.key = value;
},
},
/**
@ -145,15 +136,8 @@
secretKey: {
get: () => data.secret_key,
set: (value) => {
if (typeof value === 'string') {
if (value.trim().length) {
data.secret_key = value;
} else {
throw new ArgumentError('Value must not be empty');
}
} else {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
}
validateNotEmptyString(value);
data.secret_key = value;
},
},
/**
@ -167,14 +151,40 @@
token: {
get: () => data.session_token,
set: (value) => {
if (typeof value === 'string') {
if (value.trim().length) {
data.session_token = value;
} else {
throw new ArgumentError('Value must not be empty');
}
validateNotEmptyString(value);
data.session_token = value;
},
},
/**
* Key file path
* @name keyFilePath
* @type {string}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
keyFilePath: {
get: () => data.key_file_path,
set: (value) => {
validateNotEmptyString(value);
data.key_file_path = value;
},
},
/**
* Key file
* @name keyFile
* @type {File}
* @memberof module:API.cvat.classes.CloudStorage
* @instance
* @throws {module:API.cvat.exceptions.ArgumentError}
*/
keyFile: {
get: () => data.key_file,
set: (file) => {
if (file instanceof File) {
data.key_file = file;
} else {
throw new ArgumentError(`Value must be a string. ${typeof value} was found`);
throw new ArgumentError(`Should be a file. ${typeof file} was found`);
}
},
},
@ -189,11 +199,7 @@
resourceName: {
get: () => data.resource,
set: (value) => {
if (typeof value !== 'string') {
throw new ArgumentError(`Value must be string. ${typeof value} was found`);
} else if (!value.trim().length) {
throw new ArgumentError('Value must not be empty');
}
validateNotEmptyString(value);
data.resource = value;
},
},
@ -207,11 +213,8 @@
manifestPath: {
get: () => data.manifest_path,
set: (value) => {
if (typeof value === 'string') {
data.manifest_path = value;
} else {
throw new ArgumentError('Value must be a string');
}
validateNotEmptyString(value);
data.manifest_path = value;
},
},
/**
@ -410,7 +413,7 @@
CloudStorage.prototype.save.implementation = async function () {
function prepareOptionalFields(cloudStorageInstance) {
const data = {};
if (cloudStorageInstance.description) {
if (cloudStorageInstance.description !== undefined) {
data.description = cloudStorageInstance.description;
}
@ -430,14 +433,22 @@
data.session_token = cloudStorageInstance.token;
}
if (cloudStorageInstance.specificAttributes) {
if (cloudStorageInstance.keyFilePath) {
data.key_file_path = cloudStorageInstance.keyFilePath;
}
if (cloudStorageInstance.keyFile) {
data.key_file = cloudStorageInstance.keyFile;
}
if (cloudStorageInstance.specificAttributes !== undefined) {
data.specific_attributes = cloudStorageInstance.specificAttributes;
}
return data;
}
// update
if (typeof this.id !== 'undefined') {
// providr_type and recource should not change;
// provider_type and recource should not change;
// send to the server only the values that have changed
const initialData = {};
if (this.displayName) {

@ -340,11 +340,13 @@
* @memberof module:API.cvat.enums
* @property {string} AWS_S3 'AWS_S3_BUCKET'
* @property {string} AZURE 'AZURE_CONTAINER'
* @property {string} GOOGLE_CLOUD_STORAGE 'GOOGLE_CLOUD_STORAGE'
* @readonly
*/
const CloudStorageProviderType = Object.freeze({
AWS_S3_BUCKET: 'AWS_S3_BUCKET',
AZURE_CONTAINER: 'AZURE_CONTAINER',
GOOGLE_CLOUD_STORAGE: 'GOOGLE_CLOUD_STORAGE',
});
/**
@ -355,12 +357,14 @@
* @property {string} KEY_SECRET_KEY_PAIR 'KEY_SECRET_KEY_PAIR'
* @property {string} ACCOUNT_NAME_TOKEN_PAIR 'ACCOUNT_NAME_TOKEN_PAIR'
* @property {string} ANONYMOUS_ACCESS 'ANONYMOUS_ACCESS'
* @property {string} KEY_FILE_PATH 'KEY_FILE_PATH'
* @readonly
*/
const CloudStorageCredentialsType = Object.freeze({
KEY_SECRET_KEY_PAIR: 'KEY_SECRET_KEY_PAIR',
ACCOUNT_NAME_TOKEN_PAIR: 'ACCOUNT_NAME_TOKEN_PAIR',
ANONYMOUS_ACCESS: 'ANONYMOUS_ACCESS',
KEY_FILE_PATH: 'KEY_FILE_PATH',
});
module.exports = {

@ -45,6 +45,20 @@
return new ServerError(message, 0);
}
function prepareData(details) {
const data = new FormData();
for (const [key, value] of Object.entries(details)) {
if (Array.isArray(value)) {
value.forEach((element, idx) => {
data.append(`${key}[${idx}]`, element);
});
} else {
data.set(key, value);
}
}
return data;
}
class WorkerWrappedAxios {
constructor() {
const worker = new DownloadWorker();
@ -1181,12 +1195,10 @@
async function createCloudStorage(storageDetail) {
const { backendAPI } = config;
const storageDetailData = prepareData(storageDetail);
try {
const response = await Axios.post(`${backendAPI}/cloudstorages`, JSON.stringify(storageDetail), {
const response = await Axios.post(`${backendAPI}/cloudstorages`, storageDetailData, {
proxy: config.proxy,
headers: {
'Content-Type': 'application/json',
},
});
return response.data;
} catch (errorData) {
@ -1194,15 +1206,13 @@
}
}
async function updateCloudStorage(id, cloudStorageData) {
async function updateCloudStorage(id, storageDetail) {
const { backendAPI } = config;
const storageDetailData = prepareData(storageDetail);
try {
await Axios.patch(`${backendAPI}/cloudstorages/${id}`, JSON.stringify(cloudStorageData), {
await Axios.patch(`${backendAPI}/cloudstorages/${id}`, storageDetailData, {
proxy: config.proxy,
headers: {
'Content-Type': 'application/json',
},
});
} catch (errorData) {
throw generateError(errorData);

@ -61,24 +61,41 @@ describe('Feature: get cloud storages', () => {
});
test('get cloud storages by filters', async () => {
const filters = new Map([
['providerType', 'AWS_S3_BUCKET'],
['resourceName', 'bucket'],
['displayName', 'Demonstration bucket'],
['credentialsType', 'KEY_SECRET_KEY_PAIR'],
['description', 'It is first bucket'],
]);
const result = await window.cvat.cloudStorages.get(Object.fromEntries(filters));
const filters = [
new Map([
['providerType', 'AWS_S3_BUCKET'],
['resourceName', 'bucket'],
['displayName', 'Demonstration bucket'],
['credentialsType', 'KEY_SECRET_KEY_PAIR'],
['description', 'It is first bucket'],
]),
new Map([
['providerType', 'AZURE_CONTAINER'],
['resourceName', 'container'],
['displayName', 'Demonstration container'],
['credentialsType', 'ACCOUNT_NAME_TOKEN_PAIR'],
]),
new Map([
['providerType', 'GOOGLE_CLOUD_STORAGE'],
['resourceName', 'gcsbucket'],
['displayName', 'Demo GCS'],
['credentialsType', 'KEY_FILE_PATH'],
]),
];
const [cloudStorage] = result;
expect(Array.isArray(result)).toBeTruthy();
expect(result).toHaveLength(1);
expect(cloudStorage).toBeInstanceOf(CloudStorage);
expect(cloudStorage.id).toBe(1);
filters.forEach((value, key) => {
expect(cloudStorage[key]).toBe(value);
});
const ids = [1, 2, 3];
await Promise.all(filters.map(async (_, idx) => {
const result = await window.cvat.cloudStorages.get(Object.fromEntries(filters[idx]));
const [cloudStorage] = result;
expect(Array.isArray(result)).toBeTruthy();
expect(result).toHaveLength(1);
expect(cloudStorage).toBeInstanceOf(CloudStorage);
expect(cloudStorage.id).toBe(ids[idx]);
filters[idx].forEach((value, key) => {
expect(cloudStorage[key]).toBe(value);
});
}));
});
test('get cloud storage by invalid filters', async () => {

@ -2551,10 +2551,31 @@ const frameMetaDummyData = {
};
const cloudStoragesDummyData = {
count: 2,
count: 3,
next: null,
previous: null,
results: [
{
id: 3,
owner: {
url: 'http://localhost:7000/api/v1/users/1',
id: 1,
username: 'maya',
first_name: '',
last_name: ''
},
manifests: [
'manifest.jsonl'
],
provider_type: 'GOOGLE_CLOUD_STORAGE',
resource: 'gcsbucket',
display_name: 'Demo GCS',
created_date: '2021-09-01T09:29:47.094244Z',
updated_date: '2021-09-01T09:29:47.103264Z',
credentials_type: 'KEY_FILE_PATH',
specific_attributes: '',
description: 'It is first google cloud storage'
},
{
id: 2,
owner: {

@ -97,8 +97,8 @@ class ServerProxy {
const object = projectsDummyData.results.filter((project) => project.id === id)[0];
for (const prop in projectData) {
if (
Object.prototype.hasOwnProperty.call(projectData, prop)
&& Object.prototype.hasOwnProperty.call(object, prop)
Object.prototype.hasOwnProperty.call(projectData, prop) &&
Object.prototype.hasOwnProperty.call(object, prop)
) {
if (prop === 'labels') {
object[prop] = projectData[prop].filter((label) => !label.deleted);
@ -160,8 +160,8 @@ class ServerProxy {
const object = tasksDummyData.results.filter((task) => task.id === id)[0];
for (const prop in taskData) {
if (
Object.prototype.hasOwnProperty.call(taskData, prop)
&& Object.prototype.hasOwnProperty.call(object, prop)
Object.prototype.hasOwnProperty.call(taskData, prop) &&
Object.prototype.hasOwnProperty.call(object, prop)
) {
if (prop === 'labels') {
object[prop] = taskData[prop].filter((label) => !label.deleted);
@ -249,8 +249,8 @@ class ServerProxy {
for (const prop in jobData) {
if (
Object.prototype.hasOwnProperty.call(jobData, prop)
&& Object.prototype.hasOwnProperty.call(object, prop)
Object.prototype.hasOwnProperty.call(jobData, prop) &&
Object.prototype.hasOwnProperty.call(object, prop)
) {
object[prop] = jobData[prop];
}
@ -339,8 +339,8 @@ class ServerProxy {
if (cloudStorage) {
for (const prop in cloudStorageData) {
if (
Object.prototype.hasOwnProperty.call(cloudStorageData, prop)
&& Object.prototype.hasOwnProperty.call(cloudStorage, prop)
Object.prototype.hasOwnProperty.call(cloudStorageData, prop) &&
Object.prototype.hasOwnProperty.call(cloudStorage, prop)
) {
cloudStorage[prop] = cloudStorageData[prop];
}
@ -375,7 +375,6 @@ class ServerProxy {
}
}
Object.defineProperties(
this,
Object.freeze({

@ -1,12 +1,12 @@
{
"name": "cvat-ui",
"version": "1.27.1",
"version": "1.28.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "cvat-ui",
"version": "1.27.1",
"version": "1.28.0",
"license": "MIT",
"dependencies": {
"@ant-design/icons": "^4.6.3",

@ -1,6 +1,6 @@
{
"name": "cvat-ui",
"version": "1.27.1",
"version": "1.28.0",
"description": "CVAT single-page application",
"main": "src/index.tsx",
"scripts": {

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- The icon received from: https://github.com/gilbarbara/logos -->
<!-- License: CC0-1.0 License -->
<svg width="1em" height="1em" viewBox="0 0 256 206" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
<g>
<path d="M170.2517,56.8186 L192.5047,34.5656 L193.9877,25.1956 C153.4367,-11.6774 88.9757,-7.4964 52.4207,33.9196 C42.2667,45.4226 34.7337,59.7636 30.7167,74.5726 L38.6867,73.4496 L83.1917,66.1106 L86.6277,62.5966 C106.4247,40.8546 139.8977,37.9296 162.7557,56.4286 L170.2517,56.8186 Z" fill="#EA4335"></path>
<path d="M224.2048,73.9182 C219.0898,55.0822 208.5888,38.1492 193.9878,25.1962 L162.7558,56.4282 C175.9438,67.2042 183.4568,83.4382 183.1348,100.4652 L183.1348,106.0092 C198.4858,106.0092 210.9318,118.4542 210.9318,133.8052 C210.9318,149.1572 198.4858,161.2902 183.1348,161.2902 L127.4638,161.2902 L121.9978,167.2242 L121.9978,200.5642 L127.4638,205.7952 L183.1348,205.7952 C223.0648,206.1062 255.6868,174.3012 255.9978,134.3712 C256.1858,110.1682 244.2528,87.4782 224.2048,73.9182" fill="#4285F4"></path>
<path d="M71.8704,205.7957 L127.4634,205.7957 L127.4634,161.2897 L71.8704,161.2897 C67.9094,161.2887 64.0734,160.4377 60.4714,158.7917 L52.5844,161.2117 L30.1754,183.4647 L28.2234,191.0387 C40.7904,200.5277 56.1234,205.8637 71.8704,205.7957" fill="#34A853"></path>
<path d="M71.8704,61.4255 C31.9394,61.6635 -0.2366,94.2275 0.0014,134.1575 C0.1344,156.4555 10.5484,177.4455 28.2234,191.0385 L60.4714,158.7915 C46.4804,152.4705 40.2634,136.0055 46.5844,122.0155 C52.9044,108.0255 69.3704,101.8085 83.3594,108.1285 C89.5244,110.9135 94.4614,115.8515 97.2464,122.0155 L129.4944,89.7685 C115.7734,71.8315 94.4534,61.3445 71.8704,61.4255" fill="#FBBC05"></path>
</g>
</svg>

After

Width:  |  Height:  |  Size: 1.8 KiB

@ -85,6 +85,7 @@ export default function CloudStorageItemComponent(props: Props): JSX.Element {
size='small'
style={style}
className='cvat-cloud-storage-item'
hoverable
>
<Meta
title={(

@ -44,10 +44,14 @@
}
.cvat-cloud-storage-item {
margin-bottom: $grid-unit-size;
div.ant-typography {
margin-bottom: 0;
}
cursor: default;
.cvat-cloud-storage-item-loading-preview,
.cvat-cloud-storage-item-empty-preview {
.ant-spin {

@ -14,20 +14,25 @@ import Select from 'antd/lib/select';
import Input from 'antd/lib/input';
import TextArea from 'antd/lib/input/TextArea';
import notification from 'antd/lib/notification';
import Tooltip from 'antd/lib/tooltip';
import { CombinedState, CloudStorage } from 'reducers/interfaces';
import { createCloudStorageAsync, updateCloudStorageAsync } from 'actions/cloud-storage-actions';
import { ProviderType, CredentialsType } from 'utils/enums';
import { AzureProvider, S3Provider } from '../../icons';
import { DeleteOutlined, UploadOutlined } from '@ant-design/icons';
import Upload, { RcFile } from 'antd/lib/upload';
import { Space } from 'antd';
import { AzureProvider, S3Provider, GoogleCloudProvider } from '../../icons';
import S3Region from './s3-region';
import GCSLocation from './gcs-locatiion';
import ManifestsManager from './manifests-manager';
export interface Props {
cloudStorage?: CloudStorage;
}
type CredentialsFormNames = 'key' | 'secret_key' | 'account_name' | 'session_token';
type CredentialsCamelCaseNames = 'key' | 'secretKey' | 'accountName' | 'sessionToken';
type CredentialsFormNames = 'key' | 'secret_key' | 'account_name' | 'session_token' | 'key_file_path';
type CredentialsCamelCaseNames = 'key' | 'secretKey' | 'accountName' | 'sessionToken' | 'keyFilePath';
interface CloudStorageForm {
credentials_type: CredentialsType;
@ -39,13 +44,18 @@ interface CloudStorageForm {
key?: string;
secret_key?: string;
SAS_token?: string;
key_file_path?: string;
key_file?: File;
description?: string;
region?: string;
prefix?: string;
project_id?: string;
manifests: string[];
}
export default function CreateCloudStorageForm(props: Props): JSX.Element {
const { cloudStorage } = props;
const cloudStorageId = cloudStorage ? cloudStorage.id : null;
const dispatch = useDispatch();
const history = useHistory();
const [form] = Form.useForm();
@ -66,15 +76,22 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
sessionToken: 'X'.repeat(300),
key: 'X'.repeat(20),
secretKey: 'X'.repeat(40),
keyFilePath: 'X'.repeat(10),
};
const [keyVisibility, setKeyVisibility] = useState(false);
const [secretKeyVisibility, setSecretKeyVisibility] = useState(false);
const [sessionTokenVisibility, setSessionTokenVisibility] = useState(false);
const [accountNameVisibility, setAccountNameVisibility] = useState(false);
const [keyFilePathVisibility, setKeyFilePathVisibility] = useState(false);
const [manifestNames, setManifestNames] = useState<string[]>([]);
const [keyFilePathIsDisabled, setKeyFilePathIsDisabled] = useState(false);
const [keyFileIsDisabled, setKeyFileIsDisabled] = useState(false);
const [uploadedKeyFile, setUploadedKeyFile] = useState<File | null>(null);
function initializeFields(): void {
setManifestNames(cloudStorage.manifests);
const fieldsValue: CloudStorageForm = {
@ -95,12 +112,24 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
} else if (cloudStorage.credentialsType === CredentialsType.KEY_SECRET_KEY_PAIR) {
fieldsValue.key = fakeCredentialsData.key;
fieldsValue.secret_key = fakeCredentialsData.secretKey;
} else if (cloudStorage.credentialsType === CredentialsType.KEY_FILE_PATH) {
fieldsValue.key_file_path = fakeCredentialsData.keyFilePath;
}
if (cloudStorage.providerType === ProviderType.AWS_S3_BUCKET && cloudStorage.specificAttributes) {
const region = new URLSearchParams(cloudStorage.specificAttributes).get('region');
if (region) {
setSelectedRegion(region);
if (cloudStorage.specificAttributes) {
const parsedOptions = new URLSearchParams(cloudStorage.specificAttributes);
const location = parsedOptions.get('region') || parsedOptions.get('location');
const prefix = parsedOptions.get('prefix');
const projectId = parsedOptions.get('project_id');
if (location) {
setSelectedRegion(location);
}
if (prefix) {
fieldsValue.prefix = prefix;
}
if (projectId) {
fieldsValue.project_id = projectId;
}
}
@ -161,7 +190,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
}, [updatedCloudStorageId]);
useEffect(() => {
if (cloudStorage && cloudStorage.credentialsType !== CredentialsType.ANONYMOUS_ACCESS) {
if (cloudStorageId && cloudStorage.credentialsType !== CredentialsType.ANONYMOUS_ACCESS) {
notification.info({
message: `For security reasons, your credentials are hidden and represented by fake values
that will not be taken into account when updating the cloud storage.
@ -170,15 +199,37 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
duration: 15,
});
}
}, [cloudStorage]);
}, []);
const onSubmit = async (): Promise<void> => {
let cloudStorageData: Record<string, any> = {};
const formValues = await form.validateFields();
cloudStorageData = { ...formValues };
if (formValues.region !== undefined) {
delete cloudStorageData.region;
cloudStorageData.specific_attributes = `region=${selectedRegion}`;
// specific attributes
const specificAttributes = new URLSearchParams();
if (selectedRegion) {
if (cloudStorageData.provider_type === ProviderType.AWS_S3_BUCKET) {
delete cloudStorageData.region;
specificAttributes.append('region', selectedRegion as string);
} else if (cloudStorageData.provider_type === ProviderType.GOOGLE_CLOUD_STORAGE) {
delete cloudStorageData.location;
specificAttributes.append('location', selectedRegion as string);
}
}
if (formValues.prefix) {
delete cloudStorageData.prefix;
specificAttributes.append('prefix', formValues.prefix);
}
if (formValues.project_id) {
delete cloudStorageData.project_id;
specificAttributes.append('project_id', formValues.project_id);
}
cloudStorageData.specific_attributes = specificAttributes.toString();
if (uploadedKeyFile) {
cloudStorageData.key_file = uploadedKeyFile;
}
if (cloudStorageData.credentials_type === CredentialsType.ACCOUNT_NAME_TOKEN_PAIR) {
@ -195,6 +246,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
if (cloudStorage) {
cloudStorageData.id = cloudStorage.id;
if (cloudStorageData.account_name === fakeCredentialsData.accountName) {
delete cloudStorageData.account_name;
}
@ -207,6 +259,9 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
if (cloudStorageData.session_token === fakeCredentialsData.sessionToken) {
delete cloudStorageData.session_token;
}
if (cloudStorageData.key_file_path === fakeCredentialsData.keyFilePath) {
delete cloudStorageData.key_file_path;
}
dispatch(updateCloudStorageAsync(cloudStorageData));
} else {
dispatch(createCloudStorageAsync(cloudStorageData));
@ -219,6 +274,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
secret_key: undefined,
session_token: undefined,
account_name: undefined,
key_file_path: undefined,
});
};
@ -264,7 +320,6 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
const internalCommonProps = {
...commonProps,
labelCol: { span: 8, offset: 2 },
wrapperCol: { offset: 1 },
};
if (providerType === ProviderType.AWS_S3_BUCKET && credentialsType === CredentialsType.KEY_SECRET_KEY_PAIR) {
@ -320,8 +375,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
visibilityToggle={accountNameVisibility}
onChange={() => setAccountNameVisibility(true)}
onFocus={() => onFocusCredentialsItem('accountName', 'account_name')}
onBlur={() =>
onBlurCredentialsItem('accountName', 'account_name', setAccountNameVisibility)}
onBlur={() => onBlurCredentialsItem('accountName', 'account_name', setAccountNameVisibility)}
/>
</Form.Item>
<Form.Item
@ -335,8 +389,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
maxLength={437}
onChange={() => setSessionTokenVisibility(true)}
onFocus={() => onFocusCredentialsItem('sessionToken', 'session_token')}
onBlur={() =>
onBlurCredentialsItem('sessionToken', 'session_token', setSessionTokenVisibility)}
onBlur={() => onBlurCredentialsItem('sessionToken', 'session_token', setSessionTokenVisibility)}
/>
</Form.Item>
</>
@ -363,6 +416,72 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
);
}
if (providerType === ProviderType.GOOGLE_CLOUD_STORAGE && credentialsType === CredentialsType.KEY_FILE_PATH) {
return (
<Form.Item
{...internalCommonProps}
label={(
<Tooltip title='You can specify path to key file or upload key file.
If you leave these fields blank, the environment variable will be used.'
>
Key file
</Tooltip>
)}
>
<Space align='start' className='cvat-cloud-storage-form-item-key-file'>
<Form.Item
name='key_file_path'
noStyle
>
<Input.Password
visibilityToggle={keyFilePathVisibility}
onChange={(e) => {
setKeyFilePathVisibility(true);
const isDisabled = !!(e.target.value);
setKeyFileIsDisabled(isDisabled);
}}
onFocus={() => onFocusCredentialsItem('keyFilePath', 'key_file_path')}
onBlur={() => onBlurCredentialsItem('keyFilePath', 'key_file_path', setKeyFilePathVisibility)}
disabled={keyFilePathIsDisabled}
/>
</Form.Item>
<Tooltip title='Attach a file'>
<Upload
accept='.json, application/json'
multiple={false}
maxCount={1}
showUploadList={false}
beforeUpload={(file: RcFile): boolean => {
if (form.getFieldValue('key_file_path')) {
form.setFieldsValue({
key_file_path: undefined,
});
}
setKeyFilePathIsDisabled(true);
setUploadedKeyFile(file);
return false;
}}
>
<Button icon={<UploadOutlined />} disabled={keyFileIsDisabled} />
</Upload>
</Tooltip>
<Tooltip title='Delete an uploaded file'>
<Button
icon={<DeleteOutlined />}
disabled={keyFileIsDisabled}
onClick={() => {
setKeyFilePathIsDisabled(false);
setUploadedKeyFile(null);
}}
/>
</Tooltip>
</Space>
</Form.Item>
);
}
return <></>;
};
@ -442,6 +561,61 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
);
};
const GoogleCloudStorageConfiguration = (): JSX.Element => {
const internalCommonProps = {
...commonProps,
labelCol: { span: 6, offset: 1 },
wrapperCol: { offset: 1 },
};
return (
<>
<Form.Item
label='Bucket name'
name='resource'
rules={[{ required: true, message: 'Please, specify a bucket name' }]}
{...internalCommonProps}
>
{/* maxlength https://cloud.google.com/storage/docs/naming-buckets#requirements */}
<Input disabled={!!cloudStorage} maxLength={222} />
</Form.Item>
<Form.Item
label='Authorization type'
name='credentials_type'
rules={[{ required: true, message: 'Please, specify credentials type' }]}
{...internalCommonProps}
>
<Select onSelect={(value: CredentialsType) => onChangeCredentialsType(value)}>
<Select.Option value={CredentialsType.KEY_FILE_PATH}>
Key file
</Select.Option>
<Select.Option value={CredentialsType.ANONYMOUS_ACCESS}>Anonymous access</Select.Option>
</Select>
</Form.Item>
{credentialsBlok()}
<Form.Item
label='Prefix'
name='prefix'
{...internalCommonProps}
>
<Input />
</Form.Item>
<Form.Item
label='Project ID'
name='project_id'
{...internalCommonProps}
>
<Input />
</Form.Item>
<GCSLocation
selectedRegion={selectedRegion}
onSelectRegion={onSelectRegion}
internalCommonProps={internalCommonProps}
/>
</>
);
};
return (
<Form className='cvat-cloud-storage-form' layout='horizontal' form={form}>
<Form.Item
@ -481,10 +655,17 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
Azure Blob Container
</span>
</Select.Option>
<Select.Option value={ProviderType.GOOGLE_CLOUD_STORAGE}>
<span className='cvat-cloud-storage-select-provider'>
<GoogleCloudProvider />
Google Cloud Storage
</span>
</Select.Option>
</Select>
</Form.Item>
{providerType === ProviderType.AWS_S3_BUCKET && AWSS3Configuration()}
{providerType === ProviderType.AZURE_CONTAINER && AzureBlobStorageConfiguration()}
{providerType === ProviderType.GOOGLE_CLOUD_STORAGE && GoogleCloudStorageConfiguration()}
<ManifestsManager form={form} manifestNames={manifestNames} setManifestNames={setManifestNames} />
<Row justify='end'>
<Col>

@ -0,0 +1,31 @@
// Copyright (C) 2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
import React from 'react';
import Location from './location';
import consts from '../../consts';
interface Props {
selectedRegion: any;
onSelectRegion: any;
internalCommonProps: any;
}
export default function GCSLocation(props: Props): JSX.Element {
const {
selectedRegion,
onSelectRegion,
internalCommonProps,
} = props;
return (
<Location
selectedRegion={selectedRegion}
onSelectRegion={onSelectRegion}
internalCommonProps={internalCommonProps}
values={consts.DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS}
name='location'
label='Location'
href='https://cloud.google.com/storage/docs/locations#available-locations'
/>
);
}

@ -0,0 +1,123 @@
// Copyright (C) 2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
import React, { useState } from 'react';
import Divider from 'antd/lib/divider';
import Select from 'antd/lib/select';
import { PlusCircleOutlined, QuestionCircleOutlined } from '@ant-design/icons';
import Input from 'antd/lib/input';
import Button from 'antd/lib/button';
import Form from 'antd/lib/form';
import notification from 'antd/lib/notification';
import Tooltip from 'antd/lib/tooltip';
const { Option } = Select;
interface Props {
selectedRegion: undefined | string;
onSelectRegion: any;
internalCommonProps: any;
label: 'Location' | 'Region';
name: 'location' | 'region';
values: string[][];
href: string;
}
interface Locations {
[index: string]: string;
}
export default function Location(props: Props): JSX.Element {
const {
selectedRegion, onSelectRegion, internalCommonProps, name, values, href, label,
} = props;
const [locations, setLocations] = useState<Locations>(() => Object.fromEntries(values));
const [newRegionKey, setNewRegionKey] = useState<string>('');
const [newRegionName, setNewRegionName] = useState<string>('');
const handleAddingRegion = (): void => {
if (!newRegionKey || !newRegionName) {
notification.warning({
message: 'Incorrect region',
className: 'cvat-incorrect-add-region-notification',
});
} else if (locations[newRegionKey]) {
notification.warning({
message: 'This region already exists',
className: 'cvat-incorrect-add-region-notification',
});
} else {
setLocations({
...locations,
[newRegionKey]: newRegionName,
});
setNewRegionKey('');
setNewRegionName('');
}
};
return (
<Form.Item
label={(
<>
{label}
<Tooltip title='More information'>
<Button
className='cvat-cloud-storage-help-button'
type='link'
target='_blank'
href={href}
>
<QuestionCircleOutlined />
</Button>
</Tooltip>
</>
)}
name={name}
{...internalCommonProps}
>
<Select
placeholder={name}
defaultValue={selectedRegion ? locations[selectedRegion] : undefined}
dropdownRender={(menu) => (
<div>
{menu}
<Divider className='cvat-divider' />
<div className='cvat-cloud-storage-region-creator'>
<Input
value={newRegionKey}
onChange={(event: any) => setNewRegionKey(event.target.value)}
maxLength={14}
placeholder='key'
/>
<Input
value={newRegionName}
onChange={(event: any) => setNewRegionName(event.target.value)}
placeholder='name'
/>
<Button
type='link'
onClick={handleAddingRegion}
>
Add region
<PlusCircleOutlined />
</Button>
</div>
</div>
)}
onSelect={(_, instance) => onSelectRegion(instance.key)}
>
{
Array.from(Object.entries(locations)).map(
([key, value]): JSX.Element => (
<Option key={key} value={value}>
{value}
</Option>
),
)
}
</Select>
</Form.Item>
);
}

@ -1,119 +1,31 @@
// Copyright (C) 2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
import React, { useState } from 'react';
import Divider from 'antd/lib/divider';
import Select from 'antd/lib/select';
import { PlusCircleOutlined, QuestionCircleOutlined } from '@ant-design/icons';
import Input from 'antd/lib/input';
import Button from 'antd/lib/button';
import Form from 'antd/lib/form';
import notification from 'antd/lib/notification';
import Tooltip from 'antd/lib/tooltip';
import React from 'react';
import Location from './location';
import consts from '../../consts';
const { Option } = Select;
interface Props {
selectedRegion: undefined | string;
selectedRegion: any;
onSelectRegion: any;
internalCommonProps: any;
}
function prepareDefaultRegions(): Map<string, string> {
const temp = new Map<string, string>();
for (const [key, value] of consts.DEFAULT_AWS_S3_REGIONS) {
temp.set(key, value);
}
return temp;
}
export default function S3Region(props: Props): JSX.Element {
const { selectedRegion, onSelectRegion, internalCommonProps } = props;
const [regions, setRegions] = useState<Map<string, string>>(() => prepareDefaultRegions());
const [newRegionKey, setNewRegionKey] = useState<string>('');
const [newRegionName, setNewRegionName] = useState<string>('');
const handleAddingRegion = (): void => {
if (!newRegionKey || !newRegionName) {
notification.warning({
message: 'Incorrect region',
className: 'cvat-incorrect-add-region-notification',
});
} else if (regions.has(newRegionKey)) {
notification.warning({
message: 'This region already exists',
className: 'cvat-incorrect-add-region-notification',
});
} else {
const regionsCopy = regions;
setRegions(regionsCopy.set(newRegionKey, newRegionName));
setNewRegionKey('');
setNewRegionName('');
}
};
const {
selectedRegion,
onSelectRegion,
internalCommonProps,
} = props;
return (
<Form.Item
label={(
<>
Region
<Tooltip title='More information'>
<Button
className='cvat-cloud-storage-help-button'
type='link'
target='_blank'
href='https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions'
>
<QuestionCircleOutlined />
</Button>
</Tooltip>
</>
)}
<Location
selectedRegion={selectedRegion}
onSelectRegion={onSelectRegion}
internalCommonProps={internalCommonProps}
values={consts.DEFAULT_AWS_S3_REGIONS}
name='region'
{...internalCommonProps}
>
<Select
placeholder='Select region'
defaultValue={selectedRegion ? regions.get(selectedRegion) : undefined}
dropdownRender={(menu) => (
<div>
{menu}
<Divider className='cvat-divider' />
<div className='cvat-cloud-storage-region-creator'>
<Input
value={newRegionKey}
onChange={(event: any) => setNewRegionKey(event.target.value)}
maxLength={14}
placeholder='key'
/>
<Input
value={newRegionName}
onChange={(event: any) => setNewRegionName(event.target.value)}
placeholder='name'
/>
<Button
type='link'
onClick={handleAddingRegion}
>
Add region
<PlusCircleOutlined />
</Button>
</div>
</div>
)}
onSelect={(_, instance) => onSelectRegion(instance.key)}
>
{
Array.from(regions.entries()).map(
([key, value]): JSX.Element => (
<Option key={key} value={value}>
{value}
</Option>
),
)
}
</Select>
</Form.Item>
label='Region'
href='https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions'
/>
);
}

@ -35,6 +35,14 @@
}
}
.cvat-cloud-storage-form-item-key-file {
width: 100%;
:nth-child(1) {
flex-grow: 1;
}
}
> div:not(first-child) {
margin-top: $grid-unit-size;
}

@ -13,7 +13,7 @@ import { debounce } from 'lodash';
import Select from 'antd/lib/select';
import getCore from 'cvat-core-wrapper';
import { CloudStorage } from 'reducers/interfaces';
import { AzureProvider, S3Provider } from 'icons';
import { AzureProvider, GoogleCloudProvider, S3Provider } from 'icons';
import { ProviderType } from 'utils/enums';
import CloudStorageFiles from './cloud-storages-files';
@ -125,11 +125,12 @@ export default function CloudStorageTab(props: Props): JSX.Element {
<span
className='cvat-cloud-storage-select-provider'
>
{_cloudStorage.providerType === ProviderType.AWS_S3_BUCKET ? (
<S3Provider />
) : (
<AzureProvider />
)}
{_cloudStorage.providerType === ProviderType.AWS_S3_BUCKET && <S3Provider />}
{_cloudStorage.providerType === ProviderType.AZURE_CONTAINER && <AzureProvider />}
{
_cloudStorage.providerType === ProviderType.GOOGLE_CLOUD_STORAGE &&
<GoogleCloudProvider />
}
{_cloudStorage.displayName}
</span>
),

@ -67,6 +67,8 @@ export default function SearchTooltip(props: Props): JSX.Element {
<q>AWS_S3_BUCKET</q>
or
<q>AZURE_CONTAINER</q>
or
<q>GOOGLE_CLOUD_STORAGE</q>
</Text>
</Paragraph>
) : null}
@ -78,6 +80,8 @@ export default function SearchTooltip(props: Props): JSX.Element {
or
<q>ACCOUNT_NAME_TOKEN_PAIR</q>
or
<q>KEY_FILE_PATH</q>
or
<q>ANONYMOUS_ACCESS</q>
</Text>
</Paragraph>

@ -45,6 +45,45 @@ const DEFAULT_AWS_S3_REGIONS: string[][] = [
['sa-east-1', 'South America (São Paulo)'],
];
const DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS: string[][] = [
['NORTHAMERICA-NORTHEAST1', 'Montréal'],
['NORTHAMERICA-NORTHEAST2', 'Toronto'],
['US-CENTRAL1', 'Iowa'],
['US-EAST1', 'South Carolina'],
['US-EAST4', 'Northern Virginia'],
['US-WEST1', 'Oregon'],
['US-WEST2', 'Los Angeles'],
['US-WEST3', 'Salt Lake City'],
['US-WEST4', 'Las Vegas'],
['SOUTHAMERICA-EAST1', 'São Paulo'],
['EUROPE-CENTRAL2', 'Warsaw'],
['EUROPE-NORTH1', 'Finland'],
['EUROPE-WEST1', 'Belgium'],
['EUROPE-WEST2', 'London'],
['EUROPE-WEST3', 'Frankfurt'],
['EUROPE-WEST4', 'Netherlands'],
['EUROPE-WEST6', 'Zürich'],
['ASIA-EAST1', 'Taiwan'],
['ASIA-EAST2', 'Hong Kong'],
['ASIA-NORTHEAST1', 'Tokyo'],
['ASIA-NORTHEAST2', 'Osaka'],
['ASIA-NORTHEAST3', 'Seoul'],
['ASIA-SOUTH1', 'Mumbai'],
['ASIA-SOUTH2', 'Delhi'],
['ASIA-SOUTHEAST1', 'Singapore'],
['ASIA-SOUTHEAST2', 'Jakarta'],
['AUSTRALIA-SOUTHEAST1', 'Sydney'],
['AUSTRALIA-SOUTHEAST2', 'Melbourne'],
// Multi-regions
['ASIA', 'Data centers in Asia'],
['EU', 'Data centers within member states of the European Union'],
['US', 'Data centers in the United States'],
// Dual-regions
['ASIA1', 'ASIA-NORTHEAST1 and ASIA-NORTHEAST2'],
['EUR4', 'EUROPE-NORTH1 and EUROPE-WEST4'],
['NAM4', 'US-CENTRAL1 and US-EAST1'],
];
export default {
UNDEFINED_ATTRIBUTE_VALUE,
NO_BREAK_SPACE,
@ -67,4 +106,5 @@ export default {
INTEL_COOKIES_URL,
INTEL_PRIVACY_URL,
DEFAULT_AWS_S3_REGIONS,
DEFAULT_GOOGLE_CLOUD_STORAGE_LOCATIONS,
};

@ -52,6 +52,7 @@ import SVGOpenCV from './assets/opencv.svg';
import SVGFilterIcon from './assets/object-filter-icon.svg';
import SVGCVATAzureProvider from './assets/vscode-icons_file-type-azure.svg';
import SVGCVATS3Provider from './assets/S3.svg';
import SVGCVATGoogleCloudProvider from './assets/google-cloud.svg';
export const CVATLogo = React.memo((): JSX.Element => <SVGCVATLogo />);
export const AccountIcon = React.memo((): JSX.Element => <SVGAccountIcon />);
@ -101,3 +102,4 @@ export const OpenCVIcon = React.memo((): JSX.Element => <SVGOpenCV />);
export const FilterIcon = React.memo((): JSX.Element => <SVGFilterIcon />);
export const AzureProvider = React.memo((): JSX.Element => <SVGCVATAzureProvider />);
export const S3Provider = React.memo((): JSX.Element => <SVGCVATS3Provider />);
export const GoogleCloudProvider = React.memo((): JSX.Element => <SVGCVATGoogleCloudProvider />);

@ -5,12 +5,14 @@
export enum ProviderType {
AWS_S3_BUCKET = 'AWS_S3_BUCKET',
AZURE_CONTAINER = 'AZURE_CONTAINER',
GOOGLE_CLOUD_STORAGE = 'GOOGLE_CLOUD_STORAGE',
}
export enum CredentialsType {
KEY_SECRET_KEY_PAIR = 'KEY_SECRET_KEY_PAIR',
ACCOUNT_NAME_TOKEN_PAIR = 'ACCOUNT_NAME_TOKEN_PAIR',
ANONYMOUS_ACCESS = 'ANONYMOUS_ACCESS',
KEY_FILE_PATH = 'KEY_FILE_PATH',
}
export enum StorageStatuses {

@ -119,6 +119,7 @@ def get_cloud_storage_instance(cloud_provider, resource, credentials, specific_a
instance = GoogleCloudStorage(
bucket_name=resource,
service_account_json=credentials.key_file_path,
anonymous_access = credentials.credentials_type == CredentialsTypeChoice.ANONYMOUS_ACCESS,
prefix=specific_attributes.get('prefix'),
location=specific_attributes.get('location'),
project=specific_attributes.get('project')
@ -356,18 +357,18 @@ def _define_gcs_status(func):
class GoogleCloudStorage(_CloudStorage):
def __init__(self, bucket_name, prefix=None, service_account_json=None, project=None, location=None):
def __init__(self, bucket_name, prefix=None, service_account_json=None, anonymous_access=False, project=None, location=None):
super().__init__()
if service_account_json:
self._storage_client = storage.Client.from_service_account_json(service_account_json)
elif anonymous_access:
self._storage_client = storage.Client.create_anonymous_client()
else:
# If no credentials were provided when constructing the client, the
# client library will look for credentials in the environment.
self._storage_client = storage.Client()
bucket = self._storage_client.lookup_bucket(bucket_name)
if bucket is None:
bucket = self._storage_client.bucket(bucket_name, user_project=project)
self._bucket = bucket
self._bucket = self._storage_client.bucket(bucket_name, user_project=project)
self._bucket_location = location
self._prefix = prefix
@ -464,7 +465,6 @@ class Credentials:
elif self.credentials_type == CredentialsTypeChoice.ACCOUNT_NAME_TOKEN_PAIR:
self.account_name, self.session_token = credentials.get('value').split()
elif self.credentials_type == CredentialsTypeChoice.ANONYMOUS_ACCESS:
self.session_token, self.key, self.secret_key = ('', '', '')
# account_name will be in [some_value, '']
self.account_name = credentials.get('value')
elif self.credentials_type == CredentialsTypeChoice.KEY_FILE_PATH:
@ -472,31 +472,25 @@ class Credentials:
else:
raise NotImplementedError('Found {} not supported credentials type'.format(self.credentials_type))
def reset(self, exclusion):
for i in set(self.__slots__) - exclusion - {'credentials_type'}:
self.__setattr__(i, '')
def mapping_with_new_values(self, credentials):
self.credentials_type = credentials.get('credentials_type', self.credentials_type)
if self.credentials_type == CredentialsTypeChoice.ANONYMOUS_ACCESS:
self.key = ''
self.secret_key = ''
self.session_token = ''
self.key_file_path = ''
self.reset(exclusion={'account_name'})
self.account_name = credentials.get('account_name', self.account_name)
elif self.credentials_type == CredentialsTypeChoice.KEY_SECRET_KEY_PAIR:
self.reset(exclusion={'key', 'secret_key'})
self.key = credentials.get('key', self.key)
self.secret_key = credentials.get('secret_key', self.secret_key)
self.session_token = ''
self.account_name = ''
self.key_file_path = ''
elif self.credentials_type == CredentialsTypeChoice.ACCOUNT_NAME_TOKEN_PAIR:
self.reset(exclusion={'session_token', 'account_name'})
self.session_token = credentials.get('session_token', self.session_token)
self.account_name = credentials.get('account_name', self.account_name)
self.key = ''
self.secret_key = ''
self.key_file_path = ''
elif self.credentials_type == CredentialsTypeChoice.KEY_FILE_PATH:
self.key = ''
self.secret_key = ''
self.session_token = ''
self.account_name = ''
self.reset(exclusion={'key_file_path'})
self.key_file_path = credentials.get('key_file_path', self.key_file_path)
else:
raise NotImplementedError('Mapping credentials: unsupported credentials type')

@ -0,0 +1,23 @@
# Generated by Django 3.1.13 on 2021-11-23 08:24
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('engine', '0044_auto_20211115_0858'),
]
operations = [
migrations.AlterField(
model_name='cloudstorage',
name='resource',
field=models.CharField(max_length=222),
),
migrations.AlterField(
model_name='cloudstorage',
name='specific_attributes',
field=models.CharField(blank=True, max_length=128),
),
]

@ -585,15 +585,19 @@ class Manifest(models.Model):
class CloudStorage(models.Model):
# restrictions:
# AWS bucket name, Azure container name - 63
# AWS bucket name, Azure container name - 63, Google bucket name - 63 without dots and 222 with dots
# https://cloud.google.com/storage/docs/naming-buckets#requirements
# AWS access key id - 20
# AWS secret access key - 40
# AWS temporary session tocken - None
# The size of the security token that AWS STS API operations return is not fixed.
# We strongly recommend that you make no assumptions about the maximum size.
# The typical token size is less than 4096 bytes, but that can vary.
# specific attributes:
# location - max 23
# project ID: 6 - 30 (https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin)
provider_type = models.CharField(max_length=20, choices=CloudProviderChoice.choices())
resource = models.CharField(max_length=63)
resource = models.CharField(max_length=222)
display_name = models.CharField(max_length=63)
owner = models.ForeignKey(User, null=True, blank=True,
on_delete=models.SET_NULL, related_name="cloud_storages")
@ -601,7 +605,7 @@ class CloudStorage(models.Model):
updated_date = models.DateTimeField(auto_now=True)
credentials = models.CharField(max_length=500)
credentials_type = models.CharField(max_length=29, choices=CredentialsTypeChoice.choices())#auth_type
specific_attributes = models.CharField(max_length=50, blank=True)
specific_attributes = models.CharField(max_length=128, blank=True)
description = models.TextField(blank=True)
class Meta:
@ -625,3 +629,6 @@ class CloudStorage(models.Model):
def get_specific_attributes(self):
return parse_specific_attributes(self.specific_attributes)
def get_key_file_path(self):
return os.path.join(self.get_storage_dirname(), 'key.json')

@ -6,6 +6,8 @@ import os
import re
import shutil
from tempfile import NamedTemporaryFile
from rest_framework import serializers, exceptions
from django.contrib.auth.models import User, Group
@ -798,6 +800,7 @@ class CloudStorageSerializer(serializers.ModelSerializer):
key = serializers.CharField(max_length=20, allow_blank=True, required=False)
secret_key = serializers.CharField(max_length=40, allow_blank=True, required=False)
key_file_path = serializers.CharField(max_length=64, allow_blank=True, required=False)
key_file = serializers.FileField(required=False)
account_name = serializers.CharField(max_length=24, allow_blank=True, required=False)
manifests = ManifestSerializer(many=True, default=[])
@ -806,7 +809,8 @@ class CloudStorageSerializer(serializers.ModelSerializer):
fields = (
'provider_type', 'resource', 'display_name', 'owner', 'credentials_type',
'created_date', 'updated_date', 'session_token', 'account_name', 'key',
'secret_key', 'key_file_path', 'specific_attributes', 'description', 'id', 'manifests',
'secret_key', 'key_file_path', 'key_file', 'specific_attributes',
'description', 'id', 'manifests',
)
read_only_fields = ('created_date', 'updated_date', 'owner')
@ -820,20 +824,33 @@ class CloudStorageSerializer(serializers.ModelSerializer):
return value
def validate(self, attrs):
if attrs.get('provider_type') == models.CloudProviderChoice.AZURE_CONTAINER:
provider_type = attrs.get('provider_type')
if provider_type == models.CloudProviderChoice.AZURE_CONTAINER:
if not attrs.get('account_name', ''):
raise serializers.ValidationError('Account name for Azure container was not specified')
if attrs.get('key_file', '') and attrs.get('key_file_path', ''):
raise serializers.ValidationError('Should be specified key file or key file path')
return attrs
def create(self, validated_data):
provider_type = validated_data.get('provider_type')
should_be_created = validated_data.pop('should_be_created', None)
key_file = validated_data.pop('key_file', None)
# we need to save it to temporary file to check the granted permissions
temporary_file = ''
if key_file:
with NamedTemporaryFile(mode='wb', prefix='cvat', delete=False) as temp_key:
temp_key.write(key_file.read())
temporary_file = temp_key.name
key_file.close()
del key_file
credentials = Credentials(
account_name=validated_data.pop('account_name', ''),
key=validated_data.pop('key', ''),
secret_key=validated_data.pop('secret_key', ''),
session_token=validated_data.pop('session_token', ''),
key_file_path=validated_data.pop('key_file_path', ''),
key_file_path=validated_data.pop('key_file_path', '') or temporary_file,
credentials_type = validated_data.get('credentials_type')
)
details = {
@ -880,6 +897,15 @@ class CloudStorageSerializer(serializers.ModelSerializer):
shutil.rmtree(cloud_storage_path)
os.makedirs(db_storage.get_storage_logs_dirname(), exist_ok=True)
if temporary_file:
# so, gcs key file is valid and we need to set correct path to the file
real_path_to_key_file = db_storage.get_key_file_path()
shutil.copyfile(temporary_file, real_path_to_key_file)
os.remove(temporary_file)
credentials.key_file_path = real_path_to_key_file
db_storage.credentials = credentials.convert_to_db()
db_storage.save()
return db_storage
elif storage_status == Status.FORBIDDEN:
field = 'credentials'
@ -887,6 +913,8 @@ class CloudStorageSerializer(serializers.ModelSerializer):
else:
field = 'recource'
message = 'The resource {} not found. It may have been deleted.'.format(storage.name)
if temporary_file:
os.remove(temporary_file)
slogger.glob.error(message)
raise serializers.ValidationError({field: message})
@ -897,8 +925,23 @@ class CloudStorageSerializer(serializers.ModelSerializer):
'type': instance.credentials_type,
'value': instance.credentials,
})
tmp = {k:v for k,v in validated_data.items() if k in {'key', 'secret_key', 'account_name', 'session_token', 'key_file_path', 'credentials_type'}}
credentials.mapping_with_new_values(tmp)
credentials_dict = {k:v for k,v in validated_data.items() if k in {
'key','secret_key', 'account_name', 'session_token', 'key_file_path',
'credentials_type'
}}
key_file = validated_data.pop('key_file', None)
temporary_file = ''
if key_file:
with NamedTemporaryFile(mode='wb', prefix='cvat', delete=False) as temp_key:
temp_key.write(key_file.read())
temporary_file = temp_key.name
# pair (key_file, key_file_path) isn't supported by server, so only one value may be specified
credentials_dict['key_file_path'] = temporary_file
key_file.close()
del key_file
credentials.mapping_with_new_values(credentials_dict)
instance.credentials = credentials.convert_to_db()
instance.credentials_type = validated_data.get('credentials_type', instance.credentials_type)
instance.resource = validated_data.get('resource', instance.resource)
@ -937,6 +980,13 @@ class CloudStorageSerializer(serializers.ModelSerializer):
})
manifest_instances = [models.Manifest(filename=f, cloud_storage=instance) for f in delta_to_create]
models.Manifest.objects.bulk_create(manifest_instances)
if temporary_file:
# so, gcs key file is valid and we need to set correct path to the file
real_path_to_key_file = instance.get_key_file_path()
shutil.copyfile(temporary_file, real_path_to_key_file)
os.remove(temporary_file)
instance.credentials = real_path_to_key_file
instance.save()
return instance
elif storage_status == Status.FORBIDDEN:
@ -945,6 +995,8 @@ class CloudStorageSerializer(serializers.ModelSerializer):
else:
field = 'recource'
message = 'The resource {} not found. It may have been deleted.'.format(storage.name)
if temporary_file:
os.remove(temporary_file)
slogger.glob.error(message)
raise serializers.ValidationError({field: message})

@ -1334,6 +1334,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS
)
@action(detail=True, methods=['GET'], url_path='content')
def content(self, request, pk):
storage = None
try:
db_storage = CloudStorageModel.objects.get(pk=pk)
credentials = Credentials()
@ -1378,7 +1379,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS
return Response(data=msg, status=status.HTTP_404_NOT_FOUND)
except Exception as ex:
# check that cloud storage was not deleted
storage_status = storage.get_status()
storage_status = storage.get_status() if storage else None
if storage_status == Status.FORBIDDEN:
msg = 'The resource {} is no longer available. Access forbidden.'.format(storage.name)
elif storage_status == Status.NOT_FOUND:
@ -1397,6 +1398,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS
)
@action(detail=True, methods=['GET'], url_path='preview')
def preview(self, request, pk):
storage = None
try:
db_storage = CloudStorageModel.objects.get(pk=pk)
if not os.path.exists(db_storage.get_preview_path()):
@ -1455,7 +1457,7 @@ class CloudStorageViewSet(auth.CloudStorageGetQuerySetMixin, viewsets.ModelViewS
return HttpResponseNotFound(message)
except Exception as ex:
# check that cloud storage was not deleted
storage_status = storage.get_status()
storage_status = storage.get_status() if storage else None
if storage_status == Status.FORBIDDEN:
msg = 'The resource {} is no longer available. Access forbidden.'.format(storage.name)
elif storage_status == Status.NOT_FOUND:

Loading…
Cancel
Save