wangchunlin
/
cvat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Git different formats (#3708)

Browse Source
main
PMazarovich 4 years ago committed by GitHub
parent 7b40b84cc8
commit fb6305fff8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 110 additions and 25 deletions
Show Stats Download Patch File Download Diff File

1
cvat-ui/src/actions/tasks-actions.ts
Unescape Escape View File

@ -404,6 +404,7 @@ export function createTaskAsync(data: any): ThunkAction<Promise<void>, {}, {}, A
};
gitPlugin.data.task = taskInstance;
gitPlugin.data.repos = data.advanced.repository;
gitPlugin.data.format = data.advanced.format;
gitPlugin.data.lfs = data.advanced.lfs;
}
}

28
cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx
Unescape Escape View File

@ -6,14 +6,16 @@ import React, { RefObject } from 'react';
import { Row, Col } from 'antd/lib/grid';
import { PercentageOutlined } from '@ant-design/icons';
import Input from 'antd/lib/input';
import Select from 'antd/lib/select';
import Checkbox from 'antd/lib/checkbox';
import Form, { FormInstance, RuleObject, RuleRender } from 'antd/lib/form';
import Text from 'antd/lib/typography/Text';
import { Store } from 'antd/lib/form/interface';
import CVATTooltip from 'components/common/cvat-tooltip';
import patterns from 'utils/validation-patterns';
const { Option } = Select;
export interface AdvancedConfiguration {
bugTracker?: string;
imageQuality?: number;
@ -23,6 +25,7 @@ export interface AdvancedConfiguration {
stopFrame?: number;
frameFilter?: string;
lfs: boolean;
format?: string,
repository?: string;
useZipChunks: boolean;
dataChunkSize?: number;
@ -42,6 +45,7 @@ interface Props {
onSubmit(values: AdvancedConfiguration): void;
installedGit: boolean;
activeFileManagerTab: string;
dumpers: []
}
function validateURL(_: RuleObject, value: string): Promise<void> {
@ -276,15 +280,37 @@ class AdvancedConfigurationForm extends React.PureComponent<Props> {
);
}
private renderGitFormat(): JSX.Element {
const { dumpers } = this.props;
return (
<Form.Item
initialValue='CVAT for video 1.1'
name='format'
label='Choose format'
>
<Select style={{ width: '100%' }} initialValue='CVAT for video 1.1'>
{
dumpers.map((dumper: any) =>
<Option value={dumper.name}>{dumper.name}</Option>)
}
</Select>
</Form.Item>
);
}
private renderGit(): JSX.Element {
return (
<>
<Row>
<Col span={24}>{this.renderGitRepositoryURL()}</Col>
</Row>
<Row>
<Col span={24}>{this.renderGitFormat()}</Col>
</Row>
<Row>
<Col span={24}>{this.renderGitLFSBox()}</Col>
</Row>
</>
);
}

4
cvat-ui/src/components/create-task-page/create-task-content.tsx
Unescape Escape View File

@ -39,6 +39,7 @@ interface Props {
taskId: number | null;
projectId: number | null;
installedGit: boolean;
dumpers:[]
}
type State = CreateTaskData;
@ -311,13 +312,14 @@ class CreateTaskContent extends React.PureComponent<Props & RouteComponentProps,
}
private renderAdvancedBlock(): JSX.Element {
const { installedGit } = this.props;
const { installedGit, dumpers } = this.props;
const { activeFileManagerTab } = this.state;
return (
<Col span={24}>
<Collapse>
<Collapse.Panel key='1' header={<Text className='cvat-title'>Advanced configuration</Text>}>
<AdvancedConfigurationForm
dumpers={dumpers}
installedGit={installedGit}
activeFileManagerTab={activeFileManagerTab}
ref={this.advancedConfigurationComponent}

4
cvat-ui/src/components/create-task-page/create-task-page.tsx
Unescape Escape View File

@ -19,11 +19,12 @@ interface Props {
error: string;
taskId: number | null;
installedGit: boolean;
dumpers: []
}
export default function CreateTaskPage(props: Props): JSX.Element {
const {
error, status, taskId, onCreate, installedGit,
error, status, taskId, onCreate, installedGit, dumpers,
} = props;
const location = useLocation();
@ -79,6 +80,7 @@ export default function CreateTaskPage(props: Props): JSX.Element {
status={status}
onCreate={onCreate}
installedGit={installedGit}
dumpers={dumpers}
/>
</Col>
</Row>

15
cvat-ui/src/components/task-page/details.tsx
Unescape Escape View File

@ -12,11 +12,11 @@ import Text from 'antd/lib/typography/Text';
import Title from 'antd/lib/typography/Title';
import moment from 'moment';
import Descriptions from 'antd/lib/descriptions';
import getCore from 'cvat-core-wrapper';
import { getReposData, syncRepos } from 'utils/git-utils';
import { ActiveInference } from 'reducers/interfaces';
import AutomaticAnnotationProgress from 'components/tasks-page/automatic-annotation-progress';
import Descriptions from 'antd/lib/descriptions';
import UserSelector, { User } from './user-selector';
import BugTrackerEditor from './bug-tracker-editor';
import LabelsEditorComponent from '../labels-editor/labels-editor';
@ -39,6 +39,7 @@ interface State {
subset: string;
repository: string;
repositoryStatus: string;
format: string;
}
export default class DetailsComponent extends React.PureComponent<Props, State> {
@ -60,6 +61,7 @@ export default class DetailsComponent extends React.PureComponent<Props, State>
name: taskInstance.name,
subset: taskInstance.subset,
repository: '',
format: '',
repositoryStatus: '',
};
}
@ -100,6 +102,7 @@ export default class DetailsComponent extends React.PureComponent<Props, State>
this.setState({
repository: data.url,
format: data.format,
});
}
})
@ -203,7 +206,7 @@ export default class DetailsComponent extends React.PureComponent<Props, State>
private renderDatasetRepository(): JSX.Element | boolean {
const { taskInstance } = this.props;
const { repository, repositoryStatus } = this.state;
const { repository, repositoryStatus, format } = this.state;
return (
!!repository && (
@ -216,6 +219,14 @@ export default class DetailsComponent extends React.PureComponent<Props, State>
<a href={repository} rel='noopener noreferrer' target='_blank'>
{repository}
</a>
<br />
<p>
Using format
{' '}
<Text strong>
{format}
</Text>
</p>
{repositoryStatus === 'sync' && (
<Tag color='blue'>
<CheckCircleOutlined />

4
cvat-ui/src/containers/create-task-page/create-task-page.tsx
Unescape Escape View File

@ -1,4 +1,4 @@
// Copyright (C) 2020 Intel Corporation
// Copyright (C) 2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
@ -14,6 +14,7 @@ interface StateToProps {
status: string;
error: string;
installedGit: boolean;
dumpers:[]
}
interface DispatchToProps {
@ -31,6 +32,7 @@ function mapStateToProps(state: CombinedState): StateToProps {
return {
...creates,
installedGit: state.plugins.list.GIT_INTEGRATION,
dumpers: state.formats.annotationFormats.dumpers,
};
}

7
cvat-ui/src/utils/git-utils.ts
Unescape Escape View File

@ -1,4 +1,4 @@
// Copyright (C) 2020 Intel Corporation
// Copyright (C) 2021 Intel Corporation
//
// SPDX-License-Identifier: MIT
@ -22,6 +22,7 @@ interface GitPlugin {
};
};
data: {
format: any;
task: any;
lfs: boolean;
repos: string;
@ -37,6 +38,7 @@ interface ReposData {
value: 'sync' | '!sync' | 'merged';
error: string | null;
};
format: string
}
function waitForClone(cloneResponse: any): Promise<void> {
@ -93,6 +95,7 @@ async function cloneRepository(this: any, plugin: GitPlugin, createdTask: any):
data: JSON.stringify({
path: plugin.data.repos,
lfs: plugin.data.lfs,
format: plugin.data.format,
tid: createdTask.id,
}),
})
@ -127,6 +130,7 @@ export function registerGitPlugin(): void {
data: {
task: null,
lfs: false,
format: '',
repos: '',
},
callbacks: {
@ -152,6 +156,7 @@ export async function getReposData(tid: number): Promise<ReposData | null> {
value: response.status.value,
error: response.status.error,
},
format: response.format,
};
}

12
cvat/apps/dataset_manager/formats/registry.py
Unescape Escape View File

@ -51,7 +51,19 @@ def _wrap_format(f_or_cls, klass, name, version, ext, display_name, enabled, dim
return target
EXPORT_FORMATS = {}
def format_for(export_format, mode):
format_name = export_format
if mode == "annotation":
format_name = "CVAT for images 1.1"
elif export_format not in EXPORT_FORMATS:
format_name = "CVAT for video 1.1"
return format_name
def exporter(name, version, ext, display_name=None, enabled=True, dimension=DimensionType.DIM_2D):
assert name not in EXPORT_FORMATS, "Export format '%s' already registered" % name
def wrap_with_params(f_or_cls):

23
cvat/apps/dataset_repo/dataset_repo.py
Unescape Escape View File

@ -9,7 +9,6 @@ import os
import re
import shutil
import subprocess
from glob import glob
import zipfile
import django_rq
@ -17,11 +16,12 @@ import git
from django.db import transaction
from django.utils import timezone
from cvat.apps.dataset_manager.formats.registry import format_for
from cvat.apps.dataset_manager.task import export_task
from cvat.apps.dataset_repo.models import GitData, GitStatusChoice
from cvat.apps.engine.log import slogger
from cvat.apps.engine.models import Job, Task, User
from cvat.apps.engine.plugins import add_plugin
from cvat.apps.dataset_repo.models import GitData, GitStatusChoice
def _have_no_access_exception(ex):
@ -66,6 +66,7 @@ class Git:
self._branch_name = 'cvat_{}_{}'.format(db_task.id, self._task_name)
self._annotation_file = os.path.join(self._cwd, self._path)
self._sync_date = db_git.sync_date
self._format = db_git.format
self._lfs = db_git.lfs
@ -265,16 +266,13 @@ class Git:
# Dump an annotation
timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
if self._task_mode == "annotation":
format_name = "CVAT for images 1.1"
else:
format_name = "CVAT for video 1.1"
dump_name = os.path.join(db_task.get_task_dirname(),
"git_annotation_{}.zip".format(timestamp))
"git_annotation_{}_{}.zip".format(self._format, timestamp))
export_task(
task_id=self._tid,
dst_file=dump_name,
format_name=format_name,
format_name=self._format,
server_url=scheme + host,
save_images=False,
)
@ -353,7 +351,7 @@ class Git:
return GitStatusChoice.NON_SYNCED
def initial_create(tid, git_path, lfs, user):
def initial_create(tid, git_path, export_format, lfs, user):
try:
db_task = Task.objects.get(pk = tid)
path_pattern = r"\[(.+)\]"
@ -373,9 +371,12 @@ def initial_create(tid, git_path, lfs, user):
if len(_split) < 2 or _split[1] not in [".xml", ".zip"]:
raise Exception("Only .xml and .zip formats are supported")
format_name = format_for(export_format, db_task.mode)
db_git = GitData()
db_git.url = git_path
db_git.path = path
db_git.format = format_name
db_git.task = db_task
db_git.lfs = lfs
@ -416,7 +417,7 @@ def get(tid, user):
response = {}
response["url"] = {"value": None}
response["status"] = {"value": None, "error": None}
response["format"] = {"format": None}
db_task = Task.objects.get(pk = tid)
if GitData.objects.filter(pk = db_task).exists():
db_git = GitData.objects.select_for_update().get(pk = db_task)
@ -428,12 +429,14 @@ def get(tid, user):
if rq_job is not None and (rq_job.is_queued or rq_job.is_started):
db_git.status = GitStatusChoice.SYNCING
response['status']['value'] = str(db_git.status)
response['format'] = str(db_git.format)
else:
try:
_git = Git(db_git, db_task, user)
_git.init_repos(True)
db_git.status = _git.remote_status(db_task.updated_date)
response['status']['value'] = str(db_git.status)
response['format'] = str(db_git.format)
except git.exc.GitCommandError as ex:
_have_no_access_exception(ex)
db_git.save()

19
cvat/apps/dataset_repo/migrations/0006_gitdata_format.py
Unescape Escape View File

@ -0,0 +1,19 @@
# Generated by Django 2.1.3 on 2019-02-05 17:08
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dataset_repo', '0005_auto_20201019_1100'),
]
replaces = [('git', '0006_gitdata_format')]
operations = [
migrations.AddField(
model_name='gitdata',
name='format',
field=models.CharField(max_length=256)
),
]

1
cvat/apps/dataset_repo/models.py
Unescape Escape View File

@ -20,6 +20,7 @@ class GitData(models.Model):
task = models.OneToOneField(Task, on_delete = models.CASCADE, primary_key = True)
url = models.URLField(max_length = 2000)
path = models.CharField(max_length=256)
format = models.CharField(max_length=256)
sync_date = models.DateTimeField(auto_now_add=True)
status = models.CharField(max_length=20, default=GitStatusChoice.NON_SYNCED)
lfs = models.BooleanField(default=True)

13
cvat/apps/dataset_repo/tests.py
Unescape Escape View File

@ -23,7 +23,7 @@ from cvat.apps.dataset_repo.models import GitData, GitStatusChoice
orig_execute = git.cmd.Git.execute
GIT_URL = "https://1.2.3.4/repo/exist.git"
PARSE_GIT_URL = "git@1.2.3.4:repo/exist.git"
EXPORT_FORMAT = "CVAT for video 1.1"
def generate_image_file(filename, size=(100, 50)):
f = BytesIO()
@ -414,6 +414,7 @@ class GitDatasetRepoTest(APITestCase):
db_task = Task.objects.get(pk=task["id"])
db_git = GitData()
db_git.url = GIT_URL
db_git.format = EXPORT_FORMAT
db_git.path = "annotation.zip"
db_git.sync_date = timezone.now()
@ -429,7 +430,7 @@ class GitDatasetRepoTest(APITestCase):
@mock.patch('git.Repo.clone_from', new=GitRepo.clone)
def test_request_initial_create(self):
task = self._create_task(init_repos=False)
initial_create(task["id"], GIT_URL, 1, self.user)
initial_create(task["id"], GIT_URL, EXPORT_FORMAT, 1, self.user)
self.assertTrue(osp.isdir(osp.join(task["repos_path"], '.git')))
git_repo = git.Repo(task["repos_path"])
with git_repo.config_reader() as cw:
@ -444,7 +445,7 @@ class GitDatasetRepoTest(APITestCase):
def test_request_push(self):
task = self._create_task(init_repos=False)
tid = task["id"]
initial_create(tid, GIT_URL, 1, self.user)
initial_create(tid, GIT_URL, EXPORT_FORMAT, 1, self.user)
self.add_file(task["repos_path"], "file.txt")
push(tid, self.user, "", "")
@ -459,7 +460,7 @@ class GitDatasetRepoTest(APITestCase):
def test_push_and_request_get(self):
task = self._create_task(init_repos=False)
tid = task["id"]
initial_create(tid, GIT_URL, 1, self.user)
initial_create(tid, GIT_URL, EXPORT_FORMAT, 1, self.user)
self.add_file(task["repos_path"], "file.txt")
push(tid, self.user, "", "")
response = get(tid, self.user)
@ -474,7 +475,7 @@ class GitDatasetRepoTest(APITestCase):
def test_request_get(self):
task = self._create_task(init_repos=False)
tid = task["id"]
initial_create(tid, GIT_URL, 1, self.user)
initial_create(tid, GIT_URL, EXPORT_FORMAT, 1, self.user)
response = get(tid, self.user)
self.assertTrue(response["status"]["value"], "not sync")
@ -487,7 +488,7 @@ class GitDatasetRepoTest(APITestCase):
def test_request_on_save(self):
task = self._create_task(init_repos=False)
tid = task["id"]
initial_create(tid, GIT_URL, 1, self.user)
initial_create(tid, GIT_URL, EXPORT_FORMAT, 1, self.user)
jobs = self._get_jobs(tid)
ann = {

4
cvat/apps/dataset_repo/views.py
Unescape Escape View File

@ -40,14 +40,14 @@ def check_process(request, rq_id):
def create(request, tid):
try:
slogger.task[tid].info("create repository request")
body = json.loads(request.body.decode('utf-8'))
path = body["path"]
export_format = body["format"]
lfs = body["lfs"]
rq_id = "git.create.{}".format(tid)
queue = django_rq.get_queue("default")
queue.enqueue_call(func = CVATGit.initial_create, args = (tid, path, lfs, request.user), job_id = rq_id)
queue.enqueue_call(func = CVATGit.initial_create, args = (tid, path, export_format, lfs, request.user), job_id = rq_id)
return JsonResponse({ "rq_id": rq_id })
except Exception as ex:
slogger.glob.error("error occurred during initial cloning repository request with rq id {}".format(rq_id), exc_info=True)

Write Preview
Loading…
Cancel
Save