wangchunlin
/
cvat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Tus for project dataset (#4485)

Browse Source
main
Kirill Lakhov 4 years ago committed by GitHub
parent dd60b2d8e0
commit bceae228a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 115 additions and 65 deletions
Show Stats Download Patch File Download Diff File

1
CHANGELOG.md
Unescape Escape View File

@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Task annotations importing via chunk uploads (<https://github.com/openvinotoolkit/cvat/pull/4327>)
- Advanced filtration and sorting for a list of tasks/projects/cloudstorages (<https://github.com/openvinotoolkit/cvat/pull/4403>)
- Project dataset importing via chunk uploads (<https://github.com/openvinotoolkit/cvat/pull/4485>)
### Changed
- Added missing geos dependency into Dockerfile (<https://github.com/openvinotoolkit/cvat/pull/4451>)

4
cvat-core/package-lock.json generated
Unescape Escape View File

@ -1,12 +1,12 @@
{
"name": "cvat-core",
"version": "5.0.0",
"version": "5.0.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "cvat-core",
"version": "5.0.0",
"version": "5.0.1",
"license": "MIT",
"dependencies": {
"axios": "^0.21.4",

2
cvat-core/package.json
Unescape Escape View File

@ -1,6 +1,6 @@
{
"name": "cvat-core",
"version": "5.0.0",
"version": "5.0.1",
"description": "Part of Computer Vision Tool which presents an interface for client-side integration",
"main": "babel.config.js",
"scripts": {

83
cvat-core/src/server-proxy.js
Unescape Escape View File

@ -75,7 +75,7 @@
onProgress(bytesUploaded) {
if (onUpdate && Number.isInteger(totalSentSize) && Number.isInteger(totalSize)) {
const currentUploadedSize = totalSentSize + bytesUploaded;
const percentage = currentUploadedSize / totalSize;
const percentage = Math.round(currentUploadedSize / totalSize);
onUpdate(percentage);
}
},
@ -612,41 +612,63 @@
}
async function importDataset(id, format, file, onUpdate) {
const { backendAPI } = config;
const { backendAPI, origin } = config;
const params = {
...enableOrganization(),
format,
filename: file.name,
};
const uploadConfig = {
chunkSize: config.uploadChunkSize * 1024 * 1024,
endpoint: `${origin}${backendAPI}/projects/${id}/dataset/`,
totalSentSize: 0,
totalSize: file.size,
onUpdate: (percentage) => {
onUpdate('The dataset is being uploaded to the server', percentage);
},
};
const url = `${backendAPI}/projects/${id}/dataset`;
const formData = new FormData();
formData.append('dataset_file', file);
return new Promise((resolve, reject) => {
async function requestStatus() {
try {
const response = await Axios.get(`${url}?action=import_status`, {
proxy: config.proxy,
});
if (response.status === 202) {
if (onUpdate && response.data.message !== '') {
onUpdate(response.data.message, response.data.progress || 0);
try {
await Axios.post(url,
new FormData(), {
params,
proxy: config.proxy,
headers: { 'Upload-Start': true },
});
await chunkUpload(file, uploadConfig);
await Axios.post(url,
new FormData(), {
params,
proxy: config.proxy,
headers: { 'Upload-Finish': true },
});
return new Promise((resolve, reject) => {
async function requestStatus() {
try {
const response = await Axios.get(url, {
params: { ...params, action: 'import_status' },
proxy: config.proxy,
});
if (response.status === 202) {
if (onUpdate && response.data.message) {
onUpdate(response.data.message, response.data.progress || 0);
}
setTimeout(requestStatus, 3000);
} else if (response.status === 201) {
resolve();
} else {
reject(generateError(response));
}
setTimeout(requestStatus, 3000);
} else if (response.status === 201) {
resolve();
} else {
reject(generateError(response));
} catch (error) {
reject(generateError(error));
}
} catch (error) {
reject(generateError(error));
}
}
Axios.post(`${url}?format=${format}`, formData, {
proxy: config.proxy,
}).then(() => {
setTimeout(requestStatus, 2000);
}).catch((error) => {
reject(generateError(error));
});
});
} catch (errorData) {
throw generateError(errorData);
}
}
async function exportTask(id) {
@ -1279,8 +1301,7 @@
setTimeout(requestStatus);
});
} catch (errorData) {
generateError(errorData);
return null;
throw generateError(errorData);
}
}

3
cvat/apps/engine/models.py
Unescape Escape View File

@ -248,6 +248,9 @@ class Project(models.Model):
def get_project_logs_dirname(self):
return os.path.join(self.get_project_dirname(), 'logs')
def get_tmp_dirname(self):
return os.path.join(self.get_project_dirname(), "tmp")
def get_client_log_path(self):
return os.path.join(self.get_project_logs_dirname(), "client.log")

85
cvat/apps/engine/views.py
Unescape Escape View File

@ -250,7 +250,7 @@ class ServerViewSet(viewsets.ViewSet):
'200': ProjectSerializer,
})
)
class ProjectViewSet(viewsets.ModelViewSet):
class ProjectViewSet(viewsets.ModelViewSet, UploadMixin):
queryset = models.Project.objects.prefetch_related(Prefetch('label_set',
queryset=models.Label.objects.order_by('id')
))
@ -330,21 +330,13 @@ class ProjectViewSet(viewsets.ModelViewSet):
'400': OpenApiResponse(description='Failed to import dataset'),
'405': OpenApiResponse(description='Format is not available'),
})
@action(detail=True, methods=['GET', 'POST'], serializer_class=None,
url_path='dataset')
@action(detail=True, methods=['GET', 'POST', 'OPTIONS'], serializer_class=None,
url_path=r'dataset/?$')
def dataset(self, request, pk):
db_project = self.get_object() # force to call check_object_permissions
if request.method == 'POST':
format_name = request.query_params.get("format", "")
self._object = self.get_object() # force to call check_object_permissions
return _import_project_dataset(
request=request,
rq_id=f"/api/project/{pk}/dataset_import",
rq_func=dm.project.import_dataset_as_project,
pk=pk,
format_name=format_name,
)
if request.method == 'POST' or request.method == 'OPTIONS':
return self.upload_data(request)
else:
action = request.query_params.get("action", "").lower()
if action in ("import_status",):
@ -353,12 +345,12 @@ class ProjectViewSet(viewsets.ModelViewSet):
if rq_job is None:
return Response(status=status.HTTP_404_NOT_FOUND)
elif rq_job.is_finished:
os.close(rq_job.meta['tmp_file_descriptor'])
if rq_job.meta['tmp_file_descriptor']: os.close(rq_job.meta['tmp_file_descriptor'])
os.remove(rq_job.meta['tmp_file'])
rq_job.delete()
return Response(status=status.HTTP_201_CREATED)
elif rq_job.is_failed:
os.close(rq_job.meta['tmp_file_descriptor'])
if rq_job.meta['tmp_file_descriptor']: os.close(rq_job.meta['tmp_file_descriptor'])
os.remove(rq_job.meta['tmp_file'])
rq_job.delete()
return Response(
@ -373,7 +365,7 @@ class ProjectViewSet(viewsets.ModelViewSet):
else:
format_name = request.query_params.get("format", "")
return _export_annotations(
db_instance=db_project,
db_instance=self._object,
rq_id="/api/project/{}/dataset/{}".format(pk, format_name),
request=request,
action=action,
@ -382,6 +374,35 @@ class ProjectViewSet(viewsets.ModelViewSet):
filename=request.query_params.get("filename", "").lower(),
)
def get_upload_dir(self):
if 'dataset' in self.action:
return self._object.get_tmp_dirname()
return ""
def upload_finished(self, request):
if self.action == 'dataset':
format_name = request.query_params.get("format", "")
filename = request.query_params.get("filename", "")
tmp_dir = self._object.get_tmp_dirname()
uploaded_file = None
if os.path.isfile(os.path.join(tmp_dir, filename)):
uploaded_file = os.path.join(tmp_dir, filename)
return _import_project_dataset(
request=request,
filename=uploaded_file,
rq_id=f"/api/project/{self._object.pk}/dataset_import",
rq_func=dm.project.import_dataset_as_project,
pk=self._object.pk,
format_name=format_name,
)
return Response(data='Unknown upload was finished',
status=status.HTTP_400_BAD_REQUEST)
@action(detail=True, methods=['HEAD', 'PATCH'], url_path='dataset/'+UploadMixin.file_id_regex)
def append_dataset_chunk(self, request, pk, file_id):
self._object = self.get_object()
return self.append_tus_chunk(request, file_id)
@extend_schema(summary='Method allows to download project annotations',
parameters=[
OpenApiParameter('format', description='Desired output format name\n'
@ -1768,7 +1789,7 @@ def _export_annotations(db_instance, rq_id, request, format_name, action, callba
result_ttl=ttl, failure_ttl=ttl)
return Response(status=status.HTTP_202_ACCEPTED)
def _import_project_dataset(request, rq_id, rq_func, pk, format_name):
def _import_project_dataset(request, rq_id, rq_func, pk, format_name, filename=None):
format_desc = {f.DISPLAY_NAME: f
for f in dm.views.get_import_formats()}.get(format_name)
if format_desc is None:
@ -1781,19 +1802,21 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name):
rq_job = queue.fetch_job(rq_id)
if not rq_job:
serializer = DatasetFileSerializer(data=request.data)
if serializer.is_valid(raise_exception=True):
dataset_file = serializer.validated_data['dataset_file']
fd, filename = mkstemp(prefix='cvat_{}'.format(pk))
with open(filename, 'wb+') as f:
for chunk in dataset_file.chunks():
f.write(chunk)
rq_job = queue.enqueue_call(
func=rq_func,
args=(pk, filename, format_name),
job_id=rq_id,
meta={
fd = None
if not filename:
serializer = DatasetFileSerializer(data=request.data)
if serializer.is_valid(raise_exception=True):
dataset_file = serializer.validated_data['dataset_file']
fd, filename = mkstemp(prefix='cvat_{}'.format(pk))
with open(filename, 'wb+') as f:
for chunk in dataset_file.chunks():
f.write(chunk)
rq_job = queue.enqueue_call(
func=rq_func,
args=(pk, filename, format_name),
job_id=rq_id,
meta={
'tmp_file': filename,
'tmp_file_descriptor': fd,
},

2
cvat/apps/iam/permissions.py
Unescape Escape View File

@ -492,6 +492,8 @@ class ProjectPermission(OpenPolicyAgentPermission):
('retrieve', 'GET'): 'view',
('tasks', 'GET'): 'view',
('dataset', 'POST'): 'import:dataset',
('append_dataset_chunk', 'HEAD'): 'import:dataset',
('append_dataset_chunk', 'PATCH'): 'import:dataset',
('annotations', 'GET'): 'export:annotations',
('dataset', 'GET'): 'export:dataset',
('export_backup', 'GET'): 'export:backup',

Write Preview
Loading…
Cancel
Save