wangchunlin
/
cvat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Tensorflow annotation update (rq queues instead of CVAT statuses) (#155)

Browse Source 
* Use rq instead of cvat statuses
* Removed permissions for meta info
main
Boris Sekachev 8 years ago committed by Nikita Manovich
parent 920247dc31
commit dd6bdeaa2a
6 changed files with 124 additions and 85 deletions
Show Stats Download Patch File Download Diff File

1
cvat/apps/engine/models.py
Unescape Escape View File

@ -1,4 +1,3 @@
# Copyright (C) 2018 Intel Corporation
#
# SPDX-License-Identifier: MIT

2
cvat/apps/engine/task.py
Unescape Escape View File

@ -192,7 +192,7 @@ def save_job_status(jid, status, user):
db_task = db_job.segment.task
status = StatusChoice(status)
slogger.job[jid].info('changing job status from {} to {} by an user {}'.format(db_job.status, status, user))
slogger.job[jid].info('changing job status from {} to {} by an user {}'.format(db_job.status, str(status), user))
db_job.status = status.value
db_job.save()

50
cvat/apps/tf_annotation/static/tf_annotation/js/tf_annotation.js
Unescape Escape View File

@ -9,7 +9,7 @@ function CheckTFAnnotationRequest(taskId, tfAnnotationButton) {
let errorCount = 0;
let interval = setInterval(function() {
$.ajax ({
url: '/tf_annotation/check/task/' + taskId,
url: '/tensorflow/annotation/check/task/' + taskId,
success: function(jsonData) {
let status = jsonData["status"];
if (status == "started" || status == "queued") {
@ -40,7 +40,7 @@ function RunTFAnnotationRequest() {
let tfAnnotationButton = this;
let taskID = window.cvat.dashboard.taskID;
$.ajax ({
url: '/tf_annotation/create/task/' + taskID,
url: '/tensorflow/annotation/create/task/' + taskID,
success: function() {
showMessage('Process started.');
tfAnnotationButton.text(`Cancel TF Annotation (0%)`);
@ -57,7 +57,7 @@ function RunTFAnnotationRequest() {
function CancelTFAnnotationRequest() {
let tfAnnotationButton = this;
$.ajax ({
url: '/tf_annotation/cancel/task/' + window.cvat.dashboard.taskID,
url: '/tensorflow/annotation/cancel/task/' + window.cvat.dashboard.taskID,
success: function() {
tfAnnotationButton.prop("disabled", true);
},
@ -90,20 +90,38 @@ window.cvat.dashboard = window.cvat.dashboard || {};
window.cvat.dashboard.uiCallbacks = window.cvat.dashboard.uiCallbacks || [];
window.cvat.dashboard.uiCallbacks.push(function(newElements) {
newElements.each(function(idx) {
let elem = $(newElements[idx]);
let taskId = +elem.attr('id').split('_')[1];
let status = $.trim($(elem.find('label.dashboardStatusLabel')[0]).text());
let buttonsUI = elem.find('div.dashboardButtonsUI')[0];
let tfAnnotationButton = $('<button> Run TF Annotation </button>');
tfAnnotationButton.on('click', onTFAnnotationClick.bind(tfAnnotationButton));
tfAnnotationButton.addClass('dashboardTFAnnotationButton semiBold dashboardButtonUI');
tfAnnotationButton.appendTo(buttonsUI);
tids = [];
for (let el of newElements) {
tids.push(el.id.split('_')[1])
}
if (status == "TF Annotation") {
tfAnnotationButton.text("Cancel TF Annotation");
tfAnnotationButton.addClass("tfAnnotationProcess");
CheckTFAnnotationRequest(taskId, tfAnnotationButton);
$.ajax({
type: 'POST',
url: '/tensorflow/annotation/meta/get',
data: JSON.stringify(tids),
contentType: "application/json; charset=utf-8",
success: (data) => {
newElements.each(function(idx) {
let elem = $(newElements[idx]);
let tid = +elem.attr('id').split('_')[1];
let buttonsUI = elem.find('div.dashboardButtonsUI')[0];
let tfAnnotationButton = $('<button> Run TF Annotation </button>');
tfAnnotationButton.on('click', onTFAnnotationClick.bind(tfAnnotationButton));
tfAnnotationButton.addClass('dashboardTFAnnotationButton semiBold dashboardButtonUI');
tfAnnotationButton.appendTo(buttonsUI);
if ((tid in data) && (data[tid].active)) {
tfAnnotationButton.text("Cancel TF Annotation");
tfAnnotationButton.addClass("tfAnnotationProcess");
CheckTFAnnotationRequest(tid, tfAnnotationButton);
}
});
},
error: (data) => {
let message = `Can not get tf annotation meta info. Code: ${data.status}. Message: ${data.responseText || data.statusText}`;
showMessage(message);
throw Error(message);
}
});
});

1
cvat/apps/tf_annotation/urls.py
Unescape Escape View File

@ -10,4 +10,5 @@ urlpatterns = [
path('create/task/<int:tid>', views.create),
path('check/task/<int:tid>', views.check),
path('cancel/task/<int:tid>', views.cancel),
path('meta/get', views.get_meta_info),
]

153
cvat/apps/tf_annotation/views.py
Unescape Escape View File

@ -258,13 +258,32 @@ def create_thread(tid, labels_mapping):
# Modify data format and save
result = convert_to_cvat_format(result)
annotation.save_task(tid, result)
db_task.status = "Annotation"
db_task.save()
slogger.glob.info('tf annotation for task {} done'.format(tid))
except:
try:
slogger.task[tid].exception('exception was occured during tf annotation of the task', exc_info=True)
except:
slogger.glob.exception('exception was occured during tf annotation of the task {}'.format(tid), exc_into=True)
@login_required
def get_meta_info(request):
try:
queue = django_rq.get_queue('low')
tids = json.loads(request.body.decode('utf-8'))
result = {}
for tid in tids:
job = queue.fetch_job('tf_annotation.create/{}'.format(tid))
if job is not None:
result[tid] = {
"active": job.is_queued or job.is_started,
"success": not job.is_failed
}
return JsonResponse(result)
except Exception as ex:
slogger.glob.exception('exception was occured during tf annotation of the task {}: {}'.format(tid, ex))
db_task.status = "TF Annotation Fault"
db_task.save()
slogger.glob.exception('exception was occured during tf meta request', exc_into=True)
return HttpResponseBadRequest(str(ex))
@login_required
@permission_required(perm=['engine.view_task', 'engine.change_annotation'], raise_exception=True)
@ -272,71 +291,70 @@ def create(request, tid):
slogger.glob.info('tf annotation create request for task {}'.format(tid))
try:
db_task = TaskModel.objects.get(pk=tid)
except ObjectDoesNotExist:
slogger.glob.exception('task with id {} not found'.format(tid))
return HttpResponseBadRequest("A task with this ID was not found")
if not task.is_task_owner(request.user, tid):
slogger.glob.error('not enought of permissions for tf annotation of the task {}'.format(tid))
return HttpResponseBadRequest("You don't have permissions to tf annotation of the task.")
queue = django_rq.get_queue('low')
job = queue.fetch_job('tf_annotation.create/{}'.format(tid))
if job is not None and (job.is_started or job.is_queued):
slogger.glob.error('tf annotation for task {} already running'.format(tid))
return HttpResponseBadRequest("The process is already running")
db_labels = db_task.label_set.prefetch_related('attributespec_set').all()
db_labels = {db_label.id:db_label.name for db_label in db_labels}
tf_annotation_labels = {
"person": 1, "bicycle": 2, "car": 3, "motorcycle": 4, "airplane": 5,
"bus": 6, "train": 7, "truck": 8, "boat": 9, "traffic_light": 10,
"fire_hydrant": 11, "stop_sign": 13, "parking_meter": 14, "bench": 15,
"bird": 16, "cat": 17, "dog": 18, "horse": 19, "sheep": 20, "cow": 21,
"elephant": 22, "bear": 23, "zebra": 24, "giraffe": 25, "backpack": 27,
"umbrella": 28, "handbag": 31, "tie": 32, "suitcase": 33, "frisbee": 34,
"skis": 35, "snowboard": 36, "sports_ball": 37, "kite": 38, "baseball_bat": 39,
"baseball_glove": 40, "skateboard": 41, "surfboard": 42, "tennis_racket": 43,
"bottle": 44, "wine_glass": 46, "cup": 47, "fork": 48, "knife": 49, "spoon": 50,
"bowl": 51, "banana": 52, "apple": 53, "sandwich": 54, "orange": 55, "broccoli": 56,
"carrot": 57, "hot_dog": 58, "pizza": 59, "donut": 60, "cake": 61, "chair": 62,
"couch": 63, "potted_plant": 64, "bed": 65, "dining_table": 67, "toilet": 70,
"tv": 72, "laptop": 73, "mouse": 74, "remote": 75, "keyboard": 76, "cell_phone": 77,
"microwave": 78, "oven": 79, "toaster": 80, "sink": 81, "refrigerator": 83,
"book": 84, "clock": 85, "vase": 86, "scissors": 87, "teddy_bear": 88, "hair_drier": 89,
"toothbrush": 90
}
labels_mapping = {}
for key, labels in db_labels.items():
if labels in tf_annotation_labels.keys():
labels_mapping[tf_annotation_labels[labels]] = key
if not len(labels_mapping.values()):
slogger.glob.error('no labels found for task {} tf annotation'.format(tid))
return HttpResponseBadRequest("No labels found for tf annotation")
if not task.is_task_owner(request.user, tid):
raise Exception('Not enought of permissions for tf annotation')
db_task.status = "TF Annotation"
db_task.save()
queue = django_rq.get_queue('low')
job = queue.fetch_job('tf_annotation.create/{}'.format(tid))
if job is not None and (job.is_started or job.is_queued):
raise Exception("The process is already running")
db_labels = db_task.label_set.prefetch_related('attributespec_set').all()
db_labels = {db_label.id:db_label.name for db_label in db_labels}
tf_annotation_labels = {
"person": 1, "bicycle": 2, "car": 3, "motorcycle": 4, "airplane": 5,
"bus": 6, "train": 7, "truck": 8, "boat": 9, "traffic_light": 10,
"fire_hydrant": 11, "stop_sign": 13, "parking_meter": 14, "bench": 15,
"bird": 16, "cat": 17, "dog": 18, "horse": 19, "sheep": 20, "cow": 21,
"elephant": 22, "bear": 23, "zebra": 24, "giraffe": 25, "backpack": 27,
"umbrella": 28, "handbag": 31, "tie": 32, "suitcase": 33, "frisbee": 34,
"skis": 35, "snowboard": 36, "sports_ball": 37, "kite": 38, "baseball_bat": 39,
"baseball_glove": 40, "skateboard": 41, "surfboard": 42, "tennis_racket": 43,
"bottle": 44, "wine_glass": 46, "cup": 47, "fork": 48, "knife": 49, "spoon": 50,
"bowl": 51, "banana": 52, "apple": 53, "sandwich": 54, "orange": 55, "broccoli": 56,
"carrot": 57, "hot_dog": 58, "pizza": 59, "donut": 60, "cake": 61, "chair": 62,
"couch": 63, "potted_plant": 64, "bed": 65, "dining_table": 67, "toilet": 70,
"tv": 72, "laptop": 73, "mouse": 74, "remote": 75, "keyboard": 76, "cell_phone": 77,
"microwave": 78, "oven": 79, "toaster": 80, "sink": 81, "refrigerator": 83,
"book": 84, "clock": 85, "vase": 86, "scissors": 87, "teddy_bear": 88, "hair_drier": 89,
"toothbrush": 90
}
labels_mapping = {}
for key, labels in db_labels.items():
if labels in tf_annotation_labels.keys():
labels_mapping[tf_annotation_labels[labels]] = key
if not len(labels_mapping.values()):
raise Exception('No labels found for tf annotation')
# Run tf annotation job
queue.enqueue_call(func=create_thread,
args=(tid, labels_mapping),
job_id='tf_annotation.create/{}'.format(tid),
timeout=604800) # 7 days
slogger.task[tid].info('tensorflow annotation job enqueued with labels {}'.format(labels_mapping))
# Run tf annotation job
queue.enqueue_call(func=create_thread,
args=(tid, labels_mapping),
job_id='tf_annotation.create/{}'.format(tid),
timeout=604800) # 7 days
slogger.glob.info('tf annotation job enqueued for task {} with labels {}'.format(tid, labels_mapping))
except Exception as ex:
try:
slogger.task[tid].exception("exception was occured during tensorflow annotation request", exc_info=True)
except:
pass
return HttpResponseBadRequest(str(ex))
return HttpResponse()
@login_required
@permission_required(perm='engine.view_task', raise_exception=True)
def check(request, tid):
queue = django_rq.get_queue('low')
job = queue.fetch_job('tf_annotation.create/{}'.format(tid))
if job is not None and 'cancel' in job.meta:
return JsonResponse({'status': 'finished'})
data = {}
try:
queue = django_rq.get_queue('low')
job = queue.fetch_job('tf_annotation.create/{}'.format(tid))
if job is not None and 'cancel' in job.meta:
return JsonResponse({'status': 'finished'})
data = {}
if job is None:
data['status'] = 'unknown'
elif job.is_queued:
@ -350,6 +368,7 @@ def check(request, tid):
else:
data['status'] = 'failed'
job.delete()
except Exception:
data['status'] = 'unknown'
@ -363,14 +382,16 @@ def cancel(request, tid):
queue = django_rq.get_queue('low')
job = queue.fetch_job('tf_annotation.create/{}'.format(tid))
if job is None or job.is_finished or job.is_failed:
raise Exception('Task is not in tf annotation process')
raise Exception('Task is not being annotated currently')
elif 'cancel' not in job.meta:
job.meta['cancel'] = True
job.save()
db_task = TaskModel.objects.get(pk=tid)
db_task.status = "Annotation"
db_task.save()
except Exception as ex:
return HttpResponseBadRequest("TF annotation cancel error: {}".format(str(ex)))
try:
slogger.task[tid].exception("cannot cancel tensorflow annotation for task #{}".format(tid), exc_info=True)
except:
pass
return HttpResponseBadRequest(str(ex))
return HttpResponse()

2
cvat/urls.py
Unescape Escape View File

@ -35,7 +35,7 @@ urlpatterns = [
]
if apps.is_installed('cvat.apps.tf_annotation'):
urlpatterns.append(path('tf_annotation/', include('cvat.apps.tf_annotation.urls')))
urlpatterns.append(path('tensorflow/annotation/', include('cvat.apps.tf_annotation.urls')))
if apps.is_installed('cvat.apps.log_viewer'):
urlpatterns.append(path('analytics/', include('cvat.apps.log_viewer.urls')))

Write Preview
Loading…
Cancel
Save