wangchunlin
/
cvat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove client id info from dump file (#213)

Browse Source 
* removed shape id info from dump
* force set client id by server in save annotation for task functionality
* delete unused code
main
Andrey Zhavoronkov 7 years ago committed by Nikita Manovich
parent af894e1f5c
commit 9608b7d237
10 changed files with 146 additions and 124 deletions
Show Stats Download Patch File Download Diff File

13
cvat/apps/dashboard/static/dashboard/js/dashboard.js
Unescape Escape View File

@ -525,11 +525,14 @@ function uploadAnnotationRequest() {
url: '/get/task/' + window.cvat.dashboard.taskID,
success: function(data) {
let annotationParser = new AnnotationParser({
start: 0,
stop: data.size,
image_meta_data: data.image_meta_data,
flipped: data.flipped
}, new LabelsInfo(data.spec));
start: 0,
stop: data.size,
image_meta_data: data.image_meta_data,
flipped: data.flipped
},
new LabelsInfo(data.spec),
new ConstIdGenerator(-1),
);
let asyncParse = function() {
let parsed = null;

110
cvat/apps/engine/annotation.py
Unescape Escape View File

@ -82,7 +82,8 @@ def save_job(jid, data):
.select_for_update().get(id=jid)
annotation = _AnnotationForJob(db_job)
annotation.validate_data_from_client(data)
annotation.force_set_client_id(data['create'])
client_ids = annotation.validate_data_from_client(data)
annotation.delete_from_db(data['delete'])
annotation.save_to_db(data['create'])
@ -90,6 +91,10 @@ def save_job(jid, data):
db_job.segment.task.updated_date = timezone.now()
db_job.segment.task.save()
db_job.max_shape_id = max(db_job.max_shape_id, max(client_ids['create']) if client_ids['create'] else -1)
db_job.save()
slogger.job[jid].info("Leave save_job API: jid = {}".format(jid))
@silk_profile(name="Clear job")
@ -404,33 +409,8 @@ class _Annotation:
return non_empty
def get_max_client_id(self):
max_client_id = -1
def extract_client_id(shape):
return shape.client_id
if self.boxes:
max_client_id = max(max_client_id, (max(self.boxes, key=extract_client_id)).client_id)
if self.box_paths:
max_client_id = max(max_client_id, (max(self.box_paths, key=extract_client_id)).client_id)
if self.polygons:
max_client_id = max(max_client_id, (max(self.polygons, key=extract_client_id)).client_id)
if self.polygon_paths:
max_client_id = max(max_client_id, (max(self.polygon_paths, key=extract_client_id)).client_id)
if self.polylines:
max_client_id = max(max_client_id, (max(self.polylines, key=extract_client_id)).client_id)
if self.polyline_paths:
max_client_id = max(max_client_id, (max(self.polyline_paths, key=extract_client_id)).client_id)
if self.points:
max_client_id = max(max_client_id, (max(self.points, key=extract_client_id)).client_id)
if self.points_paths:
max_client_id = max(max_client_id, (max(self.points_paths, key=extract_client_id)).client_id)
return max_client_id
# Functions below used by dump functionality
def to_boxes(self, start_client_id):
def to_boxes(self):
boxes = []
for path in self.box_paths:
for box in path.get_interpolated_boxes():
@ -442,15 +422,13 @@ class _Annotation:
group_id=path.group_id,
occluded=box.occluded,
z_order=box.z_order,
client_id=start_client_id,
attributes=box.attributes + path.attributes,
)
boxes.append(box)
start_client_id += 1
return self.boxes + boxes, start_client_id
return self.boxes + boxes
def _to_poly_shapes(self, iter_attr_name, start_client_id):
def _to_poly_shapes(self, iter_attr_name):
shapes = []
for path in getattr(self, iter_attr_name):
for shape in path.get_interpolated_shapes():
@ -462,24 +440,22 @@ class _Annotation:
group_id=path.group_id,
occluded=shape.occluded,
z_order=shape.z_order,
client_id=start_client_id,
attributes=shape.attributes + path.attributes,
)
shapes.append(shape)
start_client_id += 1
return shapes, start_client_id
return shapes
def to_polygons(self, start_client_id):
polygons, client_id = self._to_poly_shapes('polygon_paths', start_client_id)
return polygons + self.polygons, client_id
def to_polygons(self):
polygons = self._to_poly_shapes('polygon_paths')
return polygons + self.polygons
def to_polylines(self, start_client_id):
polylines, client_id = self._to_poly_shapes('polyline_paths', start_client_id)
return polylines + self.polylines, client_id
def to_polylines(self):
polylines = self._to_poly_shapes('polyline_paths')
return polylines + self.polylines
def to_points(self, start_client_id):
points, client_id = self._to_poly_shapes('points_paths', start_client_id)
return points + self.points, client_id
def to_points(self):
points = self._to_poly_shapes('points_paths')
return points + self.points
def to_box_paths(self):
paths = []
@ -496,7 +472,6 @@ class _Annotation:
group_id=box.group_id,
boxes=[box0, box1],
attributes=box.attributes,
client_id=box.client_id,
)
paths.append(path)
@ -516,7 +491,6 @@ class _Annotation:
stop_frame=shape.frame + 1,
group_id=shape.group_id,
shapes=[shape0, shape1],
client_id=shape.client_id,
attributes=shape.attributes,
)
paths.append(path)
@ -1353,7 +1327,7 @@ class _AnnotationForJob(_Annotation):
"polylines": [],
"polyline_paths": [],
"points": [],
"points_paths": []
"points_paths": [],
}
for box in self.boxes:
@ -1429,8 +1403,8 @@ class _AnnotationForJob(_Annotation):
return data
def validate_data_from_client(self, data):
db_client_ids = self._get_client_ids_from_db()
client_ids = {
'saved': self._get_client_ids_from_db(),
'create': set(),
'update': set(),
'delete': set(),
@ -1461,18 +1435,43 @@ class _AnnotationForJob(_Annotation):
if tmp_res:
raise Exception('More than one action for shape(s) with id={}'.format(tmp_res))
tmp_res = (db_client_ids - client_ids['delete']) & client_ids['create']
tmp_res = (client_ids['saved'] - client_ids['delete']) & client_ids['create']
if tmp_res:
raise Exception('Trying to create new shape(s) with existing client id {}'.format(tmp_res))
tmp_res = client_ids['delete'] - db_client_ids
tmp_res = client_ids['delete'] - client_ids['saved']
if tmp_res:
raise Exception('Trying to delete shape(s) with nonexistent client id {}'.format(tmp_res))
tmp_res = client_ids['update'] - (db_client_ids - client_ids['delete'])
tmp_res = client_ids['update'] - (client_ids['saved'] - client_ids['delete'])
if tmp_res:
raise Exception('Trying to update shape(s) with nonexistent client id {}'.format(tmp_res))
max_id = self.db_job.max_shape_id
if any(new_client_id <= max_id for new_client_id in client_ids['create']):
raise Exception('Trying to create shape(s) with client id {} less than allowed value {}'.format(client_ids['create'], max_id))
return client_ids
def force_set_client_id(self, data):
shape_types = ['boxes', 'points', 'polygons', 'polylines', 'box_paths',
'points_paths', 'polygon_paths', 'polyline_paths']
max_id = self.db_job.max_shape_id
for shape_type in shape_types:
if not data[shape_type]:
continue
for shape in data[shape_type]:
if 'id' in shape:
max_id = max(max_id, shape['id'])
max_id += 1
for shape_type in shape_types:
for shape in data[shape_type]:
if 'id' not in shape or shape['id'] == -1:
shape['id'] = max_id
max_id += 1
class _AnnotationForSegment(_Annotation):
def __init__(self, db_segment):
super().__init__(db_segment.start_frame, db_segment.stop_frame)
@ -1962,25 +1961,25 @@ class _AnnotationForTask(_Annotation):
shapes["polygons"] = {}
shapes["polylines"] = {}
shapes["points"] = {}
boxes, max_client_id = self.to_boxes(self.get_max_client_id() + 1)
boxes = self.to_boxes()
for box in boxes:
if box.frame not in shapes["boxes"]:
shapes["boxes"][box.frame] = []
shapes["boxes"][box.frame].append(box)
polygons, max_client_id = self.to_polygons(max_client_id)
polygons = self.to_polygons()
for polygon in polygons:
if polygon.frame not in shapes["polygons"]:
shapes["polygons"][polygon.frame] = []
shapes["polygons"][polygon.frame].append(polygon)
polylines, max_client_id = self.to_polylines(max_client_id)
polylines = self.to_polylines()
for polyline in polylines:
if polyline.frame not in shapes["polylines"]:
shapes["polylines"][polyline.frame] = []
shapes["polylines"][polyline.frame].append(polyline)
points, max_client_id = self.to_points(max_client_id)
points = self.to_points()
for points in points:
if points.frame not in shapes["points"]:
shapes["points"][points.frame] = []
@ -2022,7 +2021,6 @@ class _AnnotationForTask(_Annotation):
("xbr", "{:.2f}".format(shape.xbr)),
("ybr", "{:.2f}".format(shape.ybr)),
("occluded", str(int(shape.occluded))),
("id", str(shape.client_id)),
])
if db_task.z_order:
dump_dict['z_order'] = str(shape.z_order)
@ -2042,7 +2040,6 @@ class _AnnotationForTask(_Annotation):
)) for p in shape.points.split(' '))
)),
("occluded", str(int(shape.occluded))),
("id", str(shape.client_id)),
])
if db_task.z_order:
@ -2087,7 +2084,6 @@ class _AnnotationForTask(_Annotation):
path_list = paths[shape_type]
for path in path_list:
dump_dict = OrderedDict([
("id", str(path.client_id)),
("label", path.label.name),
])
if path.group_id:

18
cvat/apps/engine/migrations/0014_job_max_shape_id.py
Unescape Escape View File

@ -0,0 +1,18 @@
# Generated by Django 2.1.3 on 2018-11-23 10:07
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('engine', '0013_auth_no_default_permissions'),
]
operations = [
migrations.AddField(
model_name='job',
name='max_shape_id',
field=models.BigIntegerField(default=-1),
),
]

1
cvat/apps/engine/models.py
Unescape Escape View File

@ -97,6 +97,7 @@ class Job(models.Model):
segment = models.ForeignKey(Segment, on_delete=models.CASCADE)
assignee = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL)
status = models.CharField(max_length=32, default=StatusChoice.ANNOTATION)
max_shape_id = models.BigIntegerField(default=-1)
class Meta:
default_permissions = ()

35
cvat/apps/engine/static/engine/js/annotationParser.js
Unescape Escape View File

@ -8,14 +8,14 @@
"use strict";
class AnnotationParser {
constructor(job, labelsInfo) {
constructor(job, labelsInfo, idGenerator) {
this._parser = new DOMParser();
this._startFrame = job.start;
this._stopFrame = job.stop;
this._flipped = job.flipped;
this._im_meta = job.image_meta_data;
this._labelsInfo = labelsInfo;
this._id_set = new Set();
this._idGen = idGenerator;
}
_xmlParseError(parsedXML) {
@ -133,7 +133,6 @@ class AnnotationParser {
for (let track of tracks) {
let label = track.getAttribute('label');
let group_id = track.getAttribute('group_id') || '0';
let id = track.getAttribute('id') || '-1';
let labelId = this._labelsInfo.labelIdOf(label);
if (labelId === null) {
throw Error(`An unknown label found in the annotation file: ${label}`);
@ -151,7 +150,6 @@ class AnnotationParser {
!+shapes[0].getAttribute('outside') && +shapes[1].getAttribute('outside')) {
shapes[0].setAttribute('label', label);
shapes[0].setAttribute('group_id', group_id);
shapes[0].setAttribute('id', id);
result.push(shapes[0]);
}
}
@ -212,15 +210,6 @@ class AnnotationParser {
throw Error('An unknown label found in the annotation file: ' + shape.getAttribute('label'));
}
let id = parseInt(shape.getAttribute('id') || '-1');
if (id !== -1) {
if (this._id_set.has(id)) {
throw Error('More than one shape has the same id attribute');
}
this._id_set.add(id);
}
let attributeList = this._getAttributeList(shape, labelId);
if (shape_type === 'boxes') {
@ -236,7 +225,7 @@ class AnnotationParser {
ybr: ybr,
z_order: z_order,
attributes: attributeList,
id: id,
id: this._idGen.next(),
});
}
else {
@ -249,7 +238,7 @@ class AnnotationParser {
occluded: occluded,
z_order: z_order,
attributes: attributeList,
id: id,
id: this._idGen.next(),
});
}
}
@ -270,7 +259,6 @@ class AnnotationParser {
for (let track of tracks) {
let labelId = this._labelsInfo.labelIdOf(track.getAttribute('label'));
let groupId = track.getAttribute('group_id') || '0';
let id = parseInt(track.getAttribute('id') || '-1');
if (labelId === null) {
throw Error('An unknown label found in the annotation file: ' + name);
}
@ -323,17 +311,9 @@ class AnnotationParser {
frame: +parsed[type][0].getAttribute('frame'),
attributes: [],
shapes: [],
id: id,
id: this._idGen.next(),
};
if (id !== -1) {
if (this._id_set.has(id)) {
throw Error('More than one shape has the same id attribute');
}
this._id_set.add(id);
}
for (let shape of parsed[type]) {
let keyFrame = +shape.getAttribute('keyframe');
let outside = +shape.getAttribute('outside');
@ -405,12 +385,7 @@ class AnnotationParser {
return data;
}
_reset() {
this._id_set.clear();
}
parse(text) {
this._reset();
let xml = this._parser.parseFromString(text, 'text/xml');
let parseerror = this._xmlParseError(xml);
if (parseerror.length) {

10
cvat/apps/engine/static/engine/js/annotationUI.js
Unescape Escape View File

@ -100,12 +100,18 @@ function buildAnnotationUI(job, shapeData, loadJobEvent) {
window.cvat.config = new Config();
// Setup components
let annotationParser = new AnnotationParser(job, window.cvat.labelsInfo);
let idGenerator = new IncrementIdGenerator(job.max_shape_id + 1);
let annotationParser = new AnnotationParser(job, window.cvat.labelsInfo, idGenerator);
let shapeCollectionModel = new ShapeCollectionModel().import(shapeData, true);
let shapeCollectionModel = new ShapeCollectionModel(idGenerator).import(shapeData, true);
let shapeCollectionController = new ShapeCollectionController(shapeCollectionModel);
let shapeCollectionView = new ShapeCollectionView(shapeCollectionModel, shapeCollectionController);
// In case of old tasks that dont provide max saved shape id properly
if (job.max_shape_id === -1) {
idGenerator.reset(shapeCollectionModel.maxId + 1);
}
window.cvat.data = {
get: () => shapeCollectionModel.exportAll(),
set: (data) => {

36
cvat/apps/engine/static/engine/js/base.js
Unescape Escape View File

@ -4,8 +4,16 @@
* SPDX-License-Identifier: MIT
*/
/* exported confirm showMessage showOverlay dumpAnnotationRequest ExportType
createExportContainer getExportTargetContainer
/* exported
ExportType
IncrementIdGenerator
ConstIdGenerator
confirm
createExportContainer
dumpAnnotationRequest
getExportTargetContainer
showMessage
showOverlay
*/
"use strict";
@ -237,6 +245,30 @@ function getExportTargetContainer(export_type, shape_type, container) {
return shape_container_target;
}
class IncrementIdGenerator {
constructor(startId=0) {
this._startId = startId;
}
next() {
return this._startId++;
}
reset(startId=0) {
this._startId = startId;
}
}
class ConstIdGenerator {
constructor(startId=-1) {
this._startId = startId;
}
next() {
return this._startId;
}
}
/* These HTTP methods do not require CSRF protection */
function csrfSafeMethod(method) {
return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));

27
cvat/apps/engine/static/engine/js/shapeCollection.js
Unescape Escape View File

@ -8,7 +8,7 @@
"use strict";
class ShapeCollectionModel extends Listener {
constructor() {
constructor(idGenereator) {
super('onCollectionUpdate', () => this);
this._annotationShapes = {};
this._groups = {};
@ -54,10 +54,7 @@ class ShapeCollectionModel extends Listener {
this._initialShapes = {};
this._exportedShapes = {};
this._shapesToDelete = createExportContainer();
}
_nextIdx() {
return this._idx++;
this._idGen = idGenereator;
}
_nextGroupIdx() {
@ -251,21 +248,10 @@ class ShapeCollectionModel extends Listener {
}
}
this._updateClientIds();
this.notify();
return this;
}
_updateClientIds() {
this._idx = Math.max(-1, ...(this._shapes.map( shape => shape.id ))) + 1;
for (const shape of this._shapes) {
if (shape.id === -1) {
shape._id = this._nextIdx();
}
}
}
confirmExportedState() {
this._initialShapes = this._exportedShapes;
this._shapesToDelete = createExportContainer();
@ -412,12 +398,11 @@ class ShapeCollectionModel extends Listener {
add(data, type) {
let id = null;
if (!('id' in data)) {
id = this._nextIdx();
if (!('id' in data) || data.id === -1) {
id = this._idGen.next();
}
else {
id = data.id;
this._idx = Math.max(this._idx, id) + 1;
}
let model = buildShapeModel(data, type, id, this.nextColor());
@ -880,6 +865,10 @@ class ShapeCollectionModel extends Listener {
get shapes() {
return this._shapes;
}
get maxId() {
return Math.max(-1, ...this._shapes.map( shape => shape.id ));
}
}
class ShapeCollectionController {

11
cvat/apps/engine/task.py
Unescape Escape View File

@ -163,7 +163,13 @@ def get(tid):
attributes[db_label.id][db_attrspec.id] = db_attrspec.text
db_segments = list(db_task.segment_set.prefetch_related('job_set').all())
segment_length = max(db_segments[0].stop_frame - db_segments[0].start_frame + 1, 1)
job_indexes = [segment.job_set.first().id for segment in db_segments]
job_indexes = []
for segment in db_segments:
db_job = segment.job_set.first()
job_indexes.append({
"job_id": db_job.id,
"max_shape_id": db_job.max_shape_id,
})
response = {
"status": db_task.status,
@ -242,7 +248,8 @@ def get_job(jid):
"attributes": attributes,
"z_order": db_task.z_order,
"flipped": db_task.flipped,
"image_meta_data": im_meta_data
"image_meta_data": im_meta_data,
"max_shape_id": db_job.max_shape_id,
}
else:
raise Exception("Cannot find the job: {}".format(jid))

9
cvat/apps/tf_annotation/views.py
Unescape Escape View File

@ -204,7 +204,6 @@ def convert_to_cvat_format(data):
'delete': create_anno_container(),
}
client_idx = 0
for label in data:
boxes = data[label]
for box in boxes:
@ -219,11 +218,9 @@ def convert_to_cvat_format(data):
"group_id": 0,
"occluded": False,
"attributes": [],
"id": client_idx,
"id": -1,
})
client_idx += 1
return result
def create_thread(tid, labels_mapping):
@ -235,9 +232,6 @@ def create_thread(tid, labels_mapping):
job.save_meta()
# Get job indexes and segment length
db_task = TaskModel.objects.get(pk=tid)
db_segments = list(db_task.segment_set.prefetch_related('job_set').all())
segment_length = max(db_segments[0].stop_frame - db_segments[0].start_frame + 1, 1)
job_indexes = [segment.job_set.first().id for segment in db_segments]
# Get image list
image_list = make_image_list(db_task.get_data_dirname())
@ -256,6 +250,7 @@ def create_thread(tid, labels_mapping):
# Modify data format and save
result = convert_to_cvat_format(result)
annotation.clear_task(tid)
annotation.save_task(tid, result)
slogger.glob.info('tf annotation for task {} done'.format(tid))
except:

Write Preview
Loading…
Cancel
Save