Don't use 'pre_erase' flag inside save request. Renamed cleint_it -> id. (#158)

* removed pre_erase flag
* used 'id' name instead 'client_id' in client code and dumped annotation

cvat/apps/engine/annotation.py

@@ -79,12 +79,10 @@ def save_job(jid, data):
 
     annotation = _AnnotationForJob(db_job)
     annotation.validate_data_from_client(data)
-    if data['pre_erase']:
-        annotation.delete_objs_from_db()
 
-    for action in ['create', 'update', 'delete']:
-        annotation.init_from_client(data[action])
-        annotation.save_to_db(action)
+    annotation.delete_from_db(data['delete'])
+    annotation.save_to_db(data['create'])
+    annotation.update_in_db(data['update'])
 
     db_job.segment.task.updated_date = timezone.now()
     db_job.segment.task.save()
@@ -855,7 +853,7 @@ class _AnnotationForJob(_Annotation):
                 group_id=int(box['group_id']),
                 occluded=strtobool(str(box['occluded'])),
                 z_order=int(box['z_order']),
-                client_id=int(box['client_id']),
+                client_id=int(box['id']),
             )
 
             for attr in box['attributes']:
@@ -878,7 +876,7 @@ class _AnnotationForJob(_Annotation):
                     group_id=int(poly_shape['group_id']),
                     occluded=poly_shape['occluded'],
                     z_order=int(poly_shape['z_order']),
-                    client_id=int(poly_shape['client_id']),
+                    client_id=int(poly_shape['id']),
                 )
 
                 for attr in poly_shape['attributes']:
@@ -946,7 +944,7 @@ class _AnnotationForJob(_Annotation):
                 stop_frame=self.stop_frame,
                 group_id=int(path['group_id']),
                 boxes=boxes,
-                client_id=int(path['client_id']),
+                client_id=int(path['id']),
                 attributes=attributes,
             )
             self.box_paths.append(box_path)
@@ -1009,7 +1007,7 @@ class _AnnotationForJob(_Annotation):
                     stop_frame=self.stop_frame + 1,
                     group_id=int(path['group_id']),
                     shapes=poly_shapes,
-                    client_id=int(path['client_id']),
+                    client_id=int(path['id']),
                     attributes=attributes,
                 )
 
@@ -1259,51 +1257,49 @@ class _AnnotationForJob(_Annotation):
             self._get_shape_attr_class(shape_type).objects.bulk_create(db_attrvals)
 
     def _update_shapes_in_db(self):
-        self._delete_paths_from_db()
-        self._save_paths_to_db()
+        client_ids_to_delete = {}
+        for shape_type in ['polygons', 'polylines', 'points', 'boxes']:
+            client_ids_to_delete[shape_type] = list(shape.client_id for shape in getattr(self, shape_type))
+        self._delete_shapes_from_db(client_ids_to_delete)
+        self._save_shapes_to_db()
 
     def _update_paths_in_db(self):
-        self._delete_shapes_from_db()
-        self._save_shapes_to_db()
+        client_ids_to_delete = {}
+        for shape_type in ['polygon_paths', 'polyline_paths', 'points_paths', 'box_paths']:
+            client_ids_to_delete[shape_type] = list(shape.client_id for shape in getattr(self, shape_type))
+        self._delete_paths_from_db(client_ids_to_delete)
+        self._save_paths_to_db()
 
-    def _delete_shapes_from_db(self):
+    def _delete_shapes_from_db(self, data):
         for shape_type in ['polygons', 'polylines', 'points', 'boxes']:
-            client_ids_to_delete = list(shape.client_id for shape in getattr(self, shape_type))
+            client_ids_to_delete = data[shape_type]
             deleted = self._get_shape_set(shape_type).filter(client_id__in=client_ids_to_delete).delete()
             class_name = 'engine.{}'.format(self._get_shape_class(shape_type).__name__)
             if not (deleted[0] == 0 and len(client_ids_to_delete) == 0) and (class_name in deleted[1] and deleted[1][class_name] != len(client_ids_to_delete)):
                 raise Exception('Number of deleted object doesn\'t match with requested number')
 
-    def _delete_paths_from_db(self):
+    def _delete_paths_from_db(self, data):
         for shape_type in ['polygon_paths', 'polyline_paths', 'points_paths', 'box_paths']:
-            client_ids_to_delete = list(shape.client_id for shape in getattr(self, shape_type))
+            client_ids_to_delete = data[shape_type]
             deleted = self.db_job.objectpath_set.filter(client_id__in=client_ids_to_delete).delete()
             class_name = 'engine.ObjectPath'
             if not (deleted[0] == 0 and len(client_ids_to_delete) == 0) and \
                (class_name in deleted[1] and deleted[1][class_name] != len(client_ids_to_delete)):
                raise Exception('Number of deleted object doesn\'t match with requested number')
 
-    def _delete_all_shapes_from_db(self):
-        for shape_type in ['polygons', 'polylines', 'points', 'boxes']:
-            self._get_shape_set(shape_type).all().delete()
-
-    def _delete_all_paths_from_db(self):
-        self.db_job.objectpath_set.all().delete()
-
-    def save_to_db(self, action):
-        if action == 'create':
-            self._save_shapes_to_db()
-            self._save_paths_to_db()
-        elif action == 'update':
-            self._update_shapes_in_db()
-            self._update_paths_in_db()
-        elif action == 'delete':
-            self._delete_shapes_from_db()
-            self._delete_paths_from_db()
-
-    def delete_objs_from_db(self):
-        self._delete_all_shapes_from_db()
-        self._delete_all_paths_from_db()
+    def delete_from_db(self, data):
+        self._delete_shapes_from_db(data)
+        self._delete_paths_from_db(data)
+
+    def update_in_db(self, data):
+        self.init_from_client(data)
+        self._update_shapes_in_db()
+        self._update_paths_in_db()
+
+    def save_to_db(self, data):
+        self.init_from_client(data)
+        self._save_shapes_to_db()
+        self._save_paths_to_db()
 
     def to_client(self):
         data = {
@@ -1319,7 +1315,7 @@ class _AnnotationForJob(_Annotation):
 
         for box in self.boxes:
             data["boxes"].append({
-                "client_id": box.client_id,
+                "id": box.client_id,
                 "label_id": box.label.id,
                 "group_id": box.group_id,
                 "xtl": box.xtl,
@@ -1335,7 +1331,7 @@ class _AnnotationForJob(_Annotation):
         for poly_type in ['polygons', 'polylines', 'points']:
             for poly in getattr(self, poly_type):
                 data[poly_type].append({
-                    "client_id": poly.client_id,
+                    "id": poly.client_id,
                     "label_id": poly.label.id,
                     "group_id": poly.group_id,
                     "points": poly.points,
@@ -1347,7 +1343,7 @@ class _AnnotationForJob(_Annotation):
 
         for box_path in self.box_paths:
             data["box_paths"].append({
-                "client_id": box_path.client_id,
+                "id": box_path.client_id,
                 "label_id": box_path.label.id,
                 "group_id": box_path.group_id,
                 "frame": box_path.frame,
@@ -1370,7 +1366,7 @@ class _AnnotationForJob(_Annotation):
         for poly_path_type in ['polygon_paths', 'polyline_paths', 'points_paths']:
             for poly_path in getattr(self, poly_path_type):
                 data[poly_path_type].append({
-                    "client_id": poly_path.client_id,
+                    "id": poly_path.client_id,
                     "label_id": poly_path.label.id,
                     "group_id": poly_path.group_id,
                     "frame": poly_path.frame,
@@ -1393,18 +1389,18 @@ class _AnnotationForJob(_Annotation):
         # check unique id for each object
         client_ids = set()
         def extract_and_check_clinet_id(shape):
-            if 'client_id' not in shape:
-                raise Exception('No client_id field in received data')
-            client_id = shape['client_id']
+            if 'id' not in shape:
+                raise Exception('No id field in received data')
+            client_id = shape['id']
             if client_id in client_ids:
-                raise Exception('More than one object has the same client_id {}'.format(client_id))
+                raise Exception('More than one object has the same id {}'.format(client_id))
             client_ids.add(client_id)
             return client_id
 
         shape_types = ['boxes', 'points', 'polygons', 'polylines', 'box_paths',
             'points_paths', 'polygon_paths', 'polyline_paths']
 
-        for action in ['create', 'update', 'delete']:
+        for action in ['create', 'update']:
             for shape_type in shape_types:
                 for shape in data[action][shape_type]:
                     extract_and_check_clinet_id(shape)
@@ -1958,7 +1954,7 @@ class _AnnotationForTask(_Annotation):
                                         ("xbr", "{:.2f}".format(shape.xbr)),
                                         ("ybr", "{:.2f}".format(shape.ybr)),
                                         ("occluded", str(int(shape.occluded))),
-                                        ("client_id", str(shape.client_id)),
+                                        ("id", str(shape.client_id)),
                                     ])
                                     if db_task.z_order:
                                         dump_dict['z_order'] = str(shape.z_order)
@@ -1978,7 +1974,7 @@ class _AnnotationForTask(_Annotation):
                                             )) for p in shape.points.split(' '))
                                         )),
                                         ("occluded", str(int(shape.occluded))),
-                                        ("client_id", str(shape.client_id)),
+                                        ("id", str(shape.client_id)),
                                     ])
 
                                     if db_task.z_order:
@@ -2019,14 +2015,12 @@ class _AnnotationForTask(_Annotation):
                 im_w = im_meta_data['original_size'][0]['width']
                 im_h = im_meta_data['original_size'][0]['height']
 
-                path_idx = 0
                 for shape_type in ["boxes", "polygons", "polylines", "points"]:
                     path_list = paths[shape_type]
                     for path in path_list:
                         dump_dict = OrderedDict([
-                            ("id", str(path_idx)),
+                            ("id", str(path.client_id)),
                             ("label", path.label.name),
-                            ("client_id", str(path.client_id)),
                         ])
                         if path.group_id:
                             dump_dict['group_id'] = str(path.group_id)
@@ -2095,6 +2089,5 @@ class _AnnotationForTask(_Annotation):
                                     dumper.close_polyline()
                                 else:
                                     dumper.close_points()
-                        path_idx += 1
                         dumper.close_track()
             dumper.close_root()

cvat/apps/engine/static/engine/js/annotationParser.js

@@ -15,7 +15,7 @@ class AnnotationParser {
         this._flipped = job.flipped;
         this._im_meta = job.image_meta_data;
         this._labelsInfo = labelsInfo;
-        this._client_id_set = new Set();
+        this._id_set = new Set();
     }
 
     _xmlParseError(parsedXML) {
@@ -133,7 +133,7 @@ class AnnotationParser {
         for (let track of tracks) {
             let label = track.getAttribute('label');
             let group_id = track.getAttribute('group_id') || '0';
-            let client_id = track.getAttribute('client_id') || '-1';
+            let id = track.getAttribute('id') || '-1';
             let labelId = this._labelsInfo.labelIdOf(label);
             if (labelId === null) {
                 throw Error(`An unknown label found in the annotation file: ${label}`);
@@ -151,7 +151,7 @@ class AnnotationParser {
                     !+shapes[0].getAttribute('outside') && +shapes[1].getAttribute('outside')) {
                     shapes[0].setAttribute('label', label);
                     shapes[0].setAttribute('group_id', group_id);
-                    shapes[0].setAttribute('client_id', client_id);
+                    shapes[0].setAttribute('id', id);
                     result.push(shapes[0]);
                 }
             }
@@ -160,17 +160,6 @@ class AnnotationParser {
         return result;
     }
 
-    _updateClientIds(data) {
-        let maxId = Math.max(-1, ...Array.from(this._client_id_set));
-        for (const shape_type in data) {
-            for (const shape of data[shape_type]) {
-                if (shape.client_id === -1) {
-                    shape.client_id = ++maxId;
-                }
-            }
-        }
-    }
-
     _parseAnnotationData(xml) {
         let data = {
             boxes: [],
@@ -223,13 +212,13 @@ class AnnotationParser {
                     throw Error('An unknown label found in the annotation file: ' + shape.getAttribute('label'));
                 }
 
-                let client_id = parseInt(shape.getAttribute('client_id') || '-1');
-                if (client_id !== -1) {
-                    if (this._client_id_set.has(client_id)) {
-                        throw Error('More than one shape has the same client_id attribute');
+                let id = parseInt(shape.getAttribute('id') || '-1');
+                if (id !== -1) {
+                    if (this._id_set.has(id)) {
+                        throw Error('More than one shape has the same id attribute');
                     }
 
-                    this._client_id_set.add(client_id);
+                    this._id_set.add(id);
                 }
 
                 let attributeList = this._getAttributeList(shape, labelId);
@@ -247,7 +236,7 @@ class AnnotationParser {
                         ybr: ybr,
                         z_order: z_order,
                         attributes: attributeList,
-                        client_id: client_id,
+                        id: id,
                     });
                 }
                 else {
@@ -260,7 +249,7 @@ class AnnotationParser {
                         occluded: occluded,
                         z_order: z_order,
                         attributes: attributeList,
-                        client_id: client_id,
+                        id: id,
                     });
                 }
             }
@@ -281,7 +270,7 @@ class AnnotationParser {
         for (let track of tracks) {
             let labelId = this._labelsInfo.labelIdOf(track.getAttribute('label'));
             let groupId = track.getAttribute('group_id') || '0';
-            let client_id = parseInt(track.getAttribute('client_id') || '-1');
+            let id = parseInt(track.getAttribute('id') || '-1');
             if (labelId === null) {
                 throw Error('An unknown label found in the annotation file: ' + name);
             }
@@ -334,15 +323,15 @@ class AnnotationParser {
                 frame: +parsed[type][0].getAttribute('frame'),
                 attributes: [],
                 shapes: [],
-                client_id: client_id,
+                id: id,
             };
 
-            if (client_id !== -1) {
-                if (this._client_id_set.has(client_id)) {
-                    throw Error('More than one shape has the same client_id attribute');
+            if (id !== -1) {
+                if (this._id_set.has(id)) {
+                    throw Error('More than one shape has the same id attribute');
                 }
 
-                this._client_id_set.add(client_id);
+                this._id_set.add(id);
             }
 
             for (let shape of parsed[type]) {
@@ -417,7 +406,7 @@ class AnnotationParser {
     }
 
     _reset() {
-        this._client_id_set.clear();
+        this._id_set.clear();
     }
 
     parse(text) {
@@ -430,8 +419,6 @@ class AnnotationParser {
 
         let interpolationData = this._parseInterpolationData(xml);
         let annotationData = this._parseAnnotationData(xml);
-        let data = Object.assign({}, annotationData, interpolationData);
-        this._updateClientIds(data);
-        return data;
+        return Object.assign({}, annotationData, interpolationData);
     }
 }

cvat/apps/engine/static/engine/js/annotationUI.js

@@ -100,8 +100,7 @@ function buildAnnotationUI(job, shapeData, loadJobEvent) {
     // Setup components
     let annotationParser = new AnnotationParser(job, window.cvat.labelsInfo);
 
-    let shapeCollectionModel = new ShapeCollectionModel().import(shapeData).updateExportedState();
-    shapeCollectionModel.confirmExportedState();
+    let shapeCollectionModel = new ShapeCollectionModel().import(shapeData, true);
     let shapeCollectionController = new ShapeCollectionController(shapeCollectionModel);
     let shapeCollectionView = new ShapeCollectionView(shapeCollectionModel, shapeCollectionController);
 
@@ -109,7 +108,7 @@ function buildAnnotationUI(job, shapeData, loadJobEvent) {
         get: () => shapeCollectionModel.exportAll(),
         set: (data) => {
             shapeCollectionModel.empty();
-            shapeCollectionModel.import(data);
+            shapeCollectionModel.import(data, false);
             shapeCollectionModel.update();
         },
         clear: () => shapeCollectionModel.empty(),
@@ -664,7 +663,7 @@ function uploadAnnotation(shapeCollectionModel, historyModel, annotationParser,
                     try {
                         historyModel.empty();
                         shapeCollectionModel.empty();
-                        shapeCollectionModel.import(data);
+                        shapeCollectionModel.import(data, false);
                         shapeCollectionModel.update();
                     }
                     finally {

cvat/apps/engine/static/engine/js/base.js

@@ -183,7 +183,6 @@ function createExportContainer() {
             "polyline_paths": [],
         };
     });
-    container.pre_erase = false;
 
     return container;
 }

cvat/apps/engine/static/engine/js/shapeCollection.js

@@ -51,9 +51,9 @@ class ShapeCollectionModel extends Listener {
         this._colorIdx = 0;
         this._filter = new FilterModel(() => this.update());
         this._splitter = new ShapeSplitter();
-        this._erased = false;
         this._initialShapes = {};
         this._exportedShapes = {};
+        this._shapesToDelete = createExportContainer();
     }
 
     _nextIdx() {
@@ -203,73 +203,102 @@ class ShapeCollectionModel extends Listener {
         }
     }
 
-    import(data) {
+    import(data, udpateInitialState=false) {
         for (let box of data.boxes) {
             this.add(box, 'annotation_box');
         }
 
-        for (let box_path of data.box_paths) {
-            this.add(box_path, 'interpolation_box');
+        for (let boxPath of data.box_paths) {
+            this.add(boxPath, 'interpolation_box');
         }
 
         for (let points of data.points) {
             this.add(points, 'annotation_points');
         }
 
-        for (let points_path of data.points_paths) {
-            this.add(points_path, 'interpolation_points');
+        for (let pointsPath of data.points_paths) {
+            this.add(pointsPath, 'interpolation_points');
         }
 
         for (let polygon of data.polygons) {
             this.add(polygon, 'annotation_polygon');
         }
 
-        for (let polygon_path of data.polygon_paths) {
-            this.add(polygon_path, 'interpolation_polygon');
+        for (let polygonPath of data.polygon_paths) {
+            this.add(polygonPath, 'interpolation_polygon');
         }
 
         for (let polyline of data.polylines) {
             this.add(polyline, 'annotation_polyline');
         }
 
-        for (let polyline_path of data.polyline_paths) {
-            this.add(polyline_path, 'interpolation_polyline');
+        for (let polylinePath of data.polyline_paths) {
+            this.add(polylinePath, 'interpolation_polyline');
         }
 
+        if (udpateInitialState) {
+            for (const shape of this._shapes) {
+                if (shape.id === -1) {
+                    const toDelete = getExportTargetContainer(ExportType.delete, shape.type, this._shapesToDelete);
+                    toDelete.push(shape.id);
+                }
+                else {
+                    this._initialShapes[shape.id] = {
+                        type: shape.type,
+                        exportedString: shape.export(),
+                    };
+                }
+            }
+        }
+
+        this._updateClientIds();
+
         this.notify();
         return this;
     }
 
+    _updateClientIds() {
+        this._idx = Math.max(-1, ...(this._shapes.map( shape => shape.id ))) + 1;
+        for (const shape of this._shapes) {
+            if (shape.id === -1) {
+                shape._id = this._nextIdx();
+            }
+        }
+    }
+
     confirmExportedState() {
-        this._erased = false;
         this._initialShapes = this._exportedShapes;
+        this._shapesToDelete = createExportContainer();
     }
 
     export() {
         const response = createExportContainer();
-        response.pre_erase = this._erased;
 
         for (const shape of this._shapes) {
-            let target_export_container = undefined;
+            let targetExportContainer = undefined;
             if (!shape._removed) {
-                if (!(shape.id in this._initialShapes) || this._erased) {
-                    target_export_container = getExportTargetContainer(ExportType.create, shape.type, response);
-                } else if (JSON.stringify(this._initialShapes[shape.id]) !== JSON.stringify(shape.export())) {
-                    target_export_container = getExportTargetContainer(ExportType.update, shape.type, response);
+                if (!(shape.id in this._initialShapes)) {
+                    targetExportContainer = getExportTargetContainer(ExportType.create, shape.type, response);
+                } else if (JSON.stringify(this._initialShapes[shape.id].exportedString) !== JSON.stringify(shape.export())) {
+                    targetExportContainer = getExportTargetContainer(ExportType.update, shape.type, response);
                 } else {
                     continue;
                 }
+                targetExportContainer.push(shape.export());
             }
-            else if (shape.id in this._initialShapes && !this._erased) {
-                // TODO in this case need push only id
-                target_export_container = getExportTargetContainer(ExportType.delete, shape.type, response);
+            else if (shape.id in this._initialShapes) {
+                targetExportContainer = getExportTargetContainer(ExportType.delete, shape.type, response);
+                targetExportContainer.push(shape.id);
             }
             else {
                 continue;
             }
-
-            target_export_container.push(shape.export());
         }
+        for (const shapeType in this._shapesToDelete.delete) {
+            const shapes = this._shapesToDelete.delete[shapeType];
+            response.delete[shapeType].push.apply(response.delete[shapeType], shapes);
+        }
+
         return response;
     }
 
@@ -348,46 +377,46 @@ class ShapeCollectionModel extends Listener {
             }
         }
 
-        return exportData.pre_erase;
+        return false;
     }
 
     updateExportedState() {
         this._exportedShapes = {};
 
-        if (this._erased) {
-            return this;
-        }
-
         for (const shape of this._shapes) {
             if (!shape.removed) {
-                this._exportedShapes[shape.id] = shape.export();
+                this._exportedShapes[shape.id] = {
+                    type: shape.type,
+                    exportedString: shape.export(),
+                };
             }
         }
         return this;
     }
 
     empty() {
+        for (const shapeId in this._initialShapes) {
+            const exportTarget = getExportTargetContainer(ExportType.delete, this._initialShapes[shapeId].type, this._shapesToDelete);
+            exportTarget.push(shapeId);
+        }
+
+        this._initialShapes = {};
         this._annotationShapes = {};
         this._interpolationShapes = [];
         this._shapes = [];
         this._idx = 0;
         this._colorIdx = 0;
-        this._erased = true;
         this._interpolate();
     }
 
     add(data, type) {
         let id = null;
 
-        if (!('client_id' in data)) {
-            id = this._nextIdx();
-        }
-        else if (data.client_id === -1 ) {
-            this._erased = true;
+        if (!('id' in data)) {
             id = this._nextIdx();
         }
         else {
-            id = data.client_id;
+            id = data.id;
             this._idx = Math.max(this._idx, id) + 1;
         }
 

cvat/apps/engine/static/engine/js/shapes.js

@@ -760,7 +760,7 @@ class BoxModel extends ShapeModel {
             }
 
             return Object.assign({}, this._positions[this._frame], {
-                client_id: this._id,
+                id: this._id,
                 attributes: immutableAttributes,
                 label_id: this._label,
                 group_id: this._groupId,
@@ -769,7 +769,7 @@ class BoxModel extends ShapeModel {
         }
         else {
             let boxPath = {
-                client_id: this._id,
+                id: this._id,
                 label_id: this._label,
                 group_id: this._groupId,
                 frame: this._frame,
@@ -955,7 +955,7 @@ class PolyShapeModel extends ShapeModel {
             }
 
             return Object.assign({}, this._positions[this._frame], {
-                client_id: this._id,
+                id: this._id,
                 attributes: immutableAttributes,
                 label_id: this._label,
                 group_id: this._groupId,
@@ -964,7 +964,7 @@ class PolyShapeModel extends ShapeModel {
         }
         else {
             let polyPath = {
-                client_id: this._id,
+                id: this._id,
                 label_id: this._label,
                 group_id: this._groupId,
                 frame: this._frame,

cvat/apps/tf_annotation/views.py

@@ -220,7 +220,7 @@ def convert_to_cvat_format(data):
                 "group_id": 0,
                 "occluded": False,
                 "attributes": [],
-                "client_id": client_idx,
+                "id": client_idx,
             })
 
             client_idx += 1