From 3db9c2d9965f97778684a465c0ec9bdefd105b9c Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 30 Aug 2022 10:06:56 +0300 Subject: [PATCH] Some fixes for skeleton patch related with merge & delete, undo/redo (#4875) * Fixed merge after changing server API scheme * Fixed some delete-undo related issues * Fixed server id assignment * Fixed selection a skeleton point * Updated versions * Applied some comments * Updated version * Fixed stylelint * Updated license headers --- cvat-canvas/package.json | 2 +- cvat-canvas/src/scss/canvas.scss | 5 + cvat-canvas/src/typescript/canvasView.ts | 13 ++- cvat-core/package.json | 2 +- cvat-core/src/annotations-collection.ts | 127 ++++++++++++++--------- cvat-core/src/annotations-objects.ts | 75 ++++++++++++- cvat-core/src/annotations-saver.ts | 5 +- 7 files changed, 170 insertions(+), 59 deletions(-) diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json index ef594a23..bd3706b4 100644 --- a/cvat-canvas/package.json +++ b/cvat-canvas/package.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "2.15.0", + "version": "2.15.1", "description": "Part of Computer Vision Annotation Tool which presents its canvas library", "main": "src/canvas.ts", "scripts": { diff --git a/cvat-canvas/src/scss/canvas.scss b/cvat-canvas/src/scss/canvas.scss index a16e76c5..7c124118 100644 --- a/cvat-canvas/src/scss/canvas.scss +++ b/cvat-canvas/src/scss/canvas.scss @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2022 CVAT.ai Corp // // SPDX-License-Identifier: MIT @@ -99,6 +100,10 @@ polyline.cvat_canvas_shape_grouping { @extend .cvat_shape_action_opacity; fill: blue; + + > circle[data-node-id] { + fill: blue; + } } polyline.cvat_canvas_shape_merging { diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts index b340675f..bf25a369 100644 --- a/cvat-canvas/src/typescript/canvasView.ts +++ b/cvat-canvas/src/typescript/canvasView.ts @@ -1,4 +1,5 @@ // Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) 2022 CVAT.ai Corp // // SPDX-License-Identifier: MIT @@ -2657,7 +2658,7 @@ export class CanvasViewImpl implements CanvasView, Listener { const mouseover = (e: MouseEvent): void => { const locked = this.drawnStates[state.clientID].lock; - if (!locked && !e.ctrlKey) { + if (!locked && !e.ctrlKey && this.mode === Mode.IDLE) { circle.attr({ 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale, }); @@ -2678,6 +2679,14 @@ export class CanvasViewImpl implements CanvasView, Listener { } }; + const mousemove = (e: MouseEvent) => { + if (this.mode === Mode.IDLE) { + // stop propagation to canvas where it calls another canvas.moved + // and does not allow to activate an element + e.stopPropagation(); + } + }; + const mouseleave = (): void => { circle.attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, @@ -2699,11 +2708,13 @@ export class CanvasViewImpl implements CanvasView, Listener { circle.on('mouseover', mouseover); circle.on('mouseleave', mouseleave); + circle.on('mousemove', mousemove); circle.on('click', click); circle.on('remove', () => { circle.off('remove'); circle.off('mouseover', mouseover); circle.off('mouseleave', mouseleave); + circle.off('mousemove', mousemove); circle.off('click', click); }); diff --git a/cvat-core/package.json b/cvat-core/package.json index ee0113da..76d656d8 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "6.0.1", + "version": "6.0.2", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "src/api.ts", "scripts": { diff --git a/cvat-core/src/annotations-collection.ts b/cvat-core/src/annotations-collection.ts index c831598e..53dfadab 100644 --- a/cvat-core/src/annotations-collection.ts +++ b/cvat-core/src/annotations-collection.ts @@ -1,4 +1,5 @@ // Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) 2022 CVAT.ai Corp // // SPDX-License-Identifier: MIT @@ -157,25 +158,9 @@ return objectStates; } - merge(objectStates) { - checkObjectType('shapes for merge', objectStates, null, Array); - if (!objectStates.length) return; - const objectsForMerge = objectStates.map((state) => { - checkObjectType('object state', state, null, ObjectState); - const object = this.objects[state.clientID]; - if (typeof object === 'undefined') { - throw new ArgumentError( - 'The object is not in collection yet. Call ObjectState.put([state]) before you can merge it', - ); - } - return object; - }); - + _mergeInternal(objectsForMerge, shapeType, label): any { const keyframes = {}; // frame: position - const { label, shapeType } = objectStates[0]; - if (!(label.id in this.labels)) { - throw new ArgumentError(`Unknown label for the task: ${label.id}`); - } + const elements = {}; // element_sublabel_id: [element], each sublabel will be merged recursively if (!Object.values(ShapeType).includes(shapeType)) { throw new ArgumentError(`Got unknown shapeType "${shapeType}"`); @@ -189,15 +174,16 @@ for (let i = 0; i < objectsForMerge.length; i++) { // For each state get corresponding object const object = objectsForMerge[i]; - const state = objectStates[i]; - if (state.label.id !== label.id) { + if (object.label.id !== label.id) { throw new ArgumentError( - `All shape labels are expected to be ${label.name}, but got ${state.label.name}`, + `All object labels are expected to be "${label.name}", but got "${object.label.name}"`, ); } - if (state.shapeType !== shapeType) { - throw new ArgumentError(`All shapes are expected to be ${shapeType}, but got ${state.shapeType}`); + if (object.shapeType !== shapeType) { + throw new ArgumentError( + `All shapes are expected to be "${shapeType}", but got "${object.shapeType}"`, + ); } // If this object is shape, get it position and save as a keyframe @@ -211,10 +197,6 @@ type: shapeType, frame: object.frame, points: object.shapeType === ShapeType.SKELETON ? undefined : [...object.points], - elements: object.shapeType === ShapeType.SKELETON ? object.elements.map((el) => { - const { id, clientID, ...rest } = el.toJSON(); - return rest; - }) : undefined, occluded: object.occluded, rotation: object.rotation, z_order: object.zOrder, @@ -232,7 +214,7 @@ }; // Push outside shape after each annotation shape - // Any not outside shape rewrites it + // Any not outside shape will rewrite it later if (!(object.frame + 1 in keyframes) && object.frame + 1 <= this.stopFrame) { keyframes[object.frame + 1] = JSON.parse(JSON.stringify(keyframes[object.frame])); keyframes[object.frame + 1].outside = true; @@ -244,15 +226,10 @@ }); } } else if (object instanceof Track) { - // If this object is track, iterate through all its + // If this object is a track, iterate through all its // keyframes and push copies to new keyframes const attributes = {}; // id:value const trackShapes = object.shapes; - const exportedShapes = object.shapeType === ShapeType.SKELETON ? - object.prepareShapesForServer().reduce((acc, val) => { - acc[val.frame] = val; - return acc; - }, {}) : {}; for (const keyframe of Object.keys(trackShapes)) { const shape = trackShapes[keyframe]; // Frame already saved and it is not outside @@ -279,11 +256,6 @@ type: shapeType, frame: +keyframe, points: object.shapeType === ShapeType.SKELETON ? undefined : [...shape.points], - elements: object.shapeType === ShapeType.SKELETON ? - exportedShapes[keyframe].elements.map((el) => { - const { id, ...rest } = el; - return rest; - }) : undefined, rotation: shape.rotation, occluded: shape.occluded, outside: shape.outside, @@ -304,6 +276,37 @@ 'Only shapes and tracks are expected.', ); } + + if (object.shapeType === ShapeType.SKELETON) { + for (const element of object.elements) { + // for each track/shape element get its first objectState and keep it + elements[element.label.id] = [ + ...(elements[element.label.id] || []), element, + ]; + } + } + } + + const mergedElements = []; + if (shapeType === ShapeType.SKELETON) { + for (const sublabel of label.structure.sublabels) { + if (!(sublabel.id in elements)) { + throw new ArgumentError( + `Merged skeleton is absent some of its elements (sublabel id: ${sublabel.id})`, + ); + } + + try { + mergedElements.push(this._mergeInternal( + elements[sublabel.id], elements[sublabel.id][0].shapeType, sublabel, + )); + } catch (error) { + throw new ArgumentError( + `Could not merge some skeleton parts (sublabel id: ${sublabel.id}). + Original error is ${error.toString()}`, + ); + } + } } let firstNonOutside = false; @@ -317,21 +320,21 @@ } } - const clientID = ++this.count; const track = { frame: Math.min.apply( null, Object.keys(keyframes).map((frame) => +frame), ), shapes: Object.values(keyframes), + elements: shapeType === ShapeType.SKELETON ? mergedElements : undefined, group: 0, - source: objectStates[0].source, + source: Source.MANUAL, label_id: label.id, - attributes: Object.keys(objectStates[0].attributes).reduce((accumulator, attrID) => { + attributes: Object.keys(objectsForMerge[0].attributes).reduce((accumulator, attrID) => { if (!labelAttributes[attrID].mutable) { accumulator.push({ spec_id: +attrID, - value: objectStates[0].attributes[attrID], + value: objectsForMerge[0].attributes[attrID], }); } @@ -339,30 +342,56 @@ }, []), }; - const trackModel = trackFactory(track, clientID, this.injection); - this.tracks.push(trackModel); - this.objects[clientID] = trackModel; + return track; + } + + merge(objectStates) { + checkObjectType('shapes for merge', objectStates, null, Array); + if (!objectStates.length) return; + const objectsForMerge = objectStates.map((state) => { + checkObjectType('object state', state, null, ObjectState); + const object = this.objects[state.clientID]; + if (typeof object === 'undefined') { + throw new ArgumentError( + 'The object is not in collection yet. Call ObjectState.put([state]) before you can merge it', + ); + } + return object; + }); + + const { label, shapeType } = objectStates[0]; + if (!(label.id in this.labels)) { + throw new ArgumentError(`Unknown label for the task: ${label.id}`); + } + + const track = this._mergeInternal(objectsForMerge, shapeType, label); + const imported = this.import({ + tracks: [track], + tags: [], + shapes: [], + }); // Remove other shapes for (const object of objectsForMerge) { object.removed = true; } + const [importedTrack] = imported.tracks; this.history.do( HistoryActions.MERGED_OBJECTS, () => { - trackModel.removed = true; + importedTrack.removed = true; for (const object of objectsForMerge) { object.removed = false; } }, () => { - trackModel.removed = false; + importedTrack.removed = false; for (const object of objectsForMerge) { object.removed = true; } }, - [...objectsForMerge.map((object) => object.clientID), trackModel.clientID], + [...objectsForMerge.map((object) => object.clientID), importedTrack.clientID], objectStates[0].frame, ); } diff --git a/cvat-core/src/annotations-objects.ts b/cvat-core/src/annotations-objects.ts index 880eeff1..9ccc1dd7 100644 --- a/cvat-core/src/annotations-objects.ts +++ b/cvat-core/src/annotations-objects.ts @@ -1,4 +1,5 @@ // Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) 2022 CVAT.ai Corp // // SPDX-License-Identifier: MIT @@ -201,7 +202,7 @@ class Annotation { protected group: number; public label: Label; protected frame: number; - protected removed: boolean; + private _removed: boolean; protected lock: boolean; protected readOnlyFields: string[]; protected color: string; @@ -223,7 +224,7 @@ class Annotation { this.group = data.group; this.label = this.taskLabels[data.label_id]; this.frame = data.frame; - this.removed = false; + this._removed = false; this.lock = false; this.readOnlyFields = injection.readOnlyFields || []; this.color = color; @@ -445,6 +446,14 @@ class Annotation { } } + _clearServerID(): void { + this.serverID = undefined; + } + + updateServerID(body: any): void { + this.serverID = body.id; + } + appendDefaultAttributes(label: Label): void { const labelAttributes = label.attributes; for (const attribute of labelAttributes) { @@ -464,11 +473,9 @@ class Annotation { delete(frame: number, force: boolean): boolean { if (!this.lock || force) { this.removed = true; - this.history.do( HistoryActions.REMOVED_OBJECT, () => { - this.serverID = undefined; this.removed = false; this.updated = Date.now(); }, @@ -495,6 +502,17 @@ class Annotation { toJSON(): void { throw new ScriptingError('Is not implemented'); } + + public get removed(): boolean { + return this._removed; + } + + public set removed(value: boolean) { + if (value) { + this._clearServerID(); + } + this._removed = value; + } } class Drawn extends Annotation { @@ -1137,6 +1155,21 @@ export class Track extends Drawn { return result; } + updateServerID(body: RawTrackData): void { + this.serverID = body.id; + for (const shape of body.shapes) { + this.shapes[shape.frame].serverID = shape.id; + } + } + + _clearServerID(): void { + /* eslint-disable-next-line no-underscore-dangle */ + Drawn.prototype._clearServerID.call(this); + for (const keyframe of Object.keys(this.shapes)) { + this.shapes[keyframe].serverID = undefined; + } + } + _saveLabel(label: Label, frame: number): void { const undoLabel = this.label; const redoLabel = label; @@ -2115,6 +2148,23 @@ export class SkeletonShape extends Shape { }; } + updateServerID(body: RawShapeData): void { + Shape.prototype.updateServerID.call(this, body); + for (const element of body.elements) { + const thisElement = this.elements.find((_element: Shape) => _element.label.id === element.label_id); + thisElement.updateServerID(element); + } + } + + _clearServerID(): void { + /* eslint-disable-next-line no-underscore-dangle */ + Shape.prototype._clearServerID.call(this); + for (const element of this.elements) { + /* eslint-disable-next-line no-underscore-dangle */ + element._clearServerID(); + } + } + _saveRotation(rotation, frame) { const undoSkeletonPoints = this.elements.map((element) => element.points); const undoSource = this.source; @@ -2690,6 +2740,23 @@ export class SkeletonTrack extends Track { )).sort((a: Annotation, b: Annotation) => a.label.id - b.label.id) as any as Track[]; } + updateServerID(body: RawTrackData): void { + Track.prototype.updateServerID.call(this, body); + for (const element of body.elements) { + const thisElement = this.elements.find((_element: Track) => _element.label.id === element.label_id); + thisElement.updateServerID(element); + } + } + + _clearServerID(): void { + /* eslint-disable-next-line no-underscore-dangle */ + Track.prototype._clearServerID.call(this); + for (const element of this.elements) { + /* eslint-disable-next-line no-underscore-dangle */ + element._clearServerID(); + } + } + _saveRotation(rotation: number, frame: number): void { const undoSkeletonShapes = this.elements.map((element) => element.shapes[frame]); const undoSource = this.source; diff --git a/cvat-core/src/annotations-saver.ts b/cvat-core/src/annotations-saver.ts index d1adbf75..fdc157aa 100644 --- a/cvat-core/src/annotations-saver.ts +++ b/cvat-core/src/annotations-saver.ts @@ -1,4 +1,5 @@ // Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) 2022 CVAT.ai Corp // // SPDX-License-Identifier: MIT @@ -151,9 +152,7 @@ _updateCreatedObjects(saved, indexes) { const savedLength = saved.tracks.length + saved.shapes.length + saved.tags.length; - const indexesLength = indexes.tracks.length + indexes.shapes.length + indexes.tags.length; - if (indexesLength !== savedLength) { throw new ScriptingError( `Number of indexes is differed by number of saved objects ${indexesLength} vs ${savedLength}`, @@ -164,7 +163,7 @@ for (const type of Object.keys(indexes)) { for (let i = 0; i < indexes[type].length; i++) { const clientID = indexes[type][i]; - this.collection.objects[clientID].serverID = saved[type][i].id; + this.collection.objects[clientID].updateServerID(saved[type][i]); } } }