From f7df747cdf6a2089be745c9e10aef5d7a2fb08da Mon Sep 17 00:00:00 2001 From: Boris Sekachev <40690378+bsekachev@users.noreply.github.com> Date: Tue, 24 Dec 2019 16:55:36 +0300 Subject: [PATCH] Automatic bordering feature during drawing/editing (#997) --- .../engine/static/engine/js/annotationUI.js | 2 +- .../engine/static/engine/js/borderSticker.js | 213 ++++++++++++++++++ .../static/engine/js/polyshapeEditor.js | 107 +++++++-- .../engine/static/engine/js/shapeCreator.js | 85 +++++-- cvat/apps/engine/static/engine/js/shapes.js | 6 +- cvat/apps/engine/static/engine/stylesheet.css | 18 ++ .../engine/templates/engine/annotation.html | 5 + 7 files changed, 399 insertions(+), 37 deletions(-) create mode 100644 cvat/apps/engine/static/engine/js/borderSticker.js diff --git a/cvat/apps/engine/static/engine/js/annotationUI.js b/cvat/apps/engine/static/engine/js/annotationUI.js index 2b5a0e95..b1469728 100644 --- a/cvat/apps/engine/static/engine/js/annotationUI.js +++ b/cvat/apps/engine/static/engine/js/annotationUI.js @@ -583,7 +583,7 @@ function buildAnnotationUI(jobData, taskData, imageMetaData, annotationData, ann const shapeCreatorController = new ShapeCreatorController(shapeCreatorModel); const shapeCreatorView = new ShapeCreatorView(shapeCreatorModel, shapeCreatorController); - const polyshapeEditorModel = new PolyshapeEditorModel(); + const polyshapeEditorModel = new PolyshapeEditorModel(shapeCollectionModel); const polyshapeEditorController = new PolyshapeEditorController(polyshapeEditorModel); const polyshapeEditorView = new PolyshapeEditorView(polyshapeEditorModel, polyshapeEditorController); diff --git a/cvat/apps/engine/static/engine/js/borderSticker.js b/cvat/apps/engine/static/engine/js/borderSticker.js new file mode 100644 index 00000000..c3ab90fe --- /dev/null +++ b/cvat/apps/engine/static/engine/js/borderSticker.js @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2019 Intel Corporation + * + * SPDX-License-Identifier: MIT + */ + +/* exported BorderSticker */ + +class BorderSticker { + constructor(currentShape, frameContent, shapes, scale) { + this._currentShape = currentShape; + this._frameContent = frameContent; + this._enabled = false; + this._groups = null; + this._scale = scale; + this._accounter = { + clicks: [], + shapeID: null, + }; + + const transformedShapes = shapes + .filter((shape) => !shape.model.removed) + .map((shape) => { + const pos = shape.interpolation.position; + // convert boxes to point sets + if (!('points' in pos)) { + return { + points: window.cvat.translate.points + .actualToCanvas(`${pos.xtl},${pos.ytl} ${pos.xbr},${pos.ytl}` + + ` ${pos.xbr},${pos.ybr} ${pos.xtl},${pos.ybr}`), + color: shape.model.color.shape, + }; + } + + return { + points: window.cvat.translate.points + .actualToCanvas(pos.points), + color: shape.model.color.shape, + }; + }); + + this._drawBorderMarkers(transformedShapes); + } + + _addRawPoint(x, y) { + this._currentShape.array().valueOf().pop(); + this._currentShape.array().valueOf().push([x, y]); + // not error, specific of the library + this._currentShape.array().valueOf().push([x, y]); + const paintHandler = this._currentShape.remember('_paintHandler'); + paintHandler.drawCircles(); + paintHandler.set.members.forEach((el) => { + el.attr('stroke-width', 1 / this._scale).attr('r', 2.5 / this._scale); + }); + this._currentShape.plot(this._currentShape.array().valueOf()); + } + + _drawBorderMarkers(shapes) { + const namespace = 'http://www.w3.org/2000/svg'; + + this._groups = shapes.reduce((acc, shape, shapeID) => { + // Group all points by inside svg groups + const group = window.document.createElementNS(namespace, 'g'); + shape.points.split(/\s/).map((point, pointID, points) => { + const [x, y] = point.split(',').map((coordinate) => +coordinate); + const circle = window.document.createElementNS(namespace, 'circle'); + circle.classList.add('shape-creator-border-point'); + circle.setAttribute('fill', shape.color); + circle.setAttribute('stroke', 'black'); + circle.setAttribute('stroke-width', 1 / this._scale); + circle.setAttribute('cx', +x); + circle.setAttribute('cy', +y); + circle.setAttribute('r', 5 / this._scale); + + circle.doubleClickListener = (e) => { + // Just for convenience (prevent screen fit feature) + e.stopPropagation(); + }; + + circle.clickListener = (e) => { + e.stopPropagation(); + // another shape was clicked + if (this._accounter.shapeID !== null && this._accounter.shapeID !== shapeID) { + this.reset(); + } + + this._accounter.shapeID = shapeID; + + if (this._accounter.clicks[1] === pointID) { + // the same point repeated two times + const [_x, _y] = point.split(',').map((coordinate) => +coordinate); + this._addRawPoint(_x, _y); + this.reset(); + return; + } + + // the first point can not be clicked twice + if (this._accounter.clicks[0] !== pointID) { + this._accounter.clicks.push(pointID); + } else { + return; + } + + // up clicked group for convenience + this._frameContent.node.appendChild(group); + + // the first click + if (this._accounter.clicks.length === 1) { + // draw and remove initial point just to initialize data structures + if (!this._currentShape.remember('_paintHandler').startPoint) { + this._currentShape.draw('point', e); + this._currentShape.draw('undo'); + } + + const [_x, _y] = point.split(',').map((coordinate) => +coordinate); + this._addRawPoint(_x, _y); + // the second click + } else if (this._accounter.clicks.length === 2) { + circle.classList.add('shape-creator-border-point-direction'); + // the third click + } else { + // sign defines bypass direction + const landmarks = this._accounter.clicks; + const sign = Math.sign(landmarks[2] - landmarks[0]) + * Math.sign(landmarks[1] - landmarks[0]) + * Math.sign(landmarks[2] - landmarks[1]); + + // go via a polygon and get vertexes + // the first vertex has been already drawn + const way = []; + for (let i = landmarks[0] + sign; ; i += sign) { + if (i < 0) { + i = points.length - 1; + } else if (i === points.length) { + i = 0; + } + + way.push(points[i]); + + if (i === this._accounter.clicks[this._accounter.clicks.length - 1]) { + // put the last element twice + // specific of svg.draw.js + // way.push(points[i]); + break; + } + } + + // remove the latest cursor position from drawing array + for (const wayPoint of way) { + const [_x, _y] = wayPoint.split(',').map((coordinate) => +coordinate); + this._addRawPoint(_x, _y); + } + + this.reset(); + } + }; + + circle.addEventListener('click', circle.clickListener); + circle.addEventListener('dblclick', circle.doubleClickListener); + + return circle; + }).forEach((circle) => group.appendChild(circle)); + + acc.push(group); + return acc; + }, []); + + this._groups + .forEach((group) => this._frameContent.node.appendChild(group)); + } + + reset() { + if (this._accounter.shapeID !== null) { + while (this._accounter.clicks.length > 0) { + const resetID = this._accounter.clicks.pop(); + this._groups[this._accounter.shapeID] + .children[resetID].classList.remove('shape-creator-border-point-direction'); + } + } + + this._accounter = { + clicks: [], + shapeID: null, + }; + } + + disable() { + if (this._groups) { + this._groups.forEach((group) => { + Array.from(group.children).forEach((circle) => { + circle.removeEventListener('click', circle.clickListener); + circle.removeEventListener('dblclick', circle.doubleClickListener); + }); + + group.remove(); + }); + + this._groups = null; + } + } + + scale(scale) { + this._scale = scale; + if (this._groups) { + this._groups.forEach((group) => { + Array.from(group.children).forEach((circle) => { + circle.setAttribute('r', 5 / scale); + circle.setAttribute('stroke-width', 1 / scale); + }); + }); + } + } +} diff --git a/cvat/apps/engine/static/engine/js/polyshapeEditor.js b/cvat/apps/engine/static/engine/js/polyshapeEditor.js index e3934580..7a30a7c1 100644 --- a/cvat/apps/engine/static/engine/js/polyshapeEditor.js +++ b/cvat/apps/engine/static/engine/js/polyshapeEditor.js @@ -12,16 +12,18 @@ PolyShapeModel:false STROKE_WIDTH:false SVG:false + BorderSticker:false */ "use strict"; class PolyshapeEditorModel extends Listener { - constructor() { + constructor(shapeCollection) { super("onPolyshapeEditorUpdate", () => this); this._modeName = 'poly_editing'; this._active = false; + this._shapeCollection = shapeCollection; this._data = { points: null, color: null, @@ -29,10 +31,12 @@ class PolyshapeEditorModel extends Listener { oncomplete: null, type: null, event: null, + startPoint: null, + id: null, }; } - edit(type, points, color, start, event, oncomplete) { + edit(type, points, color, start, startPoint, e, oncomplete, id) { if (!this._active && !window.cvat.mode) { window.cvat.mode = this._modeName; this._active = true; @@ -41,7 +45,9 @@ class PolyshapeEditorModel extends Listener { this._data.start = start; this._data.oncomplete = oncomplete; this._data.type = type; - this._data.event = event; + this._data.event = e; + this._data.startPoint = startPoint; + this._data.id = id; this.notify(); } else if (this._active) { @@ -73,6 +79,7 @@ class PolyshapeEditorModel extends Listener { this._data.oncomplete = null; this._data.type = null; this._data.event = null; + this._data.startPoint = null; this.notify(); } } @@ -84,6 +91,11 @@ class PolyshapeEditorModel extends Listener { get data() { return this._data; } + + get currentShapes() { + this._shapeCollection.update(); + return this._shapeCollection.currentShapes; + } } @@ -99,6 +111,10 @@ class PolyshapeEditorController { cancel() { this._model.cancel(); } + + get currentShapes() { + return this._model.currentShapes; + } } @@ -108,14 +124,30 @@ class PolyshapeEditorView { this._data = null; this._frameContent = SVG.adopt($('#frameContent')[0]); + this._autoBorderingCheckbox = $('#autoBorderingCheckbox'); this._originalShapePointsGroup = null; this._originalShapePoints = []; this._originalShape = null; this._correctLine = null; + this._borderSticker = null; this._scale = window.cvat.player.geometry.scale; this._frame = window.cvat.player.frames.current; + this._autoBorderingCheckbox.on('change.shapeEditor', (e) => { + if (this._correctLine) { + if (!e.target.checked) { + this._borderSticker.disable(); + this._borderSticker = null; + } else { + this._borderSticker = new BorderSticker(this._correctLine, this._frameContent, + this._controller.currentShapes + .filter((shape) => shape.model.id !== this._data.id), + this._scale); + } + } + }); + model.subscribe(this); } @@ -180,6 +212,16 @@ class PolyshapeEditorView { return offset; } + _addRawPoint(x, y) { + this._correctLine.array().valueOf().pop(); + this._correctLine.array().valueOf().push([x, y]); + // not error, specific of the library + this._correctLine.array().valueOf().push([x, y]); + this._correctLine.remember('_paintHandler').drawCircles(); + this._correctLine.plot(this._correctLine.array().valueOf()); + this._rescaleDrawPoints(); + } + _startEdit() { this._frame = window.cvat.player.frames.current; let strokeWidth = this._data.type === 'points' ? 0 : STROKE_WIDTH / this._scale; @@ -222,17 +264,24 @@ class PolyshapeEditorView { } + const [x, y] = this._data.startPoint + .split(',').map((el) => +el); let prevPoint = { - x: this._data.event.clientX, - y: this._data.event.clientY + x, + y, }; + // draw and remove initial point just to initialize data structures this._correctLine.draw('point', this._data.event); - this._rescaleDrawPoints(); + this._correctLine.draw('undo'); + + this._addRawPoint(x, y); + this._frameContent.on('mousemove.polyshapeEditor', (e) => { - if (e.shiftKey && this._data.type != 'points') { - let delta = Math.sqrt(Math.pow(e.clientX - prevPoint.x, 2) + Math.pow(e.clientY - prevPoint.y, 2)); - let deltaTreshold = 15; + if (e.shiftKey && this._data.type !== 'points') { + const delta = Math.sqrt(Math.pow(e.clientX - prevPoint.x, 2) + + Math.pow(e.clientY - prevPoint.y, 2)); + const deltaTreshold = 15; if (delta > deltaTreshold) { this._correctLine.draw('point', e); prevPoint = { @@ -246,8 +295,10 @@ class PolyshapeEditorView { this._frameContent.on('contextmenu.polyshapeEditor', (e) => { if (PolyShapeModel.convertStringToNumberArray(this._correctLine.attr('points')).length > 2) { this._correctLine.draw('undo'); - } - else { + if (this._borderSticker) { + this._borderSticker.reset(); + } + } else { // Finish without points argument is just cancel this._controller.finish(); } @@ -272,9 +323,19 @@ class PolyshapeEditorView { }).on('mouseout', () => { instance.attr('stroke-width', STROKE_WIDTH / this._scale); }).on('mousedown', (e) => { - if (e.which != 1) return; + if (e.which !== 1) { + return; + } let currentPoints = PolyShapeModel.convertStringToNumberArray(this._data.points); - let correctPoints = PolyShapeModel.convertStringToNumberArray(this._correctLine.attr('points')); + // replace the latest point from the event + // (which has not precise coordinates, to precise coordinates) + let correctPoints = this._correctLine + .attr('points') + .split(/\s/) + .slice(0, -1); + correctPoints = correctPoints.concat([`${instance.attr('cx')},${instance.attr('cy')}`]).join(' '); + correctPoints = PolyShapeModel.convertStringToNumberArray(correctPoints); + let resultPoints = []; if (this._data.type === 'polygon') { @@ -338,6 +399,14 @@ class PolyshapeEditorView { this._controller.finish(PolyShapeModel.convertNumberArrayToString(resultPoints)); }); } + + this._autoBorderingCheckbox[0].disabled = false; + $('body').on('keydown.shapeEditor', (e) => { + if (e.ctrlKey && e.keyCode === 17) { + this._autoBorderingCheckbox[0].checked = !this._borderSticker; + this._autoBorderingCheckbox.trigger('change.shapeEditor'); + } + }); } _endEdit() { @@ -361,6 +430,14 @@ class PolyshapeEditorView { this._frameContent.off('mousemove.polyshapeEditor'); this._frameContent.off('mousedown.polyshapeEditor'); this._frameContent.off('contextmenu.polyshapeEditor'); + + $('body').off('keydown.shapeEditor'); + this._autoBorderingCheckbox[0].checked = false; + this._autoBorderingCheckbox[0].disabled = true; + if (this._borderSticker) { + this._borderSticker.disable(); + this._borderSticker = null; + } } @@ -379,6 +456,10 @@ class PolyshapeEditorView { if (this._scale != scale) { this._scale = scale; + if (this._borderSticker) { + this._borderSticker.scale(this._scale); + } + let strokeWidth = this._data && this._data.type === 'points' ? 0 : STROKE_WIDTH / this._scale; let pointRadius = POINT_RADIUS / this._scale; diff --git a/cvat/apps/engine/static/engine/js/shapeCreator.js b/cvat/apps/engine/static/engine/js/shapeCreator.js index 70ad3ed5..a2b58b4f 100644 --- a/cvat/apps/engine/static/engine/js/shapeCreator.js +++ b/cvat/apps/engine/static/engine/js/shapeCreator.js @@ -16,10 +16,9 @@ showMessage:false STROKE_WIDTH:false SVG:false + BorderSticker: false */ -"use strict"; - class ShapeCreatorModel extends Listener { constructor(shapeCollection) { super('onShapeCreatorUpdate', () => this); @@ -34,8 +33,8 @@ class ShapeCreatorModel extends Listener { } finish(result) { - let data = {}; - let frame = window.cvat.player.frames.current; + const data = {}; + const frame = window.cvat.player.frames.current; data.label_id = this._defaultLabel; data.group = 0; @@ -50,7 +49,7 @@ class ShapeCreatorModel extends Listener { mode: this._defaultMode, type: this._defaultType, label: this._defaultLabel, - frame: frame, + frame, }); } @@ -64,7 +63,7 @@ class ShapeCreatorModel extends Listener { this._shapeCollection.add(data, `annotation_${this._defaultType}`); } - let model = this._shapeCollection.shapes.slice(-1)[0]; + const model = this._shapeCollection.shapes.slice(-1)[0]; // Undo/redo code window.cvat.addAction('Draw Object', () => { @@ -87,12 +86,10 @@ class ShapeCreatorModel extends Listener { if (this._createMode) { this._createEvent = Logger.addContinuedEvent(Logger.EventType.drawObject); window.cvat.mode = 'creation'; - } - else if (window.cvat.mode === 'creation') { + } else if (window.cvat.mode === 'creation') { window.cvat.mode = null; } - } - else { + } else { this._createMode = false; if (window.cvat.mode === 'creation') { window.cvat.mode = null; @@ -106,6 +103,11 @@ class ShapeCreatorModel extends Listener { this.notify(); } + get currentShapes() { + this._shapeCollection.update(); + return this._shapeCollection.currentShapes; + } + get saveCurrent() { return this._saveCurrent; } @@ -177,6 +179,10 @@ class ShapeCreatorController { finish(result) { this._model.finish(result); } + + get currentShapes() { + return this._model.currentShapes; + } } class ShapeCreatorView { @@ -188,6 +194,7 @@ class ShapeCreatorView { this._modeSelector = $('#shapeModeSelector'); this._typeSelector = $('#shapeTypeSelector'); this._polyShapeSizeInput = $('#polyShapeSize'); + this._autoBorderingCheckbox = $('#autoBorderingCheckbox'); this._frameContent = SVG.adopt($('#frameContent')[0]); this._frameText = SVG.adopt($("#frameText")[0]); this._playerFrame = $('#playerFrame'); @@ -203,6 +210,7 @@ class ShapeCreatorView { this._mode = null; this._cancel = false; this._scale = 1; + this._borderSticker = null; let shortkeys = window.cvat.config.shortkeys; this._createButton.attr('title', ` @@ -288,8 +296,19 @@ class ShapeCreatorView { } } }); - } + this._autoBorderingCheckbox.on('change.shapeCreator', (e) => { + if (this._drawInstance) { + if (!e.target.checked) { + this._borderSticker.disable(); + this._borderSticker = null; + } else { + this._borderSticker = new BorderSticker(this._drawInstance, this._frameContent, + this._controller.currentShapes, this._scale); + } + } + }); + } _createPolyEvents() { // If number of points for poly shape specified, use it. @@ -340,10 +359,21 @@ class ShapeCreatorView { numberOfPoints ++; }); + this._autoBorderingCheckbox[0].disabled = false; + $('body').on('keydown.shapeCreator', (e) => { + if (e.ctrlKey && e.keyCode === 17) { + this._autoBorderingCheckbox[0].checked = !this._borderSticker; + this._autoBorderingCheckbox.trigger('change.shapeCreator'); + } + }); + this._frameContent.on('mousedown.shapeCreator', (e) => { if (e.which === 3) { let lenBefore = this._drawInstance.array().value.length; this._drawInstance.draw('undo'); + if (this._borderSticker) { + this._borderSticker.reset(); + } let lenAfter = this._drawInstance.array().value.length; if (lenBefore != lenAfter) { numberOfPoints --; @@ -373,6 +403,13 @@ class ShapeCreatorView { this._drawInstance.on('drawstop', () => { this._frameContent.off('mousedown.shapeCreator'); this._frameContent.off('mousemove.shapeCreator'); + this._autoBorderingCheckbox[0].disabled = true; + this._autoBorderingCheckbox[0].checked = false; + $('body').off('keydown.shapeCreator'); + if (this._borderSticker) { + this._borderSticker.disable(); + this._borderSticker = null; + } }); // Also we need callback on drawdone event for get points this._drawInstance.on('drawdone', function(e) { @@ -418,7 +455,7 @@ class ShapeCreatorView { let sizeUI = null; switch(this._type) { case 'box': - this._drawInstance = this._frameContent.rect().draw({snapToGrid: 0.1}).addClass('shapeCreation').attr({ + this._drawInstance = this._frameContent.rect().draw({ snapToGrid: 0.1 }).addClass('shapeCreation').attr({ 'stroke-width': STROKE_WIDTH / this._scale, }).on('drawstop', function(e) { if (this._cancel) return; @@ -461,9 +498,10 @@ class ShapeCreatorView { }); break; case 'points': - this._drawInstance = this._frameContent.polyline().draw({snapToGrid: 0.1}).addClass('shapeCreation').attr({ - 'stroke-width': 0, - }); + this._drawInstance = this._frameContent.polyline().draw({ snapToGrid: 0.1 }) + .addClass('shapeCreation').attr({ + 'stroke-width': 0, + }); this._createPolyEvents(); break; case 'polygon': @@ -474,9 +512,10 @@ class ShapeCreatorView { this._controller.switchCreateMode(true); return; } - this._drawInstance = this._frameContent.polygon().draw({snapToGrid: 0.1}).addClass('shapeCreation').attr({ - 'stroke-width': STROKE_WIDTH / this._scale, - }); + this._drawInstance = this._frameContent.polygon().draw({ snapToGrid: 0.1 }) + .addClass('shapeCreation').attr({ + 'stroke-width': STROKE_WIDTH / this._scale, + }); this._createPolyEvents(); break; case 'polyline': @@ -487,9 +526,10 @@ class ShapeCreatorView { this._controller.switchCreateMode(true); return; } - this._drawInstance = this._frameContent.polyline().draw({snapToGrid: 0.1}).addClass('shapeCreation').attr({ - 'stroke-width': STROKE_WIDTH / this._scale, - }); + this._drawInstance = this._frameContent.polyline().draw({ snapToGrid: 0.1 }) + .addClass('shapeCreation').attr({ + 'stroke-width': STROKE_WIDTH / this._scale, + }); this._createPolyEvents(); break; default: @@ -585,6 +625,9 @@ class ShapeCreatorView { this._scale = player.geometry.scale; if (this._drawInstance) { this._rescaleDrawPoints(); + if (this._borderSticker) { + this._borderSticker.scale(this._scale); + } if (this._aim) { this._aim.x.attr('stroke-width', STROKE_WIDTH / this._scale); this._aim.y.attr('stroke-width', STROKE_WIDTH / this._scale); diff --git a/cvat/apps/engine/static/engine/js/shapes.js b/cvat/apps/engine/static/engine/js/shapes.js index 2ff5c4e9..ddfff6a1 100644 --- a/cvat/apps/engine/static/engine/js/shapes.js +++ b/cvat/apps/engine/static/engine/js/shapes.js @@ -3007,7 +3007,8 @@ class PolyShapeView extends ShapeView { // Run edit mode PolyShapeView.editor.edit(this._controller.type.split('_')[1], - this._uis.shape.attr('points'), this._color, index, e, + this._uis.shape.attr('points'), this._color, index, + this._uis.shape.attr('points').split(/\s/)[index], e, (points) => { this._uis.shape.removeClass('hidden'); if (this._uis.points) { @@ -3017,7 +3018,8 @@ class PolyShapeView extends ShapeView { this._uis.shape.attr('points', points); this._controller.updatePosition(window.cvat.player.frames.current, this._buildPosition()); } - } + }, + this._controller.id ); } } diff --git a/cvat/apps/engine/static/engine/stylesheet.css b/cvat/apps/engine/static/engine/stylesheet.css index 8dd8e8dd..6e0d7b65 100644 --- a/cvat/apps/engine/static/engine/stylesheet.css +++ b/cvat/apps/engine/static/engine/stylesheet.css @@ -224,6 +224,24 @@ cursor: w-resize; } +.shape-creator-border-point { + opacity: 0.55; +} + +.shape-creator-border-point:hover { + opacity: 1; + fill: red; +} + +.shape-creator-border-point:active { + opacity: 0.55; + fill: red; +} + +.shape-creator-border-point-direction { + fill: blueviolet; +} + .shapeText { font-size: 0.12em; fill: white; diff --git a/cvat/apps/engine/templates/engine/annotation.html b/cvat/apps/engine/templates/engine/annotation.html index c8ed37a6..f8fd6719 100644 --- a/cvat/apps/engine/templates/engine/annotation.html +++ b/cvat/apps/engine/templates/engine/annotation.html @@ -41,6 +41,7 @@ + @@ -191,6 +192,10 @@ +
+ + +