From 581649469475d9174dc6df56622e212c981b740e Mon Sep 17 00:00:00 2001 From: Dmitry Kalinin Date: Thu, 21 May 2020 15:09:24 +0300 Subject: [PATCH] React UI: cuboid interpolation and cuboid drawing from rectangles (#1560) * Added backend cuboid interpolation and cuboid drawing from rectangles * Added CHANELOG.md * Fixed cuboid front edges stroke width * PR fixes --- CHANGELOG.md | 1 + cvat-canvas/README.md | 6 +++ cvat-canvas/package-lock.json | 2 +- cvat-canvas/package.json | 2 +- cvat-canvas/src/typescript/canvas.ts | 2 + cvat-canvas/src/typescript/canvasModel.ts | 6 +++ cvat-canvas/src/typescript/consts.ts | 4 ++ cvat-canvas/src/typescript/drawHandler.ts | 36 ++++++++++++++-- cvat-canvas/src/typescript/svg.patch.ts | 11 +++-- cvat-core/package.json | 2 +- cvat-core/src/annotations-objects.js | 21 ++++++++-- cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- .../controls-side-bar/draw-shape-popover.tsx | 41 ++++++++++++++++++- .../controls-side-bar/draw-shape-popover.tsx | 17 +++++++- cvat-ui/src/cvat-canvas-wrapper.ts | 2 + cvat/apps/dataset_manager/annotation.py | 2 +- 17 files changed, 140 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e905494..4195b379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - cvat-ui: added cookie policy drawer for login page () - Added `datumaro_project` export format (https://github.com/opencv/cvat/pull/1352) - Ability to configure user agreements for the user registration form (https://github.com/opencv/cvat/pull/1464) +- Added cuboid interpolation and cuboid drawing from rectangles () ### Changed - Downloaded file name in annotations export became more informative (https://github.com/opencv/cvat/pull/1352) diff --git a/cvat-canvas/README.md b/cvat-canvas/README.md index 435d27d7..a538a9e7 100644 --- a/cvat-canvas/README.md +++ b/cvat-canvas/README.md @@ -37,6 +37,11 @@ Canvas itself handles: EXTREME_POINTS = 'By 4 points' } + enum CuboidDrawingMethod { + CLASSIC = 'From rectangle', + CORNER_POINTS = 'By 4 points', + } + enum Mode { IDLE = 'idle', DRAG = 'drag', @@ -59,6 +64,7 @@ Canvas itself handles: enabled: boolean; shapeType?: string; rectDrawingMethod?: RectDrawingMethod; + cuboidDrawingMethod?: CuboidDrawingMethod; numberOfPoints?: number; initialState?: any; crosshair?: boolean; diff --git a/cvat-canvas/package-lock.json b/cvat-canvas/package-lock.json index 30f1ef50..b05d8126 100644 --- a/cvat-canvas/package-lock.json +++ b/cvat-canvas/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json index 79603d04..7fafda29 100644 --- a/cvat-canvas/package.json +++ b/cvat-canvas/package.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "1.0.0", + "version": "1.1.0", "description": "Part of Computer Vision Annotation Tool which presents its canvas library", "main": "src/canvas.ts", "scripts": { diff --git a/cvat-canvas/src/typescript/canvas.ts b/cvat-canvas/src/typescript/canvas.ts index 04a99d5b..61f3b0c8 100644 --- a/cvat-canvas/src/typescript/canvas.ts +++ b/cvat-canvas/src/typescript/canvas.ts @@ -11,6 +11,7 @@ import { CanvasModel, CanvasModelImpl, RectDrawingMethod, + CuboidDrawingMethod, Configuration, } from './canvasModel'; @@ -159,5 +160,6 @@ export { CanvasVersion, Configuration, RectDrawingMethod, + CuboidDrawingMethod, Mode as CanvasMode, }; diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index bd93314b..55873e4a 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -46,6 +46,11 @@ export enum RectDrawingMethod { EXTREME_POINTS = 'By 4 points' } +export enum CuboidDrawingMethod { + CLASSIC = 'From rectangle', + CORNER_POINTS = 'By 4 points', +} + export interface Configuration { autoborders?: boolean; displayAllText?: boolean; @@ -57,6 +62,7 @@ export interface DrawData { enabled: boolean; shapeType?: string; rectDrawingMethod?: RectDrawingMethod; + cuboidDrawingMethod?: CuboidDrawingMethod; numberOfPoints?: number; initialState?: any; crosshair?: boolean; diff --git a/cvat-canvas/src/typescript/consts.ts b/cvat-canvas/src/typescript/consts.ts index 2945b48d..aa92a9f2 100644 --- a/cvat-canvas/src/typescript/consts.ts +++ b/cvat-canvas/src/typescript/consts.ts @@ -11,6 +11,8 @@ const SIZE_THRESHOLD = 3; const POINTS_STROKE_WIDTH = 1.5; const POINTS_SELECTED_STROKE_WIDTH = 4; const MIN_EDGE_LENGTH = 3; +const CUBOID_ACTIVE_EDGE_STROKE_WIDTH = 2.5; +const CUBOID_UNACTIVE_EDGE_STROKE_WIDTH = 1.75; const UNDEFINED_ATTRIBUTE_VALUE = '__undefined__'; export default { @@ -23,5 +25,7 @@ export default { POINTS_STROKE_WIDTH, POINTS_SELECTED_STROKE_WIDTH, MIN_EDGE_LENGTH, + CUBOID_ACTIVE_EDGE_STROKE_WIDTH, + CUBOID_UNACTIVE_EDGE_STROKE_WIDTH, UNDEFINED_ATTRIBUTE_VALUE, }; diff --git a/cvat-canvas/src/typescript/drawHandler.ts b/cvat-canvas/src/typescript/drawHandler.ts index 8aa8b551..5a52badd 100644 --- a/cvat-canvas/src/typescript/drawHandler.ts +++ b/cvat-canvas/src/typescript/drawHandler.ts @@ -22,6 +22,7 @@ import { Geometry, RectDrawingMethod, Configuration, + CuboidDrawingMethod, } from './canvasModel'; import { cuboidFrom4Points } from './cuboid'; @@ -227,7 +228,8 @@ export class DrawHandlerImpl implements DrawHandler { // Or when no drawn points, but we call cancel() drawing // We check if it is activated with remember function if (this.drawInstance.remember('_paintHandler')) { - if (this.drawData.shapeType !== 'rectangle') { + if (this.drawData.shapeType !== 'rectangle' + && this.drawData.cuboidDrawingMethod !== CuboidDrawingMethod.CLASSIC) { // Check for unsaved drawn shapes this.drawInstance.draw('done'); } @@ -451,7 +453,7 @@ export class DrawHandlerImpl implements DrawHandler { this.drawPolyshape(); } - private drawCuboid(): void { + private drawCuboidBy4Points(): void { this.drawInstance = (this.canvas as any).polyline() .addClass('cvat_canvas_shape_drawing').attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, @@ -459,6 +461,29 @@ export class DrawHandlerImpl implements DrawHandler { this.drawPolyshape(); } + private drawCuboid(): void { + this.drawInstance = this.canvas.rect(); + this.drawInstance.on('drawstop', (e: Event): void => { + const bbox = (e.target as SVGRectElement).getBBox(); + const [xtl, ytl, xbr, ybr] = this.getFinalRectCoordinates(bbox); + const { shapeType } = this.drawData; + this.release(); + + if (this.canceled) return; + if ((xbr - xtl) * (ybr - ytl) >= consts.AREA_THRESHOLD) { + const d = { x: (xbr - xtl) * 0.1, y: (ybr - ytl)*0.1} + this.onDrawDone({ + shapeType, + points: cuboidFrom4Points([xtl, ybr, xbr, ybr, xbr, ytl, xbr + d.x, ytl - d.y]), + }, Date.now() - this.startTimestamp); + } + }).on('drawupdate', (): void => { + this.shapeSizeElement.update(this.drawInstance); + }).addClass('cvat_canvas_shape_drawing').attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + }); + } + private pastePolyshape(): void { this.drawInstance.on('done', (e: CustomEvent): void => { const targetPoints = this.drawInstance @@ -679,7 +704,12 @@ export class DrawHandlerImpl implements DrawHandler { } else if (this.drawData.shapeType === 'points') { this.drawPoints(); } else if (this.drawData.shapeType === 'cuboid') { - this.drawCuboid(); + if (this.drawData.cuboidDrawingMethod === CuboidDrawingMethod.CORNER_POINTS) { + this.drawCuboidBy4Points(); + } else { + this.drawCuboid(); + this.shapeSizeElement = displayShapeSize(this.canvas, this.text); + } } this.setupDrawEvents(); } diff --git a/cvat-canvas/src/typescript/svg.patch.ts b/cvat-canvas/src/typescript/svg.patch.ts index d17356c3..f571accb 100644 --- a/cvat-canvas/src/typescript/svg.patch.ts +++ b/cvat-canvas/src/typescript/svg.patch.ts @@ -248,8 +248,8 @@ function getTopDown(edgeIndex: EdgeIndex): number[] { this.bot = this.polygon(this.cuboidModel.bot.points); this.top = this.polygon(this.cuboidModel.top.points); this.right = this.polygon(this.cuboidModel.right.points); - this.dorsal = this.polygon(this.cuboidModel.dorsal.points); this.left = this.polygon(this.cuboidModel.left.points); + this.dorsal = this.polygon(this.cuboidModel.dorsal.points); this.face = this.polygon(this.cuboidModel.front.points); }, @@ -631,6 +631,7 @@ function getTopDown(edgeIndex: EdgeIndex): number[] { this.cuboidModel.dr.points = [topPoint, botPoint]; this.updateViewAndVM(); + this.fire(new CustomEvent('resizing', event)); }).on('dragend', (event: CustomEvent) => { this.fire(new CustomEvent('resizedone', event)); }); @@ -658,6 +659,7 @@ function getTopDown(edgeIndex: EdgeIndex): number[] { this.cuboidModel.dl.points = [topPoint, botPoint]; this.updateViewAndVM(true); + this.fire(new CustomEvent('resizing', event)); }).on('dragend', (event: CustomEvent) => { this.fire(new CustomEvent('resizedone', event)); });; @@ -856,16 +858,17 @@ function getTopDown(edgeIndex: EdgeIndex): number[] { const edges = [this.frontLeftEdge, this.frontRightEdge, this.frontTopEdge, this.frontBotEdge] const width = this.attr('stroke-width'); edges.forEach((edge: SVG.Element) => { - edge.attr('stroke-width', width * (this.strokeOffset || 1.75)); + edge.attr('stroke-width', width * (this.strokeOffset || consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH)); }); this.on('mouseover', () => { edges.forEach((edge: SVG.Element) => { - this.strokeOffset = 2.5; + this.strokeOffset = this.node.classList.contains('cvat_canvas_shape_activated') + ? consts.CUBOID_ACTIVE_EDGE_STROKE_WIDTH : consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH; edge.attr('stroke-width', width * this.strokeOffset); }) }).on('mouseout', () => { edges.forEach((edge: SVG.Element) => { - this.strokeOffset = 1.75; + this.strokeOffset = consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH; edge.attr('stroke-width', width * this.strokeOffset); }) }); diff --git a/cvat-core/package.json b/cvat-core/package.json index de54337f..b2f95a0c 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "2.0.0", + "version": "2.0.1", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-core/src/annotations-objects.js b/cvat-core/src/annotations-objects.js index ecf87634..20d2e222 100644 --- a/cvat-core/src/annotations-objects.js +++ b/cvat-core/src/annotations-objects.js @@ -1402,7 +1402,7 @@ } } - class CuboidShape extends PolyShape { + class CuboidShape extends Shape { constructor(data, clientID, color, injection) { super(data, clientID, color, injection); this.shapeType = ObjectShape.CUBOID; @@ -1967,7 +1967,7 @@ } } - class CuboidTrack extends PolyTrack { + class CuboidTrack extends Track { constructor(data, clientID, color, injection) { super(data, clientID, color, injection); this.shapeType = ObjectShape.CUBOID; @@ -1976,6 +1976,22 @@ checkNumberOfPoints(this.shapeType, shape.points); } } + + interpolatePosition(leftPosition, rightPosition, offset) { + + const positionOffset = leftPosition.points.map((point, index) => ( + rightPosition.points[index] - point + )) + + return { + points: leftPosition.points.map((point ,index) => ( + point + positionOffset[index] * offset + )), + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + }; + } } RectangleTrack.distance = RectangleShape.distance; @@ -1983,7 +1999,6 @@ PolylineTrack.distance = PolylineShape.distance; PointsTrack.distance = PointsShape.distance; CuboidTrack.distance = CuboidShape.distance; - CuboidTrack.interpolatePosition = RectangleTrack.interpolatePosition; module.exports = { RectangleShape, diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index f8679372..5ec99958 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.1.3", + "version": "1.1.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 72193237..92adc5bd 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.1.3", + "version": "1.1.4", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx index 4b0d716b..ac012ade 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx @@ -11,7 +11,7 @@ import Radio, { RadioChangeEvent } from 'antd/lib/radio'; import Tooltip from 'antd/lib/tooltip'; import Text from 'antd/lib/typography/Text'; -import { RectDrawingMethod } from 'cvat-canvas-wrapper'; +import { RectDrawingMethod, CuboidDrawingMethod } from 'cvat-canvas-wrapper'; import { ShapeType } from 'reducers/interfaces'; import { clamp } from 'utils/math'; import DEXTRPlugin from './dextr-plugin'; @@ -21,12 +21,14 @@ interface Props { labels: any[]; minimumPoints: number; rectDrawingMethod?: RectDrawingMethod; + cuboidDrawingMethod?: CuboidDrawingMethod; numberOfPoints?: number; selectedLabeID: number; repeatShapeShortcut: string; onChangeLabel(value: string): void; onChangePoints(value: number | undefined): void; onChangeRectDrawingMethod(event: RadioChangeEvent): void; + onChangeCuboidDrawingMethod(event: RadioChangeEvent): void; onDrawTrack(): void; onDrawShape(): void; } @@ -39,16 +41,18 @@ function DrawShapePopoverComponent(props: Props): JSX.Element { selectedLabeID, numberOfPoints, rectDrawingMethod, + cuboidDrawingMethod, repeatShapeShortcut, onDrawTrack, onDrawShape, onChangeLabel, onChangePoints, onChangeRectDrawingMethod, + onChangeCuboidDrawingMethod, } = props; const trackDisabled = shapeType === ShapeType.POLYGON || shapeType === ShapeType.POLYLINE - || shapeType === ShapeType.CUBOID || (shapeType === ShapeType.POINTS && numberOfPoints !== 1); + || (shapeType === ShapeType.POINTS && numberOfPoints !== 1); return (
@@ -117,6 +121,39 @@ function DrawShapePopoverComponent(props: Props): JSX.Element { ) } + { + shapeType === ShapeType.CUBOID && ( + <> + + + Drawing method + + + + + + + From rectangle + + + By 4 Points + + + + + + ) + } { shapeType !== ShapeType.RECTANGLE && shapeType !== ShapeType.CUBOID && ( diff --git a/cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx b/cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx index 8633ae6c..ba10cf17 100644 --- a/cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx +++ b/cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx @@ -8,7 +8,7 @@ import { RadioChangeEvent } from 'antd/lib/radio'; import { CombinedState, ShapeType, ObjectType } from 'reducers/interfaces'; import { rememberObject } from 'actions/annotation-actions'; -import { Canvas, RectDrawingMethod } from 'cvat-canvas-wrapper'; +import { Canvas, RectDrawingMethod, CuboidDrawingMethod } from 'cvat-canvas-wrapper'; import DrawShapePopoverComponent from 'components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover'; interface OwnProps { @@ -73,6 +73,7 @@ type Props = StateToProps & DispatchToProps; interface State { rectDrawingMethod?: RectDrawingMethod; + cuboidDrawingMethod?: CuboidDrawingMethod; numberOfPoints?: number; selectedLabelID: number; } @@ -85,10 +86,13 @@ class DrawShapePopoverContainer extends React.PureComponent { const { shapeType } = props; const defaultLabelID = props.labels[0].id; const defaultRectDrawingMethod = RectDrawingMethod.CLASSIC; + const defaultCuboidDrawingMethod = CuboidDrawingMethod.CLASSIC; this.state = { selectedLabelID: defaultLabelID, rectDrawingMethod: shapeType === ShapeType.RECTANGLE ? defaultRectDrawingMethod : undefined, + cuboidDrawingMethod: shapeType === ShapeType.CUBOID + ? defaultCuboidDrawingMethod : undefined, }; if (shapeType === ShapeType.POLYGON) { @@ -111,6 +115,7 @@ class DrawShapePopoverContainer extends React.PureComponent { const { rectDrawingMethod, + cuboidDrawingMethod, numberOfPoints, selectedLabelID, } = this.state; @@ -119,6 +124,7 @@ class DrawShapePopoverContainer extends React.PureComponent { canvasInstance.draw({ enabled: true, rectDrawingMethod, + cuboidDrawingMethod, numberOfPoints, shapeType, crosshair: [ShapeType.RECTANGLE, ShapeType.CUBOID].includes(shapeType), @@ -134,6 +140,12 @@ class DrawShapePopoverContainer extends React.PureComponent { }); }; + private onChangeCuboidDrawingMethod = (event: RadioChangeEvent): void => { + this.setState({ + cuboidDrawingMethod: event.target.value, + }) + } + private onDrawShape = (): void => { this.onDraw(ObjectType.SHAPE); }; @@ -157,6 +169,7 @@ class DrawShapePopoverContainer extends React.PureComponent { public render(): JSX.Element { const { rectDrawingMethod, + cuboidDrawingMethod, selectedLabelID, numberOfPoints, } = this.state; @@ -175,10 +188,12 @@ class DrawShapePopoverContainer extends React.PureComponent { selectedLabeID={selectedLabelID} numberOfPoints={numberOfPoints} rectDrawingMethod={rectDrawingMethod} + cuboidDrawingMethod={cuboidDrawingMethod} repeatShapeShortcut={normalizedKeyMap.SWITCH_DRAW_MODE} onChangeLabel={this.onChangeLabel} onChangePoints={this.onChangePoints} onChangeRectDrawingMethod={this.onChangeRectDrawingMethod} + onChangeCuboidDrawingMethod={this.onChangeCuboidDrawingMethod} onDrawTrack={this.onDrawTrack} onDrawShape={this.onDrawShape} /> diff --git a/cvat-ui/src/cvat-canvas-wrapper.ts b/cvat-ui/src/cvat-canvas-wrapper.ts index 671273cb..1c8dfa7e 100644 --- a/cvat-ui/src/cvat-canvas-wrapper.ts +++ b/cvat-ui/src/cvat-canvas-wrapper.ts @@ -7,6 +7,7 @@ import { CanvasMode, CanvasVersion, RectDrawingMethod, + CuboidDrawingMethod, } from 'cvat-canvas/src/typescript/canvas'; function isAbleToChangeFrame(canvas: Canvas): boolean { @@ -19,5 +20,6 @@ export { CanvasMode, CanvasVersion, RectDrawingMethod, + CuboidDrawingMethod, isAbleToChangeFrame, }; diff --git a/cvat/apps/dataset_manager/annotation.py b/cvat/apps/dataset_manager/annotation.py index d0ed61f5..df66c99b 100644 --- a/cvat/apps/dataset_manager/annotation.py +++ b/cvat/apps/dataset_manager/annotation.py @@ -495,7 +495,7 @@ class TrackManager(ObjectManager): # TODO: Need to modify a client and a database (append "outside" shapes for polytracks) if not prev_shape["outside"] and (prev_shape["type"] == ShapeType.RECTANGLE - or prev_shape["type"] == ShapeType.POINTS): + or prev_shape["type"] == ShapeType.POINTS or prev_shape["type"] == ShapeType.CUBOID): shape = copy(prev_shape) shape["frame"] = end_frame shapes.extend(interpolate(prev_shape, shape))