React UI: cuboid interpolation and cuboid drawing from rectangles (#1560)

* Added backend cuboid interpolation and cuboid drawing from rectangles
* Added CHANELOG.md
* Fixed cuboid front edges stroke width
* PR fixes

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - cvat-ui: added cookie policy drawer for login page (<https://github.com/opencv/cvat/pull/1511>)
 - Added `datumaro_project` export format (https://github.com/opencv/cvat/pull/1352)
 - Ability to configure user agreements for the user registration form (https://github.com/opencv/cvat/pull/1464)
+- Added cuboid interpolation and cuboid drawing from rectangles (<https://github.com/opencv/cvat/pull/1560>)
 
 ### Changed
 - Downloaded file name in annotations export became more informative (https://github.com/opencv/cvat/pull/1352)

cvat-canvas/README.md

@@ -37,6 +37,11 @@ Canvas itself handles:
         EXTREME_POINTS = 'By 4 points'
     }
 
+    enum CuboidDrawingMethod {
+        CLASSIC = 'From rectangle',
+        CORNER_POINTS = 'By 4 points',
+    }
+
     enum Mode {
         IDLE = 'idle',
         DRAG = 'drag',
@@ -59,6 +64,7 @@ Canvas itself handles:
         enabled: boolean;
         shapeType?: string;
         rectDrawingMethod?: RectDrawingMethod;
+        cuboidDrawingMethod?: CuboidDrawingMethod;
         numberOfPoints?: number;
         initialState?: any;
         crosshair?: boolean;

cvat-canvas/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-canvas",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-canvas/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-canvas",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Part of Computer Vision Annotation Tool which presents its canvas library",
   "main": "src/canvas.ts",
   "scripts": {

cvat-canvas/src/typescript/canvas.ts

@@ -11,6 +11,7 @@ import {
     CanvasModel,
     CanvasModelImpl,
     RectDrawingMethod,
+    CuboidDrawingMethod,
     Configuration,
 } from './canvasModel';
 
@@ -159,5 +160,6 @@ export {
     CanvasVersion,
     Configuration,
     RectDrawingMethod,
+    CuboidDrawingMethod,
     Mode as CanvasMode,
 };

cvat-canvas/src/typescript/canvasModel.ts

@@ -46,6 +46,11 @@ export enum RectDrawingMethod {
     EXTREME_POINTS = 'By 4 points'
 }
 
+export enum CuboidDrawingMethod {
+    CLASSIC = 'From rectangle',
+    CORNER_POINTS = 'By 4 points',
+}
+
 export interface Configuration {
     autoborders?: boolean;
     displayAllText?: boolean;
@@ -57,6 +62,7 @@ export interface DrawData {
     enabled: boolean;
     shapeType?: string;
     rectDrawingMethod?: RectDrawingMethod;
+    cuboidDrawingMethod?: CuboidDrawingMethod;
     numberOfPoints?: number;
     initialState?: any;
     crosshair?: boolean;

cvat-canvas/src/typescript/consts.ts

@@ -11,6 +11,8 @@ const SIZE_THRESHOLD = 3;
 const POINTS_STROKE_WIDTH = 1.5;
 const POINTS_SELECTED_STROKE_WIDTH = 4;
 const MIN_EDGE_LENGTH = 3;
+const CUBOID_ACTIVE_EDGE_STROKE_WIDTH = 2.5;
+const CUBOID_UNACTIVE_EDGE_STROKE_WIDTH = 1.75;
 const UNDEFINED_ATTRIBUTE_VALUE = '__undefined__';
 
 export default {
@@ -23,5 +25,7 @@ export default {
     POINTS_STROKE_WIDTH,
     POINTS_SELECTED_STROKE_WIDTH,
     MIN_EDGE_LENGTH,
+    CUBOID_ACTIVE_EDGE_STROKE_WIDTH,
+    CUBOID_UNACTIVE_EDGE_STROKE_WIDTH,
     UNDEFINED_ATTRIBUTE_VALUE,
 };

cvat-canvas/src/typescript/drawHandler.ts

@@ -22,6 +22,7 @@ import {
     Geometry,
     RectDrawingMethod,
     Configuration,
+    CuboidDrawingMethod,
 } from './canvasModel';
 
 import { cuboidFrom4Points } from './cuboid';
@@ -227,7 +228,8 @@ export class DrawHandlerImpl implements DrawHandler {
         // Or when no drawn points, but we call cancel() drawing
         // We check if it is activated with remember function
         if (this.drawInstance.remember('_paintHandler')) {
-            if (this.drawData.shapeType !== 'rectangle') {
+            if (this.drawData.shapeType !== 'rectangle'
+            && this.drawData.cuboidDrawingMethod !== CuboidDrawingMethod.CLASSIC) {
                 // Check for unsaved drawn shapes
                 this.drawInstance.draw('done');
             }
@@ -451,7 +453,7 @@ export class DrawHandlerImpl implements DrawHandler {
         this.drawPolyshape();
     }
 
-    private drawCuboid(): void {
+    private drawCuboidBy4Points(): void {
         this.drawInstance = (this.canvas as any).polyline()
             .addClass('cvat_canvas_shape_drawing').attr({
                 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale,
@@ -459,6 +461,29 @@ export class DrawHandlerImpl implements DrawHandler {
         this.drawPolyshape();
     }
 
+    private drawCuboid(): void {
+        this.drawInstance = this.canvas.rect();
+        this.drawInstance.on('drawstop', (e: Event): void => {
+            const bbox = (e.target as SVGRectElement).getBBox();
+            const [xtl, ytl, xbr, ybr] = this.getFinalRectCoordinates(bbox);
+            const { shapeType } = this.drawData;
+            this.release();
+
+            if (this.canceled) return;
+            if ((xbr - xtl) * (ybr - ytl) >= consts.AREA_THRESHOLD) {
+                const d = { x: (xbr - xtl) * 0.1, y: (ybr - ytl)*0.1}
+                this.onDrawDone({
+                    shapeType,
+                    points: cuboidFrom4Points([xtl, ybr, xbr, ybr, xbr, ytl, xbr + d.x, ytl - d.y]),
+                }, Date.now() - this.startTimestamp);
+            }
+        }).on('drawupdate', (): void => {
+            this.shapeSizeElement.update(this.drawInstance);
+        }).addClass('cvat_canvas_shape_drawing').attr({
+            'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale,
+        });
+    }
+
     private pastePolyshape(): void {
         this.drawInstance.on('done', (e: CustomEvent): void => {
             const targetPoints = this.drawInstance
@@ -679,7 +704,12 @@ export class DrawHandlerImpl implements DrawHandler {
             } else if (this.drawData.shapeType === 'points') {
                 this.drawPoints();
             } else if (this.drawData.shapeType === 'cuboid') {
-                this.drawCuboid();
+                if (this.drawData.cuboidDrawingMethod === CuboidDrawingMethod.CORNER_POINTS) {
+                    this.drawCuboidBy4Points();
+                } else {
+                    this.drawCuboid();
+                    this.shapeSizeElement = displayShapeSize(this.canvas, this.text);
+                }
             }
             this.setupDrawEvents();
         }

cvat-canvas/src/typescript/svg.patch.ts

@@ -248,8 +248,8 @@ function getTopDown(edgeIndex: EdgeIndex): number[] {
             this.bot = this.polygon(this.cuboidModel.bot.points);
             this.top = this.polygon(this.cuboidModel.top.points);
             this.right = this.polygon(this.cuboidModel.right.points);
-            this.dorsal = this.polygon(this.cuboidModel.dorsal.points);
             this.left = this.polygon(this.cuboidModel.left.points);
+            this.dorsal = this.polygon(this.cuboidModel.dorsal.points);
             this.face = this.polygon(this.cuboidModel.front.points);
         },
 
@@ -631,6 +631,7 @@ function getTopDown(edgeIndex: EdgeIndex): number[] {
 
                 this.cuboidModel.dr.points = [topPoint, botPoint];
                 this.updateViewAndVM();
+                this.fire(new CustomEvent('resizing', event));
             }).on('dragend', (event: CustomEvent) => {
                 this.fire(new CustomEvent('resizedone', event));
             });
@@ -658,6 +659,7 @@ function getTopDown(edgeIndex: EdgeIndex): number[] {
 
                 this.cuboidModel.dl.points = [topPoint, botPoint];
                 this.updateViewAndVM(true);
+                this.fire(new CustomEvent('resizing', event));
             }).on('dragend', (event: CustomEvent) => {
                 this.fire(new CustomEvent('resizedone', event));
             });;
@@ -856,16 +858,17 @@ function getTopDown(edgeIndex: EdgeIndex): number[] {
             const edges = [this.frontLeftEdge, this.frontRightEdge, this.frontTopEdge, this.frontBotEdge]
             const width = this.attr('stroke-width');
             edges.forEach((edge: SVG.Element) => {
-                edge.attr('stroke-width', width * (this.strokeOffset || 1.75));
+                edge.attr('stroke-width', width * (this.strokeOffset || consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH));
             });
             this.on('mouseover', () => {
                 edges.forEach((edge: SVG.Element) => {
-                    this.strokeOffset = 2.5;
+                    this.strokeOffset = this.node.classList.contains('cvat_canvas_shape_activated')
+                        ? consts.CUBOID_ACTIVE_EDGE_STROKE_WIDTH : consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH;
                     edge.attr('stroke-width', width * this.strokeOffset);
                 })
             }).on('mouseout', () => {
                 edges.forEach((edge: SVG.Element) => {
-                    this.strokeOffset = 1.75;
+                    this.strokeOffset = consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH;
                     edge.attr('stroke-width', width * this.strokeOffset);
                 })
             });

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-core/src/annotations-objects.js

@@ -1402,7 +1402,7 @@
         }
     }
 
-    class CuboidShape extends PolyShape {
+    class CuboidShape extends Shape {
         constructor(data, clientID, color, injection) {
             super(data, clientID, color, injection);
             this.shapeType = ObjectShape.CUBOID;
@@ -1967,7 +1967,7 @@
         }
     }
 
-    class CuboidTrack extends PolyTrack {
+    class CuboidTrack extends Track {
         constructor(data, clientID, color, injection) {
             super(data, clientID, color, injection);
             this.shapeType = ObjectShape.CUBOID;
@@ -1976,6 +1976,22 @@
                 checkNumberOfPoints(this.shapeType, shape.points);
             }
         }
+
+        interpolatePosition(leftPosition, rightPosition, offset) {
+
+            const positionOffset = leftPosition.points.map((point, index) => (
+                rightPosition.points[index] - point
+            ))
+
+            return {
+                points: leftPosition.points.map((point ,index) => (
+                    point + positionOffset[index] * offset
+                )),
+                occluded: leftPosition.occluded,
+                outside: leftPosition.outside,
+                zOrder: leftPosition.zOrder,
+            };
+        }
     }
 
     RectangleTrack.distance = RectangleShape.distance;
@@ -1983,7 +1999,6 @@
     PolylineTrack.distance = PolylineShape.distance;
     PointsTrack.distance = PointsShape.distance;
     CuboidTrack.distance = CuboidShape.distance;
-    CuboidTrack.interpolatePosition = RectangleTrack.interpolatePosition;
 
     module.exports = {
         RectangleShape,

cvat-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.1.3",
+  "version": "1.1.4",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.1.3",
+  "version": "1.1.4",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx

@@ -11,7 +11,7 @@ import Radio, { RadioChangeEvent } from 'antd/lib/radio';
 import Tooltip from 'antd/lib/tooltip';
 import Text from 'antd/lib/typography/Text';
 
-import { RectDrawingMethod } from 'cvat-canvas-wrapper';
+import { RectDrawingMethod, CuboidDrawingMethod } from 'cvat-canvas-wrapper';
 import { ShapeType } from 'reducers/interfaces';
 import { clamp } from 'utils/math';
 import DEXTRPlugin from './dextr-plugin';
@@ -21,12 +21,14 @@ interface Props {
     labels: any[];
     minimumPoints: number;
     rectDrawingMethod?: RectDrawingMethod;
+    cuboidDrawingMethod?: CuboidDrawingMethod;
     numberOfPoints?: number;
     selectedLabeID: number;
     repeatShapeShortcut: string;
     onChangeLabel(value: string): void;
     onChangePoints(value: number | undefined): void;
     onChangeRectDrawingMethod(event: RadioChangeEvent): void;
+    onChangeCuboidDrawingMethod(event: RadioChangeEvent): void;
     onDrawTrack(): void;
     onDrawShape(): void;
 }
@@ -39,16 +41,18 @@ function DrawShapePopoverComponent(props: Props): JSX.Element {
         selectedLabeID,
         numberOfPoints,
         rectDrawingMethod,
+        cuboidDrawingMethod,
         repeatShapeShortcut,
         onDrawTrack,
         onDrawShape,
         onChangeLabel,
         onChangePoints,
         onChangeRectDrawingMethod,
+        onChangeCuboidDrawingMethod,
     } = props;
 
     const trackDisabled = shapeType === ShapeType.POLYGON || shapeType === ShapeType.POLYLINE
-        || shapeType === ShapeType.CUBOID || (shapeType === ShapeType.POINTS && numberOfPoints !== 1);
+        || (shapeType === ShapeType.POINTS && numberOfPoints !== 1);
 
     return (
         <div className='cvat-draw-shape-popover-content'>
@@ -117,6 +121,39 @@ function DrawShapePopoverComponent(props: Props): JSX.Element {
                     </>
                 )
             }
+            {
+                shapeType === ShapeType.CUBOID && (
+                    <>
+                        <Row>
+                            <Col>
+                                <Text className='cvat-text-color'> Drawing method </Text>
+                            </Col>
+                        </Row>
+                        <Row type='flex' justify='space-around'>
+                            <Col>
+                                <Radio.Group
+                                    style={{ display: 'flex' }}
+                                    value={cuboidDrawingMethod}
+                                    onChange={onChangeCuboidDrawingMethod}
+                                >
+                                    <Radio
+                                        value={CuboidDrawingMethod.CLASSIC}
+                                        style={{ width: 'auto' }}
+                                    >
+                                        From rectangle
+                                    </Radio>
+                                    <Radio
+                                        value={CuboidDrawingMethod.CORNER_POINTS}
+                                        style={{ width: 'auto' }}
+                                    >
+                                        By 4 Points
+                                    </Radio>
+                                </Radio.Group>
+                            </Col>
+                        </Row>
+                    </>
+                )
+            }
             {
                 shapeType !== ShapeType.RECTANGLE && shapeType !== ShapeType.CUBOID && (
                     <Row type='flex' justify='space-around' align='middle'>

cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx

@@ -8,7 +8,7 @@ import { RadioChangeEvent } from 'antd/lib/radio';
 
 import { CombinedState, ShapeType, ObjectType } from 'reducers/interfaces';
 import { rememberObject } from 'actions/annotation-actions';
-import { Canvas, RectDrawingMethod } from 'cvat-canvas-wrapper';
+import { Canvas, RectDrawingMethod, CuboidDrawingMethod } from 'cvat-canvas-wrapper';
 import DrawShapePopoverComponent from 'components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover';
 
 interface OwnProps {
@@ -73,6 +73,7 @@ type Props = StateToProps & DispatchToProps;
 
 interface State {
     rectDrawingMethod?: RectDrawingMethod;
+    cuboidDrawingMethod?: CuboidDrawingMethod;
     numberOfPoints?: number;
     selectedLabelID: number;
 }
@@ -85,10 +86,13 @@ class DrawShapePopoverContainer extends React.PureComponent<Props, State> {
         const { shapeType } = props;
         const defaultLabelID = props.labels[0].id;
         const defaultRectDrawingMethod = RectDrawingMethod.CLASSIC;
+        const defaultCuboidDrawingMethod = CuboidDrawingMethod.CLASSIC;
         this.state = {
             selectedLabelID: defaultLabelID,
             rectDrawingMethod: shapeType === ShapeType.RECTANGLE
                 ? defaultRectDrawingMethod : undefined,
+            cuboidDrawingMethod: shapeType === ShapeType.CUBOID
+                ? defaultCuboidDrawingMethod : undefined,
         };
 
         if (shapeType === ShapeType.POLYGON) {
@@ -111,6 +115,7 @@ class DrawShapePopoverContainer extends React.PureComponent<Props, State> {
 
         const {
             rectDrawingMethod,
+            cuboidDrawingMethod,
             numberOfPoints,
             selectedLabelID,
         } = this.state;
@@ -119,6 +124,7 @@ class DrawShapePopoverContainer extends React.PureComponent<Props, State> {
         canvasInstance.draw({
             enabled: true,
             rectDrawingMethod,
+            cuboidDrawingMethod,
             numberOfPoints,
             shapeType,
             crosshair: [ShapeType.RECTANGLE, ShapeType.CUBOID].includes(shapeType),
@@ -134,6 +140,12 @@ class DrawShapePopoverContainer extends React.PureComponent<Props, State> {
         });
     };
 
+    private onChangeCuboidDrawingMethod = (event: RadioChangeEvent): void => {
+        this.setState({
+            cuboidDrawingMethod: event.target.value,
+        })
+    }
+
     private onDrawShape = (): void => {
         this.onDraw(ObjectType.SHAPE);
     };
@@ -157,6 +169,7 @@ class DrawShapePopoverContainer extends React.PureComponent<Props, State> {
     public render(): JSX.Element {
         const {
             rectDrawingMethod,
+            cuboidDrawingMethod,
             selectedLabelID,
             numberOfPoints,
         } = this.state;
@@ -175,10 +188,12 @@ class DrawShapePopoverContainer extends React.PureComponent<Props, State> {
                 selectedLabeID={selectedLabelID}
                 numberOfPoints={numberOfPoints}
                 rectDrawingMethod={rectDrawingMethod}
+                cuboidDrawingMethod={cuboidDrawingMethod}
                 repeatShapeShortcut={normalizedKeyMap.SWITCH_DRAW_MODE}
                 onChangeLabel={this.onChangeLabel}
                 onChangePoints={this.onChangePoints}
                 onChangeRectDrawingMethod={this.onChangeRectDrawingMethod}
+                onChangeCuboidDrawingMethod={this.onChangeCuboidDrawingMethod}
                 onDrawTrack={this.onDrawTrack}
                 onDrawShape={this.onDrawShape}
             />

cvat-ui/src/cvat-canvas-wrapper.ts

@@ -7,6 +7,7 @@ import {
     CanvasMode,
     CanvasVersion,
     RectDrawingMethod,
+    CuboidDrawingMethod,
 } from 'cvat-canvas/src/typescript/canvas';
 
 function isAbleToChangeFrame(canvas: Canvas): boolean {
@@ -19,5 +20,6 @@ export {
     CanvasMode,
     CanvasVersion,
     RectDrawingMethod,
+    CuboidDrawingMethod,
     isAbleToChangeFrame,
 };

cvat/apps/dataset_manager/annotation.py

@@ -495,7 +495,7 @@ class TrackManager(ObjectManager):
 
         # TODO: Need to modify a client and a database (append "outside" shapes for polytracks)
         if not prev_shape["outside"] and (prev_shape["type"] == ShapeType.RECTANGLE
-               or prev_shape["type"] == ShapeType.POINTS):
+               or prev_shape["type"] == ShapeType.POINTS or prev_shape["type"] == ShapeType.CUBOID):
             shape = copy(prev_shape)
             shape["frame"] = end_frame
             shapes.extend(interpolate(prev_shape, shape))