From 58f30220df340e84ab6111e1688c020e925cc4de Mon Sep 17 00:00:00 2001 From: Boris Sekachev <40690378+bsekachev@users.noreply.github.com> Date: Thu, 8 Aug 2019 18:37:37 +0300 Subject: [PATCH] CVAT-Canvas: Integrated SVG.js (#629) * Canvas rotation * Integrated SVG.js. Drawing, dragging and resizing of shapes * Removed TODO list --- cvat-canvas/README.md | 7 +- cvat-canvas/dist/canvas.css | 33 ++++ cvat-canvas/package.json | 5 +- cvat-canvas/src/canvas.ts | 8 +- cvat-canvas/src/canvasController.ts | 31 ++-- cvat-canvas/src/canvasModel.ts | 58 +++++-- cvat-canvas/src/canvasView.ts | 241 +++++++++++++++++++++++++-- cvat-canvas/src/master.ts | 5 + cvat-canvas/tsconfig.json | 1 + cvat-canvas/tslint.config.js | 2 +- cvat-core/src/annotations-objects.js | 11 +- 11 files changed, 343 insertions(+), 59 deletions(-) diff --git a/cvat-canvas/README.md b/cvat-canvas/README.md index 369effa2..f68c7b88 100644 --- a/cvat-canvas/README.md +++ b/cvat-canvas/README.md @@ -14,11 +14,6 @@ npm run build npm run build -- --mode=development # without a minification ``` -- Running development server -```bash -npm run server -``` - - Updating of a module version: ```bash npm version patch # updated after minor fixes @@ -53,7 +48,7 @@ All methods are sync. html(): HTMLDivElement; setup(frameData: FrameData, objectStates: ObjectState): void; activate(clientID: number, attributeID?: number): void; - rotate(direction: Rotation): void; + rotate(rotation: Rotation, remember?: boolean): void; focus(clientID: number, padding?: number): void; fit(): void; grid(stepX: number, stepY: number): void; diff --git a/cvat-canvas/dist/canvas.css b/cvat-canvas/dist/canvas.css index cdb35074..040d834c 100644 --- a/cvat-canvas/dist/canvas.css +++ b/cvat-canvas/dist/canvas.css @@ -2,6 +2,37 @@ display: none; } +.cvat_canvas_shape { + fill-opacity: 0.1; + stroke-opacity: 1; +} + +polyline.cvat_canvas_shape { + fill-opacity: 0; + stroke-opacity: 1; +} + +.cvat_canvas_shape_activated { + +} + +.cvat_canvas_shape_grouping { + +} + +.cvat_canvas_shape_merging { + +} + +.cvat_canvas_shape_drawing { + +} + +.svg_select_boundingRect { + opacity: 0; + pointer-events: none; +} + #cvat_canvas_wrapper { width: 100%; height: 80%; @@ -41,6 +72,7 @@ pointer-events: none; width: 100%; height: 100%; + pointer-events: none; } #cvat_canvas_background { @@ -59,6 +91,7 @@ pointer-events: none; width: 100%; height: 100%; + pointer-events: none; } #cvat_canvas_grid_pattern { diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json index 99838df5..2ee27f69 100644 --- a/cvat-canvas/package.json +++ b/cvat-canvas/package.json @@ -10,7 +10,10 @@ "author": "Intel", "license": "MIT", "dependencies": { - "@svgdotjs/svg.js": "^3.0.13" + "svg.draggable.js": "^2.2.2", + "svg.js": "^2.7.1", + "svg.resize.js": "^1.4.3", + "svg.select.js": "^3.0.1" }, "devDependencies": { "@babel/cli": "^7.5.5", diff --git a/cvat-canvas/src/canvas.ts b/cvat-canvas/src/canvas.ts index f48f0155..88f1ca96 100644 --- a/cvat-canvas/src/canvas.ts +++ b/cvat-canvas/src/canvas.ts @@ -1,5 +1,5 @@ /* -* Copyright (C) 2018 Intel Corporation +* Copyright (C) 2019 Intel Corporation * SPDX-License-Identifier: MIT */ @@ -11,7 +11,7 @@ interface Canvas { html(): HTMLDivElement; setup(frameData: any, objectStates: any[]): void; activate(clientID: number, attributeID?: number): void; - rotate(direction: Rotation): void; + rotate(rotation: Rotation, remember?: boolean): void; focus(clientID: number, padding?: number): void; fit(): void; grid(stepX: number, stepY: number): void; @@ -47,8 +47,8 @@ class CanvasImpl implements Canvas { this.model.activate(clientID, attributeID); } - public rotate(direction: Rotation): void { - this.model.rotate(direction); + public rotate(rotation: Rotation, remember: boolean): void { + this.model.rotate(rotation, remember); } public focus(clientID: number, padding: number = 0): void { diff --git a/cvat-canvas/src/canvasController.ts b/cvat-canvas/src/canvasController.ts index 3c4db0a5..834bd203 100644 --- a/cvat-canvas/src/canvasController.ts +++ b/cvat-canvas/src/canvasController.ts @@ -1,5 +1,5 @@ /* -* Copyright (C) 2018 Intel Corporation +* Copyright (C) 2019 Intel Corporation * SPDX-License-Identifier: MIT */ @@ -12,6 +12,7 @@ import { export interface CanvasController { readonly geometry: Geometry; + readonly objects: any[]; canvasSize: Size; zoom(x: number, y: number, direction: number): void; @@ -31,10 +32,6 @@ export class CanvasControllerImpl implements CanvasController { this.model = model; } - public get geometry(): Geometry { - return this.model.geometry; - } - public zoom(x: number, y: number, direction: number): void { this.model.zoom(x, y, direction); } @@ -43,14 +40,6 @@ export class CanvasControllerImpl implements CanvasController { this.model.fit(); } - public set canvasSize(value: Size) { - this.model.canvasSize = value; - } - - public get canvasSize(): Size { - return this.model.canvasSize; - } - public enableDrag(x: number, y: number): void { this.lastDragPosition = { x, @@ -74,4 +63,20 @@ export class CanvasControllerImpl implements CanvasController { public disableDrag(): void { this.isDragging = false; } + + public get geometry(): Geometry { + return this.model.geometry; + } + + public get objects(): any[] { + return this.model.objects; + } + + public set canvasSize(value: Size) { + this.model.canvasSize = value; + } + + public get canvasSize(): Size { + return this.model.canvasSize; + } } diff --git a/cvat-canvas/src/canvasModel.ts b/cvat-canvas/src/canvasModel.ts index cf24e06e..62a5f5ab 100644 --- a/cvat-canvas/src/canvasModel.ts +++ b/cvat-canvas/src/canvasModel.ts @@ -1,5 +1,5 @@ /* -* Copyright (C) 2018 Intel Corporation +* Copyright (C) 2019 Intel Corporation * SPDX-License-Identifier: MIT */ @@ -22,6 +22,7 @@ export interface Geometry { left: number; scale: number; offset: number; + angle: number; } export enum FrameZoom { @@ -30,19 +31,21 @@ export enum FrameZoom { } export enum Rotation { - CLOCKWISE90, ANTICLOCKWISE90, + CLOCKWISE90, } export enum UpdateReasons { IMAGE = 'image', + OBJECTS = 'objects', ZOOM = 'zoom', FIT = 'fit', MOVE = 'move', } export interface CanvasModel extends MasterImpl { - image: string; + readonly image: string; + readonly objects: any[]; geometry: Geometry; imageSize: Size; canvasSize: Size; @@ -52,7 +55,7 @@ export interface CanvasModel extends MasterImpl { setup(frameData: any, objectStates: any[]): void; activate(clientID: number, attributeID: number): void; - rotate(direction: Rotation): void; + rotate(rotation: Rotation, remember: boolean): void; focus(clientID: number, padding: number): void; fit(): void; grid(stepX: number, stepY: number): void; @@ -68,18 +71,22 @@ export interface CanvasModel extends MasterImpl { export class CanvasModelImpl extends MasterImpl implements CanvasModel { private data: { image: string; + objects: any[]; imageSize: Size; canvasSize: Size; imageOffset: number; scale: number; top: number; left: number; + angle: number; + rememberAngle: boolean; }; public constructor() { super(); this.data = { + angle: 0, canvasSize: { height: 0, width: 0, @@ -91,6 +98,8 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { width: 0, }, left: 0, + objects: [], + rememberAngle: false, scale: 1, top: 0, }; @@ -124,8 +133,14 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { width: (frameData.width as number), }; + if (!this.data.rememberAngle) { + this.data.angle = 0; + } + this.data.image = data; this.notify(UpdateReasons.IMAGE); + this.data.objects = objectStates; + this.notify(UpdateReasons.OBJECTS); }).catch((exception: any): void => { console.log(exception.toString()); }); @@ -137,8 +152,16 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { console.log(clientID, attributeID); } - public rotate(direction: Rotation): void { - console.log(direction); + public rotate(rotation: Rotation, remember: boolean = false): void { + if (rotation === Rotation.CLOCKWISE90) { + this.data.angle += 90; + } else { + this.data.angle -= 90; + } + + this.data.angle %= 360; + this.data.rememberAngle = remember; + this.fit(); } public focus(clientID: number, padding: number): void { @@ -146,10 +169,20 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { } public fit(): void { - this.data.scale = Math.min( - this.data.canvasSize.width / this.data.imageSize.width, - this.data.canvasSize.height / this.data.imageSize.height, - ); + const { angle } = this.data; + + if ((angle / 90) % 2) { + // 90, 270, .. + this.data.scale = Math.min( + this.data.canvasSize.width / this.data.imageSize.height, + this.data.canvasSize.height / this.data.imageSize.width, + ); + } else { + this.data.scale = Math.min( + this.data.canvasSize.width / this.data.imageSize.width, + this.data.canvasSize.height / this.data.imageSize.height, + ); + } this.data.scale = Math.min( Math.max(this.data.scale, FrameZoom.MIN), @@ -196,6 +229,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { public get geometry(): Geometry { return { + angle: this.data.angle, canvas: { height: this.data.canvasSize.height, width: this.data.canvasSize.width, @@ -215,6 +249,10 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { return this.data.image; } + public get objects(): any[] { + return this.data.objects; + } + public set imageSize(value: Size) { this.data.imageSize = { height: value.height, diff --git a/cvat-canvas/src/canvasView.ts b/cvat-canvas/src/canvasView.ts index 2dd82b30..19cc34c7 100644 --- a/cvat-canvas/src/canvasView.ts +++ b/cvat-canvas/src/canvasView.ts @@ -1,8 +1,15 @@ /* -* Copyright (C) 2018 Intel Corporation +* Copyright (C) 2019 Intel Corporation * SPDX-License-Identifier: MIT */ +import * as SVG from 'svg.js'; + +// tslint:disable-next-line: ordered-imports +import 'svg.draggable.js'; +import 'svg.resize.js'; +import 'svg.select.js'; + import { CanvasController } from './canvasController'; import { CanvasModel, Geometry, UpdateReasons } from './canvasModel'; import { Listener, Master } from './master'; @@ -11,10 +18,6 @@ export interface CanvasView { html(): HTMLDivElement; } -interface HTMLAttribute { - [index: string]: string; -} - function translateToSVG(svg: SVGSVGElement, points: number[]): number[] { const output = []; const transformationMatrix = svg.getScreenCTM().inverse(); @@ -29,44 +32,56 @@ function translateToSVG(svg: SVGSVGElement, points: number[]): number[] { return output; } -function translateFromSVG(svg: SVGSVGElement, points: number[]): number[] { - const output = []; - const transformationMatrix = svg.getScreenCTM(); - let pt = svg.createSVGPoint(); - for (let i = 0; i < points.length; i += 2) { - [pt.x] = points; - [, pt.y] = points; - pt = pt.matrixTransform(transformationMatrix); - output.push(pt.x, pt.y); - } +function darker(color: string, percentage: number) { + const R = Math.round(parseInt(color.slice(1, 3), 16) * (1 - percentage / 100)); + const G = Math.round(parseInt(color.slice(3, 5), 16) * (1 - percentage / 100)); + const B = Math.round(parseInt(color.slice(5, 7), 16) * (1 - percentage / 100)); - return output; + const rHex = Math.max(0, R).toString(16); + const gHex = Math.max(0, G).toString(16); + const bHex = Math.max(0, B).toString(16); + + return `#${rHex.length === 1 ? `0${rHex}` : rHex}` + + `${gHex.length === 1 ? `0${gHex}` : gHex}` + + `${bHex.length === 1 ? `0${bHex}` : bHex}`; } export class CanvasViewImpl implements CanvasView, Listener { private loadingAnimation: SVGSVGElement; private text: SVGSVGElement; + private adoptedText: SVG.Container; private background: SVGSVGElement; private grid: SVGSVGElement; private content: SVGSVGElement; + private adoptedContent: SVG.Container; private rotationWrapper: HTMLDivElement; private canvas: HTMLDivElement; private gridPath: SVGPathElement; private controller: CanvasController; + private svgShapes: SVG.Shape[]; + private svgTexts: SVG.Text[]; + private readonly BASE_STROKE_WIDTH: number; + private readonly BASE_POINT_SIZE: number; public constructor(model: CanvasModel & Master, controller: CanvasController) { this.controller = controller; + this.BASE_STROKE_WIDTH = 2.5; + this.BASE_POINT_SIZE = 7; + this.svgShapes = []; + this.svgTexts = []; // Create HTML elements this.loadingAnimation = window.document .createElementNS('http://www.w3.org/2000/svg', 'svg'); this.text = window.document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + this.adoptedText = (SVG.adopt((this.text as any as HTMLElement)) as SVG.Container); this.background = window.document.createElementNS('http://www.w3.org/2000/svg', 'svg'); this.grid = window.document.createElementNS('http://www.w3.org/2000/svg', 'svg'); this.gridPath = window.document.createElementNS('http://www.w3.org/2000/svg', 'path'); this.content = window.document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + this.adoptedContent = (SVG.adopt((this.content as any as HTMLElement)) as SVG.Container); this.rotationWrapper = window.document.createElement('div'); this.canvas = window.document.createElement('div'); @@ -167,9 +182,40 @@ export class CanvasViewImpl implements CanvasView, Listener { public notify(model: CanvasModel & Master, reason: UpdateReasons): void { function transform(geometry: Geometry): void { + // Transform canvas for (const obj of [this.background, this.grid, this.loadingAnimation, this.content]) { obj.style.transform = `scale(${geometry.scale})`; } + + this.rotationWrapper.style.transform = `rotate(${geometry.angle}deg)`; + + // Transform all shapes + for (const element of window.document.getElementsByClassName('svg_select_points')) { + element.setAttribute( + 'stroke-width', + `${this.BASE_STROKE_WIDTH / (3 * geometry.scale)}`, + ); + element.setAttribute( + 'r', + `${this.BASE_POINT_SIZE / (2 * geometry.scale)}`, + ); + } + + for (const element of + window.document.getElementsByClassName('cvat_canvas_selected_point')) { + element.setAttribute( + 'stroke-width', + `${+element.getAttribute('stroke-width') * 2}`, + ); + } + + for (const object of this.svgShapes) { + if (object.attr('stroke-width')) { + object.attr({ + 'stroke-width': this.BASE_STROKE_WIDTH / (geometry.scale), + }); + } + } } function resize(geometry: Geometry): void { @@ -198,6 +244,17 @@ export class CanvasViewImpl implements CanvasView, Listener { this.content.style.transform = `scale(${geometry.scale})`; } + function setupObjects(objects: any[], geometry: Geometry): void { + this.adoptedContent.clear(); + const ctm = this.content.getScreenCTM() + .inverse().multiply(this.background.getScreenCTM()); + + // TODO: Compute difference + this.addObjects(ctm, objects, geometry); + // TODO: Update objects + // TODO: Delete objects + } + const { geometry } = this.controller; if (reason === UpdateReasons.IMAGE) { if (!model.image.length) { @@ -217,10 +274,162 @@ export class CanvasViewImpl implements CanvasView, Listener { transform.call(this, geometry); } else if (reason === UpdateReasons.MOVE) { move.call(this, geometry); + } else if (reason === UpdateReasons.OBJECTS) { + setupObjects.call(this, this.controller.objects, geometry); } } public html(): HTMLDivElement { return this.canvas; } + + private addObjects(ctm: SVGMatrix, objects: any[], geometry: Geometry) { + for (const object of objects) { + if (object.objectType === 'tag') { + this.addTag(object, geometry); + } else { + const points: number[] = (object.points as number[]); + const translatedPoints: number[] = []; + for (let i = 0; i <= points.length - 1; i += 2) { + let point: SVGPoint = this.background.createSVGPoint(); + point.x = points[i]; + point.y = points[i + 1]; + point = point.matrixTransform(ctm); + translatedPoints.push(point.x, point.y); + } + + // TODO: Use enums after typification cvat-core + if (object.shapeType === 'rectangle') { + this.svgShapes.push(this.addRect(translatedPoints, object, geometry)); + } else { + const stringified = translatedPoints.reduce( + (acc: string, val: number, idx: number): string => { + if (idx % 2) { + return `${acc}${val} `; + } + + return `${acc}${val},`; + }, + '' , + ); + + if (object.shapeType === 'polygon') { + this.svgShapes.push(this.addPolygon(stringified, object, geometry)); + } else if (object.shapeType === 'polyline') { + this.svgShapes.push(this.addPolyline(stringified, object, geometry)); + } else if (object.shapeType === 'points') { + this.svgShapes.push(this.addPoints(stringified, object, geometry)); + } + } + + // TODO: add text here if need + } + } + + this.activate(geometry); + } + + private activate(geometry: Geometry) { + for (const shape of this.svgShapes) { + const self = this; + (shape as any).draggable().on('dragstart', () => { + console.log('hello'); + }).on('dragend', () => { + console.log('hello'); + }); + + (shape as any).selectize({ + deepSelect: true, + pointSize: this.BASE_POINT_SIZE / geometry.scale, + rotationPoint: false, + pointType(cx: number, cy: number): SVG.Circle { + const circle: SVG.Circle = this.nested + .circle(this.options.pointSize) + .stroke('black') + .fill(shape.node.getAttribute('fill')) + .center(cx, cy) + .attr({ + 'stroke-width': self.BASE_STROKE_WIDTH / (3 * geometry.scale), + }); + + circle.node.addEventListener('mouseenter', () => { + circle.attr({ + 'stroke-width': circle.attr('stroke-width') * 2, + }); + + circle.addClass('cvat_canvas_selected_point'); + }); + + circle.node.addEventListener('mouseleave', () => { + circle.attr({ + 'stroke-width': circle.attr('stroke-width') / 2, + }); + + circle.removeClass('cvat_canvas_selected_point'); + }); + + return circle; + }, + }).resize(); + } + + // add selectable + // add draggable + // add resizable + } + + private addRect(points: number[], state: any, geometry: Geometry): SVG.Rect { + const [xtl, ytl, xbr, ybr] = points; + + return this.adoptedContent.rect().size(xbr - xtl, ybr - ytl).attr({ + client_id: state.clientID, + 'color-rendering': 'optimizeQuality', + fill: state.color, + 'shape-rendering': 'geometricprecision', + stroke: darker(state.color, 50), + 'stroke-width': this.BASE_STROKE_WIDTH / geometry.scale, + z_order: state.zOrder, + }).move(xtl, ytl).addClass('cvat_canvas_shape'); + } + + private addPolygon(points: string, state: any, geometry: Geometry): SVG.Polygon { + return this.adoptedContent.polygon(points).attr({ + client_id: state.clientID, + 'color-rendering': 'optimizeQuality', + fill: state.color, + 'shape-rendering': 'geometricprecision', + stroke: darker(state.color, 50), + 'stroke-width': this.BASE_STROKE_WIDTH / geometry.scale, + z_order: state.zOrder, + }).addClass('cvat_canvas_shape'); + } + + private addPolyline(points: string, state: any, geometry: Geometry): SVG.PolyLine { + return this.adoptedContent.polyline(points).attr({ + client_id: state.clientID, + 'color-rendering': 'optimizeQuality', + fill: state.color, + 'shape-rendering': 'geometricprecision', + stroke: darker(state.color, 50), + 'stroke-width': this.BASE_STROKE_WIDTH / geometry.scale, + z_order: state.zOrder, + }).addClass('cvat_canvas_shape'); + } + + private addPoints(points: string, state: any, geometry: Geometry): SVG.Polygon { + return this.adoptedContent.polygon(points).attr({ + client_id: state.clientID, + 'color-rendering': 'optimizeQuality', + fill: state.color, + opacity: 0, + 'shape-rendering': 'geometricprecision', + stroke: darker(state.color, 50), + 'stroke-width': this.BASE_STROKE_WIDTH / geometry.scale, + z_order: state.zOrder, + }).addClass('cvat_canvas_shape'); + } + + private addTag(state: any, geometry: Geometry): void { + // TODO: + } } diff --git a/cvat-canvas/src/master.ts b/cvat-canvas/src/master.ts index bb4dc8b1..42e9a5e0 100644 --- a/cvat-canvas/src/master.ts +++ b/cvat-canvas/src/master.ts @@ -1,3 +1,8 @@ +/* +* Copyright (C) 2019 Intel Corporation +* SPDX-License-Identifier: MIT +*/ + export interface Master { subscribe(listener: Listener): void; unsubscribe(listener: Listener): void; diff --git a/cvat-canvas/tsconfig.json b/cvat-canvas/tsconfig.json index 959ec02b..e8603b52 100644 --- a/cvat-canvas/tsconfig.json +++ b/cvat-canvas/tsconfig.json @@ -6,6 +6,7 @@ "noImplicitAny": true, "preserveConstEnums": true, "declaration": true, + "moduleResolution": "node", "declarationDir": "dist/declaration" }, "include": [ diff --git a/cvat-canvas/tslint.config.js b/cvat-canvas/tslint.config.js index c237ed26..7c33d7b2 100644 --- a/cvat-canvas/tslint.config.js +++ b/cvat-canvas/tslint.config.js @@ -1,5 +1,5 @@ /* -* Copyright (C) 2018 Intel Corporation +* Copyright (C) 2019 Intel Corporation * SPDX-License-Identifier: MIT */ diff --git a/cvat-core/src/annotations-objects.js b/cvat-core/src/annotations-objects.js index a70553a7..d76fc4d6 100644 --- a/cvat-core/src/annotations-objects.js +++ b/cvat-core/src/annotations-objects.js @@ -749,7 +749,7 @@ return Object.assign({}, this.interpolatePosition( leftPosition, rightPosition, - targetFrame, + (targetFrame - leftFrame) / (rightFrame - leftFrame), ), { keyframe: false, }); @@ -1068,9 +1068,7 @@ } } - interpolatePosition(leftPosition, rightPosition, targetFrame) { - const offset = (targetFrame - leftPosition.frame) / ( - rightPosition.frame - leftPosition.frame); + interpolatePosition(leftPosition, rightPosition, offset) { const positionOffset = [ rightPosition.points[0] - leftPosition.points[0], rightPosition.points[1] - leftPosition.points[1], @@ -1097,7 +1095,7 @@ super(data, clientID, color, injection); } - interpolatePosition(leftPosition, rightPosition, targetFrame) { + interpolatePosition(leftPosition, rightPosition, offset) { function findBox(points) { let xmin = Number.MAX_SAFE_INTEGER; let ymin = Number.MAX_SAFE_INTEGER; @@ -1356,9 +1354,6 @@ const absoluteLeftPoints = denormalize(toArray(newLeftPoints), leftBox); const absoluteRightPoints = denormalize(toArray(newRightPoints), rightBox); - const offset = (targetFrame - leftPosition.frame) / ( - rightPosition.frame - leftPosition.frame); - const interpolation = []; for (let i = 0; i < absoluteLeftPoints.length; i++) { interpolation.push(absoluteLeftPoints[i] + (