From 908e0569d80cdebce7d0bf70230c6d34e3d3d85d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 31 Aug 2020 22:20:50 +0300 Subject: [PATCH] Improved interface of interactors on UI (#2054) --- CHANGELOG.md | 2 +- cvat-canvas/README.md | 53 +- cvat-canvas/package-lock.json | 2 +- cvat-canvas/package.json | 2 +- cvat-canvas/src/typescript/canvas.ts | 26 +- .../src/typescript/canvasController.ts | 11 + cvat-canvas/src/typescript/canvasModel.ts | 49 +- cvat-canvas/src/typescript/canvasView.ts | 67 +++ cvat-canvas/src/typescript/crosshair.ts | 70 +++ cvat-canvas/src/typescript/drawHandler.ts | 33 +- cvat-canvas/src/typescript/editHandler.ts | 12 +- .../src/typescript/interactionHandler.ts | 281 +++++++++++ cvat-core/src/annotations-history.js | 6 + cvat-core/src/annotations.js | 14 + cvat-core/src/session.js | 26 + cvat-ui/src/actions/annotation-actions.ts | 27 +- cvat-ui/src/actions/models-actions.ts | 9 +- cvat-ui/src/assets/ai-tools-icon.svg | 18 + .../controls-side-bar/controls-side-bar.tsx | 11 +- .../controls-side-bar/dextr-plugin.tsx | 90 ---- .../controls-side-bar/draw-shape-popover.tsx | 2 - .../controls-side-bar/tools-control.tsx | 472 ++++++++++++++++++ .../standard-workspace/styles.scss | 24 +- .../model-runner-modal/model-runner-modal.tsx | 46 +- .../models-page/built-model-item.tsx | 49 -- .../models-page/deployed-models-list.tsx | 1 - .../components/models-page/models-page.tsx | 14 +- .../model-runner-dialog.tsx | 6 +- .../containers/models-page/models-page.tsx | 16 +- cvat-ui/src/cvat-canvas-wrapper.ts | 5 + cvat-ui/src/icons.tsx | 5 + cvat-ui/src/reducers/annotation-reducer.ts | 23 + cvat-ui/src/reducers/interfaces.ts | 10 +- cvat-ui/src/reducers/models-reducer.ts | 12 +- cvat-ui/src/reducers/notifications-reducer.ts | 17 - cvat-ui/src/reducers/plugins-reducer.ts | 6 - cvat-ui/src/utils/dextr-utils.ts | 214 -------- cvat-ui/src/utils/plugin-checker.ts | 8 - 38 files changed, 1223 insertions(+), 516 deletions(-) create mode 100644 cvat-canvas/src/typescript/crosshair.ts create mode 100644 cvat-canvas/src/typescript/interactionHandler.ts create mode 100644 cvat-ui/src/assets/ai-tools-icon.svg delete mode 100644 cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/dextr-plugin.tsx create mode 100644 cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx delete mode 100644 cvat-ui/src/components/models-page/built-model-item.tsx delete mode 100644 cvat-ui/src/utils/dextr-utils.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e35e24d8..d1534bca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ### Changed -- +- UI models (like DEXTR) were redesigned to be more interactive () ### Deprecated - diff --git a/cvat-canvas/README.md b/cvat-canvas/README.md index 808aa5ac..d6a5698b 100644 --- a/cvat-canvas/README.md +++ b/cvat-canvas/README.md @@ -46,6 +46,7 @@ Canvas itself handles: IDLE = 'idle', DRAG = 'drag', RESIZE = 'resize', + INTERACT = 'interact', DRAW = 'draw', EDIT = 'edit', MERGE = 'merge', @@ -70,6 +71,11 @@ Canvas itself handles: crosshair?: boolean; } + interface InteractionData { + shapeType: string; + minVertices?: number; + } + interface GroupData { enabled: boolean; resetGroup?: boolean; @@ -83,6 +89,12 @@ Canvas itself handles: enabled: boolean; } + interface InteractionResult { + points: number[]; + shapeType: string; + button: number; + }; + interface DrawnData { shapeType: string; points: number[]; @@ -104,6 +116,7 @@ Canvas itself handles: grid(stepX: number, stepY: number): void; draw(drawData: DrawData): void; + interact(interactionData: InteractionData): void; group(groupData: GroupData): void; split(splitData: SplitData): void; merge(mergeData: MergeData): void; @@ -146,6 +159,7 @@ Standard JS events are used. - canvas.moved => {states: ObjectState[], x: number, y: number} - canvas.find => {states: ObjectState[], x: number, y: number} - canvas.drawn => {state: DrawnData} + - canvas.interacted => {shapes: InteractionResult[]} - canvas.editstart - canvas.edited => {state: ObjectState, points: number[]} - canvas.splitted => {state: ObjectState} @@ -187,25 +201,26 @@ Standard JS events are used. ## API Reaction -| | IDLE | GROUP | SPLIT | DRAW | MERGE | EDIT | DRAG | RESIZE | ZOOM_CANVAS | DRAG_CANVAS | -|--------------|------|-------|-------|------|-------|------|------|--------|-------------|-------------| -| setup() | + | + | + | +/- | + | +/- | +/- | +/- | + | + | -| activate() | + | - | - | - | - | - | - | - | - | - | -| rotate() | + | + | + | + | + | + | + | + | + | + | -| focus() | + | + | + | + | + | + | + | + | + | + | -| fit() | + | + | + | + | + | + | + | + | + | + | -| grid() | + | + | + | + | + | + | + | + | + | + | -| draw() | + | - | - | - | - | - | - | - | - | - | -| split() | + | - | + | - | - | - | - | - | - | - | -| group() | + | + | - | - | - | - | - | - | - | - | -| merge() | + | - | - | - | + | - | - | - | - | - | -| fitCanvas() | + | + | + | + | + | + | + | + | + | + | -| dragCanvas() | + | - | - | - | - | - | + | - | - | + | -| zoomCanvas() | + | - | - | - | - | - | - | + | + | - | -| cancel() | - | + | + | + | + | + | + | + | + | + | -| configure() | + | + | + | + | + | + | + | + | + | + | -| bitmap() | + | + | + | + | + | + | + | + | + | + | -| setZLayer() | + | + | + | + | + | + | + | + | + | + | +| | IDLE | GROUP | SPLIT | DRAW | MERGE | EDIT | DRAG | RESIZE | ZOOM_CANVAS | DRAG_CANVAS | INTERACT | +|--------------|------|-------|-------|------|-------|------|------|--------|-------------|-------------|----------| +| setup() | + | + | + | +/- | + | +/- | +/- | +/- | + | + | + | +| activate() | + | - | - | - | - | - | - | - | - | - | - | +| rotate() | + | + | + | + | + | + | + | + | + | + | + | +| focus() | + | + | + | + | + | + | + | + | + | + | + | +| fit() | + | + | + | + | + | + | + | + | + | + | + | +| grid() | + | + | + | + | + | + | + | + | + | + | + | +| draw() | + | - | - | + | - | - | - | - | - | - | - | +| interact() | + | - | - | - | - | - | - | - | - | - | + | +| split() | + | - | + | - | - | - | - | - | - | - | - | +| group() | + | + | - | - | - | - | - | - | - | - | - | +| merge() | + | - | - | - | + | - | - | - | - | - | - | +| fitCanvas() | + | + | + | + | + | + | + | + | + | + | + | +| dragCanvas() | + | - | - | - | - | - | + | - | - | + | - | +| zoomCanvas() | + | - | - | - | - | - | - | + | + | - | - | +| cancel() | - | + | + | + | + | + | + | + | + | + | + | +| configure() | + | + | + | + | + | + | + | + | + | + | + | +| bitmap() | + | + | + | + | + | + | + | + | + | + | + | +| setZLayer() | + | + | + | + | + | + | + | + | + | + | + | You can call setup() during editing, dragging, and resizing only to update objects, not to change a frame. You can change frame during draw only when you do not redraw an existing object diff --git a/cvat-canvas/package-lock.json b/cvat-canvas/package-lock.json index b50b6223..5ee19968 100644 --- a/cvat-canvas/package-lock.json +++ b/cvat-canvas/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "2.0.2", + "version": "2.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json index 8792ce08..ba7f0180 100644 --- a/cvat-canvas/package.json +++ b/cvat-canvas/package.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "2.0.2", + "version": "2.1.0", "description": "Part of Computer Vision Annotation Tool which presents its canvas library", "main": "src/canvas.ts", "scripts": { diff --git a/cvat-canvas/src/typescript/canvas.ts b/cvat-canvas/src/typescript/canvas.ts index 35699766..508e5564 100644 --- a/cvat-canvas/src/typescript/canvas.ts +++ b/cvat-canvas/src/typescript/canvas.ts @@ -8,26 +8,17 @@ import { MergeData, SplitData, GroupData, + InteractionData, + InteractionResult, CanvasModel, CanvasModelImpl, RectDrawingMethod, CuboidDrawingMethod, Configuration, } from './canvasModel'; - -import { - Master, -} from './master'; - -import { - CanvasController, - CanvasControllerImpl, -} from './canvasController'; - -import { - CanvasView, - CanvasViewImpl, -} from './canvasView'; +import { Master } from './master'; +import { CanvasController, CanvasControllerImpl } from './canvasController'; +import { CanvasView, CanvasViewImpl } from './canvasView'; import '../scss/canvas.scss'; import pjson from '../../package.json'; @@ -43,6 +34,7 @@ interface Canvas { fit(): void; grid(stepX: number, stepY: number): void; + interact(interactionData: InteractionData): void; draw(drawData: DrawData): void; group(groupData: GroupData): void; split(splitData: SplitData): void; @@ -118,6 +110,10 @@ class CanvasImpl implements Canvas { this.model.grid(stepX, stepY); } + public interact(interactionData: InteractionData): void { + this.model.interact(interactionData); + } + public draw(drawData: DrawData): void { this.model.draw(drawData); } @@ -162,4 +158,6 @@ export { RectDrawingMethod, CuboidDrawingMethod, Mode as CanvasMode, + InteractionData, + InteractionResult, }; diff --git a/cvat-canvas/src/typescript/canvasController.ts b/cvat-canvas/src/typescript/canvasController.ts index 179f9b32..786836d8 100644 --- a/cvat-canvas/src/typescript/canvasController.ts +++ b/cvat-canvas/src/typescript/canvasController.ts @@ -13,6 +13,7 @@ import { SplitData, GroupData, Mode, + InteractionData, } from './canvasModel'; export interface CanvasController { @@ -21,6 +22,7 @@ export interface CanvasController { readonly focusData: FocusData; readonly activeElement: ActiveElement; readonly drawData: DrawData; + readonly interactionData: InteractionData; readonly mergeData: MergeData; readonly splitData: SplitData; readonly groupData: GroupData; @@ -30,6 +32,7 @@ export interface CanvasController { zoom(x: number, y: number, direction: number): void; draw(drawData: DrawData): void; + interact(interactionData: InteractionData): void; merge(mergeData: MergeData): void; split(splitData: SplitData): void; group(groupData: GroupData): void; @@ -84,6 +87,10 @@ export class CanvasControllerImpl implements CanvasController { this.model.draw(drawData); } + public interact(interactionData: InteractionData): void { + this.model.interact(interactionData); + } + public merge(mergeData: MergeData): void { this.model.merge(mergeData); } @@ -124,6 +131,10 @@ export class CanvasControllerImpl implements CanvasController { return this.model.drawData; } + public get interactionData(): InteractionData { + return this.model.interactionData; + } + public get mergeData(): MergeData { return this.model.mergeData; } diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index ec860e77..74d9f49c 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -69,6 +69,20 @@ export interface DrawData { redraw?: number; } +export interface InteractionData { + enabled: boolean; + shapeType?: string; + crosshair?: boolean; + minPosVertices?: number; + minNegVertices?: number; +} + +export interface InteractionResult { + points: number[]; + shapeType: string; + button: number; +} + export interface EditData { enabled: boolean; state: any; @@ -105,6 +119,7 @@ export enum UpdateReasons { FITTED_CANVAS = 'fitted_canvas', + INTERACT = 'interact', DRAW = 'draw', MERGE = 'merge', SPLIT = 'split', @@ -126,6 +141,7 @@ export enum Mode { MERGE = 'merge', SPLIT = 'split', GROUP = 'group', + INTERACT = 'interact', DRAG_CANVAS = 'drag_canvas', ZOOM_CANVAS = 'zoom_canvas', } @@ -139,6 +155,7 @@ export interface CanvasModel { readonly focusData: FocusData; readonly activeElement: ActiveElement; readonly drawData: DrawData; + readonly interactionData: InteractionData; readonly mergeData: MergeData; readonly splitData: SplitData; readonly groupData: GroupData; @@ -162,6 +179,7 @@ export interface CanvasModel { split(splitData: SplitData): void; merge(mergeData: MergeData): void; select(objectState: any): void; + interact(interactionData: InteractionData): void; fitCanvas(width: number, height: number): void; bitmap(enabled: boolean): void; @@ -192,6 +210,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { top: number; zLayer: number | null; drawData: DrawData; + interactionData: InteractionData; mergeData: MergeData; groupData: GroupData; splitData: SplitData; @@ -242,6 +261,9 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { enabled: false, initialState: null, }, + interactionData: { + enabled: false, + }, mergeData: { enabled: false, }, @@ -490,6 +512,27 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { this.notify(UpdateReasons.DRAW); } + public interact(interactionData: InteractionData): void { + if (![Mode.IDLE, Mode.INTERACT].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (interactionData.enabled) { + if (this.data.interactionData.enabled) { + throw new Error('Interaction has been already started'); + } else if (!interactionData.shapeType) { + throw new Error('A shape type was not specified'); + } + } + + this.data.interactionData = interactionData; + if (typeof (this.data.interactionData.crosshair) !== 'boolean') { + this.data.interactionData.crosshair = true; + } + + this.notify(UpdateReasons.INTERACT); + } + public split(splitData: SplitData): void { if (![Mode.IDLE, Mode.SPLIT].includes(this.data.mode)) { throw Error(`Canvas is busy. Action: ${this.data.mode}`); @@ -567,7 +610,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { } public isAbleToChangeFrame(): boolean { - const isUnable = [Mode.DRAG, Mode.EDIT, Mode.RESIZE].includes(this.data.mode) + const isUnable = [Mode.DRAG, Mode.EDIT, Mode.RESIZE, Mode.INTERACT].includes(this.data.mode) || (this.data.mode === Mode.DRAW && typeof (this.data.drawData.redraw) === 'number'); return !isUnable; @@ -647,6 +690,10 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { return { ...this.data.drawData }; } + public get interactionData(): InteractionData { + return { ...this.data.interactionData }; + } + public get mergeData(): MergeData { return { ...this.data.mergeData }; } diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts index 4eb95a8f..3cf37f04 100644 --- a/cvat-canvas/src/typescript/canvasView.ts +++ b/cvat-canvas/src/typescript/canvasView.ts @@ -16,6 +16,7 @@ import { MergeHandler, MergeHandlerImpl } from './mergeHandler'; import { SplitHandler, SplitHandlerImpl } from './splitHandler'; import { GroupHandler, GroupHandlerImpl } from './groupHandler'; import { ZoomHandler, ZoomHandlerImpl } from './zoomHandler'; +import { InteractionHandler, InteractionHandlerImpl } from './interactionHandler'; import { AutoborderHandler, AutoborderHandlerImpl } from './autoborderHandler'; import consts from './consts'; import { @@ -42,6 +43,8 @@ import { Mode, Size, Configuration, + InteractionResult, + InteractionData, } from './canvasModel'; export interface CanvasView { @@ -72,6 +75,7 @@ export class CanvasViewImpl implements CanvasView, Listener { private groupHandler: GroupHandler; private zoomHandler: ZoomHandler; private autoborderHandler: AutoborderHandler; + private interactionHandler: InteractionHandler; private activeElement: ActiveElement; private configuration: Configuration; private serviceFlags: { @@ -127,6 +131,41 @@ export class CanvasViewImpl implements CanvasView, Listener { } } + private onInteraction( + shapes: InteractionResult[] | null, + shapesUpdated: boolean = true, + isDone: boolean = false, + ): void { + const { zLayer } = this.controller; + if (Array.isArray(shapes)) { + const event: CustomEvent = new CustomEvent('canvas.interacted', { + bubbles: false, + cancelable: true, + detail: { + shapesUpdated, + isDone, + shapes, + zOrder: zLayer || 0, + }, + }); + + this.canvas.dispatchEvent(event); + } + + if (shapes === null || isDone) { + const event: CustomEvent = new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }); + + this.canvas.dispatchEvent(event); + this.mode = Mode.IDLE; + this.controller.interact({ + enabled: false, + }); + } + } + private onDrawDone(data: object | null, duration: number, continueDraw?: boolean): void { const hiddenBecauseOfDraw = Object.keys(this.serviceFlags.drawHidden) .map((_clientID): number => +_clientID); @@ -373,6 +412,8 @@ export class CanvasViewImpl implements CanvasView, Listener { this.drawHandler.transform(this.geometry); this.editHandler.transform(this.geometry); this.zoomHandler.transform(this.geometry); + this.autoborderHandler.transform(this.geometry); + this.interactionHandler.transform(this.geometry); } private transformCanvas(): void { @@ -438,7 +479,9 @@ export class CanvasViewImpl implements CanvasView, Listener { // Transform handlers this.drawHandler.transform(this.geometry); this.editHandler.transform(this.geometry); + this.zoomHandler.transform(this.geometry); this.autoborderHandler.transform(this.geometry); + this.interactionHandler.transform(this.geometry); } private resizeCanvas(): void { @@ -846,6 +889,11 @@ export class CanvasViewImpl implements CanvasView, Listener { this.adoptedContent, this.geometry, ); + this.interactionHandler = new InteractionHandlerImpl( + this.onInteraction.bind(this), + this.adoptedContent, + this.geometry, + ); // Setup event handlers this.content.addEventListener('dblclick', (e: MouseEvent): void => { @@ -1063,6 +1111,18 @@ export class CanvasViewImpl implements CanvasView, Listener { this.drawHandler.draw(data, this.geometry); } } + } else if (reason === UpdateReasons.INTERACT) { + const data: InteractionData = this.controller.interactionData; + if (data.enabled && this.mode === Mode.IDLE) { + this.canvas.style.cursor = 'crosshair'; + this.mode = Mode.INTERACT; + this.interactionHandler.interact(data); + } else { + this.canvas.style.cursor = ''; + if (this.mode !== Mode.IDLE) { + this.interactionHandler.interact(data); + } + } } else if (reason === UpdateReasons.MERGE) { const data: MergeData = this.controller.mergeData; if (data.enabled) { @@ -1101,6 +1161,8 @@ export class CanvasViewImpl implements CanvasView, Listener { } else if (reason === UpdateReasons.CANCEL) { if (this.mode === Mode.DRAW) { this.drawHandler.cancel(); + } else if (this.mode === Mode.INTERACT) { + this.interactionHandler.cancel(); } else if (this.mode === Mode.MERGE) { this.mergeHandler.cancel(); } else if (this.mode === Mode.SPLIT) { @@ -1405,6 +1467,11 @@ export class CanvasViewImpl implements CanvasView, Listener { [state, +state.getAttribute('data-z-order')] )); + const crosshair = Array.from(this.content.getElementsByClassName('cvat_canvas_crosshair')); + crosshair.forEach((line: SVGLineElement): void => this.content.append(line)); + const interaction = Array.from(this.content.getElementsByClassName('cvat_interaction_point')); + interaction.forEach((circle: SVGCircleElement): void => this.content.append(circle)); + const needSort = states.some((pair): boolean => pair[1] !== states[0][1]); if (!states.length || !needSort) { return; diff --git a/cvat-canvas/src/typescript/crosshair.ts b/cvat-canvas/src/typescript/crosshair.ts new file mode 100644 index 00000000..27d25569 --- /dev/null +++ b/cvat-canvas/src/typescript/crosshair.ts @@ -0,0 +1,70 @@ +// Copyright (C) 2020 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import consts from './consts'; + +export default class Crosshair { + private x: SVG.Line | null; + private y: SVG.Line | null; + private canvas: SVG.Container | null; + + public constructor() { + this.x = null; + this.y = null; + this.canvas = null; + } + + public show(canvas: SVG.Container, x: number, y: number, scale: number): void { + if (this.canvas && this.canvas !== canvas) { + if (this.x) this.x.remove(); + if (this.y) this.y.remove(); + this.x = null; + this.y = null; + } + + this.canvas = canvas; + this.x = this.canvas.line(0, y, this.canvas.node.clientWidth, y).attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * scale), + }).addClass('cvat_canvas_crosshair'); + + this.y = this.canvas.line(x, 0, x, this.canvas.node.clientHeight).attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * scale), + }).addClass('cvat_canvas_crosshair'); + } + + public hide(): void { + if (this.x) { + this.x.remove(); + this.x = null; + } + + if (this.y) { + this.y.remove(); + this.y = null; + } + + this.canvas = null; + } + + public move(x: number, y: number): void { + if (this.x) { + this.x.attr({ y1: y, y2: y }); + } + + if (this.y) { + this.y.attr({ x1: x, x2: x }); + } + } + + public scale(scale: number): void { + if (this.x) { + this.x.attr('stroke-width', consts.BASE_STROKE_WIDTH / (2 * scale)); + } + + if (this.y) { + this.y.attr('stroke-width', consts.BASE_STROKE_WIDTH / (2 * scale)); + } + } +} diff --git a/cvat-canvas/src/typescript/drawHandler.ts b/cvat-canvas/src/typescript/drawHandler.ts index 404157bd..6716ead0 100644 --- a/cvat-canvas/src/typescript/drawHandler.ts +++ b/cvat-canvas/src/typescript/drawHandler.ts @@ -16,6 +16,7 @@ import { BBox, Box, } from './shared'; +import Crosshair from './crosshair'; import consts from './consts'; import { DrawData, @@ -44,10 +45,7 @@ export class DrawHandlerImpl implements DrawHandler { x: number; y: number; }; - private crosshair: { - x: SVG.Line; - y: SVG.Line; - }; + private crosshair: Crosshair; private drawData: DrawData; private geometry: Geometry; private autoborderHandler: AutoborderHandler; @@ -188,22 +186,11 @@ export class DrawHandlerImpl implements DrawHandler { private addCrosshair(): void { const { x, y } = this.cursorPosition; - this.crosshair = { - x: this.canvas.line(0, y, this.canvas.node.clientWidth, y).attr({ - 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * this.geometry.scale), - zOrder: Number.MAX_SAFE_INTEGER, - }).addClass('cvat_canvas_crosshair'), - y: this.canvas.line(x, 0, x, this.canvas.node.clientHeight).attr({ - 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * this.geometry.scale), - zOrder: Number.MAX_SAFE_INTEGER, - }).addClass('cvat_canvas_crosshair'), - }; + this.crosshair.show(this.canvas, x, y, this.geometry.scale); } private removeCrosshair(): void { - this.crosshair.x.remove(); - this.crosshair.y.remove(); - this.crosshair = null; + this.crosshair.hide(); } private release(): void { @@ -741,7 +728,7 @@ export class DrawHandlerImpl implements DrawHandler { this.canceled = false; this.drawData = null; this.geometry = null; - this.crosshair = null; + this.crosshair = new Crosshair(); this.drawInstance = null; this.pointsGroup = null; this.cursorPosition = { @@ -756,8 +743,7 @@ export class DrawHandlerImpl implements DrawHandler { ); this.cursorPosition = { x, y }; if (this.crosshair) { - this.crosshair.x.attr({ y1: y, y2: y }); - this.crosshair.y.attr({ x1: x, x2: x }); + this.crosshair.move(x, y); } }); } @@ -787,12 +773,7 @@ export class DrawHandlerImpl implements DrawHandler { } if (this.crosshair) { - this.crosshair.x.attr({ - 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * geometry.scale), - }); - this.crosshair.y.attr({ - 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * geometry.scale), - }); + this.crosshair.scale(this.geometry.scale); } if (this.pointsGroup) { diff --git a/cvat-canvas/src/typescript/editHandler.ts b/cvat-canvas/src/typescript/editHandler.ts index 8cea079d..413beda7 100644 --- a/cvat-canvas/src/typescript/editHandler.ts +++ b/cvat-canvas/src/typescript/editHandler.ts @@ -115,7 +115,7 @@ export class EditHandlerImpl implements EditHandler { (this.editLine as any).addClass('cvat_canvas_shape_drawing').style({ 'pointer-events': 'none', 'fill-opacity': 0, - 'stroke': strokeColor, + stroke: strokeColor, }).attr({ 'data-origin-client-id': this.editData.state.clientID, }).on('drawstart drawpoint', (e: CustomEvent): void => { @@ -213,20 +213,20 @@ export class EditHandlerImpl implements EditHandler { const cutIndexes2 = oldPoints.reduce((acc: string[], _: string, i: number) => i <= stop && i >= start ? [...acc, i] : acc, []); - const curveLength = (indexes: number[]) => { + const curveLength = (indexes: number[]): number => { const points = indexes.map((index: number): string => oldPoints[index]) .map((point: string): string[] => point.split(',')) .map((point: string[]): number[] => [+point[0], +point[1]]); let length = 0; for (let i = 1; i < points.length; i++) { length += Math.sqrt( - (points[i][0] - points[i - 1][0]) ** 2 - + (points[i][1] - points[i - 1][1]) ** 2, + ((points[i][0] - points[i - 1][0]) ** 2) + + ((points[i][1] - points[i - 1][1]) ** 2), ); } return length; - } + }; const pointsCriteria = cutIndexes1.length > cutIndexes2.length; const lengthCriteria = curveLength(cutIndexes1) > curveLength(cutIndexes2); @@ -278,8 +278,6 @@ export class EditHandlerImpl implements EditHandler { }); } } - - return; } private setupPoints(enabled: boolean): void { diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts new file mode 100644 index 00000000..76237cf0 --- /dev/null +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -0,0 +1,281 @@ +// Copyright (C) 2020 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import consts from './consts'; +import Crosshair from './crosshair'; +import { translateToSVG } from './shared'; +import { InteractionData, InteractionResult, Geometry } from './canvasModel'; + +export interface InteractionHandler { + transform(geometry: Geometry): void; + interact(interactData: InteractionData): void; + cancel(): void; +} + +export class InteractionHandlerImpl implements InteractionHandler { + private onInteraction: ( + shapes: InteractionResult[] | null, + shapesUpdated?: boolean, + isDone?: boolean, + ) => void; + private geometry: Geometry; + private canvas: SVG.Container; + private interactionData: InteractionData; + private cursorPosition: { x: number; y: number }; + private shapesWereUpdated: boolean; + private interactionShapes: SVG.Shape[]; + private currentInteractionShape: SVG.Shape | null; + private crosshair: Crosshair; + + private prepareResult(): InteractionResult[] { + return this.interactionShapes.map((shape: SVG.Shape): InteractionResult => { + if (shape.type === 'circle') { + const points = [(shape as SVG.Circle).cx(), (shape as SVG.Circle).cy()]; + return { + points: points.map((coord: number): number => coord - this.geometry.offset), + shapeType: 'points', + button: shape.attr('stroke') === 'green' ? 0 : 2, + }; + } + + const bbox = (shape.node as any as SVGRectElement).getBBox(); + const points = [bbox.x, bbox.y, bbox.x + bbox.width, bbox.y + bbox.height]; + return { + points: points.map((coord: number): number => coord - this.geometry.offset), + shapeType: 'rectangle', + button: 0, + }; + }); + } + + private shouldRaiseEvent(ctrlKey: boolean): boolean { + const { interactionData, interactionShapes, shapesWereUpdated } = this; + const { minPosVertices, minNegVertices, enabled } = interactionData; + + const positiveShapes = interactionShapes + .filter((shape: SVG.Shape): boolean => (shape as any).attr('stroke') === 'green'); + const negativeShapes = interactionShapes + .filter((shape: SVG.Shape): boolean => (shape as any).attr('stroke') !== 'green'); + + if (interactionData.shapeType === 'rectangle') { + return enabled && !ctrlKey && !!interactionShapes.length; + } + + const minimumVerticesAchieved = (typeof (minPosVertices) === 'undefined' + || minPosVertices <= positiveShapes.length) && (typeof (minNegVertices) === 'undefined' + || minPosVertices <= negativeShapes.length); + return enabled && !ctrlKey && minimumVerticesAchieved && shapesWereUpdated; + } + + private addCrosshair(): void { + const { x, y } = this.cursorPosition; + this.crosshair.show(this.canvas, x, y, this.geometry.scale); + } + + private removeCrosshair(): void { + this.crosshair.hide(); + } + + private interactPoints(): void { + const eventListener = (e: MouseEvent): void => { + if ((e.button === 0 || e.button === 2) && !e.altKey) { + e.preventDefault(); + const [cx, cy] = translateToSVG( + this.canvas.node as any as SVGSVGElement, + [e.clientX, e.clientY], + ); + this.currentInteractionShape = this.canvas + .circle(consts.BASE_POINT_SIZE * 2 / this.geometry.scale).center(cx, cy) + .fill('white') + .stroke(e.button === 0 ? 'green' : 'red') + .addClass('cvat_interaction_point') + .attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / this.geometry.scale, + }); + + this.interactionShapes.push(this.currentInteractionShape); + this.shapesWereUpdated = true; + if (this.shouldRaiseEvent(e.ctrlKey)) { + this.onInteraction(this.prepareResult(), true, false); + } + + const self = this.currentInteractionShape; + self.on('mouseenter', (): void => { + self.attr({ + 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale, + }); + + self.on('mousedown', (_e: MouseEvent): void => { + _e.preventDefault(); + _e.stopPropagation(); + self.remove(); + this.interactionShapes = this.interactionShapes.filter( + (shape: SVG.Shape): boolean => shape !== self, + ); + this.shapesWereUpdated = true; + if (this.shouldRaiseEvent(_e.ctrlKey)) { + this.onInteraction(this.prepareResult(), true, false); + } + }); + }); + + self.on('mouseleave', (): void => { + self.attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / this.geometry.scale, + }); + + self.off('mousedown'); + }); + } + }; + + // clear this listener in relese() + this.canvas.on('mousedown.interaction', eventListener); + } + + private interactRectangle(): void { + let initialized = false; + const eventListener = (e: MouseEvent): void => { + if (e.button === 0 && !e.altKey) { + if (!initialized) { + (this.currentInteractionShape as any).draw(e, { snapToGrid: 0.1 }); + initialized = true; + } else { + (this.currentInteractionShape as any).draw(e); + } + } + }; + + this.currentInteractionShape = this.canvas.rect(); + this.canvas.on('mousedown.interaction', eventListener); + this.currentInteractionShape.on('drawstop', (): void => { + this.interactionShapes.push(this.currentInteractionShape); + this.shapesWereUpdated = true; + + this.canvas.off('mousedown.interaction', eventListener); + if (this.shouldRaiseEvent(false)) { + this.onInteraction(this.prepareResult(), true, false); + } + + this.interact({ enabled: false }); + }).addClass('cvat_canvas_shape_drawing').attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + }); + } + + private initInteraction(): void { + if (this.interactionData.crosshair) { + this.addCrosshair(); + } + } + + private startInteraction(): void { + if (this.interactionData.shapeType === 'rectangle') { + this.interactRectangle(); + } else if (this.interactionData.shapeType === 'points') { + this.interactPoints(); + } else { + throw new Error('Interactor implementation supports only rectangle and points'); + } + } + + private release(): void { + if (this.crosshair) { + this.removeCrosshair(); + } + + this.canvas.off('mousedown.interaction'); + this.interactionShapes.forEach((shape: SVG.Shape): SVG.Shape => shape.remove()); + this.interactionShapes = []; + if (this.currentInteractionShape) { + this.currentInteractionShape.remove(); + this.currentInteractionShape = null; + } + } + + public constructor( + onInteraction: ( + shapes: InteractionResult[] | null, + shapesUpdated?: boolean, + isDone?: boolean, + ) => void, + canvas: SVG.Container, + geometry: Geometry, + ) { + this.onInteraction = ( + shapes: InteractionResult[] | null, + shapesUpdated?: boolean, + isDone?: boolean, + ): void => { + this.shapesWereUpdated = false; + onInteraction(shapes, shapesUpdated, isDone); + }; + this.canvas = canvas; + this.geometry = geometry; + this.shapesWereUpdated = false; + this.interactionShapes = []; + this.interactionData = { enabled: false }; + this.currentInteractionShape = null; + this.crosshair = new Crosshair(); + this.cursorPosition = { + x: 0, + y: 0, + }; + + this.canvas.on('mousemove.interaction', (e: MouseEvent): void => { + const [x, y] = translateToSVG( + this.canvas.node as any as SVGSVGElement, + [e.clientX, e.clientY], + ); + this.cursorPosition = { x, y }; + if (this.crosshair) { + this.crosshair.move(x, y); + } + }); + + document.body.addEventListener('keyup', (e: KeyboardEvent): void => { + if (e.keyCode === 17 && this.shouldRaiseEvent(false)) { // 17 is ctrl + this.onInteraction(this.prepareResult(), true, false); + } + }); + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + + if (this.crosshair) { + this.crosshair.scale(this.geometry.scale); + } + + const shapesToBeScaled = this.currentInteractionShape + ? [...this.interactionShapes, this.currentInteractionShape] + : [...this.interactionShapes]; + for (const shape of shapesToBeScaled) { + if (shape.type === 'circle') { + (shape as SVG.Circle).radius(consts.BASE_POINT_SIZE / this.geometry.scale); + shape.attr('stroke-width', consts.POINTS_STROKE_WIDTH / this.geometry.scale); + } else { + shape.attr('stroke-width', consts.BASE_STROKE_WIDTH / this.geometry.scale); + } + } + } + + public interact(interactionData: InteractionData): void { + if (interactionData.enabled) { + this.interactionData = interactionData; + this.initInteraction(); + this.startInteraction(); + } else { + this.onInteraction(this.prepareResult(), this.shouldRaiseEvent(false), true); + this.release(); + this.interactionData = interactionData; + } + } + + public cancel(): void { + this.release(); + this.onInteraction(null); + } +} diff --git a/cvat-core/src/annotations-history.js b/cvat-core/src/annotations-history.js index 4fdbf34c..ee84fddc 100644 --- a/cvat-core/src/annotations-history.js +++ b/cvat-core/src/annotations-history.js @@ -7,9 +7,14 @@ const MAX_HISTORY_LENGTH = 128; class AnnotationHistory { constructor() { + this.frozen = false; this.clear(); } + freeze(frozen) { + this.frozen = frozen; + } + get() { return { undo: this._undo.map((undo) => [undo.action, undo.frame]), @@ -18,6 +23,7 @@ class AnnotationHistory { } do(action, undo, redo, clientIDs, frame) { + if (this.frozen) return; const actionItem = { clientIDs, action, diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 9d7eadbf..63a316ac 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -327,6 +327,19 @@ ); } + function freezeHistory(session, frozen) { + const sessionType = session instanceof Task ? 'task' : 'job'; + const cache = getCache(sessionType); + + if (cache.has(session)) { + return cache.get(session).history.freeze(frozen); + } + + throw new DataError( + 'Collection has not been initialized yet. Call annotations.get() or annotations.clear(true) before', + ); + } + function clearActions(session) { const sessionType = session instanceof Task ? 'task' : 'job'; const cache = getCache(sessionType); @@ -372,6 +385,7 @@ exportDataset, undoActions, redoActions, + freezeHistory, clearActions, getActions, closeSession, diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index 6a994e0a..45187896 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -170,6 +170,11 @@ .apiWrapper.call(this, prototype.actions.redo, count); return result; }, + async freeze(frozen) { + const result = await PluginRegistry + .apiWrapper.call(this, prototype.actions.freeze, frozen); + return result; + }, async clear() { const result = await PluginRegistry .apiWrapper.call(this, prototype.actions.clear); @@ -545,6 +550,14 @@ * @instance * @async */ + /** + * Freeze history (do not save new actions) + * @method freeze + * @memberof Session.actions + * @throws {module:API.cvat.exceptions.PluginError} + * @instance + * @async + */ /** * Remove all actions from history * @method clear @@ -745,6 +758,7 @@ this.actions = { undo: Object.getPrototypeOf(this).actions.undo.bind(this), redo: Object.getPrototypeOf(this).actions.redo.bind(this), + freeze: Object.getPrototypeOf(this).actions.freeze.bind(this), clear: Object.getPrototypeOf(this).actions.clear.bind(this), get: Object.getPrototypeOf(this).actions.get.bind(this), }; @@ -1299,6 +1313,7 @@ this.actions = { undo: Object.getPrototypeOf(this).actions.undo.bind(this), redo: Object.getPrototypeOf(this).actions.redo.bind(this), + freeze: Object.getPrototypeOf(this).actions.freeze.bind(this), clear: Object.getPrototypeOf(this).actions.clear.bind(this), get: Object.getPrototypeOf(this).actions.get.bind(this), }; @@ -1390,6 +1405,7 @@ exportDataset, undoActions, redoActions, + freezeHistory, clearActions, getActions, closeSession, @@ -1582,6 +1598,11 @@ return result; }; + Job.prototype.actions.freeze.implementation = function (frozen) { + const result = freezeHistory(this, frozen); + return result; + }; + Job.prototype.actions.clear.implementation = function () { const result = clearActions(this); return result; @@ -1846,6 +1867,11 @@ return result; }; + Task.prototype.actions.freeze.implementation = function (frozen) { + const result = freezeHistory(this, frozen); + return result; + }; + Task.prototype.actions.clear.implementation = function () { const result = clearActions(this); return result; diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 3bea5b6c..efeaa81e 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -20,6 +20,7 @@ import { Rotation, ContextMenuType, Workspace, + Model, } from 'reducers/interfaces'; import getCore from 'cvat-core-wrapper'; @@ -187,6 +188,7 @@ export enum AnnotationActionTypes { CHANGE_WORKSPACE = 'CHANGE_WORKSPACE', SAVE_LOGS_SUCCESS = 'SAVE_LOGS_SUCCESS', SAVE_LOGS_FAILED = 'SAVE_LOGS_FAILED', + INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS', } export function saveLogsAsync(): ThunkAction { @@ -1385,6 +1387,16 @@ export function pasteShapeAsync(): ThunkAction { }; } +export function interactWithCanvas(activeInteractor: Model, activeLabelID: number): AnyAction { + return { + type: AnnotationActionTypes.INTERACT_WITH_CANVAS, + payload: { + activeInteractor, + activeLabelID, + }, + }; +} + export function repeatDrawShapeAsync(): ThunkAction { return async (dispatch: ActionCreator): Promise => { const { @@ -1401,6 +1413,7 @@ export function repeatDrawShapeAsync(): ThunkAction { }, }, drawing: { + activeInteractor, activeObjectType, activeLabelID, activeShapeType, @@ -1410,6 +1423,16 @@ export function repeatDrawShapeAsync(): ThunkAction { } = getStore().getState().annotation; let activeControl = ActiveControl.CURSOR; + if (activeInteractor) { + canvasInstance.interact({ + enabled: true, + shapeType: 'points', + minPosVertices: 4, // TODO: Add parameter to interactor + }); + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + return; + } + if (activeShapeType === ShapeType.RECTANGLE) { activeControl = ActiveControl.DRAW_RECTANGLE; } else if (activeShapeType === ShapeType.POINTS) { @@ -1443,7 +1466,7 @@ export function repeatDrawShapeAsync(): ThunkAction { rectDrawingMethod: activeRectDrawingMethod, numberOfPoints: activeNumOfPoints, shapeType: activeShapeType, - crosshair: activeShapeType === ShapeType.RECTANGLE, + crosshair: [ShapeType.RECTANGLE, ShapeType.CUBOID].includes(activeShapeType), }); } }; @@ -1490,7 +1513,7 @@ export function redrawShapeAsync(): ThunkAction { enabled: true, redraw: activatedStateID, shapeType: state.shapeType, - crosshair: state.shapeType === ShapeType.RECTANGLE, + crosshair: [ShapeType.RECTANGLE, ShapeType.CUBOID].includes(state.shapeType), }); } } diff --git a/cvat-ui/src/actions/models-actions.ts b/cvat-ui/src/actions/models-actions.ts index 1f2afe14..2627e5ff 100644 --- a/cvat-ui/src/actions/models-actions.ts +++ b/cvat-ui/src/actions/models-actions.ts @@ -10,11 +10,6 @@ export enum ModelsActionTypes { GET_MODELS = 'GET_MODELS', GET_MODELS_SUCCESS = 'GET_MODELS_SUCCESS', GET_MODELS_FAILED = 'GET_MODELS_FAILED', - DELETE_MODEL = 'DELETE_MODEL', - CREATE_MODEL = 'CREATE_MODEL', - CREATE_MODEL_SUCCESS = 'CREATE_MODEL_SUCCESS', - CREATE_MODEL_FAILED = 'CREATE_MODEL_FAILED', - CREATE_MODEL_STATUS_UPDATED = 'CREATE_MODEL_STATUS_UPDATED', START_INFERENCE_FAILED = 'START_INFERENCE_FAILED', GET_INFERENCE_STATUS_SUCCESS = 'GET_INFERENCE_STATUS_SUCCESS', GET_INFERENCE_STATUS_FAILED = 'GET_INFERENCE_STATUS_FAILED', @@ -84,8 +79,7 @@ export function getModelsAsync(): ThunkAction { dispatch(modelsActions.getModels()); try { - const models = (await core.lambda.list()) - .filter((model: Model) => ['detector', 'reid'].includes(model.type)); + const models = await core.lambda.list(); dispatch(modelsActions.getModelsSuccess(models)); } catch (error) { dispatch(modelsActions.getModelsFailed(error)); @@ -162,7 +156,6 @@ export function startInferenceAsync( return async (dispatch): Promise => { try { const requestID: string = await core.lambda.run(taskInstance, model, body); - const dispatchCallback = (action: ModelsActions): void => { dispatch(action); }; diff --git a/cvat-ui/src/assets/ai-tools-icon.svg b/cvat-ui/src/assets/ai-tools-icon.svg new file mode 100644 index 00000000..c8b4f304 --- /dev/null +++ b/cvat-ui/src/assets/ai-tools-icon.svg @@ -0,0 +1,18 @@ + + + + + diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx index f6ffeb2a..a202d5c9 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx @@ -14,6 +14,7 @@ import CursorControl from './cursor-control'; import MoveControl from './move-control'; import FitControl from './fit-control'; import ResizeControl from './resize-control'; +import ToolsControl from './tools-control'; import DrawRectangleControl from './draw-rectangle-control'; import DrawPolygonControl from './draw-polygon-control'; import DrawPolylineControl from './draw-polyline-control'; @@ -84,7 +85,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { preventDefault(event); const drawing = [ActiveControl.DRAW_POINTS, ActiveControl.DRAW_POLYGON, ActiveControl.DRAW_POLYLINE, ActiveControl.DRAW_RECTANGLE, - ActiveControl.DRAW_CUBOID].includes(activeControl); + ActiveControl.DRAW_CUBOID, ActiveControl.INTERACTION].includes(activeControl); if (!drawing) { canvasInstance.cancel(); @@ -97,6 +98,12 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { repeatDrawShape(); } } else { + if (activeControl === ActiveControl.INTERACTION) { + // separated API method + canvasInstance.interact({ enabled: false }); + return; + } + canvasInstance.draw({ enabled: false }); } }, @@ -178,7 +185,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
- + - { - setActivated(event.target.checked); - if (event.target.checked) { - activate(canvasInstance); - } else { - deactivate(canvasInstance); - } - }} - > - Make AI polygon - - - ) : null - ); -} - -export default connect( - mapStateToProps, - mapDispatchToProps, -)(DEXTRPlugin); - -// TODO: Add dialog window with cancel button diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx index 5edfc758..d8ce4d33 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx @@ -14,7 +14,6 @@ import Text from 'antd/lib/typography/Text'; import { RectDrawingMethod, CuboidDrawingMethod } from 'cvat-canvas-wrapper'; import { ShapeType } from 'reducers/interfaces'; import { clamp } from 'utils/math'; -import DEXTRPlugin from './dextr-plugin'; interface Props { shapeType: ShapeType; @@ -91,7 +90,6 @@ function DrawShapePopoverComponent(props: Props): JSX.Element { - { shapeType === ShapeType.POLYGON && } { shapeType === ShapeType.RECTANGLE && ( <> diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx new file mode 100644 index 00000000..5bdae3d4 --- /dev/null +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -0,0 +1,472 @@ +// Copyright (C) 2020 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import React from 'react'; +import { connect } from 'react-redux'; +import Icon from 'antd/lib/icon'; +import Popover from 'antd/lib/popover'; +import Select, { OptionProps } from 'antd/lib/select'; +import Button from 'antd/lib/button'; +import Modal from 'antd/lib/modal'; +import Text from 'antd/lib/typography/Text'; +import { Row, Col } from 'antd/lib/grid'; +import notification from 'antd/lib/notification'; + +import { AIToolsIcon } from 'icons'; +import { Canvas } from 'cvat-canvas-wrapper'; +import getCore from 'cvat-core-wrapper'; +import { + CombinedState, + ActiveControl, + Model, + ObjectType, + ShapeType, +} from 'reducers/interfaces'; +import { interactWithCanvas, fetchAnnotationsAsync, updateAnnotationsAsync } from 'actions/annotation-actions'; +import { InteractionResult } from 'cvat-canvas/src/typescript/canvas'; + +interface StateToProps { + canvasInstance: Canvas; + labels: any[]; + states: any[]; + activeLabelID: number; + jobInstance: any; + isInteraction: boolean; + frame: number; + interactors: Model[]; +} + +interface DispatchToProps { + onInteractionStart(activeInteractor: Model, activeLabelID: number): void; + updateAnnotations(statesToUpdate: any[]): void; + fetchAnnotations(): void; +} + +const core = getCore(); + +function mapStateToProps(state: CombinedState): StateToProps { + const { annotation } = state; + const { number: frame } = annotation.player.frame; + const { instance: jobInstance } = annotation.job; + const { instance: canvasInstance, activeControl } = annotation.canvas; + const { models } = state; + const { interactors } = models; + + return { + interactors, + isInteraction: activeControl === ActiveControl.INTERACTION, + activeLabelID: annotation.drawing.activeLabelID, + labels: annotation.job.labels, + states: annotation.annotations.states, + canvasInstance, + jobInstance, + frame, + }; +} + +function mapDispatchToProps(dispatch: any): DispatchToProps { + return { + onInteractionStart(activeInteractor: Model, activeLabelID: number): void { + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + }, + updateAnnotations(statesToUpdate: any[]): void { + dispatch(updateAnnotationsAsync(statesToUpdate)); + }, + fetchAnnotations(): void { + dispatch(fetchAnnotationsAsync()); + }, + }; +} + +function convertShapesForInteractor(shapes: InteractionResult[]): number[][] { + const reducer = (acc: number[][], _: number, index: number, array: number[]): number[][] => { + if (!(index % 2)) { // 0, 2, 4 + acc.push([ + array[index], + array[index + 1], + ]); + } + return acc; + }; + + return shapes.filter((shape: InteractionResult): boolean => shape.shapeType === 'points' && shape.button === 0) + .map((shape: InteractionResult): number[] => shape.points) + .flat().reduce(reducer, []); +} + +type Props = StateToProps & DispatchToProps; +interface State { + activeInteractor: Model | null; + activeLabelID: number; + interactiveStateID: number | null; + fetching: boolean; +} + +class ToolsControlComponent extends React.PureComponent { + private interactionIsAborted: boolean; + private interactionIsDone: boolean; + + public constructor(props: Props) { + super(props); + this.state = { + activeInteractor: props.interactors.length ? props.interactors[0] : null, + activeLabelID: props.labels[0].id, + interactiveStateID: null, + fetching: false, + }; + + this.interactionIsAborted = false; + this.interactionIsDone = false; + } + + public componentDidMount(): void { + const { canvasInstance } = this.props; + canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener); + canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener); + } + + public componentDidUpdate(prevProps: Props): void { + const { isInteraction } = this.props; + if (prevProps.isInteraction && !isInteraction) { + window.removeEventListener('contextmenu', this.contextmenuDisabler); + } else if (!prevProps.isInteraction && isInteraction) { + this.interactionIsDone = false; + this.interactionIsAborted = false; + window.addEventListener('contextmenu', this.contextmenuDisabler); + } + } + + public componentWillUnmount(): void { + const { canvasInstance } = this.props; + canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener); + canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener); + } + + private getInteractiveState(): any | null { + const { states } = this.props; + const { interactiveStateID } = this.state; + return states + .filter((_state: any): boolean => _state.clientID === interactiveStateID)[0] || null; + } + + private contextmenuDisabler = (e: MouseEvent): void => { + if (e.target && (e.target as Element).classList + && (e.target as Element).classList.toString().includes('ant-modal')) { + e.preventDefault(); + } + }; + + private cancelListener = async (): Promise => { + const { + isInteraction, + jobInstance, + frame, + fetchAnnotations, + } = this.props; + const { interactiveStateID, fetching } = this.state; + + if (isInteraction) { + if (fetching && !this.interactionIsDone) { + // user pressed ESC + this.setState({ fetching: false }); + this.interactionIsAborted = true; + } + + if (interactiveStateID !== null) { + const state = this.getInteractiveState(); + this.setState({ interactiveStateID: null }); + await state.delete(frame); + fetchAnnotations(); + } + + await jobInstance.actions.freeze(false); + } + }; + + private interactionListener = async (e: Event): Promise => { + const { + frame, + labels, + jobInstance, + isInteraction, + activeLabelID, + fetchAnnotations, + updateAnnotations, + } = this.props; + const { activeInteractor, interactiveStateID, fetching } = this.state; + + try { + if (!isInteraction) { + throw Error('Canvas raises event "canvas.interacted" when interaction is off'); + } + + if (fetching) { + this.interactionIsDone = (e as CustomEvent).detail.isDone; + return; + } + + const interactor = activeInteractor as Model; + + let result = []; + if ((e as CustomEvent).detail.shapesUpdated) { + this.setState({ fetching: true }); + try { + result = await core.lambda.call(jobInstance.task, interactor, { + task: jobInstance.task, + frame, + points: convertShapesForInteractor((e as CustomEvent).detail.shapes), + }); + + if (this.interactionIsAborted) { + // while the server request + // user has cancelled interaction (for example pressed ESC) + return; + } + } finally { + this.setState({ fetching: false }); + } + } + + if (this.interactionIsDone) { + // while the server request, user has done interaction (for example pressed N) + const object = new core.classes.ObjectState({ + frame, + objectType: ObjectType.SHAPE, + label: labels + .filter((label: any) => label.id === activeLabelID)[0], + shapeType: ShapeType.POLYGON, + points: result.flat(), + occluded: false, + zOrder: (e as CustomEvent).detail.zOrder, + }); + + await jobInstance.annotations.put([object]); + fetchAnnotations(); + } else { + // no shape yet, then create it and save to collection + if (interactiveStateID === null) { + // freeze history for interaction time + // (points updating shouldn't cause adding new actions to history) + await jobInstance.actions.freeze(true); + const object = new core.classes.ObjectState({ + frame, + objectType: ObjectType.SHAPE, + label: labels + .filter((label: any) => label.id === activeLabelID)[0], + shapeType: ShapeType.POLYGON, + points: result.flat(), + occluded: false, + zOrder: (e as CustomEvent).detail.zOrder, + }); + // need a clientID of a created object to interact with it further + // so, we do not use createAnnotationAction + const [clientID] = await jobInstance.annotations.put([object]); + + // update annotations on a canvas + fetchAnnotations(); + this.setState({ interactiveStateID: clientID }); + return; + } + + const state = this.getInteractiveState(); + if ((e as CustomEvent).detail.isDone) { + const finalObject = new core.classes.ObjectState({ + frame: state.frame, + objectType: state.objectType, + label: state.label, + shapeType: state.shapeType, + points: result.length ? result.flat() : state.points, + occluded: state.occluded, + zOrder: state.zOrder, + }); + this.setState({ interactiveStateID: null }); + await state.delete(frame); + await jobInstance.actions.freeze(false); + await jobInstance.annotations.put([finalObject]); + fetchAnnotations(); + } else { + state.points = result.flat(); + updateAnnotations([state]); + fetchAnnotations(); + } + } + } catch (err) { + notification.error({ + description: err.toString(), + message: 'Interaction error occured', + }); + } + }; + + private setActiveInteractor = (key: string): void => { + const { interactors } = this.props; + this.setState({ + activeInteractor: interactors.filter( + (interactor: Model) => interactor.id === key, + )[0], + }); + }; + + private renderLabelBlock(): JSX.Element { + const { labels } = this.props; + const { activeLabelID } = this.state; + return ( + <> + + + Label + + + + + + + + + ); + } + + private renderInteractorBlock(): JSX.Element { + const { interactors, canvasInstance, onInteractionStart } = this.props; + const { activeInteractor, activeLabelID, fetching } = this.state; + + return ( + <> + + + Interactor + + + + + + + + + + + + + + ); + } + + private renderPopoverContent(): JSX.Element { + return ( +
+ + + AI Tools + + + { this.renderLabelBlock() } + { this.renderInteractorBlock() } +
+ ); + } + + public render(): JSX.Element | null { + const { interactors, isInteraction, canvasInstance } = this.props; + const { fetching } = this.state; + + if (!interactors.length) return null; + + const dynamcPopoverPros = isInteraction ? { + overlayStyle: { + display: 'none', + }, + } : {}; + + const dynamicIconProps = isInteraction ? { + className: 'cvat-active-canvas-control cvat-tools-control', + onClick: (): void => { + canvasInstance.interact({ enabled: false }); + }, + } : { + className: 'cvat-tools-control', + }; + + return ( + <> + + Waiting for a server response.. + + + + + + + ); + } +} + +export default connect( + mapStateToProps, + mapDispatchToProps, +)(ToolsControlComponent); diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss index a9514add..71eb7e49 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss +++ b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss @@ -83,17 +83,31 @@ padding: 0; } -.cvat-draw-shape-popover > -.ant-popover-content > -.ant-popover-inner > div > -.ant-popover-inner-content { - padding: 0; +.cvat-draw-shape-popover, +.cvat-tools-control-popover { + > .ant-popover-content > + .ant-popover-inner > div > + .ant-popover-inner-content { + padding: 0; + } +} + +.cvat-tools-interact-button { + width: 100%; + margin-top: 10px; } .cvat-draw-shape-popover-points-selector { width: 100%; } +.cvat-tools-control-popover-content { + padding: 10px; + border-radius: 5px; + background: $background-color-2; + width: 270px; +} + .cvat-draw-shape-popover-content { padding: 10px; border-radius: 5px; diff --git a/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx b/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx index 234c7fd0..d8fd959e 100644 --- a/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx +++ b/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx @@ -21,7 +21,8 @@ import { } from 'reducers/interfaces'; interface Props { - models: Model[]; + reid: Model[]; + detectors: Model[]; activeProcesses: StringObject; visible: boolean; taskInstance: any; @@ -88,14 +89,14 @@ export default class ModelRunnerModalComponent extends React.PureComponent @@ -166,10 +168,7 @@ export default class ModelRunnerModalComponent extends React.PureComponent @@ -203,11 +202,7 @@ export default class ModelRunnerModalComponent extends React.PureComponent @@ -346,16 +338,10 @@ export default class ModelRunnerModalComponent extends React.PureComponent _model.name === selectedModel)[0]; @@ -414,13 +400,15 @@ export default class ModelRunnerModalComponent extends React.PureComponent model.name === selectedModel, )[0]; diff --git a/cvat-ui/src/components/models-page/built-model-item.tsx b/cvat-ui/src/components/models-page/built-model-item.tsx deleted file mode 100644 index f100dfe5..00000000 --- a/cvat-ui/src/components/models-page/built-model-item.tsx +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Tag from 'antd/lib/tag'; -import Select from 'antd/lib/select'; -import Text from 'antd/lib/typography/Text'; - -import { Model } from 'reducers/interfaces'; - -interface Props { - model: Model; -} - -export default function BuiltModelItemComponent(props: Props): JSX.Element { - const { model } = props; - - return ( - - - {model.framework} - - - - {model.name} - - - - - - - - ); -} diff --git a/cvat-ui/src/components/models-page/deployed-models-list.tsx b/cvat-ui/src/components/models-page/deployed-models-list.tsx index 93e301be..45d62aa8 100644 --- a/cvat-ui/src/components/models-page/deployed-models-list.tsx +++ b/cvat-ui/src/components/models-page/deployed-models-list.tsx @@ -9,7 +9,6 @@ import Text from 'antd/lib/typography/Text'; import { Model } from 'reducers/interfaces'; import DeployedModelItem from './deployed-model-item'; - interface Props { models: Model[]; } diff --git a/cvat-ui/src/components/models-page/models-page.tsx b/cvat-ui/src/components/models-page/models-page.tsx index a5606a2a..3b4fe521 100644 --- a/cvat-ui/src/components/models-page/models-page.tsx +++ b/cvat-ui/src/components/models-page/models-page.tsx @@ -12,11 +12,21 @@ import FeedbackComponent from '../feedback/feedback'; import { Model } from '../../reducers/interfaces'; interface Props { - deployedModels: Model[]; + interactors: Model[]; + detectors: Model[]; + trackers: Model[]; + reid: Model[]; } export default function ModelsPageComponent(props: Props): JSX.Element { - const { deployedModels } = props; + const { + interactors, + detectors, + trackers, + reid, + } = props; + + const deployedModels = [...detectors, ...interactors, ...trackers, ...reid]; return (
diff --git a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx index b956876c..3f63bd7f 100644 --- a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx +++ b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx @@ -9,7 +9,8 @@ import { Model, CombinedState } from 'reducers/interfaces'; import { startInferenceAsync, modelsActions } from 'actions/models-actions'; interface StateToProps { - models: Model[]; + reid: Model[]; + detectors: Model[]; activeProcesses: { [index: string]: string; }; @@ -30,7 +31,8 @@ function mapStateToProps(state: CombinedState): StateToProps { const { models } = state; return { - models: models.models, + reid: models.reid, + detectors: models.detectors, activeProcesses: {}, taskInstance: models.activeRunTask, visible: models.visibleRunWindows, diff --git a/cvat-ui/src/containers/models-page/models-page.tsx b/cvat-ui/src/containers/models-page/models-page.tsx index 8282d41d..734ad0dc 100644 --- a/cvat-ui/src/containers/models-page/models-page.tsx +++ b/cvat-ui/src/containers/models-page/models-page.tsx @@ -11,14 +11,26 @@ import { } from 'reducers/interfaces'; interface StateToProps { - deployedModels: Model[]; + interactors: Model[]; + detectors: Model[]; + trackers: Model[]; + reid: Model[]; } function mapStateToProps(state: CombinedState): StateToProps { const { models } = state; + const { + interactors, + detectors, + trackers, + reid, + } = models; return { - deployedModels: models.models, + interactors, + detectors, + trackers, + reid, }; } diff --git a/cvat-ui/src/cvat-canvas-wrapper.ts b/cvat-ui/src/cvat-canvas-wrapper.ts index 631a52a9..2dc70b8d 100644 --- a/cvat-ui/src/cvat-canvas-wrapper.ts +++ b/cvat-ui/src/cvat-canvas-wrapper.ts @@ -8,8 +8,13 @@ import { CanvasVersion, RectDrawingMethod, CuboidDrawingMethod, + InteractionData as InteractionDataType, + InteractionResult as InteractionResultType, } from 'cvat-canvas/src/typescript/canvas'; +export type InteractionData = InteractionDataType; +export type InteractionResult = InteractionResultType; + export { Canvas, CanvasMode, diff --git a/cvat-ui/src/icons.tsx b/cvat-ui/src/icons.tsx index fbf48607..a3e92e92 100644 --- a/cvat-ui/src/icons.tsx +++ b/cvat-ui/src/icons.tsx @@ -42,6 +42,8 @@ import SVGForegroundIcon from './assets/foreground-icon.svg'; import SVGCubeIcon from './assets/cube-icon.svg'; import SVGResetPerspectiveIcon from './assets/reset-perspective.svg'; import SVGColorizeIcon from './assets/colorize-icon.svg'; +import SVGAITools from './assets/ai-tools-icon.svg'; + export const CVATLogo = React.memo( (): JSX.Element => , @@ -154,6 +156,9 @@ export const CubeIcon = React.memo( export const ResetPerspectiveIcon = React.memo( (): JSX.Element => , ); +export const AIToolsIcon = React.memo( + (): JSX.Element => , +); export const ColorizeIcon = React.memo( (): JSX.Element => , ); diff --git a/cvat-ui/src/reducers/annotation-reducer.ts b/cvat-ui/src/reducers/annotation-reducer.ts index 4976977d..b37e3cb6 100644 --- a/cvat-ui/src/reducers/annotation-reducer.ts +++ b/cvat-ui/src/reducers/annotation-reducer.ts @@ -428,6 +428,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { activeControl, }, drawing: { + activeInteractor: undefined, activeLabelID: labelID, activeNumOfPoints: points, activeObjectType: objectType, @@ -1039,8 +1040,30 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { }, }; } + case AnnotationActionTypes.INTERACT_WITH_CANVAS: { + return { + ...state, + annotations: { + ...state.annotations, + activatedStateID: null, + }, + drawing: { + ...state.drawing, + activeInteractor: action.payload.activeInteractor, + activeLabelID: action.payload.activeLabelID, + }, + canvas: { + ...state.canvas, + activeControl: ActiveControl.INTERACTION, + }, + }; + } case AnnotationActionTypes.CHANGE_WORKSPACE: { const { workspace } = action.payload; + if (state.canvas.activeControl !== ActiveControl.CURSOR) { + return state; + } + return { ...state, workspace, diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index fbc4b5ed..9a48b52e 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -76,7 +76,6 @@ export interface FormatsState { // eslint-disable-next-line import/prefer-default-export export enum SupportedPlugins { GIT_INTEGRATION = 'GIT_INTEGRATION', - DEXTR_SEGMENTATION = 'DEXTR_SEGMENTATION', ANALYTICS = 'ANALYTICS', } @@ -161,7 +160,10 @@ export interface ModelsState { initialized: boolean; fetching: boolean; creatingStatus: string; - models: Model[]; + interactors: Model[]; + detectors: Model[]; + trackers: Model[]; + reid: Model[]; inferences: { [index: number]: ActiveInference; }; @@ -206,9 +208,7 @@ export interface NotificationsState { fetching: null | ErrorState; }; models: { - creating: null | ErrorState; starting: null | ErrorState; - deleting: null | ErrorState; fetching: null | ErrorState; canceling: null | ErrorState; metaFetching: null | ErrorState; @@ -270,6 +270,7 @@ export enum ActiveControl { GROUP = 'group', SPLIT = 'split', EDIT = 'edit', + INTERACTION = 'interaction', } export enum ShapeType { @@ -342,6 +343,7 @@ export interface AnnotationState { frameAngles: number[]; }; drawing: { + activeInteractor?: Model; activeShapeType: ShapeType; activeRectDrawingMethod?: RectDrawingMethod; activeNumOfPoints?: number; diff --git a/cvat-ui/src/reducers/models-reducer.ts b/cvat-ui/src/reducers/models-reducer.ts index 3e369584..18371aea 100644 --- a/cvat-ui/src/reducers/models-reducer.ts +++ b/cvat-ui/src/reducers/models-reducer.ts @@ -5,13 +5,16 @@ import { boundariesActions, BoundariesActionTypes } from 'actions/boundaries-actions'; import { ModelsActionTypes, ModelsActions } from 'actions/models-actions'; import { AuthActionTypes, AuthActions } from 'actions/auth-actions'; -import { ModelsState } from './interfaces'; +import { ModelsState, Model } from './interfaces'; const defaultState: ModelsState = { initialized: false, fetching: false, creatingStatus: '', - models: [], + interactors: [], + detectors: [], + trackers: [], + reid: [], visibleRunWindows: false, activeRunTask: null, inferences: {}, @@ -32,7 +35,10 @@ export default function ( case ModelsActionTypes.GET_MODELS_SUCCESS: { return { ...state, - models: action.payload.models, + interactors: action.payload.models.filter((model: Model) => ['interactor'].includes(model.type)), + detectors: action.payload.models.filter((model: Model) => ['detector'].includes(model.type)), + trackers: action.payload.models.filter((model: Model) => ['tracker'].includes(model.type)), + reid: action.payload.models.filter((model: Model) => ['reid'].includes(model.type)), initialized: true, fetching: false, }; diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index dfbfdbf5..e87d22b8 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -51,9 +51,7 @@ const defaultState: NotificationsState = { fetching: null, }, models: { - creating: null, starting: null, - deleting: null, fetching: null, canceling: null, metaFetching: null, @@ -414,21 +412,6 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } - case ModelsActionTypes.CREATE_MODEL_FAILED: { - return { - ...state, - errors: { - ...state.errors, - models: { - ...state.errors.models, - creating: { - message: 'Could not create the model', - reason: action.payload.error.toString(), - }, - }, - }, - }; - } case ModelsActionTypes.GET_INFERENCE_STATUS_SUCCESS: { if (action.payload.activeInference.status === 'finished') { const { taskID } = action.payload; diff --git a/cvat-ui/src/reducers/plugins-reducer.ts b/cvat-ui/src/reducers/plugins-reducer.ts index 71cfc3ab..b18db987 100644 --- a/cvat-ui/src/reducers/plugins-reducer.ts +++ b/cvat-ui/src/reducers/plugins-reducer.ts @@ -4,7 +4,6 @@ import { PluginsActionTypes, PluginActions } from 'actions/plugins-actions'; import { registerGitPlugin } from 'utils/git-utils'; -import { registerDEXTRPlugin } from 'utils/dextr-utils'; import { PluginsState } from './interfaces'; const defaultState: PluginsState = { @@ -12,7 +11,6 @@ const defaultState: PluginsState = { initialized: false, list: { GIT_INTEGRATION: false, - DEXTR_SEGMENTATION: false, ANALYTICS: false, }, }; @@ -36,10 +34,6 @@ export default function ( registerGitPlugin(); } - if (!state.list.DEXTR_SEGMENTATION && list.DEXTR_SEGMENTATION) { - registerDEXTRPlugin(); - } - return { ...state, initialized: true, diff --git a/cvat-ui/src/utils/dextr-utils.ts b/cvat-ui/src/utils/dextr-utils.ts deleted file mode 100644 index a9d92abf..00000000 --- a/cvat-ui/src/utils/dextr-utils.ts +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import getCore from 'cvat-core-wrapper'; -import { Canvas } from 'cvat-canvas-wrapper'; -import { ShapeType, CombinedState } from 'reducers/interfaces'; -import { getCVATStore } from 'cvat-store'; - -const core = getCore(); - -interface DEXTRPlugin { - name: string; - description: string; - cvat: { - classes: { - Job: { - prototype: { - annotations: { - put: { - enter(self: any, objects: any[]): Promise; - }; - }; - }; - }; - }; - }; - data: { - canceled: boolean; - enabled: boolean; - }; -} - -interface Point { - x: number; - y: number; -} - -const antModalRoot = document.createElement('div'); -const antModalMask = document.createElement('div'); -antModalMask.classList.add('ant-modal-mask'); -const antModalWrap = document.createElement('div'); -antModalWrap.classList.add('ant-modal-wrap'); -antModalWrap.setAttribute('role', 'dialog'); -const antModal = document.createElement('div'); -antModal.classList.add('ant-modal'); -antModal.style.width = '300px'; -antModal.style.top = '40%'; -antModal.setAttribute('role', 'document'); -const antModalContent = document.createElement('div'); -antModalContent.classList.add('ant-modal-content'); -const antModalBody = document.createElement('div'); -antModalBody.classList.add('ant-modal-body'); -antModalBody.style.textAlign = 'center'; -const antModalSpan = document.createElement('span'); -antModalSpan.innerText = 'Segmentation request is being processed'; -antModalSpan.style.display = 'block'; -const antModalButton = document.createElement('button'); -antModalButton.disabled = true; -antModalButton.classList.add('ant-btn', 'ant-btn-primary'); -antModalButton.style.width = '100px'; -antModalButton.style.margin = '10px auto'; -const antModalButtonSpan = document.createElement('span'); -antModalButtonSpan.innerText = 'Cancel'; - -antModalBody.append(antModalSpan, antModalButton); -antModalButton.append(antModalButtonSpan); -antModalContent.append(antModalBody); -antModal.append(antModalContent); -antModalWrap.append(antModal); -antModalRoot.append(antModalMask, antModalWrap); - -async function serverRequest( - taskInstance: any, - frame: number, - points: number[], -): Promise { - const reducer = (acc: number[][], - _: number, index: number, - array: number[]): number[][] => { - if (!(index % 2)) { // 0, 2, 4 - acc.push([ - array[index], - array[index + 1], - ]); - } - return acc; - }; - - const reducedPoints = points.reduce(reducer, []); - const models = await core.lambda.list(); - const model = models.filter((func: any): boolean => func.id === 'openvino.dextr')[0]; - const result = await core.lambda.call(taskInstance, model, { - task: taskInstance, - frame, - points: reducedPoints, - }); - - return result.flat(); -} - -async function enter(this: any, self: DEXTRPlugin, objects: any[]): Promise { - try { - if (self.data.enabled && objects.length === 1) { - const state = (getCVATStore().getState() as CombinedState); - const isPolygon = state.annotation - .drawing.activeShapeType === ShapeType.POLYGON; - if (!isPolygon) return; - - document.body.append(antModalRoot); - const promises: Record> = {}; - for (let i = 0; i < objects.length; i++) { - if (objects[i].points.length >= 8) { - promises[i] = serverRequest( - this.task, - objects[i].frame, - objects[i].points, - ); - } else { - promises[i] = new Promise((resolve) => { - resolve(objects[i].points); - }); - } - } - - const transformed = await Promise - .all(Object.values(promises)); - for (let i = 0; i < objects.length; i++) { - // eslint-disable-next-line no-param-reassign - objects[i] = new core.classes.ObjectState({ - frame: objects[i].frame, - objectType: objects[i].objectType, - label: objects[i].label, - shapeType: ShapeType.POLYGON, - points: transformed[i], - occluded: objects[i].occluded, - zOrder: objects[i].zOrder, - }); - } - } - - return; - } catch (error) { - throw new core.exceptions.PluginError(error.toString()); - } finally { - // eslint-disable-next-line no-param-reassign - self.data.canceled = false; - antModalButton.disabled = true; - if (antModalRoot.parentElement === document.body) { - document.body.removeChild(antModalRoot); - } - } -} - -const plugin: DEXTRPlugin = { - name: 'Deep extreme cut', - description: 'Plugin allows to get a polygon from extreme points using AI', - cvat: { - classes: { - Job: { - prototype: { - annotations: { - put: { - enter, - }, - }, - }, - }, - }, - }, - data: { - canceled: false, - enabled: false, - }, -}; - - -antModalButton.onclick = () => { - plugin.data.canceled = true; -}; - -export function activate(canvasInstance: Canvas): void { - if (!plugin.data.enabled) { - // eslint-disable-next-line no-param-reassign - canvasInstance.draw = (drawData: any): void => { - if (drawData.enabled && drawData.shapeType === ShapeType.POLYGON - && (typeof (drawData.numberOfPoints) === 'undefined' || drawData.numberOfPoints >= 4) - && (typeof (drawData.initialState) === 'undefined') - ) { - const patchedData = { ...drawData }; - patchedData.shapeType = ShapeType.POINTS; - patchedData.crosshair = true; - Object.getPrototypeOf(canvasInstance) - .draw.call(canvasInstance, patchedData); - } else { - Object.getPrototypeOf(canvasInstance) - .draw.call(canvasInstance, drawData); - } - }; - plugin.data.enabled = true; - } -} - -export function deactivate(canvasInstance: Canvas): void { - if (plugin.data.enabled) { - // eslint-disable-next-line no-param-reassign - canvasInstance.draw = Object.getPrototypeOf(canvasInstance).draw; - plugin.data.enabled = false; - } -} - -export function registerDEXTRPlugin(): void { - core.plugins.register(plugin); -} diff --git a/cvat-ui/src/utils/plugin-checker.ts b/cvat-ui/src/utils/plugin-checker.ts index 38c29575..e4150610 100644 --- a/cvat-ui/src/utils/plugin-checker.ts +++ b/cvat-ui/src/utils/plugin-checker.ts @@ -17,14 +17,6 @@ class PluginChecker { case SupportedPlugins.GIT_INTEGRATION: { return isReachable(`${serverHost}/git/repository/meta/get`, 'OPTIONS'); } - case SupportedPlugins.DEXTR_SEGMENTATION: { - try { - const list = await core.lambda.list(); - return list.map((func: any): boolean => func.id).includes('openvino.dextr'); - } catch (_) { - return false; - } - } case SupportedPlugins.ANALYTICS: { return isReachable(`${serverHost}/analytics/app/kibana`, 'GET'); }