diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3ca8e06c..4fbefd3f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -168,7 +168,11 @@ jobs: npx cypress run --headless --browser chrome --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json else - npx cypress run --headless --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' + if [[ ${{ matrix.specs }} != 'canvas3d_functionality' ]]; then + npx cypress run --headless --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' + else + npx cypress run --browser chrome --env coverage=false --spec 'cypress/integration/${{ matrix.specs }}/**/*.js' + fi fi - name: Creating a log file from "cvat" container logs if: failure() diff --git a/README.md b/README.md index cdcac6a7..f08c2357 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ Other ways to ask questions and get our support: vision AI platform that fully integrates CVAT with scalable data processing and parallelized training pipelines. - [DataIsKey](https://dataiskey.eu/annotation-tool/) uses CVAT as their prime data labeling tool -to offer annotation services for projects of any size. + to offer annotation services for projects of any size. diff --git a/cvat-canvas3d/README.md b/cvat-canvas3d/README.md index 7ce318f7..17566e61 100644 --- a/cvat-canvas3d/README.md +++ b/cvat-canvas3d/README.md @@ -26,11 +26,20 @@ npm run build -- --mode=development # without a minification ```ts interface Canvas3d { - html(): HTMLDivElement; - setup(frameData: any): void; - mode(): Mode; + html(): ViewsDOM; + setup(frameData: any, objectStates: any[]): void; isAbleToChangeFrame(): boolean; + mode(): Mode; render(): void; + keyControls(keys: KeyboardEvent): void; + draw(drawData: DrawData): void; + cancel(): void; + dragCanvas(enable: boolean): void; + activate(clientID: number | null, attributeID?: number): void; + configureShapes(shapeProperties: ShapeProperties): void; + fitCanvas(): void; + fit(): void; + group(groupData: GroupData): void; } ``` @@ -44,5 +53,9 @@ console.log('Version ', window.canvas.CanvasVersion); console.log('Current mode is ', window.canvas.mode()); // Put canvas to a html container -htmlContainer.appendChild(canvas.html()); +const views = canvas.html(); +htmlContainer.appendChild(views.perspective); +htmlContainer.appendChild(views.top); +htmlContainer.appendChild(views.side); +htmlContainer.appendChild(views.front); ``` diff --git a/cvat-canvas3d/src/typescript/canvas3d.ts b/cvat-canvas3d/src/typescript/canvas3d.ts index 139321fe..02a375cd 100644 --- a/cvat-canvas3d/src/typescript/canvas3d.ts +++ b/cvat-canvas3d/src/typescript/canvas3d.ts @@ -5,10 +5,17 @@ import pjson from '../../package.json'; import { Canvas3dController, Canvas3dControllerImpl } from './canvas3dController'; import { - Canvas3dModel, Canvas3dModelImpl, Mode, DrawData, ViewType, MouseInteraction, + Canvas3dModel, + Canvas3dModelImpl, + Mode, + DrawData, + ViewType, + MouseInteraction, + ShapeProperties, + GroupData, } from './canvas3dModel'; import { - Canvas3dView, Canvas3dViewImpl, ViewsDOM, CAMERA_ACTION, + Canvas3dView, Canvas3dViewImpl, ViewsDOM, CameraAction, } from './canvas3dView'; import { Master } from './master'; @@ -16,19 +23,24 @@ const Canvas3dVersion = pjson.version; interface Canvas3d { html(): ViewsDOM; - setup(frameData: any): void; + setup(frameData: any, objectStates: any[]): void; isAbleToChangeFrame(): boolean; mode(): Mode; render(): void; keyControls(keys: KeyboardEvent): void; - mouseControls(type: string, event: MouseEvent): void; draw(drawData: DrawData): void; cancel(): void; + dragCanvas(enable: boolean): void; + activate(clientID: number | null, attributeID?: number): void; + configureShapes(shapeProperties: ShapeProperties): void; + fitCanvas(): void; + fit(): void; + group(groupData: GroupData): void; } class Canvas3dImpl implements Canvas3d { - private model: Canvas3dModel & Master; - private controller: Canvas3dController; + private readonly model: Canvas3dModel & Master; + private readonly controller: Canvas3dController; private view: Canvas3dView; public constructor() { @@ -45,10 +57,6 @@ class Canvas3dImpl implements Canvas3d { this.view.keyControls(keys); } - public mouseControls(type: MouseInteraction, event: MouseEvent): void { - this.view.mouseControls(type, event); - } - public render(): void { this.view.render(); } @@ -57,14 +65,18 @@ class Canvas3dImpl implements Canvas3d { this.model.draw(drawData); } - public setup(frameData: any): void { - this.model.setup(frameData); + public setup(frameData: any, objectStates: any[]): void { + this.model.setup(frameData, objectStates); } public mode(): Mode { return this.model.mode; } + public group(groupData: GroupData): void { + this.model.group(groupData); + } + public isAbleToChangeFrame(): boolean { return this.model.isAbleToChangeFrame(); } @@ -72,8 +84,28 @@ class Canvas3dImpl implements Canvas3d { public cancel(): void { this.model.cancel(); } + + public dragCanvas(enable: boolean): void { + this.model.dragCanvas(enable); + } + + public configureShapes(shapeProperties: ShapeProperties): void { + this.model.configureShapes(shapeProperties); + } + + public activate(clientID: number | null, attributeID: number | null = null): void { + this.model.activate(String(clientID), attributeID); + } + + public fit(): void { + this.model.fit(); + } + + public fitCanvas(): void { + this.model.fit(); + } } export { - Canvas3dImpl as Canvas3d, Canvas3dVersion, ViewType, MouseInteraction, CAMERA_ACTION, + Canvas3dImpl as Canvas3d, Canvas3dVersion, ViewType, MouseInteraction, CameraAction, ViewsDOM, }; diff --git a/cvat-canvas3d/src/typescript/canvas3dController.ts b/cvat-canvas3d/src/typescript/canvas3dController.ts index 5320fc03..00b08f6c 100644 --- a/cvat-canvas3d/src/typescript/canvas3dController.ts +++ b/cvat-canvas3d/src/typescript/canvas3dController.ts @@ -2,11 +2,18 @@ // // SPDX-License-Identifier: MIT -import { Canvas3dModel, Mode, DrawData } from './canvas3dModel'; +import { + Canvas3dModel, Mode, DrawData, ActiveElement, FocusData, GroupData, +} from './canvas3dModel'; export interface Canvas3dController { readonly drawData: DrawData; + readonly activeElement: ActiveElement; + readonly selected: any; + readonly focused: FocusData; + readonly groupData: GroupData; mode: Mode; + group(groupData: GroupData): void; } export class Canvas3dControllerImpl implements Canvas3dController { @@ -27,4 +34,24 @@ export class Canvas3dControllerImpl implements Canvas3dController { public get drawData(): DrawData { return this.model.data.drawData; } + + public get activeElement(): ActiveElement { + return this.model.data.activeElement; + } + + public get selected(): any { + return this.model.data.selected; + } + + public get focused(): any { + return this.model.data.focusData; + } + + public get groupData(): GroupData { + return this.model.groupData; + } + + public group(groupData: GroupData): void { + this.model.group(groupData); + } } diff --git a/cvat-canvas3d/src/typescript/canvas3dModel.ts b/cvat-canvas3d/src/typescript/canvas3dModel.ts index 0b0aef59..359d64a5 100644 --- a/cvat-canvas3d/src/typescript/canvas3dModel.ts +++ b/cvat-canvas3d/src/typescript/canvas3dModel.ts @@ -9,6 +9,16 @@ export interface Size { height: number; } +export interface ActiveElement { + clientID: string | null; + attributeID: number | null; +} + +export interface GroupData { + enabled: boolean; + grouped?: []; +} + export interface Image { renderWidth: number; renderHeight: number; @@ -19,6 +29,7 @@ export interface DrawData { enabled: boolean; initialState?: any; redraw?: number; + shapeType?: string; } export enum FrameZoom { @@ -26,6 +37,13 @@ export enum FrameZoom { MAX = 10, } +export enum Planes { + TOP = 'topPlane', + SIDE = 'sidePlane', + FRONT = 'frontPlane', + PERSPECTIVE = 'perspectivePlane', +} + export enum ViewType { PERSPECTIVE = 'perspective', TOP = 'top', @@ -39,14 +57,29 @@ export enum MouseInteraction { HOVER = 'hover', } +export interface FocusData { + clientID: string | null; +} + +export interface ShapeProperties { + opacity: number; + outlined: boolean; + outlineColor: string; + selectedOpacity: number; + colorBy: string; +} + export enum UpdateReasons { IMAGE_CHANGED = 'image_changed', OBJECTS_UPDATED = 'objects_updated', - FITTED_CANVAS = 'fitted_canvas', DRAW = 'draw', SELECT = 'select', CANCEL = 'cancel', DATA_FAILED = 'data_failed', + DRAG_CANVAS = 'drag_canvas', + SHAPE_ACTIVATED = 'shape_activated', + GROUP = 'group', + FITTED_CANVAS = 'fitted_canvas', } export enum Mode { @@ -56,9 +89,12 @@ export enum Mode { DRAW = 'draw', EDIT = 'edit', INTERACT = 'interact', + DRAG_CANVAS = 'drag_canvas', + GROUP = 'group', } export interface Canvas3dDataModel { + activeElement: ActiveElement; canvasSize: Size; image: Image | null; imageID: number | null; @@ -67,15 +103,27 @@ export interface Canvas3dDataModel { drawData: DrawData; mode: Mode; exception: Error | null; + objects: any[]; + groupedObjects: any[]; + focusData: FocusData; + selected: any; + shapeProperties: ShapeProperties; + groupData: GroupData; } export interface Canvas3dModel { mode: Mode; data: Canvas3dDataModel; - setup(frameData: any): void; + readonly groupData: GroupData; + setup(frameData: any, objectStates: any[]): void; isAbleToChangeFrame(): boolean; draw(drawData: DrawData): void; cancel(): void; + dragCanvas(enable: boolean): void; + activate(clientID: string | null, attributeID: number | null): void; + configureShapes(shapeProperties: any): void; + fit(): void; + group(groupData: GroupData): void; } export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { @@ -84,10 +132,16 @@ export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { public constructor() { super(); this.data = { + activeElement: { + clientID: null, + attributeID: null, + }, canvasSize: { height: 0, width: 0, }, + objects: [], + groupedObjects: [], image: null, imageID: null, imageOffset: 0, @@ -101,37 +155,68 @@ export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { }, mode: Mode.IDLE, exception: null, + focusData: { + clientID: null, + }, + groupData: { + enabled: false, + grouped: [], + }, + selected: null, + shapeProperties: { + opacity: 40, + outlined: false, + outlineColor: '#000000', + selectedOpacity: 60, + colorBy: 'Label', + }, }; } - public setup(frameData: any): void { + public setup(frameData: any, objectStates: any[]): void { if (this.data.imageID !== frameData.number) { - this.data.imageID = frameData.number; - frameData - .data((): void => { - this.data.image = null; - this.notify(UpdateReasons.IMAGE_CHANGED); - }) - .then((data: Image): void => { - if (frameData.number !== this.data.imageID) { - // already another image - return; - } - - this.data.imageSize = { - height: frameData.height as number, - width: frameData.width as number, - }; - - this.data.image = data; - this.notify(UpdateReasons.IMAGE_CHANGED); - }) - .catch((exception: any): void => { - this.data.exception = exception; - this.notify(UpdateReasons.DATA_FAILED); - throw exception; - }); + if ([Mode.EDIT, Mode.DRAG, Mode.RESIZE].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + } + + if ([Mode.EDIT].includes(this.data.mode)) { + return; + } + + if (frameData.number === this.data.imageID) { + this.data.objects = objectStates; + this.notify(UpdateReasons.OBJECTS_UPDATED); + return; } + + this.data.imageID = frameData.number; + frameData + .data((): void => { + this.data.image = null; + this.notify(UpdateReasons.IMAGE_CHANGED); + }) + .then((data: Image): void => { + if (frameData.number !== this.data.imageID) { + // already another image + return; + } + + this.data.imageSize = { + height: frameData.height as number, + width: frameData.width as number, + }; + + this.data.image = data; + this.notify(UpdateReasons.IMAGE_CHANGED); + this.data.objects = objectStates; + this.notify(UpdateReasons.OBJECTS_UPDATED); + }) + .catch((exception: any): void => { + this.data.exception = exception; + this.notify(UpdateReasons.DATA_FAILED); + throw exception; + }); } public set mode(value: Mode) { @@ -145,7 +230,6 @@ export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { public isAbleToChangeFrame(): boolean { const isUnable = [Mode.DRAG, Mode.EDIT, Mode.RESIZE, Mode.INTERACT].includes(this.data.mode) || (this.data.mode === Mode.DRAW && typeof this.data.drawData.redraw === 'number'); - return !isUnable; } @@ -153,13 +237,98 @@ export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { if (drawData.enabled && this.data.drawData.enabled) { throw new Error('Drawing has been already started'); } + if ([Mode.DRAW, Mode.EDIT].includes(this.data.mode)) { + return; + } this.data.drawData.enabled = drawData.enabled; this.data.mode = Mode.DRAW; + if (typeof drawData.redraw === 'number') { + const clientID = drawData.redraw; + const [state] = this.data.objects.filter((_state: any): boolean => _state.clientID === clientID); + + if (state) { + this.data.drawData = { ...drawData }; + this.data.drawData.initialState = { ...this.data.drawData.initialState, label: state.label }; + this.data.drawData.shapeType = state.shapeType; + } else { + return; + } + } else { + this.data.drawData = { ...drawData }; + if (this.data.drawData.initialState) { + this.data.drawData.shapeType = this.data.drawData.initialState.shapeType; + } + } this.notify(UpdateReasons.DRAW); } public cancel(): void { this.notify(UpdateReasons.CANCEL); } + + public dragCanvas(enable: boolean): void { + if (enable && this.data.mode !== Mode.IDLE) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (!enable && this.data.mode !== Mode.DRAG_CANVAS) { + throw Error(`Canvas is not in the drag mode. Action: ${this.data.mode}`); + } + + this.data.mode = enable ? Mode.DRAG_CANVAS : Mode.IDLE; + this.notify(UpdateReasons.DRAG_CANVAS); + } + + public activate(clientID: string, attributeID: number | null): void { + if (this.data.activeElement.clientID === clientID && this.data.activeElement.attributeID === attributeID) { + return; + } + if (this.data.mode !== Mode.IDLE) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + if (typeof clientID === 'number') { + const [state] = this.data.objects.filter((_state: any): boolean => _state.clientID === clientID); + if (!state || state.objectType === 'tag') { + return; + } + } + this.data.activeElement = { + clientID, + attributeID, + }; + this.notify(UpdateReasons.SHAPE_ACTIVATED); + } + + public group(groupData: GroupData): void { + if (![Mode.IDLE, Mode.GROUP].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (this.data.groupData.enabled && groupData.enabled) { + return; + } + + if (!this.data.groupData.enabled && !groupData.enabled) { + return; + } + this.data.mode = groupData.enabled ? Mode.GROUP : Mode.IDLE; + this.data.groupData = { ...this.data.groupData, ...groupData }; + this.notify(UpdateReasons.GROUP); + } + + public configureShapes(shapeProperties: ShapeProperties): void { + this.data.shapeProperties = { + ...shapeProperties, + }; + this.notify(UpdateReasons.OBJECTS_UPDATED); + } + + public fit(): void { + this.notify(UpdateReasons.FITTED_CANVAS); + } + + public get groupData(): GroupData { + return { ...this.data.groupData }; + } } diff --git a/cvat-canvas3d/src/typescript/canvas3dView.ts b/cvat-canvas3d/src/typescript/canvas3dView.ts index 866e7cc2..d3703f66 100644 --- a/cvat-canvas3d/src/typescript/canvas3dView.ts +++ b/cvat-canvas3d/src/typescript/canvas3dView.ts @@ -9,18 +9,19 @@ import { Canvas3dController } from './canvas3dController'; import { Listener, Master } from './master'; import CONST from './consts'; import { - Canvas3dModel, UpdateReasons, Mode, DrawData, ViewType, MouseInteraction, + Canvas3dModel, DrawData, Mode, Planes, UpdateReasons, ViewType, } from './canvas3dModel'; -import { CuboidModel } from './cuboid'; +import { + createRotationHelper, CuboidModel, setEdges, setTranslationHelper, +} from './cuboid'; export interface Canvas3dView { html(): ViewsDOM; render(): void; keyControls(keys: KeyboardEvent): void; - mouseControls(type: MouseInteraction, event: MouseEvent): void; } -export enum CAMERA_ACTION { +export enum CameraAction { ZOOM_IN = 'KeyI', MOVE_UP = 'KeyU', MOVE_DOWN = 'KeyO', @@ -75,6 +76,9 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { private cube: CuboidModel; private highlighted: boolean; private selected: CubeObject; + private model: Canvas3dModel & Master; + private action: any; + private globalHelpers: any; private set mode(value: Mode) { this.controller.mode = value; @@ -88,9 +92,74 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { this.controller = controller; this.clock = new THREE.Clock(); this.speed = CONST.MOVEMENT_FACTOR; - this.cube = new CuboidModel(); + this.cube = new CuboidModel('line', '#ffffff'); this.highlighted = false; this.selected = this.cube; + this.model = model; + this.globalHelpers = { + top: { + resize: [], + rotate: [], + }, + side: { + resize: [], + rotate: [], + }, + front: { + resize: [], + rotate: [], + }, + }; + this.action = { + scan: null, + selectable: true, + frameCoordinates: { + x: 0, + y: 0, + z: 0, + }, + detected: false, + initialMouseVector: new THREE.Vector2(), + detachCam: false, + detachCamRef: 'null', + translation: { + status: false, + helper: null, + coordinates: null, + offset: new THREE.Vector3(), + inverseMatrix: new THREE.Matrix4(), + }, + rotation: { + status: false, + helper: null, + recentMouseVector: new THREE.Vector2(0, 0), + screenInit: { + x: 0, + y: 0, + }, + screenMove: { + x: 0, + y: 0, + }, + }, + resize: { + status: false, + helper: null, + recentMouseVector: new THREE.Vector2(0, 0), + initScales: { + x: 1, + y: 1, + z: 1, + }, + memScales: { + x: 1, + y: 1, + z: 1, + }, + resizeVector: new THREE.Vector3(0, 0, 0), + frontBool: false, + }, + }; this.views = { perspective: { @@ -104,18 +173,182 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { top: { renderer: new THREE.WebGLRenderer({ antialias: true }), scene: new THREE.Scene(), + rayCaster: { + renderer: new THREE.Raycaster(), + mouseVector: new THREE.Vector2(), + }, }, side: { renderer: new THREE.WebGLRenderer({ antialias: true }), scene: new THREE.Scene(), + rayCaster: { + renderer: new THREE.Raycaster(), + mouseVector: new THREE.Vector2(), + }, }, front: { renderer: new THREE.WebGLRenderer({ antialias: true }), scene: new THREE.Scene(), + rayCaster: { + renderer: new THREE.Raycaster(), + mouseVector: new THREE.Vector2(), + }, }, }; CameraControls.install({ THREE }); + const canvasPerspectiveView = this.views.perspective.renderer.domElement; + const canvasTopView = this.views.top.renderer.domElement; + const canvasSideView = this.views.side.renderer.domElement; + const canvasFrontView = this.views.front.renderer.domElement; + + canvasPerspectiveView.addEventListener('contextmenu', (e: MouseEvent): void => { + if (this.controller.focused.clientID !== null) { + this.dispatchEvent( + new CustomEvent('canvas.contextmenu', { + bubbles: false, + cancelable: true, + detail: { + clientID: Number(this.controller.focused.clientID), + clientX: e.clientX, + clientY: e.clientY, + }, + }), + ); + } + }); + + canvasTopView.addEventListener('mousedown', this.startAction.bind(this, 'top')); + canvasSideView.addEventListener('mousedown', this.startAction.bind(this, 'side')); + canvasFrontView.addEventListener('mousedown', this.startAction.bind(this, 'front')); + + canvasTopView.addEventListener('mousemove', this.moveAction.bind(this, 'top')); + canvasSideView.addEventListener('mousemove', this.moveAction.bind(this, 'side')); + canvasFrontView.addEventListener('mousemove', this.moveAction.bind(this, 'front')); + + canvasTopView.addEventListener('mouseup', this.completeActions.bind(this)); + canvasTopView.addEventListener('mouseleave', this.completeActions.bind(this)); + canvasSideView.addEventListener('mouseup', this.completeActions.bind(this)); + canvasSideView.addEventListener('mouseleave', this.completeActions.bind(this)); + canvasFrontView.addEventListener('mouseup', this.completeActions.bind(this)); + canvasFrontView.addEventListener('mouseleave', this.completeActions.bind(this)); + + canvasPerspectiveView.addEventListener('mousemove', (event: MouseEvent): void => { + event.preventDefault(); + if (this.mode === Mode.DRAG_CANVAS) return; + const canvas = this.views.perspective.renderer.domElement; + const rect = canvas.getBoundingClientRect(); + const { mouseVector } = this.views.perspective.rayCaster as { mouseVector: THREE.Vector2 }; + mouseVector.x = ((event.clientX - (canvas.offsetLeft + rect.left)) / canvas.clientWidth) * 2 - 1; + mouseVector.y = -((event.clientY - (canvas.offsetTop + rect.top)) / canvas.clientHeight) * 2 + 1; + }); + + canvasPerspectiveView.addEventListener('click', (e: MouseEvent): void => { + e.preventDefault(); + if (e.detail !== 1) return; + if (![Mode.GROUP, Mode.IDLE].includes(this.mode) || !this.views.perspective.rayCaster) return; + const intersects = this.views.perspective.rayCaster.renderer.intersectObjects( + this.views.perspective.scene.children[0].children, + false, + ); + if (intersects.length !== 0 && this.mode === Mode.GROUP && this.model.data.groupData.grouped) { + const item = this.model.data.groupData.grouped.filter( + (_state: any): boolean => _state.clientID === Number(intersects[0].object.name), + ); + if (item.length !== 0) { + // @ts-ignore + this.model.data.groupData.grouped = this.model.data.groupData.grouped.filter( + (_state: any): boolean => _state.clientID !== Number(intersects[0].object.name), + ); + intersects[0].object.material.color.set(intersects[0].object.originalColor); + } else { + const [state] = this.model.data.objects.filter( + (_state: any): boolean => _state.clientID === Number(intersects[0].object.name), + ); + this.model.data.groupData.grouped.push(state); + intersects[0].object.material.color.set('#ffffff'); + } + } else if (this.mode === Mode.IDLE) { + if (intersects.length === 0) { + this.setHelperVisibility(false); + } + this.dispatchEvent( + new CustomEvent('canvas.selected', { + bubbles: false, + cancelable: true, + detail: { + clientID: intersects.length !== 0 ? Number(intersects[0].object.name) : null, + }, + }), + ); + } + }); + + canvasPerspectiveView.addEventListener('dblclick', (e: MouseEvent): void => { + e.preventDefault(); + if (this.mode !== Mode.DRAW) { + const { perspective: viewType } = this.views; + viewType.rayCaster.renderer.setFromCamera(viewType.rayCaster.mouseVector, viewType.camera); + const intersects = viewType.rayCaster.renderer.intersectObjects( + this.views.perspective.scene.children[0].children, + false, + ); + if (intersects.length !== 0) { + this.setDefaultZoom(); + } else { + const { x, y, z } = this.action.frameCoordinates; + this.positionAllViews(x, y, z, true); + } + return; + } + this.controller.drawData.enabled = false; + this.mode = Mode.IDLE; + const { x, y, z } = this.cube.perspective.position; + const { x: width, y: height, z: depth } = this.cube.perspective.scale; + const { x: rotationX, y: rotationY, z: rotationZ } = this.cube.perspective.rotation; + const points = [x, y, z, rotationX, rotationY, rotationZ, width, height, depth, 0, 0, 0, 0, 0, 0, 0]; + const initState = this.model.data.drawData.initialState; + let label; + if (initState) { + ({ label } = initState); + } + + if (typeof this.model.data.drawData.redraw === 'number') { + const [state] = this.model.data.objects.filter( + (_state: any): boolean => _state.clientID === Number(this.model.data.selected.perspective.name), + ); + this.dispatchEvent( + new CustomEvent('canvas.edited', { + bubbles: false, + cancelable: true, + detail: { + state, + points, + }, + }), + ); + } else { + this.dispatchEvent( + new CustomEvent('canvas.drawn', { + bubbles: false, + cancelable: true, + detail: { + state: { + ...initState, + shapeType: 'cuboid', + frame: this.model.data.imageID, + points, + label, + }, + continue: undefined, + duration: 0, + }, + }), + ); + } + this.dispatchEvent(new CustomEvent('canvas.canceled')); + }); + this.mode = Mode.IDLE; Object.keys(this.views).forEach((view: string): void => { @@ -132,84 +365,436 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { this.views.perspective.camera.position.set(-15, 0, 4); this.views.perspective.camera.up.set(0, 0, 1); this.views.perspective.camera.lookAt(10, 0, 0); + this.views.perspective.camera.name = 'cameraPerspective'; this.views.top.camera = new THREE.OrthographicCamera( (-aspectRatio * viewSize) / 2 - 2, (aspectRatio * viewSize) / 2 + 2, viewSize / 2 + 2, -viewSize / 2 - 2, - -10, - 10, + -50, + 50, ); + this.views.top.camera.position.set(0, 0, 5); + this.views.top.camera.lookAt(0, 0, 0); + this.views.top.camera.up.set(0, 0, 1); + this.views.top.camera.name = 'cameraTop'; + this.views.side.camera = new THREE.OrthographicCamera( (-aspectRatio * viewSize) / 2, (aspectRatio * viewSize) / 2, viewSize / 2, -viewSize / 2, - -10, - 10, + -50, + 50, ); this.views.side.camera.position.set(0, 5, 0); this.views.side.camera.lookAt(0, 0, 0); this.views.side.camera.up.set(0, 0, 1); + this.views.side.camera.name = 'cameraSide'; this.views.front.camera = new THREE.OrthographicCamera( (-aspectRatio * viewSize) / 2, (aspectRatio * viewSize) / 2, viewSize / 2, -viewSize / 2, - -10, - 10, + -50, + 50, ); - this.views.front.camera.position.set(-7, 0, 0); + this.views.front.camera.position.set(3, 0, 0); this.views.front.camera.up.set(0, 0, 1); this.views.front.camera.lookAt(0, 0, 0); + this.views.front.camera.name = 'cameraFront'; Object.keys(this.views).forEach((view: string): void => { const viewType = this.views[view as keyof Views]; - viewType.renderer.setSize(width, height); - if (view !== ViewType.PERSPECTIVE) { - viewType.controls = new CameraControls(viewType.camera, viewType.renderer.domElement); - viewType.controls.mouseButtons.left = CameraControls.ACTION.NONE; - viewType.controls.mouseButtons.right = CameraControls.ACTION.NONE; - } else { - viewType.controls = new CameraControls(viewType.camera, viewType.renderer.domElement); + if (viewType.camera) { + viewType.renderer.setSize(width, height); + if (view !== ViewType.PERSPECTIVE) { + viewType.controls = new CameraControls(viewType.camera, viewType.renderer.domElement); + viewType.controls.mouseButtons.left = CameraControls.ACTION.NONE; + viewType.controls.mouseButtons.right = CameraControls.ACTION.NONE; + } else { + viewType.controls = new CameraControls(viewType.camera, viewType.renderer.domElement); + } + viewType.controls.minDistance = CONST.MIN_DISTANCE; + viewType.controls.maxDistance = CONST.MAX_DISTANCE; } - viewType.controls.minDistance = CONST.MIN_DISTANCE; - viewType.controls.maxDistance = CONST.MAX_DISTANCE; + }); + this.views.top.controls.enabled = false; + this.views.side.controls.enabled = false; + this.views.front.controls.enabled = false; + + [ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view: ViewType): void => { + this.views[view].renderer.domElement.addEventListener( + 'wheel', + (event: WheelEvent): void => { + event.preventDefault(); + const { camera } = this.views[view]; + if (event.deltaY < CONST.FOV_MIN && camera.zoom < CONST.FOV_MAX) { + camera.zoom += CONST.FOV_INC; + } else if (event.deltaY > CONST.FOV_MIN && camera.zoom > CONST.FOV_MIN + 0.1) { + camera.zoom -= CONST.FOV_INC; + } + this.setHelperSize(view); + }, + { passive: false }, + ); }); model.subscribe(this); } + private setDefaultZoom(): void { + if (this.model.data.activeElement === 'null') { + Object.keys(this.views).forEach((view: string): void => { + const viewType = this.views[view as keyof Views]; + if (view !== ViewType.PERSPECTIVE) { + viewType.camera.zoom = CONST.FOV_DEFAULT; + viewType.camera.updateProjectionMatrix(); + } + }); + } else { + const canvasTop = this.views.top.renderer.domElement; + const bboxtop = new THREE.Box3().setFromObject(this.model.data.selected.top); + const x1 = Math.min( + canvasTop.offsetWidth / (bboxtop.max.x - bboxtop.min.x), + canvasTop.offsetHeight / (bboxtop.max.y - bboxtop.min.y), + ) * 0.4; + this.views.top.camera.zoom = x1 / 100; + this.views.top.camera.updateProjectionMatrix(); + this.views.top.camera.updateMatrix(); + this.setHelperSize(ViewType.TOP); + + const canvasFront = this.views.top.renderer.domElement; + const bboxfront = new THREE.Box3().setFromObject(this.model.data.selected.front); + const x2 = Math.min( + canvasFront.offsetWidth / (bboxfront.max.y - bboxfront.min.y), + canvasFront.offsetHeight / (bboxfront.max.z - bboxfront.min.z), + ) * 0.4; + this.views.front.camera.zoom = x2 / 100; + this.views.front.camera.updateProjectionMatrix(); + this.views.front.camera.updateMatrix(); + this.setHelperSize(ViewType.FRONT); + + const canvasSide = this.views.side.renderer.domElement; + const bboxside = new THREE.Box3().setFromObject(this.model.data.selected.side); + const x3 = Math.min( + canvasSide.offsetWidth / (bboxside.max.x - bboxside.min.x), + canvasSide.offsetHeight / (bboxside.max.z - bboxside.min.z), + ) * 0.4; + this.views.side.camera.zoom = x3 / 100; + this.views.side.camera.updateProjectionMatrix(); + this.views.side.camera.updateMatrix(); + this.setHelperSize(ViewType.SIDE); + } + } + + private startAction(view: any, event: MouseEvent): void { + if (event.detail !== 1) return; + const { clientID } = this.model.data.activeElement; + if (clientID === 'null') return; + const canvas = this.views[view as keyof Views].renderer.domElement; + const rect = canvas.getBoundingClientRect(); + const { mouseVector } = this.views[view as keyof Views].rayCaster as { mouseVector: THREE.Vector2 }; + const diffX = event.clientX - rect.left; + const diffY = event.clientY - rect.top; + mouseVector.x = (diffX / canvas.clientWidth) * 2 - 1; + mouseVector.y = -(diffY / canvas.clientHeight) * 2 + 1; + this.action.rotation.screenInit = { x: diffX, y: diffY }; + this.action.rotation.screenMove = { x: diffX, y: diffY }; + if ( + this.model.data.selected + && !this.model.data.selected.perspective.userData.lock + && !this.model.data.selected.perspective.userData.hidden + ) { + this.action.scan = view; + this.model.mode = Mode.EDIT; + this.action.selectable = false; + } + } + + private moveAction(view: any, event: MouseEvent): void { + event.preventDefault(); + const { clientID } = this.model.data.activeElement; + if (clientID === 'null') return; + const canvas = this.views[view as keyof Views].renderer.domElement; + const rect = canvas.getBoundingClientRect(); + const { mouseVector } = this.views[view as keyof Views].rayCaster as { mouseVector: THREE.Vector2 }; + const diffX = event.clientX - rect.left; + const diffY = event.clientY - rect.top; + mouseVector.x = (diffX / canvas.clientWidth) * 2 - 1; + mouseVector.y = -(diffY / canvas.clientHeight) * 2 + 1; + this.action.rotation.screenMove = { x: diffX, y: diffY }; + } + + private translateReferencePlane(coordinates: any): void { + const topPlane = this.views.top.scene.getObjectByName(Planes.TOP); + if (topPlane) { + topPlane.position.x = coordinates.x; + topPlane.position.y = coordinates.y; + topPlane.position.z = coordinates.z; + } + const sidePlane = this.views.side.scene.getObjectByName(Planes.SIDE); + if (sidePlane) { + sidePlane.position.x = coordinates.x; + sidePlane.position.y = coordinates.y; + sidePlane.position.z = coordinates.z; + } + const frontPlane = this.views.front.scene.getObjectByName(Planes.FRONT); + if (frontPlane) { + frontPlane.position.x = coordinates.x; + frontPlane.position.y = coordinates.y; + frontPlane.position.z = coordinates.z; + } + } + + private resetActions(): void { + this.action = { + ...this.action, + scan: null, + detected: false, + translation: { + status: false, + helper: null, + }, + rotation: { + status: false, + helper: null, + recentMouseVector: new THREE.Vector2(0, 0), + }, + resize: { + ...this.action.resize, + status: false, + helper: null, + recentMouseVector: new THREE.Vector2(0, 0), + }, + }; + } + + private completeActions(): void { + const { scan, detected } = this.action; + if (!detected) { + this.resetActions(); + return; + } + + const { x, y, z } = this.model.data.selected[scan].position; + const { x: width, y: height, z: depth } = this.model.data.selected[scan].scale; + const { x: rotationX, y: rotationY, z: rotationZ } = this.model.data.selected[scan].rotation; + const points = [x, y, z, rotationX, rotationY, rotationZ, width, height, depth, 0, 0, 0, 0, 0, 0, 0]; + const [state] = this.model.data.objects.filter( + (_state: any): boolean => _state.clientID === Number(this.model.data.selected[scan].name), + ); + this.dispatchEvent( + new CustomEvent('canvas.edited', { + bubbles: false, + cancelable: true, + detail: { + state, + points, + }, + }), + ); + if (this.action.rotation.status) { + this.detachCamera(scan); + } + + this.adjustPerspectiveCameras(); + this.translateReferencePlane(new THREE.Vector3(x, y, z)); + this.resetActions(); + this.model.mode = Mode.IDLE; + this.action.selectable = true; + } + + private onGroupDone(objects?: any[]): void { + if (objects && objects.length !== 0) { + this.dispatchEvent( + new CustomEvent('canvas.groupped', { + bubbles: false, + cancelable: true, + detail: { + states: objects, + }, + }), + ); + } else { + this.dispatchEvent( + new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }), + ); + } + + this.controller.group({ + enabled: false, + grouped: [], + }); + + this.mode = Mode.IDLE; + } + + private setupObject(object: any, addToScene: boolean): CuboidModel { + const { + opacity, outlined, outlineColor, selectedOpacity, colorBy, + } = this.model.data.shapeProperties; + const clientID = String(object.clientID); + const cuboid = new CuboidModel(object.occluded ? 'dashed' : 'line', outlined ? outlineColor : '#ffffff'); + + cuboid.setName(clientID); + cuboid.perspective.userData = object; + let color = ''; + if (colorBy === 'Label') { + ({ color } = object.label); + } else if (colorBy === 'Instance') { + ({ color } = object); + } else { + ({ color } = object.group); + } + cuboid.setOriginalColor(color); + cuboid.setColor(color); + cuboid.setOpacity(opacity); + + if ( + this.model.data.activeElement.clientID === clientID + && ![Mode.DRAG_CANVAS, Mode.GROUP].includes(this.mode) + ) { + cuboid.setOpacity(selectedOpacity); + if (!object.lock) { + createRotationHelper(cuboid.top, ViewType.TOP); + createRotationHelper(cuboid.side, ViewType.SIDE); + createRotationHelper(cuboid.front, ViewType.FRONT); + setTranslationHelper(cuboid.top); + setTranslationHelper(cuboid.side); + setTranslationHelper(cuboid.front); + } + setEdges(cuboid.top); + setEdges(cuboid.side); + setEdges(cuboid.front); + this.translateReferencePlane(new THREE.Vector3(object.points[0], object.points[1], object.points[2])); + this.model.data.selected = cuboid; + if (object.hidden) { + this.setHelperVisibility(false); + return cuboid; + } + } else { + cuboid.top.visible = false; + cuboid.side.visible = false; + cuboid.front.visible = false; + } + if (object.hidden) { + return cuboid; + } + cuboid.setPosition(object.points[0], object.points[1], object.points[2]); + cuboid.setScale(object.points[6], object.points[7], object.points[8]); + cuboid.setRotation(object.points[3], object.points[4], object.points[5]); + if (addToScene) { + this.addSceneChildren(cuboid); + } + if (this.model.data.activeElement.clientID === clientID) { + cuboid.attachCameraReference(); + this.rotatePlane(null, null); + this.action.detachCam = true; + this.action.detachCamRef = this.model.data.activeElement.clientID; + if (!object.lock) { + this.setSelectedChildScale(1 / cuboid.top.scale.x, 1 / cuboid.top.scale.y, 1 / cuboid.top.scale.z); + this.setHelperVisibility(true); + this.updateRotationHelperPos(); + this.updateResizeHelperPos(); + } else { + this.setHelperVisibility(false); + } + } + return cuboid; + } + + private setupObjects(): void { + if (this.views.perspective.scene.children[0]) { + this.clearSceneObjects(); + this.setHelperVisibility(false); + for (let i = 0; i < this.model.data.objects.length; i++) { + const object = this.model.data.objects[i]; + this.setupObject(object, true); + } + } + } + + private addSceneChildren(shapeObject: CuboidModel): void { + this.views.perspective.scene.children[0].add(shapeObject.perspective); + this.views.top.scene.children[0].add(shapeObject.top); + this.views.side.scene.children[0].add(shapeObject.side); + this.views.front.scene.children[0].add(shapeObject.front); + } + + private dispatchEvent(event: CustomEvent): void { + this.views.perspective.renderer.domElement.dispatchEvent(event); + } + public notify(model: Canvas3dModel & Master, reason: UpdateReasons): void { if (reason === UpdateReasons.IMAGE_CHANGED) { + if (!model.data.image) return; const loader = new PCDLoader(); - this.clearScene(); const objectURL = URL.createObjectURL(model.data.image.imageData); + this.clearScene(); loader.load(objectURL, this.addScene.bind(this)); URL.revokeObjectURL(objectURL); - const event: CustomEvent = new CustomEvent('canvas.setup'); - this.views.perspective.renderer.domElement.dispatchEvent(event); + this.dispatchEvent(new CustomEvent('canvas.setup')); + } else if (reason === UpdateReasons.SHAPE_ACTIVATED) { + const { clientID } = this.model.data.activeElement; + this.setupObjects(); + if (clientID !== 'null') { + this.setDefaultZoom(); + } } else if (reason === UpdateReasons.DRAW) { const data: DrawData = this.controller.drawData; - if (data.enabled && this.mode === Mode.IDLE) { - this.mode = Mode.DRAW; - this.cube = new CuboidModel(); - } else if (this.mode !== Mode.IDLE) { - this.cube = new CuboidModel(); + this.cube = new CuboidModel('line', '#ffffff'); + if (data.redraw) { + const object = this.views.perspective.scene.getObjectByName(String(data.redraw)); + if (object) { + this.cube.perspective = object.clone() as THREE.Mesh; + object.visible = false; + } + } else if (data.initialState) { + this.model.data.activeElement.clientID = 'null'; + this.setupObjects(); + this.cube = this.setupObject(data.initialState, false); } + this.setHelperVisibility(false); + } else if (reason === UpdateReasons.OBJECTS_UPDATED) { + this.setupObjects(); + } else if (reason === UpdateReasons.DRAG_CANVAS) { + this.dispatchEvent( + new CustomEvent(this.mode === Mode.DRAG_CANVAS ? 'canvas.dragstart' : 'canvas.dragstop', { + bubbles: false, + cancelable: true, + }), + ); + this.model.data.activeElement.clientID = 'null'; + this.setupObjects(); } else if (reason === UpdateReasons.CANCEL) { if (this.mode === Mode.DRAW) { this.controller.drawData.enabled = false; + this.controller.drawData.redraw = undefined; Object.keys(this.views).forEach((view: string): void => { this.views[view as keyof Views].scene.children[0].remove(this.cube[view as keyof Views]); }); } + this.model.data.groupData.grouped = []; + this.setHelperVisibility(false); this.mode = Mode.IDLE; - const event: CustomEvent = new CustomEvent('canvas.canceled'); - this.views.perspective.renderer.domElement.dispatchEvent(event); + this.dispatchEvent(new CustomEvent('canvas.canceled')); + } else if (reason === UpdateReasons.FITTED_CANVAS) { + this.dispatchEvent(new CustomEvent('canvas.fit')); + } else if (reason === UpdateReasons.GROUP) { + if (!this.model.groupData.enabled) { + this.onGroupDone(this.model.data.groupData.grouped); + } else { + this.model.data.groupData.grouped = []; + this.model.data.activeElement.clientID = 'null'; + this.setupObjects(); + } } } @@ -219,13 +804,102 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { }); } + private clearSceneObjects(): void { + Object.keys(this.views).forEach((view: string): void => { + this.views[view as keyof Views].scene.children[0].children = []; + }); + } + + private setHelperVisibility(visibility: boolean): void { + [ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((viewType: ViewType): void => { + const globalRotationObject = this.views[viewType].scene.getObjectByName('globalRotationHelper'); + if (globalRotationObject) { + globalRotationObject.visible = visibility; + } + for (let i = 0; i < 8; i++) { + const resizeObject = this.views[viewType].scene.getObjectByName(`globalResizeHelper${i}`); + if (resizeObject) { + resizeObject.visible = visibility; + } + } + }); + } + + private static setupRotationHelper(): THREE.Mesh { + const sphereGeometry = new THREE.SphereGeometry(0.15); + const sphereMaterial = new THREE.MeshBasicMaterial({ color: '#ffffff', opacity: 1, visible: true }); + const rotationHelper = new THREE.Mesh(sphereGeometry, sphereMaterial); + rotationHelper.name = 'globalRotationHelper'; + return rotationHelper; + } + + private updateRotationHelperPos(): void { + [ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view: ViewType): void => { + const point = new THREE.Vector3(0, 0, 0); + this.model.data.selected[view].getObjectByName('rotationHelper').getWorldPosition(point); + const globalRotationObject = this.views[view].scene.getObjectByName('globalRotationHelper'); + if (globalRotationObject) { + globalRotationObject.position.set(point.x, point.y, point.z); + } + }); + } + + private setHelperSize(viewType: ViewType): void { + if ([ViewType.TOP, ViewType.SIDE, ViewType.FRONT].includes(viewType)) { + const { camera } = this.views[viewType]; + if (!camera || camera instanceof THREE.PerspectiveCamera) return; + const factor = (camera.top - camera.bottom) / camera.zoom; + const rotationObject = this.views[viewType].scene.getObjectByName('globalRotationHelper'); + if (rotationObject) { + rotationObject.scale.set(1, 1, 1).multiplyScalar(factor / 10); + } + for (let i = 0; i < 8; i++) { + const resizeObject = this.views[viewType].scene.getObjectByName(`globalResizeHelper${i}`); + if (resizeObject) { + resizeObject.scale.set(1, 1, 1).multiplyScalar(factor / 10); + } + } + } + } + + private setupResizeHelper(viewType: ViewType): void { + const sphereGeometry = new THREE.SphereGeometry(0.15); + const sphereMaterial = new THREE.MeshBasicMaterial({ color: '#ffffff', opacity: 1, visible: true }); + const helpers = []; + for (let i = 0; i < 8; i++) { + helpers[i] = new THREE.Mesh(sphereGeometry.clone(), sphereMaterial.clone()); + helpers[i].name = `globalResizeHelper${i}`; + this.globalHelpers[viewType].resize.push(helpers[i]); + this.views[viewType].scene.add(helpers[i]); + } + } + + private updateResizeHelperPos(): void { + [ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view: ViewType): void => { + let i = 0; + this.model.data.selected[view].children.forEach((element: any): void => { + if (element.name === 'resizeHelper') { + const p = new THREE.Vector3(0, 0, 0); + element.getWorldPosition(p); + const name = `globalResizeHelper${i}`; + const object = this.views[view].scene.getObjectByName(name); + if (object) { + object.position.set(p.x, p.y, p.z); + } + i++; + } + }); + }); + } + private addScene(points: any): void { // eslint-disable-next-line no-param-reassign - points.material.size = 0.08; - // eslint-disable-next-line no-param-reassign - points.material.color = new THREE.Color(0x0000ff); + points.material.size = 0.05; + points.material.color.set(new THREE.Color(0xffffff)); + const material = points.material.clone(); const sphereCenter = points.geometry.boundingSphere.center; const { radius } = points.geometry.boundingSphere; + if (!this.views.perspective.camera) return; const xRange = -radius / 2 < this.views.perspective.camera.position.x - sphereCenter.x && radius / 2 > this.views.perspective.camera.position.x - sphereCenter.x; const yRange = -radius / 2 < this.views.perspective.camera.position.y - sphereCenter.y @@ -245,32 +919,138 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { newZ = sphereCenter.z; } if (newX || newY || newZ) { - this.positionAllViews(newX, newY, newZ); + this.action.frameCoordinates = { x: newX, y: newY, z: newZ }; + this.positionAllViews(newX, newY, newZ, false); } - this.views.perspective.scene.add(points); + + [ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view: ViewType): void => { + this.globalHelpers[view].resize = []; + this.globalHelpers[view].rotation = []; + }); + + this.views.perspective.scene.add(points.clone()); + // Setup TopView + const canvasTopView = this.views.top.renderer.domElement; + const topScenePlane = new THREE.Mesh( + new THREE.PlaneBufferGeometry( + canvasTopView.offsetHeight, + canvasTopView.offsetWidth, + canvasTopView.offsetHeight, + canvasTopView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0xffffff, + alphaTest: 0, + visible: false, + transparent: true, + opacity: 0, + }), + ); + topScenePlane.position.set(0, 0, 0); + topScenePlane.name = Planes.TOP; + (topScenePlane.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + (topScenePlane as any).verticesNeedUpdate = true; + // eslint-disable-next-line no-param-reassign + points.material = material; + material.size = 0.5; this.views.top.scene.add(points.clone()); + this.views.top.scene.add(topScenePlane); + const topRotationHelper = Canvas3dViewImpl.setupRotationHelper(); + this.globalHelpers.top.rotation.push(topRotationHelper); + this.views.top.scene.add(topRotationHelper); + this.setupResizeHelper(ViewType.TOP); + // Setup Side View + const canvasSideView = this.views.side.renderer.domElement; + const sideScenePlane = new THREE.Mesh( + new THREE.PlaneBufferGeometry( + canvasSideView.offsetHeight, + canvasSideView.offsetWidth, + canvasSideView.offsetHeight, + canvasSideView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0xffffff, + alphaTest: 0, + visible: false, + transparent: true, + opacity: 0, + }), + ); + sideScenePlane.position.set(0, 0, 0); + sideScenePlane.rotation.set(-Math.PI / 2, Math.PI / 2000, Math.PI); + sideScenePlane.name = Planes.SIDE; + (sideScenePlane.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + (sideScenePlane as any).verticesNeedUpdate = true; this.views.side.scene.add(points.clone()); + this.views.side.scene.add(sideScenePlane); + const sideRotationHelper = Canvas3dViewImpl.setupRotationHelper(); + this.globalHelpers.side.rotation.push(sideRotationHelper); + this.views.side.scene.add(sideRotationHelper); + this.setupResizeHelper(ViewType.SIDE); + // Setup front View + const canvasFrontView = this.views.front.renderer.domElement; + const frontScenePlane = new THREE.Mesh( + new THREE.PlaneBufferGeometry( + canvasFrontView.offsetHeight, + canvasFrontView.offsetWidth, + canvasFrontView.offsetHeight, + canvasFrontView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0xffffff, + alphaTest: 0, + visible: false, + transparent: true, + opacity: 0, + }), + ); + frontScenePlane.position.set(0, 0, 0); + frontScenePlane.rotation.set(0, Math.PI / 2, 0); + frontScenePlane.name = Planes.FRONT; + (frontScenePlane.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + (frontScenePlane as any).verticesNeedUpdate = true; this.views.front.scene.add(points.clone()); + this.views.front.scene.add(frontScenePlane); + const frontRotationHelper = Canvas3dViewImpl.setupRotationHelper(); + this.globalHelpers.front.rotation.push(frontRotationHelper); + this.views.front.scene.add(frontRotationHelper); + this.setupResizeHelper(ViewType.FRONT); + this.setHelperVisibility(false); + this.setupObjects(); } - private positionAllViews(x: number, y: number, z: number): void { - this.views.perspective.controls.setLookAt(x - 8, y - 8, z + 3, x, y, z, false); - this.views.top.controls.setLookAt(x, y, z + 8, x, y, z, false); - this.views.side.controls.setLookAt(x, y + 8, z, x, y, z, false); - this.views.front.controls.setLookAt(x + 8, y, z, x, y, z, false); + private positionAllViews(x: number, y: number, z: number, animation: boolean): void { + if ( + this.views.perspective.controls + && this.views.top.controls + && this.views.side.controls + && this.views.front.controls + ) { + this.views.perspective.controls.setLookAt(x - 8, y - 8, z + 3, x, y, z, animation); + this.views.top.camera.position.set(x, y, z + 8); + this.views.top.camera.lookAt(x, y, z); + this.views.top.camera.zoom = CONST.FOV_DEFAULT; + this.views.side.camera.position.set(x, y + 8, z); + this.views.side.camera.lookAt(x, y, z); + this.views.side.camera.zoom = CONST.FOV_DEFAULT; + this.views.front.camera.position.set(x + 8, y, z); + this.views.front.camera.lookAt(x, y, z); + this.views.front.camera.zoom = CONST.FOV_DEFAULT; + } } private static resizeRendererToDisplaySize(viewName: string, view: RenderView): void { const { camera, renderer } = view; const canvas = renderer.domElement; + if (!canvas.parentElement) return; const width = canvas.parentElement.clientWidth; const height = canvas.parentElement.clientHeight; const needResize = canvas.clientWidth !== width || canvas.clientHeight !== height; - if (needResize) { + if (needResize && camera && view.camera) { if (camera instanceof THREE.PerspectiveCamera) { camera.aspect = width / height; } else { - const topViewFactor = 0; // viewName === ViewType.TOP ? 2 : 0; + const topViewFactor = 0; const viewSize = CONST.ZOOM_FACTOR; const aspectRatio = width / height; if (!(camera instanceof THREE.PerspectiveCamera)) { @@ -279,8 +1059,8 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { camera.top = viewSize / 2 + topViewFactor; camera.bottom = -viewSize / 2 - topViewFactor; } - camera.near = -10; - camera.far = 10; + camera.near = -50; + camera.far = 50; } view.renderer.setSize(width, height); view.camera.updateProjectionMatrix(); @@ -298,86 +1078,714 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { this.views.perspective.scene.children[0].add(this.cube.perspective); const newPoints = intersects[0].point; this.cube.perspective.position.copy(newPoints); + this.views.perspective.renderer.domElement.style.cursor = 'default'; } } else if (this.mode === Mode.IDLE) { - const intersects = this.views.perspective.rayCaster.renderer.intersectObjects( - this.views.perspective.scene.children[0].children, - false, - ); + const { children } = this.views.perspective.scene.children[0]; + const { renderer } = this.views.perspective.rayCaster; + const intersects = renderer.intersectObjects(children, false); if (intersects.length !== 0) { - this.views.perspective.scene.children[0].children.forEach((sceneItem: THREE.Mesh): void => { - if (this.selected.perspective !== sceneItem) { - // eslint-disable-next-line no-param-reassign - sceneItem.material.color = new THREE.Color(0xff0000); - } - }); - const selectedObject = intersects[0].object as THREE.Mesh; - if (this.selected.perspective !== selectedObject) { - selectedObject.material.color = new THREE.Color(0xffff00); - this.highlighted = true; + const clientID = intersects[0].object.name; + if (clientID === undefined || clientID === '' || this.model.data.focusData.clientID === clientID) { + return; } - } else { - if (this.highlighted) { - this.views.perspective.scene.children[0].children.forEach((sceneItem: THREE.Mesh): void => { - if (this.selected.perspective !== sceneItem) { - // eslint-disable-next-line no-param-reassign - sceneItem.material.color = new THREE.Color(0xff0000); - } - }); - } - this.highlighted = false; + if (!this.action.selectable) return; + this.resetColor(); + const object = this.views.perspective.scene.getObjectByName(clientID); + if (object === undefined) return; + this.model.data.focusData.clientID = clientID; + this.dispatchEvent( + new CustomEvent('canvas.selected', { + bubbles: false, + cancelable: true, + detail: { + clientID: Number(intersects[0].object.name), + }, + }), + ); + } else if (this.model.data.focusData.clientID !== null) { + this.resetColor(); + this.model.data.focusData.clientID = null; } } }; + private resetColor(): void { + this.model.data.objects.forEach((object: any): void => { + const { clientID } = object; + const target = this.views.perspective.scene.getObjectByName(String(clientID)); + if (target) { + ((target as THREE.Mesh).material as THREE.MeshBasicMaterial).color.set((target as any).originalColor); + } + }); + } + public render(): void { Object.keys(this.views).forEach((view: string): void => { const viewType = this.views[view as keyof Views]; + if (!(viewType.controls && viewType.camera && viewType.rayCaster)) return; Canvas3dViewImpl.resizeRendererToDisplaySize(view, viewType); - viewType.controls.update(this.clock.getDelta()); + if (viewType.controls.enabled) { + viewType.controls.update(this.clock.getDelta()); + } else { + viewType.camera.updateProjectionMatrix(); + } viewType.renderer.render(viewType.scene, viewType.camera); if (view === ViewType.PERSPECTIVE && viewType.scene.children.length !== 0) { this.renderRayCaster(viewType); } + const { clientID } = this.model.data.activeElement; + if (clientID !== 'null' && view !== ViewType.PERSPECTIVE) { + viewType.rayCaster.renderer.setFromCamera(viewType.rayCaster.mouseVector, viewType.camera); + // First Scan + if (this.action.scan === view) { + if (!(this.action.translation.status || this.action.resize.status || this.action.rotation.status)) { + this.initiateAction(view, viewType); + } + // Action Operations + if (this.action.detected) { + if (this.action.translation.status) { + this.renderTranslateAction(view as ViewType, viewType); + } else if (this.action.resize.status) { + this.renderResizeAction(view as ViewType, viewType); + } else { + this.renderRotateAction(view as ViewType, viewType); + } + this.updateRotationHelperPos(); + this.updateResizeHelperPos(); + } + } + } + }); + if (this.action.detachCam && this.action.detachCamRef === this.model.data.activeElement.clientID) { + try { + this.detachCamera(null); + // eslint-disable-next-line no-empty + } catch (e) { + } finally { + this.action.detachCam = false; + } + } + } + + private adjustPerspectiveCameras(): void { + const coordinatesTop = this.model.data.selected.getReferenceCoordinates(ViewType.TOP); + const sphericalTop = new THREE.Spherical(); + sphericalTop.setFromVector3(coordinatesTop); + this.views.top.camera.position.setFromSpherical(sphericalTop); + this.views.top.camera.updateProjectionMatrix(); + + const coordinatesSide = this.model.data.selected.getReferenceCoordinates(ViewType.SIDE); + const sphericalSide = new THREE.Spherical(); + sphericalSide.setFromVector3(coordinatesSide); + this.views.side.camera.position.setFromSpherical(sphericalSide); + this.views.side.camera.updateProjectionMatrix(); + + const coordinatesFront = this.model.data.selected.getReferenceCoordinates(ViewType.FRONT); + const sphericalFront = new THREE.Spherical(); + sphericalFront.setFromVector3(coordinatesFront); + this.views.front.camera.position.setFromSpherical(sphericalFront); + this.views.front.camera.updateProjectionMatrix(); + } + + private renderTranslateAction(view: ViewType, viewType: any): void { + if ( + this.action.translation.helper.x === this.views[view].rayCaster.mouseVector.x + && this.action.translation.helper.y === this.views[view].rayCaster.mouseVector.y + ) { + return; + } + const intersects = viewType.rayCaster.renderer.intersectObjects( + [viewType.scene.getObjectByName(`${view}Plane`)], + true, + ); + + if (intersects.length !== 0 && intersects[0].point) { + const coordinates = intersects[0].point; + this.action.translation.coordinates = coordinates; + this.moveObject(coordinates); + } + } + + private moveObject(coordinates: THREE.Vector3): void { + const { + perspective, top, side, front, + } = this.model.data.selected; + let localCoordinates = coordinates; + if (this.action.translation.status) { + localCoordinates = coordinates + .clone() + .sub(this.action.translation.offset) + .applyMatrix4(this.action.translation.inverseMatrix); + } + perspective.position.copy(localCoordinates.clone()); + top.position.copy(localCoordinates.clone()); + side.position.copy(localCoordinates.clone()); + front.position.copy(localCoordinates.clone()); + } + + private setSelectedChildScale(x: number, y: number, z: number): void { + [ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view: ViewType): void => { + this.model.data.selected[view].children.forEach((element: any): void => { + if (element.name !== CONST.CUBOID_EDGE_NAME) { + element.scale.set( + x == null ? element.scale.x : x, + y == null ? element.scale.y : y, + z == null ? element.scale.z : z, + ); + } + }); }); } + private renderResizeAction(view: ViewType, viewType: any): void { + const intersects = viewType.rayCaster.renderer.intersectObjects( + [viewType.scene.getObjectByName(`${view}Plane`)], + true, + ); + // Return if no intersection with the reference plane + if (intersects.length === 0) return; + const { x: scaleInitX, y: scaleInitY, z: scaleInitZ } = this.action.resize.initScales; + const { x: scaleMemX, y: scaleMemY, z: scaleMemZ } = this.action.resize.memScales; + const { x: initPosX, y: initPosY } = this.action.resize.helper; + const { x: currentPosX, y: currentPosY } = viewType.rayCaster.mouseVector; + const { resizeVector } = this.action.resize; + + if (this.action.resize.helper.x === currentPosX && this.action.resize.helper.y === currentPosY) { + return; + } + + if ( + this.action.resize.recentMouseVector.x === currentPosX + && this.action.resize.recentMouseVector.y === currentPosY + ) { + return; + } + this.action.resize.recentMouseVector = viewType.rayCaster.mouseVector.clone(); + switch (view) { + case ViewType.TOP: { + let y = scaleInitX * (currentPosX / initPosX); + let x = scaleInitY * (currentPosY / initPosY); + if (x < 0) x = 0.2; + if (y < 0) y = 0.2; + this.model.data.selected.setScale(y, x, this.model.data.selected.top.scale.z); + this.setSelectedChildScale(1 / y, 1 / x, null); + const differenceX = y / 2 - scaleMemX / 2; + const differenceY = x / 2 - scaleMemY / 2; + + if (currentPosX > 0 && currentPosY < 0) { + resizeVector.x += differenceX; + resizeVector.y -= differenceY; + } else if (currentPosX > 0 && currentPosY > 0) { + resizeVector.x += differenceX; + resizeVector.y += differenceY; + } else if (currentPosX < 0 && currentPosY < 0) { + resizeVector.x -= differenceX; + resizeVector.y -= differenceY; + } else if (currentPosX < 0 && currentPosY > 0) { + resizeVector.x -= differenceX; + resizeVector.y += differenceY; + } + + this.action.resize.memScales.x = y; + this.action.resize.memScales.y = x; + break; + } + case ViewType.SIDE: { + let x = scaleInitX * (currentPosX / initPosX); + let z = scaleInitZ * (currentPosY / initPosY); + if (x < 0) x = 0.2; + if (z < 0) z = 0.2; + this.model.data.selected.setScale(x, this.model.data.selected.top.scale.y, z); + this.setSelectedChildScale(1 / x, null, 1 / z); + const differenceX = x / 2 - scaleMemX / 2; + const differenceY = z / 2 - scaleMemZ / 2; + + if (currentPosX > 0 && currentPosY < 0) { + resizeVector.x += differenceX; + resizeVector.y -= differenceY; + } else if (currentPosX > 0 && currentPosY > 0) { + resizeVector.x += differenceX; + resizeVector.y += differenceY; + } else if (currentPosX < 0 && currentPosY < 0) { + resizeVector.x -= differenceX; + resizeVector.y -= differenceY; + } else if (currentPosX < 0 && currentPosY > 0) { + resizeVector.x -= differenceX; + resizeVector.y += differenceY; + } + + this.action.resize.memScales = { ...this.action.resize.memScales, x, z }; + break; + } + case ViewType.FRONT: { + let y = scaleInitY * (currentPosX / initPosX); + let z = scaleInitZ * (currentPosY / initPosY); + if (y < 0) y = 0.2; + if (z < 0) z = 0.2; + this.model.data.selected.setScale(this.model.data.selected.top.scale.x, y, z); + this.setSelectedChildScale(null, 1 / y, 1 / z); + let differenceX; + let differenceY; + + if (!this.action.resize.frontBool) { + differenceX = z / 2 - scaleMemZ / 2; + differenceY = y / 2 - scaleMemY / 2; + this.action.resize.frontBool = true; + } else { + differenceX = z / 2 - scaleMemY / 2; + differenceY = y / 2 - scaleMemZ / 2; + } + if (currentPosX > 0 && currentPosY < 0) { + resizeVector.x += differenceX; + resizeVector.y += differenceY; + } else if (currentPosX > 0 && currentPosY > 0) { + resizeVector.x -= differenceX; + resizeVector.y += differenceY; + } else if (currentPosX < 0 && currentPosY < 0) { + resizeVector.x += differenceX; + resizeVector.y -= differenceY; + } else if (currentPosX < 0 && currentPosY > 0) { + resizeVector.x -= differenceX; + resizeVector.y -= differenceY; + } + + this.action.resize.memScales.y = z; + this.action.resize.memScales.z = y; + break; + } + default: + } + const coordinates = resizeVector.clone(); + intersects[0].object.localToWorld(coordinates); + this.moveObject(coordinates); + this.adjustPerspectiveCameras(); + } + + private static isLeft(a: any, b: any, c: any): boolean { + // For reference + // A + // |\ // A = Rotation Center + // | \ // B = Previous Frame Position + // | C // C = Current Frame Position + // B + return (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x) > 0; + } + + private rotateCube(instance: CuboidModel, direction: number, view: ViewType): void { + switch (view) { + case ViewType.TOP: + instance.perspective.rotateZ(direction); + instance.top.rotateZ(direction); + instance.side.rotateZ(direction); + instance.front.rotateZ(direction); + this.rotateCamera(direction, view); + break; + case ViewType.FRONT: + instance.perspective.rotateX(direction); + instance.top.rotateX(direction); + instance.side.rotateX(direction); + instance.front.rotateX(direction); + this.rotateCamera(direction, view); + break; + case ViewType.SIDE: + instance.perspective.rotateY(direction); + instance.top.rotateY(direction); + instance.side.rotateY(direction); + instance.front.rotateY(direction); + this.rotateCamera(direction, view); + break; + default: + } + } + + private rotateCamera(direction: any, view: ViewType): void { + switch (view) { + case ViewType.TOP: + this.views.top.camera.rotateZ(direction); + break; + case ViewType.FRONT: + this.views.front.camera.rotateZ(direction); + break; + case ViewType.SIDE: + this.views.side.camera.rotateZ(direction); + break; + default: + } + } + + private attachCamera(view: ViewType): void { + switch (view) { + case ViewType.TOP: + this.model.data.selected.side.attach(this.views.side.camera); + this.model.data.selected.front.attach(this.views.front.camera); + break; + case ViewType.SIDE: + this.model.data.selected.front.attach(this.views.front.camera); + this.model.data.selected.top.attach(this.views.top.camera); + break; + case ViewType.FRONT: + this.model.data.selected.side.attach(this.views.side.camera); + this.model.data.selected.top.attach(this.views.top.camera); + break; + default: + } + } + + private detachCamera(view: ViewType): void { + const coordTop = this.model.data.selected.getReferenceCoordinates(ViewType.TOP); + const sphericaltop = new THREE.Spherical(); + sphericaltop.setFromVector3(coordTop); + + const coordSide = this.model.data.selected.getReferenceCoordinates(ViewType.SIDE); + const sphericalside = new THREE.Spherical(); + sphericalside.setFromVector3(coordSide); + + const coordFront = this.model.data.selected.getReferenceCoordinates(ViewType.FRONT); + const sphericalfront = new THREE.Spherical(); + sphericalfront.setFromVector3(coordFront); + + const { side: objectSideView, front: objectFrontView, top: objectTopView } = this.model.data.selected; + const { camera: sideCamera } = this.views.side; + const { camera: frontCamera } = this.views.front; + const { camera: topCamera } = this.views.top; + + switch (view) { + case ViewType.TOP: { + const camRotationSide = objectSideView + .getObjectByName('cameraSide') + .getWorldQuaternion(new THREE.Quaternion()); + objectSideView.remove(sideCamera); + sideCamera.position.setFromSpherical(sphericalside); + sideCamera.lookAt(objectSideView.position.x, objectSideView.position.y, objectSideView.position.z); + sideCamera.setRotationFromQuaternion(camRotationSide); + sideCamera.scale.set(1, 1, 1); + + const camRotationFront = objectFrontView + .getObjectByName('cameraFront') + .getWorldQuaternion(new THREE.Quaternion()); + objectFrontView.remove(frontCamera); + frontCamera.position.setFromSpherical(sphericalfront); + frontCamera.lookAt(objectFrontView.position.x, objectFrontView.position.y, objectFrontView.position.z); + frontCamera.setRotationFromQuaternion(camRotationFront); + frontCamera.scale.set(1, 1, 1); + break; + } + case ViewType.SIDE: { + const camRotationFront = objectFrontView + .getObjectByName('cameraFront') + .getWorldQuaternion(new THREE.Quaternion()); + objectFrontView.remove(frontCamera); + frontCamera.position.setFromSpherical(sphericalfront); + frontCamera.lookAt(objectFrontView.position.x, objectFrontView.position.y, objectFrontView.position.z); + frontCamera.setRotationFromQuaternion(camRotationFront); + frontCamera.scale.set(1, 1, 1); + + objectTopView.remove(topCamera); + topCamera.position.setFromSpherical(sphericaltop); + topCamera.lookAt(objectTopView.position.x, objectTopView.position.y, objectTopView.position.z); + topCamera.setRotationFromEuler(objectTopView.rotation); + topCamera.scale.set(1, 1, 1); + break; + } + case ViewType.FRONT: { + const camRotationSide = objectSideView + .getObjectByName('cameraSide') + .getWorldQuaternion(new THREE.Quaternion()); + objectSideView.remove(sideCamera); + sideCamera.position.setFromSpherical(sphericalside); + sideCamera.lookAt(objectSideView.position.x, objectSideView.position.y, objectSideView.position.z); + sideCamera.setRotationFromQuaternion(camRotationSide); + sideCamera.scale.set(1, 1, 1); + + objectTopView.remove(topCamera); + topCamera.position.setFromSpherical(sphericaltop); + topCamera.lookAt(objectTopView.position.x, objectTopView.position.y, objectTopView.position.z); + topCamera.setRotationFromEuler(objectTopView.rotation); + topCamera.scale.set(1, 1, 1); + break; + } + default: { + sideCamera.position.setFromSpherical(sphericalside); + sideCamera.lookAt(objectSideView.position.x, objectSideView.position.y, objectSideView.position.z); + sideCamera.rotation.z = this.views.side.scene.getObjectByName(Planes.SIDE).rotation.z; + sideCamera.scale.set(1, 1, 1); + + topCamera.position.setFromSpherical(sphericaltop); + topCamera.lookAt(objectTopView.position.x, objectTopView.position.y, objectTopView.position.z); + topCamera.setRotationFromEuler(objectTopView.rotation); + topCamera.scale.set(1, 1, 1); + + const camFrontRotate = objectFrontView + .getObjectByName('camRefRot') + .getWorldQuaternion(new THREE.Quaternion()); + frontCamera.position.setFromSpherical(sphericalfront); + frontCamera.lookAt(objectFrontView.position.x, objectFrontView.position.y, objectFrontView.position.z); + frontCamera.setRotationFromQuaternion(camFrontRotate); + frontCamera.scale.set(1, 1, 1); + } + } + } + + private rotatePlane(direction: number, view: ViewType): void { + const sceneTopPlane = this.views.top.scene.getObjectByName(Planes.TOP); + const sceneSidePlane = this.views.side.scene.getObjectByName(Planes.SIDE); + const sceneFrontPlane = this.views.front.scene.getObjectByName(Planes.FRONT); + switch (view) { + case ViewType.TOP: + sceneTopPlane.rotateZ(direction); + sceneSidePlane.rotateY(direction); + sceneFrontPlane.rotateX(-direction); + break; + case ViewType.SIDE: + sceneTopPlane.rotateY(direction); + sceneSidePlane.rotateZ(direction); + sceneFrontPlane.rotateY(direction); + break; + case ViewType.FRONT: + sceneTopPlane.rotateX(direction); + sceneSidePlane.rotateX(-direction); + sceneFrontPlane.rotateZ(direction); + break; + default: { + const { top: objectTopView, side: objectSideView, front: objectFrontView } = this.model.data.selected; + objectTopView.add(sceneTopPlane); + objectSideView.add(sceneSidePlane); + objectFrontView.add(sceneFrontPlane); + objectTopView.getObjectByName(Planes.TOP).rotation.set(0, 0, 0); + objectSideView.getObjectByName(Planes.SIDE).rotation.set(-Math.PI / 2, Math.PI / 2000, Math.PI); + objectFrontView.getObjectByName(Planes.FRONT).rotation.set(0, Math.PI / 2, 0); + + const quaternionSide = new THREE.Quaternion(); + objectSideView.getObjectByName(Planes.SIDE).getWorldQuaternion(quaternionSide); + const rotationSide = new THREE.Euler(); + rotationSide.setFromQuaternion(quaternionSide); + + const quaternionFront = new THREE.Quaternion(); + objectFrontView.getObjectByName(Planes.FRONT).getWorldQuaternion(quaternionFront); + const rotationFront = new THREE.Euler(); + rotationFront.setFromQuaternion(quaternionFront); + + const quaternionTop = new THREE.Quaternion(); + objectTopView.getObjectByName(Planes.TOP).getWorldQuaternion(quaternionTop); + const rotationTop = new THREE.Euler(); + rotationTop.setFromQuaternion(quaternionTop); + + objectTopView.remove(sceneTopPlane); + objectSideView.remove(sceneSidePlane); + objectFrontView.remove(sceneFrontPlane); + + const canvasTopView = this.views.top.renderer.domElement; + const planeTop = new THREE.Mesh( + new THREE.PlaneBufferGeometry( + canvasTopView.offsetHeight, + canvasTopView.offsetWidth, + canvasTopView.offsetHeight, + canvasTopView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0xff0000, + alphaTest: 0, + visible: false, + transparent: true, + opacity: 0.1, + }), + ); + planeTop.name = Planes.TOP; + (planeTop.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + + const canvasSideView = this.views.side.renderer.domElement; + const planeSide = new THREE.Mesh( + new THREE.PlaneBufferGeometry( + canvasSideView.offsetHeight, + canvasSideView.offsetWidth, + canvasSideView.offsetHeight, + canvasSideView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0x00ff00, + alphaTest: 0, + visible: false, + transparent: true, + opacity: 0.1, + }), + ); + planeSide.name = Planes.SIDE; + (planeSide.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + + const canvasFrontView = this.views.front.renderer.domElement; + const planeFront = new THREE.Mesh( + new THREE.PlaneBufferGeometry( + canvasFrontView.offsetHeight, + canvasFrontView.offsetWidth, + canvasFrontView.offsetHeight, + canvasFrontView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0x0000ff, + alphaTest: 0, + visible: false, + transparent: true, + opacity: 0.5, + }), + ); + planeFront.name = Planes.FRONT; + (planeFront.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + + const coordinates = { + x: objectTopView.position.x, + y: objectTopView.position.y, + z: objectTopView.position.z, + }; + + planeTop.rotation.set(rotationTop.x, rotationTop.y, rotationTop.z); + planeSide.rotation.set(rotationSide.x, rotationSide.y, rotationSide.z); + planeFront.rotation.set(rotationFront.x, rotationFront.y, rotationFront.z); + this.views.top.scene.add(planeTop); + this.views.side.scene.add(planeSide); + this.views.front.scene.add(planeFront); + + this.translateReferencePlane(coordinates); + } + } + } + + private renderRotateAction(view: ViewType, viewType: any): void { + const rotationSpeed = Math.PI / CONST.ROTATION_SPEED; + const { renderer } = viewType; + const canvas = renderer.domElement; + if (!canvas) return; + const canvasCentre = { + x: canvas.offsetLeft + canvas.offsetWidth / 2, + y: canvas.offsetTop + canvas.offsetHeight / 2, + }; + if ( + this.action.rotation.screenInit.x === this.action.rotation.screenMove.x + && this.action.rotation.screenInit.y === this.action.rotation.screenMove.y + ) { + return; + } + + if ( + this.action.rotation.recentMouseVector.x === this.views[view].rayCaster.mouseVector.x + && this.action.rotation.recentMouseVector.y === this.views[view].rayCaster.mouseVector.y + ) { + return; + } + this.action.rotation.recentMouseVector = this.views[view].rayCaster.mouseVector.clone(); + if (Canvas3dViewImpl.isLeft(canvasCentre, this.action.rotation.screenInit, this.action.rotation.screenMove)) { + this.rotateCube(this.model.data.selected, -rotationSpeed, view); + this.rotatePlane(-rotationSpeed, view); + } else { + this.rotateCube(this.model.data.selected, rotationSpeed, view); + this.rotatePlane(rotationSpeed, view); + } + this.action.rotation.screenInit.x = this.action.rotation.screenMove.x; + this.action.rotation.screenInit.y = this.action.rotation.screenMove.y; + } + + private initiateAction(view: string, viewType: any): void { + const intersectsHelperResize = viewType.rayCaster.renderer.intersectObjects( + this.globalHelpers[view].resize, + false, + ); + const [state] = this.model.data.objects.filter( + (_state: any): boolean => _state.clientID === Number(this.model.data.selected[view].name), + ); + if (state.lock) return; + + if (intersectsHelperResize.length !== 0) { + this.action.resize.helper = viewType.rayCaster.mouseVector.clone(); + this.action.resize.status = true; + this.action.detected = true; + this.views.top.controls.enabled = false; + this.views.side.controls.enabled = false; + this.views.front.controls.enabled = false; + const { x, y, z } = this.model.data.selected[view].scale; + this.action.resize.initScales = { x, y, z }; + this.action.resize.memScales = { x, y, z }; + this.action.resize.frontBool = false; + this.action.resize.resizeVector = new THREE.Vector3(0, 0, 0); + return; + } + const intersectsHelperRotation = viewType.rayCaster.renderer.intersectObjects( + this.globalHelpers[view].rotation, + false, + ); + if (intersectsHelperRotation.length !== 0) { + this.action.rotation.helper = viewType.rayCaster.mouseVector.clone(); + this.action.rotation.status = true; + this.action.detected = true; + this.views.top.controls.enabled = false; + this.views.side.controls.enabled = false; + this.views.front.controls.enabled = false; + this.attachCamera(view as ViewType); + return; + } + + const intersectsBox = viewType.rayCaster.renderer.intersectObjects([this.model.data.selected[view]], false); + const intersectsPointCloud = viewType.rayCaster.renderer.intersectObjects( + [viewType.scene.getObjectByName(`${view}Plane`)], + true, + ); + if (intersectsBox.length !== 0) { + // const [state] = this.model.data.objects.filter( + // (_state: any): boolean => _state.clientID === Number(this.model.data.selected[view].name), + // ); + if (state.pinned) return; + this.action.translation.helper = viewType.rayCaster.mouseVector.clone(); + this.action.translation.inverseMatrix = intersectsBox[0].object.parent.matrixWorld.invert(); + this.action.translation.offset = intersectsPointCloud[0].point.sub( + new THREE.Vector3().setFromMatrixPosition(intersectsBox[0].object.matrixWorld), + ); + this.action.translation.status = true; + this.action.detected = true; + this.views.top.controls.enabled = false; + this.views.side.controls.enabled = false; + this.views.front.controls.enabled = false; + } + } + public keyControls(key: any): void { const { controls } = this.views.perspective; + if (!controls) return; switch (key.code) { - case CAMERA_ACTION.ROTATE_RIGHT: + case CameraAction.ROTATE_RIGHT: controls.rotate(0.1 * THREE.MathUtils.DEG2RAD * this.speed, 0, true); break; - case CAMERA_ACTION.ROTATE_LEFT: + case CameraAction.ROTATE_LEFT: controls.rotate(-0.1 * THREE.MathUtils.DEG2RAD * this.speed, 0, true); break; - case CAMERA_ACTION.TILT_UP: + case CameraAction.TILT_UP: controls.rotate(0, -0.05 * THREE.MathUtils.DEG2RAD * this.speed, true); break; - case CAMERA_ACTION.TILT_DOWN: + case CameraAction.TILT_DOWN: controls.rotate(0, 0.05 * THREE.MathUtils.DEG2RAD * this.speed, true); break; + case 'ControlLeft': + this.action.selectable = !key.ctrlKey; + break; default: break; } if (key.altKey === true) { switch (key.code) { - case CAMERA_ACTION.ZOOM_IN: + case CameraAction.ZOOM_IN: controls.dolly(CONST.DOLLY_FACTOR, true); break; - case CAMERA_ACTION.ZOOM_OUT: + case CameraAction.ZOOM_OUT: controls.dolly(-CONST.DOLLY_FACTOR, true); break; - case CAMERA_ACTION.MOVE_LEFT: + case CameraAction.MOVE_LEFT: controls.truck(-0.01 * this.speed, 0, true); break; - case CAMERA_ACTION.MOVE_RIGHT: + case CameraAction.MOVE_RIGHT: controls.truck(0.01 * this.speed, 0, true); break; - case CAMERA_ACTION.MOVE_DOWN: + case CameraAction.MOVE_DOWN: controls.truck(0, -0.01 * this.speed, true); break; - case CAMERA_ACTION.MOVE_UP: + case CameraAction.MOVE_UP: controls.truck(0, 0.01 * this.speed, true); break; default: @@ -386,46 +1794,6 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { } } - public mouseControls(type: MouseInteraction, event: MouseEvent): void { - event.preventDefault(); - if (type === MouseInteraction.DOUBLE_CLICK && this.mode === Mode.DRAW) { - this.controller.drawData.enabled = false; - this.mode = Mode.IDLE; - const cancelEvent: CustomEvent = new CustomEvent('canvas.canceled'); - this.views.perspective.renderer.domElement.dispatchEvent(cancelEvent); - } else { - const canvas = this.views.perspective.renderer.domElement; - const rect = canvas.getBoundingClientRect(); - const { mouseVector } = this.views.perspective.rayCaster; - mouseVector.x = ((event.clientX - (canvas.offsetLeft + rect.left)) / canvas.clientWidth) * 2 - 1; - mouseVector.y = -((event.clientY - (canvas.offsetTop + rect.top)) / canvas.clientHeight) * 2 + 1; - - if (type === MouseInteraction.CLICK && this.mode === Mode.IDLE) { - const intersects = this.views.perspective.rayCaster.renderer.intersectObjects( - this.views.perspective.scene.children[0].children, - false, - ); - if (intersects.length !== 0) { - this.views.perspective.scene.children[0].children.forEach((sceneItem: THREE.Mesh): void => { - // eslint-disable-next-line no-param-reassign - sceneItem.material.color = new THREE.Color(0xff0000); - }); - const selectedObject = intersects[0].object; - selectedObject.material.color = new THREE.Color(0x00ffff); - Object.keys(this.views).forEach((view: string): void => { - if (view !== ViewType.PERSPECTIVE) { - this.views[view as keyof Views].scene.children[0].children = [selectedObject.clone()]; - this.views[view as keyof Views].controls.fitToBox(selectedObject, false); - this.views[view as keyof Views].controls.zoom(view === ViewType.TOP ? -5 : -5, false); - } - this.views[view as keyof Views].scene.background = new THREE.Color(0x000000); - }); - this.selected.perspective = selectedObject as THREE.Mesh; - } - } - } - } - public html(): ViewsDOM { return { perspective: this.views.perspective.renderer.domElement, diff --git a/cvat-canvas3d/src/typescript/consts.ts b/cvat-canvas3d/src/typescript/consts.ts index ea0b97c7..3caf834a 100644 --- a/cvat-canvas3d/src/typescript/consts.ts +++ b/cvat-canvas3d/src/typescript/consts.ts @@ -8,6 +8,15 @@ const DOLLY_FACTOR = 5; const MAX_DISTANCE = 100; const MIN_DISTANCE = 0; const ZOOM_FACTOR = 7; +const ROTATION_HELPER_OFFSET = 0.1; +const CAMERA_REFERENCE = 'camRef'; +const CUBOID_EDGE_NAME = 'edges'; +const ROTATION_HELPER = 'rotationHelper'; +const ROTATION_SPEED = 80; +const FOV_DEFAULT = 1; +const FOV_MAX = 2; +const FOV_MIN = 0; +const FOV_INC = 0.08; export default { BASE_GRID_WIDTH, @@ -16,4 +25,13 @@ export default { MAX_DISTANCE, MIN_DISTANCE, ZOOM_FACTOR, + ROTATION_HELPER_OFFSET, + CAMERA_REFERENCE, + CUBOID_EDGE_NAME, + ROTATION_HELPER, + ROTATION_SPEED, + FOV_DEFAULT, + FOV_MAX, + FOV_MIN, + FOV_INC, }; diff --git a/cvat-canvas3d/src/typescript/cuboid.ts b/cvat-canvas3d/src/typescript/cuboid.ts index 47a37e33..d293e40e 100644 --- a/cvat-canvas3d/src/typescript/cuboid.ts +++ b/cvat-canvas3d/src/typescript/cuboid.ts @@ -2,6 +2,13 @@ // // SPDX-License-Identifier: MIT import * as THREE from 'three'; +import { BufferGeometryUtils } from 'three/examples/jsm/utils/BufferGeometryUtils'; +import { ViewType } from './canvas3dModel'; +import constants from './consts'; + +export interface Indexable { + [key: string]: any; +} export class CuboidModel { public perspective: THREE.Mesh; @@ -9,12 +16,174 @@ export class CuboidModel { public side: THREE.Mesh; public front: THREE.Mesh; - public constructor() { + public constructor(outline: string, outlineColor: string) { const geometry = new THREE.BoxGeometry(1, 1, 1); - const material = new THREE.MeshBasicMaterial({ color: 0x00ff00, wireframe: true }); + const material = new THREE.MeshBasicMaterial({ + color: 0x00ff00, + wireframe: false, + transparent: true, + opacity: 0.4, + }); this.perspective = new THREE.Mesh(geometry, material); + const geo = new THREE.EdgesGeometry(this.perspective.geometry); + const wireframe = new THREE.LineSegments( + geo, + outline === 'line' + ? new THREE.LineBasicMaterial({ color: outlineColor, linewidth: 4 }) + : new THREE.LineDashedMaterial({ + color: outlineColor, + dashSize: 0.05, + gapSize: 0.05, + }), + ); + wireframe.computeLineDistances(); + wireframe.renderOrder = 1; + this.perspective.add(wireframe); + this.top = new THREE.Mesh(geometry, material); this.side = new THREE.Mesh(geometry, material); this.front = new THREE.Mesh(geometry, material); + + const camRotateHelper = new THREE.Object3D(); + camRotateHelper.translateX(-2); + camRotateHelper.name = 'camRefRot'; + camRotateHelper.up = new THREE.Vector3(0, 0, 1); + camRotateHelper.lookAt(new THREE.Vector3(0, 0, 0)); + this.front.add(camRotateHelper.clone()); + } + + public setPosition(x: number, y: number, z: number): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + (this as Indexable)[view].position.set(x, y, z); + }); + } + + public setScale(x: number, y: number, z: number): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + (this as Indexable)[view].scale.set(x, y, z); + }); + } + + public setRotation(x: number, y: number, z: number): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + (this as Indexable)[view].rotation.set(x, y, z); + }); + } + + public attachCameraReference(): void { + // Attach Cam Reference + const topCameraReference = new THREE.Object3D(); + topCameraReference.translateZ(2); + topCameraReference.name = constants.CAMERA_REFERENCE; + this.top.add(topCameraReference); + this.top.userData = { ...this.top.userData, camReference: topCameraReference }; + + const sideCameraReference = new THREE.Object3D(); + sideCameraReference.translateY(2); + sideCameraReference.name = constants.CAMERA_REFERENCE; + this.side.add(sideCameraReference); + this.side.userData = { ...this.side.userData, camReference: sideCameraReference }; + + const frontCameraReference = new THREE.Object3D(); + frontCameraReference.translateX(2); + frontCameraReference.name = constants.CAMERA_REFERENCE; + this.front.add(frontCameraReference); + this.front.userData = { ...this.front.userData, camReference: frontCameraReference }; + } + + public getReferenceCoordinates(viewType: string): THREE.Vector3 { + const { elements } = (this as Indexable)[viewType].getObjectByName(constants.CAMERA_REFERENCE).matrixWorld; + return new THREE.Vector3(elements[12], elements[13], elements[14]); + } + + public setName(clientId: any): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + (this as Indexable)[view].name = clientId; + }); + } + + public setOriginalColor(color: string): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + ((this as Indexable)[view] as any).originalColor = color; + }); + } + + public setColor(color: string): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + ((this as Indexable)[view].material as THREE.MeshBasicMaterial).color.set(color); + }); + } + + public setOpacity(opacity: number): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + ((this as Indexable)[view].material as THREE.MeshBasicMaterial).opacity = opacity / 100; + }); + } +} + +export function setEdges(instance: THREE.Mesh): THREE.LineSegments { + const edges = new THREE.EdgesGeometry(instance.geometry); + const line = new THREE.LineSegments(edges, new THREE.LineBasicMaterial({ color: '#ffffff', linewidth: 3 })); + line.name = constants.CUBOID_EDGE_NAME; + instance.add(line); + return line; +} + +export function setTranslationHelper(instance: THREE.Mesh): void { + const sphereGeometry = new THREE.SphereGeometry(0.1); + const sphereMaterial = new THREE.MeshBasicMaterial({ color: '#ffffff', opacity: 1 }); + instance.geometry.deleteAttribute('normal'); + instance.geometry.deleteAttribute('uv'); + // eslint-disable-next-line no-param-reassign + instance.geometry = BufferGeometryUtils.mergeVertices(instance.geometry); + const vertices = []; + const positionAttribute = instance.geometry.getAttribute('position'); + for (let i = 0; i < positionAttribute.count; i++) { + const vertex = new THREE.Vector3(); + vertex.fromBufferAttribute(positionAttribute, i); + vertices.push(vertex); + } + const helpers = []; + for (let i = 0; i < vertices.length; i++) { + helpers[i] = new THREE.Mesh(sphereGeometry.clone(), sphereMaterial.clone()); + helpers[i].position.set(vertices[i].x, vertices[i].y, vertices[i].z); + helpers[i].up.set(0, 0, 1); + helpers[i].name = 'resizeHelper'; + instance.add(helpers[i]); + helpers[i].scale.set(1 / instance.scale.x, 1 / instance.scale.y, 1 / instance.scale.z); + } + // eslint-disable-next-line no-param-reassign + instance.userData = { ...instance.userData, resizeHelpers: helpers }; +} + +export function createRotationHelper(instance: THREE.Mesh, viewType: ViewType): void { + const sphereGeometry = new THREE.SphereGeometry(0.1); + const sphereMaterial = new THREE.MeshBasicMaterial({ color: '#ffffff', opacity: 1 }); + const rotationHelper = new THREE.Mesh(sphereGeometry, sphereMaterial); + rotationHelper.name = constants.ROTATION_HELPER; + switch (viewType) { + case ViewType.TOP: + rotationHelper.position.set( + (instance.geometry as THREE.BoxGeometry).parameters.height / 2 + constants.ROTATION_HELPER_OFFSET, + instance.position.y, + instance.position.z, + ); + instance.add(rotationHelper.clone()); + // eslint-disable-next-line no-param-reassign + instance.userData = { ...instance.userData, rotationHelpers: rotationHelper.clone() }; + break; + case ViewType.SIDE: + case ViewType.FRONT: + rotationHelper.position.set( + instance.position.x, + instance.position.y, + (instance.geometry as THREE.BoxGeometry).parameters.depth / 2 + constants.ROTATION_HELPER_OFFSET, + ); + instance.add(rotationHelper.clone()); + // eslint-disable-next-line no-param-reassign + instance.userData = { ...instance.userData, rotationHelpers: rotationHelper.clone() }; + break; + default: + break; } } diff --git a/cvat-core/src/annotations-objects.js b/cvat-core/src/annotations-objects.js index 7f5910e4..21166756 100644 --- a/cvat-core/src/annotations-objects.js +++ b/cvat-core/src/annotations-objects.js @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -324,11 +324,16 @@ checkObjectType('points', data.points, null, Array); checkNumberOfPoints(this.shapeType, data.points); // cut points - const { width, height } = this.frameMeta[frame]; + const { width, height, filename } = this.frameMeta[frame]; fittedPoints = fitPoints(this.shapeType, data.points, width, height); - - if (!checkShapeArea(this.shapeType, fittedPoints) || checkOutside(fittedPoints, width, height)) { - fittedPoints = []; + let check = true; + if (filename && filename.slice(filename.length - 3) === 'pcd') { + check = false; + } + if (check) { + if (!checkShapeArea(this.shapeType, fittedPoints) || checkOutside(fittedPoints, width, height)) { + fittedPoints = []; + } } } diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index d2b63a9b..cf31969a 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -1082,9 +1082,7 @@ const closureId = Date.now(); predictAnnotations.latestRequest.id = closureId; - const predicate = () => ( - !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId - ); + const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId; if (predictAnnotations.latestRequest.fetching) { waitFor(5, predicate).then(() => { if (predictAnnotations.latestRequest.id !== closureId) { diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 0dd6eb55..4e346c13 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -7,7 +7,7 @@ import { ActionCreator, AnyAction, Dispatch, Store, } from 'redux'; import { ThunkAction } from 'utils/redux'; -import { RectDrawingMethod } from 'cvat-canvas-wrapper'; +import { RectDrawingMethod, Canvas } from 'cvat-canvas-wrapper'; import getCore from 'cvat-core-wrapper'; import logger, { LogType } from 'cvat-logger'; import { getCVATStore } from 'cvat-store'; @@ -1435,8 +1435,9 @@ export function pasteShapeAsync(): ThunkAction { activeControl, }, }); - - canvasInstance.cancel(); + if (canvasInstance instanceof Canvas) { + canvasInstance.cancel(); + } if (initialState.objectType === ObjectType.TAG) { const objectState = new cvat.classes.ObjectState({ objectType: ObjectType.TAG, @@ -1493,7 +1494,7 @@ export function repeatDrawShapeAsync(): ThunkAction { } = getStore().getState().annotation; let activeControl = ActiveControl.CURSOR; - if (activeInteractor) { + if (activeInteractor && canvasInstance instanceof Canvas) { if (activeInteractor.type === 'tracker') { canvasInstance.interact({ enabled: true, @@ -1511,7 +1512,6 @@ export function repeatDrawShapeAsync(): ThunkAction { return; } - if (activeShapeType === ShapeType.RECTANGLE) { activeControl = ActiveControl.DRAW_RECTANGLE; } else if (activeShapeType === ShapeType.POINTS) { @@ -1523,7 +1523,6 @@ export function repeatDrawShapeAsync(): ThunkAction { } else if (activeShapeType === ShapeType.CUBOID) { activeControl = ActiveControl.DRAW_CUBOID; } - dispatch({ type: AnnotationActionTypes.REPEAT_DRAW_SHAPE, payload: { @@ -1531,7 +1530,9 @@ export function repeatDrawShapeAsync(): ThunkAction { }, }); - canvasInstance.cancel(); + if (canvasInstance instanceof Canvas) { + canvasInstance.cancel(); + } if (activeObjectType === ObjectType.TAG) { const objectState = new cvat.classes.ObjectState({ objectType: ObjectType.TAG, @@ -1580,8 +1581,9 @@ export function redrawShapeAsync(): ThunkAction { activeControl, }, }); - - canvasInstance.cancel(); + if (canvasInstance instanceof Canvas) { + canvasInstance.cancel(); + } canvasInstance.draw({ enabled: true, redraw: activatedStateID, diff --git a/cvat-ui/src/components/annotation-page/appearance-block.tsx b/cvat-ui/src/components/annotation-page/appearance-block.tsx index a00a4937..a2adabab 100644 --- a/cvat-ui/src/components/annotation-page/appearance-block.tsx +++ b/cvat-ui/src/components/annotation-page/appearance-block.tsx @@ -14,7 +14,7 @@ import Button from 'antd/lib/button'; import ColorPicker from 'components/annotation-page/standard-workspace/objects-side-bar/color-picker'; import { ColorizeIcon } from 'icons'; -import { ColorBy, CombinedState } from 'reducers/interfaces'; +import { ColorBy, CombinedState, DimensionType } from 'reducers/interfaces'; import { collapseAppearance as collapseAppearanceAction, updateTabContentHeight as updateTabContentHeightAction, @@ -37,6 +37,7 @@ interface StateToProps { outlineColor: string; showBitmap: boolean; showProjections: boolean; + jobInstance: any; } interface DispatchToProps { @@ -66,7 +67,10 @@ export function computeHeight(): number { function mapStateToProps(state: CombinedState): StateToProps { const { - annotation: { appearanceCollapsed }, + annotation: { + appearanceCollapsed, + job: { instance: jobInstance }, + }, settings: { shapes: { colorBy, opacity, selectedOpacity, outlined, outlineColor, showBitmap, showProjections, @@ -83,6 +87,7 @@ function mapStateToProps(state: CombinedState): StateToProps { outlineColor, showBitmap, showProjections, + jobInstance, }; } @@ -144,8 +149,11 @@ function AppearanceBlock(props: Props): JSX.Element { changeShapesOutlinedBorders, changeShowBitmap, changeShowProjections, + jobInstance, } = props; + const is2D = jobInstance.task.dimension === DimensionType.DIM_2D; + return ( - - Show bitmap - - - Show projections - + {is2D && ( + + Show bitmap + + )} + {is2D && ( + + Show projections + + )} diff --git a/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx b/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx index 595613d5..fb21708f 100644 --- a/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx @@ -14,6 +14,7 @@ import { } from 'reducers/interfaces'; import { LogType } from 'cvat-logger'; import { Canvas } from 'cvat-canvas-wrapper'; +import { Canvas3d } from 'cvat-canvas3d-wrapper'; import getCore from 'cvat-core-wrapper'; import consts from 'consts'; import CVATTooltip from 'components/common/cvat-tooltip'; @@ -26,7 +27,7 @@ const MAX_DISTANCE_TO_OPEN_SHAPE = 50; interface Props { sidebarCollapsed: boolean; - canvasInstance: Canvas; + canvasInstance: Canvas | Canvas3d; jobInstance: any; activatedStateID: number | null; activatedAttributeID: number | null; @@ -103,10 +104,10 @@ export default class CanvasWrapperComponent extends React.PureComponent { automaticBordering, intelligentPolygonCrop, showObjectsTextAlways, - canvasInstance, workspace, showProjections, } = this.props; + const { canvasInstance } = this.props as { canvasInstance: Canvas }; // It's awful approach from the point of view React // But we do not have another way because cvat-canvas returns regular DOM element @@ -139,7 +140,6 @@ export default class CanvasWrapperComponent extends React.PureComponent { frameData, frameAngle, annotations, - canvasInstance, sidebarCollapsed, activatedStateID, curZLayer, @@ -161,7 +161,7 @@ export default class CanvasWrapperComponent extends React.PureComponent { canvasBackgroundColor, onFetchAnnotation, } = this.props; - + const { canvasInstance } = this.props as { canvasInstance: Canvas }; if ( prevProps.showObjectsTextAlways !== showObjectsTextAlways || prevProps.automaticBordering !== automaticBordering || @@ -306,7 +306,7 @@ export default class CanvasWrapperComponent extends React.PureComponent { } public componentWillUnmount(): void { - const { canvasInstance } = this.props; + const { canvasInstance } = this.props as { canvasInstance: Canvas }; canvasInstance.html().removeEventListener('mousedown', this.onCanvasMouseDown); canvasInstance.html().removeEventListener('click', this.onCanvasClicked); @@ -432,7 +432,8 @@ export default class CanvasWrapperComponent extends React.PureComponent { }; private onCanvasClicked = (): void => { - const { canvasInstance, onUpdateContextMenu } = this.props; + const { onUpdateContextMenu } = this.props; + const { canvasInstance } = this.props as { canvasInstance: Canvas }; onUpdateContextMenu(false, 0, 0, ContextMenuType.CANVAS_SHAPE); if (!canvasInstance.html().contains(document.activeElement) && document.activeElement instanceof HTMLElement) { document.activeElement.blur(); @@ -562,7 +563,8 @@ export default class CanvasWrapperComponent extends React.PureComponent { }; private onCanvasFindObject = async (e: any): Promise => { - const { jobInstance, canvasInstance } = this.props; + const { jobInstance } = this.props; + const { canvasInstance } = this.props as { canvasInstance: Canvas }; const result = await jobInstance.annotations.select(e.detail.states, e.detail.x, e.detail.y); @@ -596,12 +598,12 @@ export default class CanvasWrapperComponent extends React.PureComponent { const { activatedStateID, activatedAttributeID, - canvasInstance, selectedOpacity, aamZoomMargin, workspace, annotations, } = this.props; + const { canvasInstance } = this.props as { canvasInstance: Canvas }; if (activatedStateID !== null) { const [activatedState] = annotations.filter((state: any): boolean => state.clientID === activatedStateID); @@ -653,7 +655,8 @@ export default class CanvasWrapperComponent extends React.PureComponent { } private updateIssueRegions(): void { - const { canvasInstance, frameIssues } = this.props; + const { frameIssues } = this.props; + const { canvasInstance } = this.props as { canvasInstance: Canvas }; if (frameIssues === null) { canvasInstance.setupIssueRegions({}); } else { @@ -688,12 +691,12 @@ export default class CanvasWrapperComponent extends React.PureComponent { gridSize, gridColor, gridOpacity, - canvasInstance, brightnessLevel, contrastLevel, saturationLevel, canvasBackgroundColor, } = this.props; + const { canvasInstance } = this.props as { canvasInstance: Canvas }; // Size window.addEventListener('resize', this.fitCanvas); diff --git a/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx b/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx index 823a62fa..af0bd291 100644 --- a/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx @@ -10,29 +10,55 @@ import { ArrowDownOutlined, ArrowLeftOutlined, ArrowRightOutlined, ArrowUpOutlined, } from '@ant-design/icons'; import { ResizableBox } from 'react-resizable'; -import { Workspace } from 'reducers/interfaces'; import { - CAMERA_ACTION, Canvas3d, MouseInteraction, ViewType, + ColorBy, ContextMenuType, ObjectType, Workspace, +} from 'reducers/interfaces'; +import { + CameraAction, Canvas3d, ViewType, ViewsDOM, } from 'cvat-canvas3d-wrapper'; +import { Canvas } from 'cvat-canvas-wrapper'; import ContextImage from 'components/annotation-page/standard-workspace/context-image/context-image'; import CVATTooltip from 'components/common/cvat-tooltip'; +import { LogType } from 'cvat-logger'; +import getCore from 'cvat-core-wrapper'; + +const cvat = getCore(); interface Props { - canvasInstance: Canvas3d; + opacity: number; + selectedOpacity: number; + outlined: boolean; + outlineColor: string; + colorBy: ColorBy; + canvasInstance: Canvas3d | Canvas; jobInstance: any; frameData: any; curZLayer: number; annotations: any[]; + contextMenuVisibility: boolean; + activeLabelID: number; + activatedStateID: number | null; + activeObjectType: ObjectType; onSetupCanvas: () => void; + onGroupObjects: (enabled: boolean) => void; onResetCanvas(): void; + onCreateAnnotations(sessionInstance: any, frame: number, states: any[]): void; + onActivateObject(activatedStateID: number | null): void; + onUpdateAnnotations(states: any[]): void; + onUpdateContextMenu(visible: boolean, left: number, top: number, type: ContextMenuType, pointID?: number): void; + onGroupAnnotations(sessionInstance: any, frame: number, states: any[]): void; + onEditShape: (enabled: boolean) => void; + onDragCanvas: (enabled: boolean) => void; + onShapeDrawn: () => void; workspace: Workspace; - animateID: any; automaticBordering: boolean; showObjectsTextAlways: boolean; + frame: number; } interface ViewSize { fullHeight: number; + fullWidth: number; vertical: number; top: number; side: number; @@ -41,7 +67,7 @@ interface ViewSize { function viewSizeReducer( state: ViewSize, - action: { type: ViewType | 'set'; e?: SyntheticEvent; data?: ViewSize }, + action: { type: ViewType | 'set' | 'resize'; e?: SyntheticEvent; data?: ViewSize }, ): ViewSize { const event = (action.e as unknown) as MouseEvent; const canvas3dContainer = document.getElementById('canvas3d-container'); @@ -94,6 +120,33 @@ function viewSizeReducer( }; case 'set': return action.data as ViewSize; + case 'resize': { + const canvasPerspectiveContainer = document.getElementById('cvat-canvas3d-perspective'); + let midState = { ...state }; + if (canvasPerspectiveContainer) { + if (state.fullHeight !== canvas3dContainer.clientHeight) { + const diff = canvas3dContainer.clientHeight - state.fullHeight; + midState = { + ...midState, + fullHeight: canvas3dContainer.clientHeight, + vertical: state.vertical + diff, + }; + } + if (state.fullWidth !== canvasPerspectiveContainer.clientWidth) { + const oldWidth = state.fullWidth; + const width = canvasPerspectiveContainer.clientWidth; + midState = { + ...midState, + fullWidth: width, + top: (state.top / oldWidth) * width, + side: (state.side / oldWidth) * width, + front: (state.front / oldWidth) * width, + }; + } + return midState; + } + return state; + } default: throw new Error(); } @@ -105,6 +158,7 @@ const CanvasWrapperComponent = (props: Props): ReactElement => { const animateId = useRef(0); const [viewSize, setViewSize] = useReducer(viewSizeReducer, { fullHeight: 0, + fullWidth: 0, vertical: 0, top: 0, side: 0, @@ -115,70 +169,122 @@ const CanvasWrapperComponent = (props: Props): ReactElement => { const sideView = useRef(null); const frontView = useRef(null); - const { frameData, annotations, curZLayer } = props; + const { + opacity, + outlined, + outlineColor, + selectedOpacity, + colorBy, + contextMenuVisibility, + frameData, + onResetCanvas, + onSetupCanvas, + annotations, + frame, + jobInstance, + activeLabelID, + activeObjectType, + onShapeDrawn, + onCreateAnnotations, + } = props; + const { canvasInstance } = props as { canvasInstance: Canvas3d }; const onCanvasSetup = (): void => { - const { onSetupCanvas } = props; onSetupCanvas(); }; - const animateCanvas = (): void => { - const { canvasInstance } = props; + const onCanvasDragStart = (): void => { + const { onDragCanvas } = props; + onDragCanvas(true); + }; + const onCanvasDragDone = (): void => { + const { onDragCanvas } = props; + onDragCanvas(false); + }; + + const animateCanvas = (): void => { canvasInstance.render(); animateId.current = requestAnimationFrame(animateCanvas); }; const updateCanvas = (): void => { - const { canvasInstance } = props; - if (frameData !== null) { - canvasInstance.setup(frameData); + canvasInstance.setup( + frameData, + annotations.filter((e) => e.objectType !== ObjectType.TAG), + ); } }; - const onMouseClick = (event: MouseEvent): void => { - const { canvasInstance } = props; - canvasInstance.mouseControls(MouseInteraction.CLICK, event); + const onCanvasCancel = (): void => { + onResetCanvas(); }; - const onMouseDoubleClick = (event: MouseEvent): void => { - const { canvasInstance } = props; - canvasInstance.mouseControls(MouseInteraction.DOUBLE_CLICK, event); - }; + const onCanvasShapeDrawn = (event: any): void => { + if (!event.detail.continue) { + onShapeDrawn(); + } + + const { state, duration } = event.detail; + const isDrawnFromScratch = !state.label; + if (isDrawnFromScratch) { + jobInstance.logger.log(LogType.drawObject, { count: 1, duration }); + } else { + jobInstance.logger.log(LogType.pasteObject, { count: 1, duration }); + } - const onMouseHover = (event: MouseEvent): void => { - const { canvasInstance } = props; - canvasInstance.mouseControls(MouseInteraction.HOVER, event); + state.objectType = state.objectType || activeObjectType; + state.label = state.label || jobInstance.task.labels.filter((label: any) => label.id === activeLabelID)[0]; + state.occluded = state.occluded || false; + state.frame = frame; + state.zOrder = 0; + const objectState = new cvat.classes.ObjectState(state); + onCreateAnnotations(jobInstance, frame, [objectState]); }; - const onCanvasCancel = (): void => { - const { onResetCanvas } = props; - onResetCanvas(); + const onCanvasClick = (e: MouseEvent): void => { + const { onUpdateContextMenu } = props; + if (contextMenuVisibility) { + onUpdateContextMenu(false, e.clientX, e.clientY, ContextMenuType.CANVAS_SHAPE); + } }; const initialSetup = (): void => { - const { canvasInstance } = props; - - const canvasInstanceDOM = canvasInstance.html(); - // Events + const canvasInstanceDOM = canvasInstance.html() as ViewsDOM; canvasInstanceDOM.perspective.addEventListener('canvas.setup', onCanvasSetup); - canvasInstanceDOM.perspective.addEventListener('mousemove', onMouseHover); canvasInstanceDOM.perspective.addEventListener('canvas.canceled', onCanvasCancel); - canvasInstanceDOM.perspective.addEventListener(MouseInteraction.DOUBLE_CLICK, onMouseDoubleClick); - canvasInstanceDOM.perspective.addEventListener(MouseInteraction.CLICK, onMouseClick); + canvasInstanceDOM.perspective.addEventListener('canvas.dragstart', onCanvasDragStart); + canvasInstanceDOM.perspective.addEventListener('canvas.dragstop', onCanvasDragDone); }; - const keyControls = (key: KeyboardEvent): void => { - const { canvasInstance } = props; + const keyControlsKeyDown = (key: KeyboardEvent): void => { canvasInstance.keyControls(key); }; - useEffect(() => { - const { canvasInstance } = props; + const keyControlsKeyUp = (key: KeyboardEvent): void => { + if (key.code === 'ControlLeft') { + canvasInstance.keyControls(key); + } + }; - const canvasInstanceDOM = canvasInstance.html(); + const onCanvasShapeSelected = (event: any): void => { + const { onActivateObject } = props; + const { clientID } = event.detail; + onActivateObject(clientID); + canvasInstance.activate(clientID); + }; + const onCanvasEditDone = (event: any): void => { + const { onEditShape, onUpdateAnnotations } = props; + onEditShape(false); + const { state, points } = event.detail; + state.points = points; + onUpdateAnnotations([state]); + }; + + useEffect(() => { + const canvasInstanceDOM = canvasInstance.html(); if ( perspectiveView && perspectiveView.current && @@ -200,6 +306,7 @@ const CanvasWrapperComponent = (props: Props): ReactElement => { type: 'set', data: { fullHeight: canvas3dContainer.clientHeight, + fullWidth: canvas3dContainer.clientWidth, vertical: canvas3dContainer.clientHeight / 2, top: width, side: width, @@ -209,7 +316,8 @@ const CanvasWrapperComponent = (props: Props): ReactElement => { } } - document.addEventListener('keydown', keyControls); + document.addEventListener('keydown', keyControlsKeyDown); + document.addEventListener('keyup', keyControlsKeyUp); initialSetup(); updateCanvas(); @@ -217,21 +325,80 @@ const CanvasWrapperComponent = (props: Props): ReactElement => { return () => { canvasInstanceDOM.perspective.removeEventListener('canvas.setup', onCanvasSetup); - canvasInstanceDOM.perspective.removeEventListener('mousemove', onMouseHover); canvasInstanceDOM.perspective.removeEventListener('canvas.canceled', onCanvasCancel); - canvasInstanceDOM.perspective.removeEventListener(MouseInteraction.DOUBLE_CLICK, onMouseDoubleClick); - canvasInstanceDOM.perspective.removeEventListener(MouseInteraction.CLICK, onMouseClick); - document.removeEventListener('keydown', keyControls); + canvasInstanceDOM.perspective.removeEventListener('canvas.dragstart', onCanvasDragStart); + canvasInstanceDOM.perspective.removeEventListener('canvas.dragstop', onCanvasDragDone); + document.removeEventListener('keydown', keyControlsKeyDown); + document.removeEventListener('keyup', keyControlsKeyUp); cancelAnimationFrame(animateId.current); }; }, []); + const updateShapesView = (): void => { + (canvasInstance as Canvas3d).configureShapes({ + opacity, + outlined, + outlineColor, + selectedOpacity, + colorBy, + }); + }; + + const onContextMenu = (event: any): void => { + const { onUpdateContextMenu, onActivateObject } = props; + onActivateObject(event.detail.clientID); + onUpdateContextMenu( + event.detail.clientID !== null, + event.detail.clientX, + event.detail.clientY, + ContextMenuType.CANVAS_SHAPE, + ); + }; + + const onResize = (): void => { + setViewSize({ + type: 'resize', + }); + }; + + const onCanvasObjectsGroupped = (event: any): void => { + const { onGroupAnnotations, onGroupObjects } = props; + + onGroupObjects(false); + + const { states } = event.detail; + onGroupAnnotations(jobInstance, frame, states); + }; + useEffect(() => { + updateShapesView(); + }, [opacity, outlined, outlineColor, selectedOpacity, colorBy]); + + useEffect(() => { + const canvasInstanceDOM = canvasInstance.html() as ViewsDOM; updateCanvas(); - }, [frameData, annotations, curZLayer]); + canvasInstanceDOM.perspective.addEventListener('canvas.drawn', onCanvasShapeDrawn); + canvasInstanceDOM.perspective.addEventListener('canvas.selected', onCanvasShapeSelected); + canvasInstanceDOM.perspective.addEventListener('canvas.edited', onCanvasEditDone); + canvasInstanceDOM.perspective.addEventListener('canvas.contextmenu', onContextMenu); + canvasInstanceDOM.perspective.addEventListener('click', onCanvasClick); + canvasInstanceDOM.perspective.addEventListener('canvas.fit', onResize); + canvasInstanceDOM.perspective.addEventListener('canvas.groupped', onCanvasObjectsGroupped); + window.addEventListener('resize', onResize); + + return () => { + canvasInstanceDOM.perspective.removeEventListener('canvas.drawn', onCanvasShapeDrawn); + canvasInstanceDOM.perspective.removeEventListener('canvas.selected', onCanvasShapeSelected); + canvasInstanceDOM.perspective.removeEventListener('canvas.edited', onCanvasEditDone); + canvasInstanceDOM.perspective.removeEventListener('canvas.contextmenu', onContextMenu); + canvasInstanceDOM.perspective.removeEventListener('click', onCanvasClick); + canvasInstanceDOM.perspective.removeEventListener('canvas.fit', onResize); + canvasInstanceDOM.perspective.removeEventListener('canvas.groupped', onCanvasObjectsGroupped); + window.removeEventListener('resize', onResize); + }; + }, [frameData, annotations, activeLabelID, contextMenuVisibility]); - const screenKeyControl = (code: CAMERA_ACTION): void => { - const { canvasInstance } = props; + const screenKeyControl = (code: CameraAction): void => { canvasInstance.keyControls(new KeyboardEvent('keydown', { code, altKey: true })); }; @@ -239,7 +406,8 @@ const CanvasWrapperComponent = (props: Props): ReactElement => {