Intelligent scissors with OpenCV javascript (#2689)
* Some UI implementations * Added opencv wrapper * Updated Opencv wrapper * Moved initialization stub * Added threshold * Setup interaction with canvas * Fixed couple of issues * Added threshold, changing size via ctrl * tmp * Aborted host change * Fixed threshold * Aborted host * Some fixes * Using ready label selector * Raw implementation * Added additional arguments * Fixed some minor issues * Removed unused file * Fixed tool reset * Added short instructions to update opencv.js * Fixed corner case * Added error handler, opencv version, updated cvat_proxy & mod_wsgi * OpenCV returned back * Using dinamic function instead of script * Updated changelog & versionmain
Boris Sekachev
5 years ago
committed by
GitHub
parent
3d4fad4c1b
commit
e0fc323a4d
@ -0,0 +1,10 @@
|
|||||||
|
<!--
|
||||||
|
The file has been downloaded from: https://icon-icons.com/ru/%D0%B7%D0%BD%D0%B0%D1%87%D0%BE%D0%BA/%D0%92-%D1%84%D0%BE%D1%80%D0%BC%D0%B0%D1%82%D0%B5-OpenCV/132129
|
||||||
|
License: Attribution 4.0 International (CC BY 4.0) https://creativecommons.org/licenses/by/4.0/
|
||||||
|
The file has been modified
|
||||||
|
-->
|
||||||
|
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="40" height="40">
|
||||||
|
<g style="transform: scale(0.078)">
|
||||||
|
<path d="M148.6458282,81.0641403C191.8570709-0.3458452,307.612915-4.617764,356.5062561,73.3931732c37.8880615,60.4514771,13.7960815,135.4847717-41.8233948,167.7876129l-36.121521-62.5643005c22.1270447-12.8510284,31.7114563-42.7013397,16.6385498-66.750618c-19.4511414-31.034935-65.5021057-29.3354645-82.692749,3.0517044c-12.7206879,23.9658356-2.6391449,51.5502472,18.3088379,63.7294922l-36.1482544,62.6105804C142.0118256,210.643219,116.6704254,141.3057709,148.6458282,81.0641403z M167.9667206,374.4708557c-0.0435791,24.2778625-18.934967,46.8978271-46.092804,47.9000549c-36.6418304,1.3522339-61.0877724-37.6520386-43.8971252-70.0392151c13.2918015-25.0418091,43.8297424-31.7192383,65.9928284-19.1222839l36.2165222-62.7288513c-55.7241974-31.7991638-132.6246796-15.0146027-166.0706635,47.9976501c-43.2111893,81.4099731,18.2372913,179.4530945,110.3418884,176.0540161c68.1375427-2.5146179,115.5750122-59.1652527,115.8612366-120.0613708H167.9667206z M451.714386,270.7571411l-36.1215515,62.5642395c22.2027588,12.816864,31.8418274,42.7249451,16.744751,66.8127441c-19.4511414,31.0349426-65.5021057,29.3354797-82.692688-3.0516968c-12.742218-24.0063782-2.6048279-51.643219,18.4154358-63.7908325l-36.1482544-62.6105652c-52.7280579,30.5827942-78.1254272,99.9726562-46.128479,160.2548218c43.2111816,81.4099731,158.9670105,85.6818848,207.8603821,7.6710205C531.5561523,378.1168213,507.4096069,303.0259705,451.714386,270.7571411z"/>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 1.8 KiB |
@ -0,0 +1,417 @@
|
|||||||
|
// Copyright (C) 2021 Intel Corporation
|
||||||
|
//
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
import React from 'react';
|
||||||
|
import { connect } from 'react-redux';
|
||||||
|
import { Row, Col } from 'antd/lib/grid';
|
||||||
|
import Tooltip from 'antd/lib/tooltip';
|
||||||
|
import Popover from 'antd/lib/popover';
|
||||||
|
import Icon, { ScissorOutlined } from '@ant-design/icons';
|
||||||
|
import Text from 'antd/lib/typography/Text';
|
||||||
|
import Tabs from 'antd/lib/tabs';
|
||||||
|
import Button from 'antd/lib/button';
|
||||||
|
import Progress from 'antd/lib/progress';
|
||||||
|
import notification from 'antd/lib/notification';
|
||||||
|
|
||||||
|
import { OpenCVIcon } from 'icons';
|
||||||
|
import { Canvas, convertShapesForInteractor } from 'cvat-canvas-wrapper';
|
||||||
|
import getCore from 'cvat-core-wrapper';
|
||||||
|
import openCVWrapper from 'utils/opencv-wrapper/opencv-wrapper';
|
||||||
|
import { IntelligentScissors } from 'utils/opencv-wrapper/intelligent-scissors';
|
||||||
|
import {
|
||||||
|
CombinedState, ActiveControl, OpenCVTool, ObjectType,
|
||||||
|
} from 'reducers/interfaces';
|
||||||
|
import {
|
||||||
|
interactWithCanvas,
|
||||||
|
fetchAnnotationsAsync,
|
||||||
|
updateAnnotationsAsync,
|
||||||
|
createAnnotationsAsync,
|
||||||
|
} from 'actions/annotation-actions';
|
||||||
|
import LabelSelector from 'components/label-selector/label-selector';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
labels: any[];
|
||||||
|
canvasInstance: Canvas;
|
||||||
|
jobInstance: any;
|
||||||
|
isActivated: boolean;
|
||||||
|
states: any[];
|
||||||
|
frame: number;
|
||||||
|
curZOrder: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DispatchToProps {
|
||||||
|
onInteractionStart(activeInteractor: OpenCVTool, activeLabelID: number): void;
|
||||||
|
updateAnnotations(statesToUpdate: any[]): void;
|
||||||
|
createAnnotations(sessionInstance: any, frame: number, statesToCreate: any[]): void;
|
||||||
|
fetchAnnotations(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface State {
|
||||||
|
libraryInitialized: boolean;
|
||||||
|
initializationError: boolean;
|
||||||
|
initializationProgress: number;
|
||||||
|
activeLabelID: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const core = getCore();
|
||||||
|
|
||||||
|
function mapStateToProps(state: CombinedState): Props {
|
||||||
|
const {
|
||||||
|
annotation: {
|
||||||
|
annotations: {
|
||||||
|
states,
|
||||||
|
zLayer: { cur: curZOrder },
|
||||||
|
},
|
||||||
|
job: { instance: jobInstance, labels },
|
||||||
|
canvas: { activeControl, instance: canvasInstance },
|
||||||
|
player: {
|
||||||
|
frame: { number: frame },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} = state;
|
||||||
|
|
||||||
|
return {
|
||||||
|
isActivated: activeControl === ActiveControl.OPENCV_TOOLS,
|
||||||
|
canvasInstance,
|
||||||
|
jobInstance,
|
||||||
|
curZOrder,
|
||||||
|
labels,
|
||||||
|
states,
|
||||||
|
frame,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const mapDispatchToProps = {
|
||||||
|
onInteractionStart: interactWithCanvas,
|
||||||
|
updateAnnotations: updateAnnotationsAsync,
|
||||||
|
fetchAnnotations: fetchAnnotationsAsync,
|
||||||
|
createAnnotations: createAnnotationsAsync,
|
||||||
|
};
|
||||||
|
|
||||||
|
class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps, State> {
|
||||||
|
private activeTool: IntelligentScissors | null;
|
||||||
|
private interactiveStateID: number | null;
|
||||||
|
private interactionIsDone: boolean;
|
||||||
|
|
||||||
|
public constructor(props: Props & DispatchToProps) {
|
||||||
|
super(props);
|
||||||
|
const { labels } = props;
|
||||||
|
|
||||||
|
this.activeTool = null;
|
||||||
|
this.interactiveStateID = null;
|
||||||
|
this.interactionIsDone = false;
|
||||||
|
|
||||||
|
this.state = {
|
||||||
|
libraryInitialized: openCVWrapper.isInitialized,
|
||||||
|
initializationError: false,
|
||||||
|
initializationProgress: -1,
|
||||||
|
activeLabelID: labels[0].id,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public componentDidMount(): void {
|
||||||
|
const { canvasInstance } = this.props;
|
||||||
|
canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener);
|
||||||
|
canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener);
|
||||||
|
}
|
||||||
|
|
||||||
|
public componentDidUpdate(prevProps: Props): void {
|
||||||
|
const { isActivated } = this.props;
|
||||||
|
if (!prevProps.isActivated && isActivated) {
|
||||||
|
// reset flags when before using a tool
|
||||||
|
if (this.activeTool) {
|
||||||
|
this.activeTool.reset();
|
||||||
|
}
|
||||||
|
this.interactiveStateID = null;
|
||||||
|
this.interactionIsDone = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public componentWillUnmount(): void {
|
||||||
|
const { canvasInstance } = this.props;
|
||||||
|
canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener);
|
||||||
|
canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener);
|
||||||
|
}
|
||||||
|
|
||||||
|
private getInteractiveState(): any | null {
|
||||||
|
const { states } = this.props;
|
||||||
|
return states.filter((_state: any): boolean => _state.clientID === this.interactiveStateID)[0] || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private cancelListener = async (): Promise<void> => {
|
||||||
|
const {
|
||||||
|
fetchAnnotations, isActivated, jobInstance, frame,
|
||||||
|
} = this.props;
|
||||||
|
|
||||||
|
if (isActivated) {
|
||||||
|
if (this.interactiveStateID !== null) {
|
||||||
|
const state = this.getInteractiveState();
|
||||||
|
this.interactiveStateID = null;
|
||||||
|
await state.delete(frame);
|
||||||
|
fetchAnnotations();
|
||||||
|
}
|
||||||
|
|
||||||
|
await jobInstance.actions.freeze(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
private interactionListener = async (e: Event): Promise<void> => {
|
||||||
|
const {
|
||||||
|
fetchAnnotations, updateAnnotations, isActivated, jobInstance, frame, labels, curZOrder,
|
||||||
|
} = this.props;
|
||||||
|
const { activeLabelID } = this.state;
|
||||||
|
if (!isActivated || !this.activeTool) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const {
|
||||||
|
shapesUpdated, isDone, threshold, shapes,
|
||||||
|
} = (e as CustomEvent).detail;
|
||||||
|
const pressedPoints = convertShapesForInteractor(shapes).flat();
|
||||||
|
this.interactionIsDone = isDone;
|
||||||
|
|
||||||
|
try {
|
||||||
|
let points: number[] = [];
|
||||||
|
if (shapesUpdated) {
|
||||||
|
points = await this.runCVAlgorithm(pressedPoints, threshold);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.interactiveStateID === null) {
|
||||||
|
if (!this.interactionIsDone) {
|
||||||
|
await jobInstance.actions.freeze(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
const object = new core.classes.ObjectState({
|
||||||
|
...this.activeTool.params.shape,
|
||||||
|
frame,
|
||||||
|
objectType: ObjectType.SHAPE,
|
||||||
|
label: labels.filter((label: any) => label.id === activeLabelID)[0],
|
||||||
|
points,
|
||||||
|
occluded: false,
|
||||||
|
zOrder: curZOrder,
|
||||||
|
});
|
||||||
|
// need a clientID of a created object to interact with it further
|
||||||
|
// so, we do not use createAnnotationAction
|
||||||
|
const [clientID] = await jobInstance.annotations.put([object]);
|
||||||
|
this.interactiveStateID = clientID;
|
||||||
|
|
||||||
|
// update annotations on a canvas
|
||||||
|
fetchAnnotations();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const state = this.getInteractiveState();
|
||||||
|
if ((e as CustomEvent).detail.isDone) {
|
||||||
|
const finalObject = new core.classes.ObjectState({
|
||||||
|
frame: state.frame,
|
||||||
|
objectType: state.objectType,
|
||||||
|
label: state.label,
|
||||||
|
shapeType: state.shapeType,
|
||||||
|
// need to recalculate without the latest sliding point
|
||||||
|
points: points = await this.runCVAlgorithm(pressedPoints, threshold),
|
||||||
|
occluded: state.occluded,
|
||||||
|
zOrder: state.zOrder,
|
||||||
|
});
|
||||||
|
this.interactiveStateID = null;
|
||||||
|
await state.delete(frame);
|
||||||
|
await jobInstance.actions.freeze(false);
|
||||||
|
await jobInstance.annotations.put([finalObject]);
|
||||||
|
fetchAnnotations();
|
||||||
|
} else {
|
||||||
|
state.points = points;
|
||||||
|
updateAnnotations([state]);
|
||||||
|
fetchAnnotations();
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
notification.error({
|
||||||
|
description: error.toString(),
|
||||||
|
message: 'Processing error occured',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
private async runCVAlgorithm(pressedPoints: number[], threshold: number): Promise<number[]> {
|
||||||
|
// Getting image data
|
||||||
|
const canvas: HTMLCanvasElement | undefined = window.document.getElementById('cvat_canvas_background') as
|
||||||
|
| HTMLCanvasElement
|
||||||
|
| undefined;
|
||||||
|
if (!canvas) {
|
||||||
|
throw new Error('Element #cvat_canvas_background was not found');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { width, height } = canvas;
|
||||||
|
const context = canvas.getContext('2d');
|
||||||
|
if (!context) {
|
||||||
|
throw new Error('Canvas context is empty');
|
||||||
|
}
|
||||||
|
|
||||||
|
const [x, y] = pressedPoints.slice(-2);
|
||||||
|
const startX = Math.round(Math.max(0, x - threshold));
|
||||||
|
const startY = Math.round(Math.max(0, y - threshold));
|
||||||
|
const segmentWidth = Math.min(2 * threshold, width - startX);
|
||||||
|
const segmentHeight = Math.min(2 * threshold, height - startY);
|
||||||
|
const imageData = context.getImageData(startX, startY, segmentWidth, segmentHeight);
|
||||||
|
|
||||||
|
if (!this.activeTool) return [];
|
||||||
|
|
||||||
|
// Handling via OpenCV.js
|
||||||
|
const points = await this.activeTool.run(pressedPoints, imageData, startX, startY);
|
||||||
|
|
||||||
|
// Increasing number of points artificially
|
||||||
|
let minNumberOfPoints = 1;
|
||||||
|
// eslint-disable-next-line: eslintdot-notation
|
||||||
|
if (this.activeTool.params.shape.shapeType === 'polyline') {
|
||||||
|
minNumberOfPoints = 2;
|
||||||
|
} else if (this.activeTool.params.shape.shapeType === 'polygon') {
|
||||||
|
minNumberOfPoints = 3;
|
||||||
|
}
|
||||||
|
while (points.length < minNumberOfPoints * 2) {
|
||||||
|
points.push(...points.slice(points.length - 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
return points;
|
||||||
|
}
|
||||||
|
|
||||||
|
private renderDrawingContent(): JSX.Element {
|
||||||
|
const { activeLabelID } = this.state;
|
||||||
|
const { labels, canvasInstance, onInteractionStart } = this.props;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Row justify='center'>
|
||||||
|
<Col span={24}>
|
||||||
|
<LabelSelector
|
||||||
|
style={{ width: '100%' }}
|
||||||
|
labels={labels}
|
||||||
|
value={activeLabelID}
|
||||||
|
onChange={(label: any) => this.setState({ activeLabelID: label.id })}
|
||||||
|
/>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
<Row justify='start' className='cvat-opencv-drawing-tools'>
|
||||||
|
<Col>
|
||||||
|
<Tooltip title='Intelligent scissors' className='cvat-opencv-drawing-tool'>
|
||||||
|
<Button
|
||||||
|
onClick={() => {
|
||||||
|
this.activeTool = openCVWrapper.segmentation.intelligentScissorsFactory();
|
||||||
|
canvasInstance.cancel();
|
||||||
|
onInteractionStart(this.activeTool, activeLabelID);
|
||||||
|
canvasInstance.interact({
|
||||||
|
enabled: true,
|
||||||
|
...this.activeTool.params.canvas,
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<ScissorOutlined />
|
||||||
|
</Button>
|
||||||
|
</Tooltip>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private renderContent(): JSX.Element {
|
||||||
|
const { libraryInitialized, initializationProgress, initializationError } = this.state;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className='cvat-opencv-control-popover-content'>
|
||||||
|
<Row justify='start'>
|
||||||
|
<Col>
|
||||||
|
<Text className='cvat-text-color' strong>
|
||||||
|
OpenCV
|
||||||
|
</Text>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
{libraryInitialized ? (
|
||||||
|
<Tabs tabBarGutter={8}>
|
||||||
|
<Tabs.TabPane key='drawing' tab='Drawing' className='cvat-opencv-control-tabpane'>
|
||||||
|
{this.renderDrawingContent()}
|
||||||
|
</Tabs.TabPane>
|
||||||
|
<Tabs.TabPane disabled key='image' tab='Image' className='cvat-opencv-control-tabpane' />
|
||||||
|
</Tabs>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Row justify='start' align='middle'>
|
||||||
|
<Col span={initializationProgress >= 0 ? 17 : 24}>
|
||||||
|
<Button
|
||||||
|
disabled={initializationProgress !== -1}
|
||||||
|
className='cvat-opencv-initialization-button'
|
||||||
|
onClick={async () => {
|
||||||
|
try {
|
||||||
|
this.setState({
|
||||||
|
initializationError: false,
|
||||||
|
initializationProgress: 0,
|
||||||
|
});
|
||||||
|
await openCVWrapper.initialize((progress: number) => {
|
||||||
|
this.setState({ initializationProgress: progress });
|
||||||
|
});
|
||||||
|
this.setState({ libraryInitialized: true });
|
||||||
|
} catch (error) {
|
||||||
|
notification.error({
|
||||||
|
description: error.toString(),
|
||||||
|
message: 'Could not initialize OpenCV library',
|
||||||
|
});
|
||||||
|
this.setState({
|
||||||
|
initializationError: true,
|
||||||
|
initializationProgress: -1,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Load OpenCV
|
||||||
|
</Button>
|
||||||
|
</Col>
|
||||||
|
{initializationProgress >= 0 && (
|
||||||
|
<Col span={6} offset={1}>
|
||||||
|
<Progress
|
||||||
|
width={8 * 5}
|
||||||
|
percent={initializationProgress}
|
||||||
|
type='circle'
|
||||||
|
status={initializationError ? 'exception' : undefined}
|
||||||
|
/>
|
||||||
|
</Col>
|
||||||
|
)}
|
||||||
|
</Row>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public render(): JSX.Element {
|
||||||
|
const { isActivated, canvasInstance } = this.props;
|
||||||
|
const dynamcPopoverPros = isActivated ?
|
||||||
|
{
|
||||||
|
overlayStyle: {
|
||||||
|
display: 'none',
|
||||||
|
},
|
||||||
|
} :
|
||||||
|
{};
|
||||||
|
|
||||||
|
const dynamicIconProps = isActivated ?
|
||||||
|
{
|
||||||
|
className: 'cvat-active-canvas-control cvat-opencv-control',
|
||||||
|
onClick: (): void => {
|
||||||
|
canvasInstance.interact({ enabled: false });
|
||||||
|
},
|
||||||
|
} :
|
||||||
|
{
|
||||||
|
className: 'cvat-tools-control',
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Popover
|
||||||
|
{...dynamcPopoverPros}
|
||||||
|
placement='right'
|
||||||
|
overlayClassName='cvat-opencv-control-popover'
|
||||||
|
content={this.renderContent()}
|
||||||
|
>
|
||||||
|
<Icon {...dynamicIconProps} component={OpenCVIcon} />
|
||||||
|
</Popover>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default connect(mapStateToProps, mapDispatchToProps)(OpenCVControlComponent);
|
||||||
@ -0,0 +1,176 @@
|
|||||||
|
// Copyright (C) 2021 Intel Corporation
|
||||||
|
//
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
import { numberArrayToPoints, pointsToNumberArray, Point } from '../math';
|
||||||
|
|
||||||
|
export interface IntelligentScissorsParams {
|
||||||
|
shape: {
|
||||||
|
shapeType: 'polygon' | 'polyline';
|
||||||
|
};
|
||||||
|
canvas: {
|
||||||
|
shapeType: 'points';
|
||||||
|
enableThreshold: boolean;
|
||||||
|
enableSliding: boolean;
|
||||||
|
allowRemoveOnlyLast: boolean;
|
||||||
|
minPosVertices: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IntelligentScissors {
|
||||||
|
reset(): void;
|
||||||
|
run(points: number[], image: ImageData, offsetX: number, offsetY: number): number[];
|
||||||
|
params: IntelligentScissorsParams;
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyOffset(points: Point[], offsetX: number, offsetY: number): Point[] {
|
||||||
|
return points.map(
|
||||||
|
(point: Point): Point => ({
|
||||||
|
x: point.x - offsetX,
|
||||||
|
y: point.y - offsetY,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default class IntelligentScissorsImplementation implements IntelligentScissors {
|
||||||
|
private cv: any;
|
||||||
|
private scissors: {
|
||||||
|
tool: any;
|
||||||
|
state: {
|
||||||
|
path: number[];
|
||||||
|
anchors: Record<
|
||||||
|
number,
|
||||||
|
{
|
||||||
|
point: Point;
|
||||||
|
start: number;
|
||||||
|
}
|
||||||
|
>; // point index : start index in path
|
||||||
|
image: any | null;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
public constructor(cv: any) {
|
||||||
|
this.cv = cv;
|
||||||
|
this.reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
public reset(): void {
|
||||||
|
if (this.scissors && this.scissors.tool) {
|
||||||
|
this.scissors.tool.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.scissors = {
|
||||||
|
// eslint-disable-next-line new-cap
|
||||||
|
tool: new this.cv.segmentation_IntelligentScissorsMB(),
|
||||||
|
state: {
|
||||||
|
path: [],
|
||||||
|
anchors: {},
|
||||||
|
image: null,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
this.scissors.tool.setEdgeFeatureCannyParameters(32, 100);
|
||||||
|
this.scissors.tool.setGradientMagnitudeMaxLimit(200);
|
||||||
|
}
|
||||||
|
|
||||||
|
public run(coordinates: number[], image: ImageData, offsetX: number, offsetY: number): number[] {
|
||||||
|
if (!Array.isArray(coordinates)) {
|
||||||
|
throw new Error('Coordinates is expected to be an array');
|
||||||
|
}
|
||||||
|
if (!coordinates.length) {
|
||||||
|
throw new Error('At least one point is expected');
|
||||||
|
}
|
||||||
|
if (!(image instanceof ImageData)) {
|
||||||
|
throw new Error('Image is expected to be an instance of ImageData');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { cv, scissors } = this;
|
||||||
|
const { tool, state } = scissors;
|
||||||
|
|
||||||
|
const points = applyOffset(numberArrayToPoints(coordinates), offsetX, offsetY);
|
||||||
|
|
||||||
|
if (points.length > 1) {
|
||||||
|
let matImage = null;
|
||||||
|
const contour = new cv.Mat();
|
||||||
|
const approx = new cv.Mat();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const [prev, cur] = points.slice(-2);
|
||||||
|
const { x: prevX, y: prevY } = prev;
|
||||||
|
const { x: curX, y: curY } = cur;
|
||||||
|
|
||||||
|
const latestPointRemoved = points.length < Object.keys(state.anchors).length;
|
||||||
|
const latestPointReplaced = points.length === Object.keys(state.anchors).length;
|
||||||
|
|
||||||
|
if (latestPointRemoved) {
|
||||||
|
for (const i of Object.keys(state.anchors).sort((a, b) => +b - +a)) {
|
||||||
|
if (+i >= points.length) {
|
||||||
|
state.path = state.path.slice(0, state.anchors[points.length].start);
|
||||||
|
delete state.anchors[+i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...state.path];
|
||||||
|
}
|
||||||
|
|
||||||
|
matImage = cv.matFromImageData(image);
|
||||||
|
|
||||||
|
if (latestPointReplaced) {
|
||||||
|
state.path = state.path.slice(0, state.anchors[points.length - 1].start);
|
||||||
|
delete state.anchors[points.length - 1];
|
||||||
|
}
|
||||||
|
|
||||||
|
tool.applyImage(matImage);
|
||||||
|
tool.buildMap(new cv.Point(prevX, prevY));
|
||||||
|
tool.getContour(new cv.Point(curX, curY), contour);
|
||||||
|
cv.approxPolyDP(contour, approx, 2, false);
|
||||||
|
|
||||||
|
const pathSegment = [];
|
||||||
|
for (let row = 0; row < approx.rows; row++) {
|
||||||
|
pathSegment.push(approx.intAt(row, 0) + offsetX, approx.intAt(row, 1) + offsetY);
|
||||||
|
}
|
||||||
|
state.anchors[points.length - 1] = {
|
||||||
|
point: cur,
|
||||||
|
start: state.path.length,
|
||||||
|
};
|
||||||
|
state.path.push(...pathSegment);
|
||||||
|
} finally {
|
||||||
|
if (matImage) {
|
||||||
|
matImage.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
contour.delete();
|
||||||
|
approx.delete();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
state.path.push(...pointsToNumberArray(applyOffset(points.slice(-1), -offsetX, -offsetY)));
|
||||||
|
state.anchors[0] = {
|
||||||
|
point: points[0],
|
||||||
|
start: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...state.path];
|
||||||
|
}
|
||||||
|
|
||||||
|
// eslint-disable-next-line class-methods-use-this
|
||||||
|
public get type(): string {
|
||||||
|
return 'opencv_intelligent_scissors';
|
||||||
|
}
|
||||||
|
|
||||||
|
// eslint-disable-next-line class-methods-use-this
|
||||||
|
public get params(): IntelligentScissorsParams {
|
||||||
|
return {
|
||||||
|
shape: {
|
||||||
|
shapeType: 'polygon',
|
||||||
|
},
|
||||||
|
canvas: {
|
||||||
|
shapeType: 'points',
|
||||||
|
enableThreshold: true,
|
||||||
|
enableSliding: true,
|
||||||
|
allowRemoveOnlyLast: true,
|
||||||
|
minPosVertices: 1,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,94 @@
|
|||||||
|
// Copyright (C) 2020-2021 Intel Corporation
|
||||||
|
//
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
import getCore from 'cvat-core-wrapper';
|
||||||
|
import waitFor from '../wait-for';
|
||||||
|
|
||||||
|
import IntelligentScissorsImplementation, { IntelligentScissors } from './intelligent-scissors';
|
||||||
|
|
||||||
|
const core = getCore();
|
||||||
|
const baseURL = core.config.backendAPI.slice(0, -7);
|
||||||
|
|
||||||
|
export interface Segmentation {
|
||||||
|
intelligentScissorsFactory: () => IntelligentScissors;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class OpenCVWrapper {
|
||||||
|
private initialized: boolean;
|
||||||
|
private cv: any;
|
||||||
|
|
||||||
|
public constructor() {
|
||||||
|
this.initialized = false;
|
||||||
|
this.cv = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async initialize(onProgress: (percent: number) => void): Promise<void> {
|
||||||
|
const response = await fetch(`${baseURL}/opencv/opencv.js`);
|
||||||
|
if (response.status !== 200) {
|
||||||
|
throw new Error(`Response status ${response.status}. ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentLength = response.headers.get('Content-Length');
|
||||||
|
const { body } = response;
|
||||||
|
|
||||||
|
if (contentLength === null) {
|
||||||
|
throw new Error('Content length is null, but necessary');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (body === null) {
|
||||||
|
throw new Error('Response body is null, but necessary');
|
||||||
|
}
|
||||||
|
|
||||||
|
const decoder = new TextDecoder('utf-8');
|
||||||
|
const reader = (body as ReadableStream<Uint8Array>).getReader();
|
||||||
|
let recieved = false;
|
||||||
|
let receivedLength = 0;
|
||||||
|
let decodedScript = '';
|
||||||
|
|
||||||
|
while (!recieved) {
|
||||||
|
// await in the loop is necessary here
|
||||||
|
// eslint-disable-next-line
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
recieved = done;
|
||||||
|
|
||||||
|
if (value instanceof Uint8Array) {
|
||||||
|
decodedScript += decoder.decode(value);
|
||||||
|
receivedLength += value.length;
|
||||||
|
const percentage = (receivedLength * 100) / +(contentLength as string);
|
||||||
|
onProgress(+percentage.toFixed(0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inject opencv to DOM
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-implied-eval
|
||||||
|
const OpenCVConstructor = new Function(decodedScript);
|
||||||
|
OpenCVConstructor();
|
||||||
|
|
||||||
|
const global = window as any;
|
||||||
|
await waitFor(
|
||||||
|
100,
|
||||||
|
() =>
|
||||||
|
typeof global.cv !== 'undefined' && typeof global.cv.segmentation_IntelligentScissorsMB !== 'undefined',
|
||||||
|
);
|
||||||
|
|
||||||
|
this.cv = global.cv;
|
||||||
|
this.initialized = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public get isInitialized(): boolean {
|
||||||
|
return this.initialized;
|
||||||
|
}
|
||||||
|
|
||||||
|
public get segmentation(): Segmentation {
|
||||||
|
if (!this.initialized) {
|
||||||
|
throw new Error('Need to initialize OpenCV first');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
intelligentScissorsFactory: () => new IntelligentScissorsImplementation(this.cv),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default new OpenCVWrapper();
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
// Copyright (C) 2021 Intel Corporation
|
||||||
|
//
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
export default function waitFor(frequencyHz: number, predicate: CallableFunction): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
if (typeof predicate !== 'function') {
|
||||||
|
reject(new Error(`Predicate must be a function, got ${typeof predicate}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
const internalWait = (): void => {
|
||||||
|
let result = false;
|
||||||
|
try {
|
||||||
|
result = predicate();
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result) {
|
||||||
|
resolve();
|
||||||
|
} else {
|
||||||
|
setTimeout(internalWait, 1000 / frequencyHz);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
setTimeout(internalWait);
|
||||||
|
});
|
||||||
|
}
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
# Updating OpenCV.js
|
||||||
|
|
||||||
|
The latest version of OpenCV JavaScript library can be pulled from [OpenCV site](https://docs.opencv.org/master/opencv.js).
|
||||||
|
|
||||||
|
To install it just push `opencv.js` to <b>cvat/apps/opencv/static/opencv/js</b>
|
||||||
|
|
||||||
|
If develop locally, do not forget update static files after pushing `python manage.py collectstatic`
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
# Copyright (C) 2021 Intel Corporation
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
# Copyright (C) 2021 Intel Corporation
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
from django.apps import AppConfig
|
||||||
|
|
||||||
|
class OpencvConfig(AppConfig):
|
||||||
|
name = 'opencv'
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
# Copyright (C) 2021 Intel Corporation
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,10 @@
|
|||||||
|
|
||||||
|
# Copyright (C) 2018-2020 Intel Corporation
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
from django.urls import path
|
||||||
|
from . import views
|
||||||
|
|
||||||
|
urlpatterns = [
|
||||||
|
path('opencv.js', views.OpenCVLibrary)
|
||||||
|
]
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
import os
|
||||||
|
import glob
|
||||||
|
from django.conf import settings
|
||||||
|
from sendfile import sendfile
|
||||||
|
|
||||||
|
def OpenCVLibrary(request):
|
||||||
|
dirname = os.path.join(settings.STATIC_ROOT, 'opencv', 'js')
|
||||||
|
pattern = os.path.join(dirname, 'opencv_*.js')
|
||||||
|
path = glob.glob(pattern)[0]
|
||||||
|
return sendfile(request, path)
|
||||||
@ -1,3 +1,4 @@
|
|||||||
LoadModule xsendfile_module /usr/lib/apache2/modules/mod_xsendfile.so
|
LoadModule xsendfile_module /usr/lib/apache2/modules/mod_xsendfile.so
|
||||||
XSendFile On
|
XSendFile On
|
||||||
XSendFilePath ${HOME}/data/
|
XSendFilePath ${HOME}/data/
|
||||||
|
XSendFilePath ${HOME}/static/
|
||||||
|
|||||||
Loading…
Reference in New Issue