Semi-automatic tools enhancements (Client-side points minimizer) (#3450)

* First stage for points minimizer

* Fixed issue with correct opencv initialization status

* Displaying points during interaction

* Added releasing memory

* Initial version for on-the-fly optimization

* Redesigned accuracy

* Updated version & changelog

* Fixed opencv scissors

* Clean up some intermediate state

* Fixed scss

* Redesigned slider a bit

* Added errored shape

* Keep slider hidden while didn't recieve first points

* Adjusted settings slider

* Updated label

* A couple of fixes for trackers & detectors

* Updated default value

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add a tutorial for semi-automatic/automatic annotation (<https://github.com/openvinotoolkit/cvat/pull/3124>)
 - Explicit "Done" button when drawing any polyshapes (<https://github.com/openvinotoolkit/cvat/pull/3417>)
 - Histogram equalization with OpenCV javascript (<https://github.com/openvinotoolkit/cvat/pull/3447>)
+- Client-side polyshapes approximation when using semi-automatic interactors & scissors (<https://github.com/openvinotoolkit/cvat/pull/3450>)
 
 ### Changed
 

cvat-canvas/src/scss/canvas.scss

@@ -161,11 +161,16 @@ polyline.cvat_canvas_shape_splitting {
 
 .cvat_canvas_removable_interaction_point {
     cursor:
-        url('data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAiIGhlaWdodD0iMjAiIHZpZXdCb3g9IjAgMCAxMCAxMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTEgMUw5IDlNMSA5TDkgMSIgc3Ryb2tlPSJibGFjayIvPgo8L3N2Zz4K')
-        10 10,
+        url(
+            'data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAiIGhlaWdodD0iMjAiIHZpZXdCb3g9IjAgMCAxMCAxMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTEgMUw5IDlNMSA5TDkgMSIgc3Ryb2tlPSJibGFjayIvPgo8L3N2Zz4K'
+        ) 10 10,
         auto;
 }
 
+.cvat_canvas_interact_intermediate_shape_point {
+    pointer-events: none;
+}
+
 .svg_select_boundingRect {
     opacity: 0;
     pointer-events: none;

cvat-canvas/src/typescript/interactionHandler.ts

@@ -225,6 +225,7 @@ export class InteractionHandlerImpl implements InteractionHandler {
 
     private release(): void {
         if (this.drawnIntermediateShape) {
+            this.selectize(false, this.drawnIntermediateShape);
             this.drawnIntermediateShape.remove();
             this.drawnIntermediateShape = null;
         }
@@ -270,21 +271,25 @@ export class InteractionHandlerImpl implements InteractionHandler {
     private updateIntermediateShape(): void {
         const { intermediateShape, geometry } = this;
         if (this.drawnIntermediateShape) {
+            this.selectize(false, this.drawnIntermediateShape);
             this.drawnIntermediateShape.remove();
         }
 
         if (!intermediateShape) return;
         const { shapeType, points } = intermediateShape;
         if (shapeType === 'polygon') {
+            const erroredShape = shapeType === 'polygon' && points.length < 3 * 2;
             this.drawnIntermediateShape = this.canvas
                 .polygon(stringifyPoints(translateToCanvas(geometry.offset, points)))
                 .attr({
                     'color-rendering': 'optimizeQuality',
                     'shape-rendering': 'geometricprecision',
                     'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale,
+                    stroke: erroredShape ? 'red' : 'black',
                     fill: 'none',
                 })
                 .addClass('cvat_canvas_interact_intermediate_shape');
+            this.selectize(true, this.drawnIntermediateShape, erroredShape);
         } else {
             throw new Error(
                 `Shape type "${shapeType}" was not implemented at interactionHandler::updateIntermediateShape`,
@@ -292,6 +297,39 @@ export class InteractionHandlerImpl implements InteractionHandler {
         }
     }
 
+    private selectize(value: boolean, shape: SVG.Element, erroredShape = false): void {
+        const self = this;
+
+        if (value) {
+            (shape as any).selectize(value, {
+                deepSelect: true,
+                pointSize: consts.BASE_POINT_SIZE / self.geometry.scale,
+                rotationPoint: false,
+                classPoints: 'cvat_canvas_interact_intermediate_shape_point',
+                pointType(cx: number, cy: number): SVG.Circle {
+                    return this.nested
+                        .circle(this.options.pointSize)
+                        .stroke(erroredShape ? 'red' : 'black')
+                        .fill('black')
+                        .center(cx, cy)
+                        .attr({
+                            'fill-opacity': 1,
+                            'stroke-width': consts.POINTS_STROKE_WIDTH / self.geometry.scale,
+                        });
+                },
+            });
+        } else {
+            (shape as any).selectize(false, {
+                deepSelect: true,
+            });
+        }
+
+        const handler = shape.remember('_selectHandler');
+        if (handler && handler.nested) {
+            handler.nested.fill(shape.attr('fill'));
+        }
+    }
+
     public constructor(
         onInteraction: (
             shapes: InteractionResult[] | null,
@@ -398,6 +436,15 @@ export class InteractionHandlerImpl implements InteractionHandler {
                 shape.attr('stroke-width', consts.BASE_STROKE_WIDTH / this.geometry.scale);
             }
         }
+
+        for (const element of window.document.getElementsByClassName('cvat_canvas_interact_intermediate_shape_point')) {
+            element.setAttribute('stroke-width', `${consts.POINTS_STROKE_WIDTH / (2 * this.geometry.scale)}`);
+            element.setAttribute('r', `${consts.BASE_POINT_SIZE / this.geometry.scale}`);
+        }
+
+        if (this.drawnIntermediateShape) {
+            this.drawnIntermediateShape.stroke({ width: consts.BASE_STROKE_WIDTH / this.geometry.scale });
+        }
     }
 
     public interact(interactionData: InteractionData): void {

cvat-ui/src/actions/settings-actions.ts

@@ -26,6 +26,7 @@ export enum SettingsActionTypes {
     SWITCH_AUTO_SAVE = 'SWITCH_AUTO_SAVE',
     CHANGE_AUTO_SAVE_INTERVAL = 'CHANGE_AUTO_SAVE_INTERVAL',
     CHANGE_AAM_ZOOM_MARGIN = 'CHANGE_AAM_ZOOM_MARGIN',
+    CHANGE_DEFAULT_APPROX_POLY_THRESHOLD = 'CHANGE_DEFAULT_APPROX_POLY_THRESHOLD',
     SWITCH_AUTOMATIC_BORDERING = 'SWITCH_AUTOMATIC_BORDERING',
     SWITCH_INTELLIGENT_POLYGON_CROP = 'SWITCH_INTELLIGENT_POLYGON_CROP',
     SWITCH_SHOWNIG_INTERPOLATED_TRACKS = 'SWITCH_SHOWNIG_INTERPOLATED_TRACKS',
@@ -270,6 +271,15 @@ export function switchSettingsDialog(show?: boolean): AnyAction {
     };
 }
 
+export function changeDefaultApproxPolyAccuracy(approxPolyAccuracy: number): AnyAction {
+    return {
+        type: SettingsActionTypes.CHANGE_DEFAULT_APPROX_POLY_THRESHOLD,
+        payload: {
+            approxPolyAccuracy,
+        },
+    };
+}
+
 export function setSettings(settings: Partial<SettingsState>): AnyAction {
     return {
         type: SettingsActionTypes.SET_SETTINGS,

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy.tsx

@@ -0,0 +1,76 @@
+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import React, { CSSProperties } from 'react';
+import ReactDOM from 'react-dom';
+import Text from 'antd/lib/typography/Text';
+import Slider from 'antd/lib/slider';
+import { Col, Row } from 'antd/lib/grid';
+
+interface Props {
+    approxPolyAccuracy: number;
+    onChange(value: number): void;
+}
+
+export const MAX_ACCURACY = 13;
+
+export const marks: Record<number, { style: CSSProperties; label: JSX.Element }> = {};
+marks[0] = {
+    style: {
+        color: '#1890ff',
+    },
+    label: <strong>less</strong>,
+};
+marks[MAX_ACCURACY] = {
+    style: {
+        color: '#61c200',
+    },
+    label: <strong>more</strong>,
+};
+
+export function thresholdFromAccuracy(approxPolyAccuracy: number): number {
+    const approxPolyMaxDistance = MAX_ACCURACY - approxPolyAccuracy;
+    let threshold = 0;
+    if (approxPolyMaxDistance > 0) {
+        if (approxPolyMaxDistance <= 8) {
+            // −2.75x+7y+1=0 linear made from two points (1; 0.25) and (8; 3)
+            threshold = (2.75 * approxPolyMaxDistance - 1) / 7;
+        } else {
+            // 4 for 9, 8 for 10, 16 for 11, 32 for 12, 64 for 13
+            threshold = 2 ** (approxPolyMaxDistance - 7);
+        }
+    }
+
+    return threshold;
+}
+
+function ApproximationAccuracy(props: Props): React.ReactPortal | null {
+    const { approxPolyAccuracy, onChange } = props;
+    const target = window.document.getElementsByClassName('cvat-canvas-container')[0];
+
+    return target ?
+        ReactDOM.createPortal(
+            <Row align='middle' className='cvat-approx-poly-threshold-wrapper'>
+                <Col span={5}>
+                    <Text>Points: </Text>
+                </Col>
+                <Col offset={1} span={18}>
+                    <Slider
+                        value={approxPolyAccuracy}
+                        min={0}
+                        max={MAX_ACCURACY}
+                        step={1}
+                        dots
+                        tooltipVisible={false}
+                        onChange={onChange}
+                        marks={marks}
+                    />
+                </Col>
+            </Row>,
+            target,
+        ) :
+        null;
+}
+
+export default React.memo(ApproximationAccuracy);

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx

@@ -30,6 +30,9 @@ import {
 } from 'actions/annotation-actions';
 import LabelSelector from 'components/label-selector/label-selector';
 import CVATTooltip from 'components/common/cvat-tooltip';
+import ApproximationAccuracy, {
+    thresholdFromAccuracy,
+} from 'components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy';
 import { ImageProcessing } from 'utils/opencv-wrapper/opencv-interfaces';
 import withVisibilityHandling from './handle-popover-visibility';
 
@@ -41,6 +44,7 @@ interface Props {
     states: any[];
     frame: number;
     curZOrder: number;
+    defaultApproxPolyAccuracy: number;
     frameData: any;
 }
 
@@ -57,6 +61,7 @@ interface State {
     initializationError: boolean;
     initializationProgress: number;
     activeLabelID: number;
+    approxPolyAccuracy: number;
     activeImageModifiers: ImageModifier[];
 }
 
@@ -81,11 +86,15 @@ function mapStateToProps(state: CombinedState): Props {
                 frame: { number: frame, data: frameData },
             },
         },
+        settings: {
+            workspace: { defaultApproxPolyAccuracy },
+        },
     } = state;
 
     return {
         isActivated: activeControl === ActiveControl.OPENCV_TOOLS,
         canvasInstance: canvasInstance as Canvas,
+        defaultApproxPolyAccuracy,
         jobInstance,
         curZOrder,
         labels,
@@ -105,18 +114,21 @@ const mapDispatchToProps = {
 
 class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps, State> {
     private activeTool: IntelligentScissors | null;
+    private latestPoints: number[];
     private canvasForceUpdateWasEnabled: boolean;
 
     public constructor(props: Props & DispatchToProps) {
         super(props);
-        const { labels } = props;
+        const { labels, defaultApproxPolyAccuracy } = props;
         this.activeTool = null;
+        this.latestPoints = [];
         this.canvasForceUpdateWasEnabled = false;
 
         this.state = {
             libraryInitialized: openCVWrapper.isInitialized,
             initializationError: false,
             initializationProgress: -1,
+            approxPolyAccuracy: defaultApproxPolyAccuracy,
             activeLabelID: labels.length ? labels[0].id : null,
             activeImageModifiers: [],
         };
@@ -128,14 +140,35 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
         canvasInstance.html().addEventListener('canvas.setup', this.runImageModifier);
     }
 
-    public componentDidUpdate(prevProps: Props): void {
-        const { isActivated } = this.props;
+    public componentDidUpdate(prevProps: Props, prevState: State): void {
+        const { approxPolyAccuracy } = this.state;
+        const { isActivated, defaultApproxPolyAccuracy, canvasInstance } = this.props;
         if (!prevProps.isActivated && isActivated) {
-            // reset flags when before using a tool
+            // reset flags & states before using a tool
+            this.latestPoints = [];
+            this.setState({
+                approxPolyAccuracy: defaultApproxPolyAccuracy,
+            });
             if (this.activeTool) {
                 this.activeTool.reset();
             }
         }
+
+        if (prevState.approxPolyAccuracy !== approxPolyAccuracy) {
+            if (isActivated) {
+                const approx = openCVWrapper.contours.approxPoly(
+                    this.latestPoints,
+                    thresholdFromAccuracy(approxPolyAccuracy),
+                );
+                canvasInstance.interact({
+                    enabled: true,
+                    intermediateShape: {
+                        shapeType: ShapeType.POLYGON,
+                        points: approx.flat(),
+                    },
+                });
+            }
+        }
     }
 
     public componentWillUnmount(): void {
@@ -145,6 +178,7 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
     }
 
     private interactionListener = async (e: Event): Promise<void> => {
+        const { approxPolyAccuracy } = this.state;
         const {
             createAnnotations, isActivated, jobInstance, frame, labels, curZOrder, canvasInstance,
         } = this.props;
@@ -160,24 +194,32 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
 
         try {
             if (shapesUpdated) {
-                const result = await this.runCVAlgorithm(pressedPoints, threshold);
+                this.latestPoints = await this.runCVAlgorithm(pressedPoints, threshold);
+                const approx = openCVWrapper.contours.approxPoly(
+                    this.latestPoints,
+                    thresholdFromAccuracy(approxPolyAccuracy),
+                    false,
+                );
                 canvasInstance.interact({
                     enabled: true,
                     intermediateShape: {
                         shapeType: ShapeType.POLYGON,
-                        points: result,
+                        points: approx.flat(),
                     },
                 });
             }
 
             if (isDone) {
+                // need to recalculate without the latest sliding point
+                const finalPoints = await this.runCVAlgorithm(pressedPoints, threshold);
                 const finalObject = new core.classes.ObjectState({
                     frame,
                     objectType: ObjectType.SHAPE,
                     shapeType: ShapeType.POLYGON,
                     label: labels.filter((label: any) => label.id === activeLabelID)[0],
-                    // need to recalculate without the latest sliding point
-                    points: await this.runCVAlgorithm(pressedPoints, threshold),
+                    points: openCVWrapper.contours
+                        .approxPoly(finalPoints, thresholdFromAccuracy(approxPolyAccuracy))
+                        .flat(),
                     occluded: false,
                     zOrder: curZOrder,
                 });
@@ -253,19 +295,6 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
 
         // Handling via OpenCV.js
         const points = await this.activeTool.run(pressedPoints, imageData, startX, startY);
-
-        // Increasing number of points artificially
-        let minNumberOfPoints = 1;
-        // eslint-disable-next-line: eslintdot-notation
-        if (this.activeTool.params.shape.shapeType === 'polyline') {
-            minNumberOfPoints = 2;
-        } else if (this.activeTool.params.shape.shapeType === 'polygon') {
-            minNumberOfPoints = 3;
-        }
-        while (points.length < minNumberOfPoints * 2) {
-            points.push(...points.slice(points.length - 2));
-        }
-
         return points;
     }
 
@@ -449,6 +478,7 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
 
     public render(): JSX.Element {
         const { isActivated, canvasInstance, labels } = this.props;
+        const { libraryInitialized, approxPolyAccuracy } = this.state;
         const dynamcPopoverPros = isActivated ?
             {
                 overlayStyle: {
@@ -471,14 +501,31 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
         return !labels.length ? (
             <Icon className='cvat-opencv-control cvat-disabled-canvas-control' component={OpenCVIcon} />
         ) : (
-            <CustomPopover
-                {...dynamcPopoverPros}
-                placement='right'
-                overlayClassName='cvat-opencv-control-popover'
-                content={this.renderContent()}
-            >
-                <Icon {...dynamicIconProps} component={OpenCVIcon} />
-            </CustomPopover>
+            <>
+                <CustomPopover
+                    {...dynamcPopoverPros}
+                    placement='right'
+                    overlayClassName='cvat-opencv-control-popover'
+                    content={this.renderContent()}
+                    afterVisibleChange={() => {
+                        if (libraryInitialized !== openCVWrapper.isInitialized) {
+                            this.setState({
+                                libraryInitialized: openCVWrapper.isInitialized,
+                            });
+                        }
+                    }}
+                >
+                    <Icon {...dynamicIconProps} component={OpenCVIcon} />
+                </CustomPopover>
+                {isActivated ? (
+                    <ApproximationAccuracy
+                        approxPolyAccuracy={approxPolyAccuracy}
+                        onChange={(value: number) => {
+                            this.setState({ approxPolyAccuracy: value });
+                        }}
+                    />
+                ) : null}
+            </>
         );
     }
 }

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx

@@ -13,6 +13,7 @@ import Text from 'antd/lib/typography/Text';
 import Tabs from 'antd/lib/tabs';
 import { Row, Col } from 'antd/lib/grid';
 import notification from 'antd/lib/notification';
+import message from 'antd/lib/message';
 import Progress from 'antd/lib/progress';
 import InputNumber from 'antd/lib/input-number';
 
@@ -20,6 +21,7 @@ import { AIToolsIcon } from 'icons';
 import { Canvas, convertShapesForInteractor } from 'cvat-canvas-wrapper';
 import range from 'utils/range';
 import getCore from 'cvat-core-wrapper';
+import openCVWrapper from 'utils/opencv-wrapper/opencv-wrapper';
 import {
     CombinedState, ActiveControl, Model, ObjectType, ShapeType,
 } from 'reducers/interfaces';
@@ -31,6 +33,9 @@ import {
 } from 'actions/annotation-actions';
 import DetectorRunner from 'components/model-runner-modal/detector-runner';
 import LabelSelector from 'components/label-selector/label-selector';
+import ApproximationAccuracy, {
+    thresholdFromAccuracy,
+} from 'components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy';
 import withVisibilityHandling from './handle-popover-visibility';
 
 interface StateToProps {
@@ -46,6 +51,7 @@ interface StateToProps {
     trackers: Model[];
     curZOrder: number;
     aiToolsRef: MutableRefObject<any>;
+    defaultApproxPolyAccuracy: number;
 }
 
 interface DispatchToProps {
@@ -60,6 +66,7 @@ const CustomPopover = withVisibilityHandling(Popover, 'tools-control');
 
 function mapStateToProps(state: CombinedState): StateToProps {
     const { annotation } = state;
+    const { settings } = state;
     const { number: frame } = annotation.player.frame;
     const { instance: jobInstance } = annotation.job;
     const { instance: canvasInstance, activeControl } = annotation.canvas;
@@ -79,6 +86,7 @@ function mapStateToProps(state: CombinedState): StateToProps {
         frame,
         curZOrder: annotation.annotations.zLayer.cur,
         aiToolsRef: annotation.aiToolsRef,
+        defaultApproxPolyAccuracy: settings.workspace.defaultApproxPolyAccuracy,
     };
 }
 
@@ -97,14 +105,16 @@ interface State {
     trackingProgress: number | null;
     trackingFrames: number;
     fetching: boolean;
+    pointsRecieved: boolean;
+    approxPolyAccuracy: number;
     mode: 'detection' | 'interaction' | 'tracking';
 }
 
 export class ToolsControlComponent extends React.PureComponent<Props, State> {
     private interactionIsAborted: boolean;
-
     private interactionIsDone: boolean;
-    private latestResult: number[];
+    private latestResponseResult: number[][];
+    private latestResult: number[][];
 
     public constructor(props: Props) {
         super(props);
@@ -112,12 +122,15 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
             activeInteractor: props.interactors.length ? props.interactors[0] : null,
             activeTracker: props.trackers.length ? props.trackers[0] : null,
             activeLabelID: props.labels.length ? props.labels[0].id : null,
+            approxPolyAccuracy: props.defaultApproxPolyAccuracy,
             trackingProgress: null,
             trackingFrames: 10,
             fetching: false,
+            pointsRecieved: false,
             mode: 'interaction',
         };
 
+        this.latestResponseResult = [];
         this.latestResult = [];
         this.interactionIsAborted = false;
         this.interactionIsDone = false;
@@ -130,16 +143,39 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
         canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener);
     }
 
-    public componentDidUpdate(prevProps: Props): void {
-        const { isActivated } = this.props;
+    public componentDidUpdate(prevProps: Props, prevState: State): void {
+        const { isActivated, defaultApproxPolyAccuracy, canvasInstance } = this.props;
+        const { approxPolyAccuracy, activeInteractor } = this.state;
+
         if (prevProps.isActivated && !isActivated) {
             window.removeEventListener('contextmenu', this.contextmenuDisabler);
         } else if (!prevProps.isActivated && isActivated) {
             // reset flags when start interaction/tracking
+            this.setState({
+                approxPolyAccuracy: defaultApproxPolyAccuracy,
+                pointsRecieved: false,
+            });
+            this.latestResult = [];
+            this.latestResponseResult = [];
             this.interactionIsDone = false;
             this.interactionIsAborted = false;
             window.addEventListener('contextmenu', this.contextmenuDisabler);
         }
+
+        if (prevState.approxPolyAccuracy !== approxPolyAccuracy) {
+            if (isActivated && activeInteractor !== null && this.latestResponseResult.length) {
+                this.approximateResponsePoints(this.latestResponseResult).then((points: number[][]) => {
+                    this.latestResult = points;
+                    canvasInstance.interact({
+                        enabled: true,
+                        intermediateShape: {
+                            shapeType: ShapeType.POLYGON,
+                            points: this.latestResult.flat(),
+                        },
+                    });
+                });
+            }
+        }
     }
 
     public componentWillUnmount(): void {
@@ -197,20 +233,26 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
             if ((e as CustomEvent).detail.shapesUpdated) {
                 this.setState({ fetching: true });
                 try {
-                    this.latestResult = await core.lambda.call(jobInstance.task, interactor, {
+                    this.latestResponseResult = await core.lambda.call(jobInstance.task, interactor, {
                         frame,
                         pos_points: convertShapesForInteractor((e as CustomEvent).detail.shapes, 0),
                         neg_points: convertShapesForInteractor((e as CustomEvent).detail.shapes, 2),
                     });
 
+                    this.latestResult = this.latestResponseResult;
+
                     if (this.interactionIsAborted) {
                         // while the server request
                         // user has cancelled interaction (for example pressed ESC)
+                        // need to clean variables that have been just set
                         this.latestResult = [];
+                        this.latestResponseResult = [];
                         return;
                     }
+
+                    this.latestResult = await this.approximateResponsePoints(this.latestResponseResult);
                 } finally {
-                    this.setState({ fetching: false });
+                    this.setState({ fetching: false, pointsRecieved: !!this.latestResult.length });
                 }
             }
 
@@ -259,7 +301,8 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
         const { activeLabelID } = this.state;
         const [label] = jobInstance.task.labels.filter((_label: any): boolean => _label.id === activeLabelID);
 
-        if (!(e as CustomEvent).detail.isDone) {
+        const { isDone, shapesUpdated } = (e as CustomEvent).detail;
+        if (!isDone || !shapesUpdated) {
             return;
         }
 
@@ -320,8 +363,27 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
         });
     };
 
+    private async approximateResponsePoints(points: number[][]): Promise<number[][]> {
+        const { approxPolyAccuracy } = this.state;
+        if (points.length > 3) {
+            if (!openCVWrapper.isInitialized) {
+                const hide = message.loading('OpenCV.js initialization..');
+                try {
+                    await openCVWrapper.initialize(() => {});
+                } finally {
+                    hide();
+                }
+            }
+
+            const threshold = thresholdFromAccuracy(approxPolyAccuracy);
+            return openCVWrapper.contours.approxPoly(points, threshold);
+        }
+
+        return points;
+    }
+
     public async trackState(state: any): Promise<void> {
-        const { jobInstance, frame } = this.props;
+        const { jobInstance, frame, fetchAnnotations } = this.props;
         const { activeTracker, trackingFrames } = this.state;
         const { clientID, points } = state;
 
@@ -373,6 +435,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
             }
         } finally {
             this.setState({ trackingProgress: null, fetching: false });
+            fetchAnnotations();
         }
     }
 
@@ -577,7 +640,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
 
     private renderDetectorBlock(): JSX.Element {
         const {
-            jobInstance, detectors, curZOrder, frame,
+            jobInstance, detectors, curZOrder, frame, createAnnotations,
         } = this.props;
 
         if (!detectors.length) {
@@ -616,7 +679,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
                                 }),
                         );
 
-                        createAnnotationsAsync(jobInstance, frame, states);
+                        createAnnotations(jobInstance, frame, states);
                     } catch (error) {
                         notification.error({
                             description: error.toString(),
@@ -661,7 +724,9 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
         const {
             interactors, detectors, trackers, isActivated, canvasInstance, labels,
         } = this.props;
-        const { fetching, trackingProgress } = this.state;
+        const {
+            fetching, trackingProgress, approxPolyAccuracy, activeInteractor, pointsRecieved,
+        } = this.state;
 
         if (![...interactors, ...detectors, ...trackers].length) return null;
 
@@ -701,6 +766,14 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
                         <Progress percent={+(trackingProgress * 100).toFixed(0)} status='active' />
                     )}
                 </Modal>
+                {isActivated && activeInteractor !== null && pointsRecieved ? (
+                    <ApproximationAccuracy
+                        approxPolyAccuracy={approxPolyAccuracy}
+                        onChange={(value: number) => {
+                            this.setState({ approxPolyAccuracy: value });
+                        }}
+                    />
+                ) : null}
                 <CustomPopover {...dynamcPopoverPros} placement='right' content={this.renderPopoverContent()}>
                     <Icon {...dynamicIconProps} component={AIToolsIcon} />
                 </CustomPopover>

cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/styles.scss

@@ -397,3 +397,29 @@
 .cvat-objects-sidebar-label-item-disabled {
     opacity: 0.5;
 }
+
+.cvat-workspace-settings-approx-poly-threshold {
+    .ant-slider-track {
+        background: linear-gradient(90deg, #1890ff 0%, #61c200 100%);
+    }
+}
+
+.cvat-approx-poly-threshold-wrapper {
+    @extend .cvat-workspace-settings-approx-poly-threshold;
+
+    width: $grid-unit-size * 32;
+    position: absolute;
+    background: $background-color-2;
+    top: 8px;
+    left: 50%;
+    border-radius: 6px;
+    border: 1px solid $border-color-3;
+    z-index: 100;
+    padding: $grid-unit-size / 2 $grid-unit-size * 2 $grid-unit-size / 2 $grid-unit-size / 2;
+
+    .ant-slider-mark {
+        position: static;
+        margin-top: 4px;
+        pointer-events: none;
+    }
+}

cvat-ui/src/components/header/settings-modal/styles.scss

@@ -27,7 +27,9 @@
 .cvat-workspace-settings-autoborders,
 .cvat-workspace-settings-intelligent-polygon-cropping,
 .cvat-workspace-settings-show-text-always,
-.cvat-workspace-settings-show-interpolated {
+.cvat-workspace-settings-show-interpolated,
+.cvat-workspace-settings-approx-poly-threshold,
+.cvat-workspace-settings-aam-zoom-margin {
     margin-bottom: 25px;
 
     > div:first-child {
@@ -35,6 +37,10 @@
     }
 }
 
+.cvat-workspace-settings-approx-poly-threshold {
+    user-select: none;
+}
+
 .cvat-player-settings-step,
 .cvat-player-settings-speed,
 .cvat-player-settings-reset-zoom,

cvat-ui/src/components/header/settings-modal/workspace-settings.tsx

@@ -8,7 +8,12 @@ import { Row, Col } from 'antd/lib/grid';
 import Checkbox, { CheckboxChangeEvent } from 'antd/lib/checkbox';
 import InputNumber from 'antd/lib/input-number';
 import Text from 'antd/lib/typography/Text';
+import Slider from 'antd/lib/slider';
 
+import {
+    MAX_ACCURACY,
+    marks,
+} from 'components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy';
 import { clamp } from 'utils/math';
 
 interface Props {
@@ -19,16 +24,18 @@ interface Props {
     showObjectsTextAlways: boolean;
     automaticBordering: boolean;
     intelligentPolygonCrop: boolean;
+    defaultApproxPolyAccuracy: number;
     onSwitchAutoSave(enabled: boolean): void;
     onChangeAutoSaveInterval(interval: number): void;
     onChangeAAMZoomMargin(margin: number): void;
+    onChangeDefaultApproxPolyAccuracy(approxPolyAccuracy: number): void;
     onSwitchShowingInterpolatedTracks(enabled: boolean): void;
     onSwitchShowingObjectsTextAlways(enabled: boolean): void;
     onSwitchAutomaticBordering(enabled: boolean): void;
     onSwitchIntelligentPolygonCrop(enabled: boolean): void;
 }
 
-export default function WorkspaceSettingsComponent(props: Props): JSX.Element {
+function WorkspaceSettingsComponent(props: Props): JSX.Element {
     const {
         autoSave,
         autoSaveInterval,
@@ -37,6 +44,7 @@ export default function WorkspaceSettingsComponent(props: Props): JSX.Element {
         showObjectsTextAlways,
         automaticBordering,
         intelligentPolygonCrop,
+        defaultApproxPolyAccuracy,
         onSwitchAutoSave,
         onChangeAutoSaveInterval,
         onChangeAAMZoomMargin,
@@ -44,6 +52,7 @@ export default function WorkspaceSettingsComponent(props: Props): JSX.Element {
         onSwitchShowingObjectsTextAlways,
         onSwitchAutomaticBordering,
         onSwitchIntelligentPolygonCrop,
+        onChangeDefaultApproxPolyAccuracy,
     } = props;
 
     const minAutoSaveInterval = 1;
@@ -168,6 +177,27 @@ export default function WorkspaceSettingsComponent(props: Props): JSX.Element {
                     />
                 </Col>
             </Row>
+            <Row className='cvat-workspace-settings-approx-poly-threshold'>
+                <Col>
+                    <Text className='cvat-text-color'>Default number of points in polygon approximation</Text>
+                </Col>
+                <Col span={7} offset={1}>
+                    <Slider
+                        min={0}
+                        max={MAX_ACCURACY}
+                        step={1}
+                        value={defaultApproxPolyAccuracy}
+                        dots
+                        onChange={onChangeDefaultApproxPolyAccuracy}
+                        marks={marks}
+                    />
+                </Col>
+                <Col span={24}>
+                    <Text type='secondary'>Works for serverless interactors and OpenCV scissors</Text>
+                </Col>
+            </Row>
         </div>
     );
 }
+
+export default React.memo(WorkspaceSettingsComponent);

cvat-ui/src/containers/header/settings-modal/workspace-settings.tsx

@@ -13,6 +13,7 @@ import {
     switchShowingObjectsTextAlways,
     switchAutomaticBordering,
     switchIntelligentPolygonCrop,
+    changeDefaultApproxPolyAccuracy,
 } from 'actions/settings-actions';
 
 import { CombinedState } from 'reducers/interfaces';
@@ -25,6 +26,7 @@ interface StateToProps {
     aamZoomMargin: number;
     showAllInterpolationTracks: boolean;
     showObjectsTextAlways: boolean;
+    defaultApproxPolyAccuracy: number;
     automaticBordering: boolean;
     intelligentPolygonCrop: boolean;
 }
@@ -37,6 +39,7 @@ interface DispatchToProps {
     onSwitchShowingObjectsTextAlways(enabled: boolean): void;
     onSwitchAutomaticBordering(enabled: boolean): void;
     onSwitchIntelligentPolygonCrop(enabled: boolean): void;
+    onChangeDefaultApproxPolyAccuracy(approxPolyAccuracy: number): void;
 }
 
 function mapStateToProps(state: CombinedState): StateToProps {
@@ -49,6 +52,7 @@ function mapStateToProps(state: CombinedState): StateToProps {
         showObjectsTextAlways,
         automaticBordering,
         intelligentPolygonCrop,
+        defaultApproxPolyAccuracy,
     } = workspace;
 
     return {
@@ -59,6 +63,7 @@ function mapStateToProps(state: CombinedState): StateToProps {
         showObjectsTextAlways,
         automaticBordering,
         intelligentPolygonCrop,
+        defaultApproxPolyAccuracy,
     };
 }
 
@@ -85,6 +90,9 @@ function mapDispatchToProps(dispatch: any): DispatchToProps {
         onSwitchIntelligentPolygonCrop(enabled: boolean): void {
             dispatch(switchIntelligentPolygonCrop(enabled));
         },
+        onChangeDefaultApproxPolyAccuracy(threshold: number): void {
+            dispatch(changeDefaultApproxPolyAccuracy(threshold));
+        },
     };
 }
 

cvat-ui/src/reducers/interfaces.ts

@@ -556,6 +556,7 @@ export interface WorkspaceSettingsState {
     showObjectsTextAlways: boolean;
     showAllInterpolationTracks: boolean;
     intelligentPolygonCrop: boolean;
+    defaultApproxPolyAccuracy: number;
 }
 
 export interface ShapesSettingsState {

cvat-ui/src/reducers/settings-reducer.ts

@@ -31,6 +31,7 @@ const defaultState: SettingsState = {
         showObjectsTextAlways: false,
         showAllInterpolationTracks: false,
         intelligentPolygonCrop: true,
+        defaultApproxPolyAccuracy: 9,
     },
     player: {
         canvasBackgroundColor: '#ffffff',
@@ -277,6 +278,15 @@ export default (state = defaultState, action: AnyAction): SettingsState => {
                 },
             };
         }
+        case SettingsActionTypes.CHANGE_DEFAULT_APPROX_POLY_THRESHOLD: {
+            return {
+                ...state,
+                workspace: {
+                    ...state.workspace,
+                    defaultApproxPolyAccuracy: action.payload.approxPolyAccuracy,
+                },
+            };
+        }
         case SettingsActionTypes.SWITCH_SETTINGS_DIALOG: {
             return {
                 ...state,

cvat-ui/src/utils/opencv-wrapper/intelligent-scissors.ts

@@ -92,7 +92,6 @@ export default class IntelligentScissorsImplementation implements IntelligentSci
         if (points.length > 1) {
             let matImage = null;
             const contour = new cv.Mat();
-            const approx = new cv.Mat();
 
             try {
                 const [prev, cur] = points.slice(-2);
@@ -123,11 +122,10 @@ export default class IntelligentScissorsImplementation implements IntelligentSci
                 tool.applyImage(matImage);
                 tool.buildMap(new cv.Point(prevX, prevY));
                 tool.getContour(new cv.Point(curX, curY), contour);
-                cv.approxPolyDP(contour, approx, 2, false);
 
                 const pathSegment = [];
-                for (let row = 0; row < approx.rows; row++) {
-                    pathSegment.push(approx.intAt(row, 0) + offsetX, approx.intAt(row, 1) + offsetY);
+                for (let row = 0; row < contour.rows; row++) {
+                    pathSegment.push(contour.intAt(row, 0) + offsetX, contour.intAt(row, 1) + offsetY);
                 }
                 state.anchors[points.length - 1] = {
                     point: cur,
@@ -140,7 +138,6 @@ export default class IntelligentScissorsImplementation implements IntelligentSci
                 }
 
                 contour.delete();
-                approx.delete();
             }
         } else {
             state.path.push(...pointsToNumberArray(applyOffset(points.slice(-1), -offsetX, -offsetY)));

cvat-ui/src/utils/opencv-wrapper/opencv-wrapper.ts

@@ -15,6 +15,10 @@ export interface Segmentation {
     intelligentScissorsFactory: () => IntelligentScissors;
 }
 
+export interface Contours {
+    approxPoly: (points: number[] | any, threshold: number, closed?: boolean) => number[][];
+}
+
 export interface ImgProc {
     hist: () => HistogramEqualization
 }
@@ -85,6 +89,39 @@ export class OpenCVWrapper {
         return this.initialized;
     }
 
+    public get contours(): Contours {
+        if (!this.initialized) {
+            throw new Error('Need to initialize OpenCV first');
+        }
+
+        const { cv } = this;
+        return {
+            approxPoly: (points: number[] | number[][], threshold: number, closed = true): number[][] => {
+                const isArrayOfArrays = Array.isArray(points[0]);
+                if (points.length < 3) {
+                    // one pair of coordinates [x, y], approximation not possible
+                    return (isArrayOfArrays ? points : [points]) as number[][];
+                }
+                const rows = isArrayOfArrays ? points.length : points.length / 2;
+                const cols = 2;
+
+                const approx = new cv.Mat();
+                const contour = cv.matFromArray(rows, cols, cv.CV_32FC1, points.flat());
+                try {
+                    cv.approxPolyDP(contour, approx, threshold, closed); // approx output type is CV_32F
+                    const result = [];
+                    for (let row = 0; row < approx.rows; row++) {
+                        result.push([approx.floatAt(row, 0), approx.floatAt(row, 1)]);
+                    }
+                    return result;
+                } finally {
+                    approx.delete();
+                    contour.delete();
+                }
+            },
+        };
+    }
+
     public get segmentation(): Segmentation {
         if (!this.initialized) {
             throw new Error('Need to initialize OpenCV first');