UI Tracking with serverless functions (#2136)

* tmp

* Refactored

* Refactoring & added button to context menu

* Updated changelog, updated versions

* Improved styles

* Removed outdated code

* Updated icon

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007)
 - Annotation in process outline color wheel (<https://github.com/opencv/cvat/pull/2084>)
 - On the fly annotation using DL detectors (<https://github.com/opencv/cvat/pull/2102>)
+- Automatic tracking of bounding boxes using serverless functions (<https://github.com/opencv/cvat/pull/2136>)
 - [Datumaro] CLI command for dataset equality comparison (<https://github.com/opencv/cvat/pull/1989>)
 - [Datumaro] Merging of datasets with different labels (<https://github.com/opencv/cvat/pull/2098>)
 

cvat-canvas/src/typescript/interactionHandler.ts

@@ -155,10 +155,6 @@ export class InteractionHandlerImpl implements InteractionHandler {
             this.shapesWereUpdated = true;
 
             this.canvas.off('mousedown.interaction', eventListener);
-            if (this.shouldRaiseEvent(false)) {
-                this.onInteraction(this.prepareResult(), true, false);
-            }
-
             this.interact({ enabled: false });
         }).addClass('cvat_canvas_shape_drawing').attr({
             'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale,

cvat-core/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "3.6.0",
+  "version": "3.6.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "3.6.0",
+  "version": "3.6.1",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.9.2",
+  "version": "1.9.3",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.9.2",
+  "version": "1.9.3",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/actions/annotation-actions.ts

@@ -27,6 +27,7 @@ import getCore from 'cvat-core-wrapper';
 import logger, { LogType } from 'cvat-logger';
 import { RectDrawingMethod } from 'cvat-canvas-wrapper';
 import { getCVATStore } from 'cvat-store';
+import { MutableRefObject } from 'react';
 
 interface AnnotationsParameters {
     filters: string[];
@@ -189,6 +190,7 @@ export enum AnnotationActionTypes {
     SAVE_LOGS_SUCCESS = 'SAVE_LOGS_SUCCESS',
     SAVE_LOGS_FAILED = 'SAVE_LOGS_FAILED',
     INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS',
+    SET_AI_TOOLS_REF = 'SET_AI_TOOLS_REF',
 }
 
 export function saveLogsAsync(): ThunkAction {
@@ -1397,6 +1399,16 @@ export function interactWithCanvas(activeInteractor: Model, activeLabelID: numbe
     };
 }
 
+export function setAIToolsRef(ref: MutableRefObject<any>): AnyAction {
+    return {
+        type: AnnotationActionTypes.SET_AI_TOOLS_REF,
+        payload: {
+            aiToolsRef: ref,
+        },
+    };
+}
+
+
 export function repeatDrawShapeAsync(): ThunkAction {
     return async (dispatch: ActionCreator<Dispatch>): Promise<void> => {
         const {
@@ -1424,12 +1436,21 @@ export function repeatDrawShapeAsync(): ThunkAction {
 
         let activeControl = ActiveControl.CURSOR;
         if (activeInteractor) {
-            canvasInstance.interact({
+            if (activeInteractor.type === 'tracker') {
-                enabled: true,
+                canvasInstance.interact({
-                shapeType: 'points',
+                    enabled: true,
-                minPosVertices: 4, // TODO: Add parameter to interactor
+                    shapeType: 'rectangle',
-            });
+                });
-            dispatch(interactWithCanvas(activeInteractor, activeLabelID));
+                dispatch(interactWithCanvas(activeInteractor, activeLabelID));
+            } else {
+                canvasInstance.interact({
+                    enabled: true,
+                    shapeType: 'points',
+                    minPosVertices: 4, // TODO: Add parameter to interactor
+                });
+                dispatch(interactWithCanvas(activeInteractor, activeLabelID));
+            }
+
             return;
         }
 

cvat-ui/src/actions/plugins-actions.ts

@@ -30,19 +30,16 @@ export function checkPluginsAsync(): ThunkAction {
         const plugins: PluginObjects = {
             ANALYTICS: false,
             GIT_INTEGRATION: false,
-            DEXTR_SEGMENTATION: false,
         };
 
         const promises: Promise<boolean>[] = [
             // check must return true/false with no exceptions
             PluginChecker.check(SupportedPlugins.ANALYTICS),
             PluginChecker.check(SupportedPlugins.GIT_INTEGRATION),
-            PluginChecker.check(SupportedPlugins.DEXTR_SEGMENTATION),
         ];
 
         const values = await Promise.all(promises);
-        [plugins.ANALYTICS, plugins.GIT_INTEGRATION,
+        [plugins.ANALYTICS, plugins.GIT_INTEGRATION] = values;
-            plugins.DEXTR_SEGMENTATION] = values;
         dispatch(pluginActions.checkedAllPlugins(plugins));
     };
 }

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx

@@ -85,7 +85,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
             preventDefault(event);
             const drawing = [ActiveControl.DRAW_POINTS, ActiveControl.DRAW_POLYGON,
                 ActiveControl.DRAW_POLYLINE, ActiveControl.DRAW_RECTANGLE,
-                ActiveControl.DRAW_CUBOID, ActiveControl.INTERACTION].includes(activeControl);
+                ActiveControl.DRAW_CUBOID, ActiveControl.AI_TOOLS].includes(activeControl);
 
             if (!drawing) {
                 canvasInstance.cancel();
@@ -98,7 +98,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
                     repeatDrawShape();
                 }
             } else {
-                if (activeControl === ActiveControl.INTERACTION) {
+                if (activeControl === ActiveControl.AI_TOOLS) {
                     // separated API method
                     canvasInstance.interact({ enabled: false });
                     return;

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: MIT
 
-import React from 'react';
+import React, { MutableRefObject } from 'react';
 import { connect } from 'react-redux';
 import Icon from 'antd/lib/icon';
 import Popover from 'antd/lib/popover';
@@ -13,9 +13,11 @@ import Text from 'antd/lib/typography/Text';
 import Tabs from 'antd/lib/tabs';
 import { Row, Col } from 'antd/lib/grid';
 import notification from 'antd/lib/notification';
+import Progress from 'antd/lib/progress';
 
 import { AIToolsIcon } from 'icons';
 import { Canvas } from 'cvat-canvas-wrapper';
+import range from 'utils/range';
 import getCore from 'cvat-core-wrapper';
 import {
     CombinedState,
@@ -32,6 +34,7 @@ import {
 } from 'actions/annotation-actions';
 import { InteractionResult } from 'cvat-canvas/src/typescript/canvas';
 import DetectorRunner from 'components/model-runner-modal/detector-runner';
+import InputNumber from 'antd/lib/input-number';
 
 interface StateToProps {
     canvasInstance: Canvas;
@@ -39,10 +42,13 @@ interface StateToProps {
     states: any[];
     activeLabelID: number;
     jobInstance: any;
-    isInteraction: boolean;
+    isActivated: boolean;
     frame: number;
     interactors: Model[];
     detectors: Model[];
+    trackers: Model[];
+    curZOrder: number;
+    aiToolsRef: MutableRefObject<any>;
 }
 
 interface DispatchToProps {
@@ -60,18 +66,21 @@ function mapStateToProps(state: CombinedState): StateToProps {
     const { instance: jobInstance } = annotation.job;
     const { instance: canvasInstance, activeControl } = annotation.canvas;
     const { models } = state;
-    const { interactors, detectors } = models;
+    const { interactors, detectors, trackers } = models;
 
     return {
         interactors,
         detectors,
-        isInteraction: activeControl === ActiveControl.INTERACTION,
+        trackers,
+        isActivated: activeControl === ActiveControl.AI_TOOLS,
         activeLabelID: annotation.drawing.activeLabelID,
         labels: annotation.job.labels,
         states: annotation.annotations.states,
         canvasInstance,
         jobInstance,
         frame,
+        curZOrder: annotation.annotations.zLayer.cur,
+        aiToolsRef: annotation.aiToolsRef,
     };
 }
 
@@ -103,10 +112,14 @@ interface State {
     activeInteractor: Model | null;
     activeLabelID: number;
     interactiveStateID: number | null;
+    activeTracker: Model | null;
+    trackingProgress: number | null;
+    trackingFrames: number;
     fetching: boolean;
+    mode: 'detection' | 'interaction' | 'tracking';
 }
 
-class ToolsControlComponent extends React.PureComponent<Props, State> {
+export class ToolsControlComponent extends React.PureComponent<Props, State> {
     private interactionIsAborted: boolean;
     private interactionIsDone: boolean;
 
@@ -114,9 +127,13 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         super(props);
         this.state = {
             activeInteractor: props.interactors.length ? props.interactors[0] : null,
+            activeTracker: props.trackers.length ? props.trackers[0] : null,
             activeLabelID: props.labels[0].id,
             interactiveStateID: null,
+            trackingProgress: null,
+            trackingFrames: 10,
             fetching: false,
+            mode: 'interaction',
         };
 
         this.interactionIsAborted = false;
@@ -124,16 +141,18 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
     }
 
     public componentDidMount(): void {
-        const { canvasInstance } = this.props;
+        const { canvasInstance, aiToolsRef } = this.props;
+        aiToolsRef.current = this;
         canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener);
         canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener);
     }
 
     public componentDidUpdate(prevProps: Props): void {
-        const { isInteraction } = this.props;
+        const { isActivated } = this.props;
-        if (prevProps.isInteraction && !isInteraction) {
+        if (prevProps.isActivated && !isActivated) {
             window.removeEventListener('contextmenu', this.contextmenuDisabler);
-        } else if (!prevProps.isInteraction && isInteraction) {
+        } else if (!prevProps.isActivated && isActivated) {
+            // reset flags when start interaction/tracking
             this.interactionIsDone = false;
             this.interactionIsAborted = false;
             window.addEventListener('contextmenu', this.contextmenuDisabler);
@@ -141,7 +160,8 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
     }
 
     public componentWillUnmount(): void {
-        const { canvasInstance } = this.props;
+        const { canvasInstance, aiToolsRef } = this.props;
+        aiToolsRef.current = undefined;
         canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener);
         canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener);
     }
@@ -162,14 +182,14 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
 
     private cancelListener = async (): Promise<void> => {
         const {
-            isInteraction,
+            isActivated,
             jobInstance,
             frame,
             fetchAnnotations,
         } = this.props;
         const { interactiveStateID, fetching } = this.state;
 
-        if (isInteraction) {
+        if (isActivated) {
             if (fetching && !this.interactionIsDone) {
                 // user pressed ESC
                 this.setState({ fetching: false });
@@ -187,12 +207,13 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         }
     };
 
-    private interactionListener = async (e: Event): Promise<void> => {
+    private onInteraction = async (e: Event): Promise<void> => {
         const {
             frame,
             labels,
+            curZOrder,
             jobInstance,
-            isInteraction,
+            isActivated,
             activeLabelID,
             fetchAnnotations,
             updateAnnotations,
@@ -200,8 +221,8 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         const { activeInteractor, interactiveStateID, fetching } = this.state;
 
         try {
-            if (!isInteraction) {
+            if (!isActivated) {
-                throw Error('Canvas raises event "canvas.interacted" when interaction is off');
+                throw Error('Canvas raises event "canvas.interacted" when interaction with it is off');
             }
 
             if (fetching) {
@@ -216,7 +237,6 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                 this.setState({ fetching: true });
                 try {
                     result = await core.lambda.call(jobInstance.task, interactor, {
-                        task: jobInstance.task,
                         frame,
                         points: convertShapesForInteractor((e as CustomEvent).detail.shapes),
                     });
@@ -241,7 +261,7 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                     shapeType: ShapeType.POLYGON,
                     points: result.flat(),
                     occluded: false,
-                    zOrder: (e as CustomEvent).detail.zOrder,
+                    zOrder: curZOrder,
                 });
 
                 await jobInstance.annotations.put([object]);
@@ -260,7 +280,7 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                         shapeType: ShapeType.POLYGON,
                         points: result.flat(),
                         occluded: false,
-                        zOrder: (e as CustomEvent).detail.zOrder,
+                        zOrder: curZOrder,
                     });
                     // need a clientID of a created object to interact with it further
                     // so, we do not use createAnnotationAction
@@ -302,6 +322,71 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         }
     };
 
+    private onTracking = async (e: Event): Promise<void> => {
+        const {
+            isActivated,
+            jobInstance,
+            frame,
+            curZOrder,
+            fetchAnnotations,
+        } = this.props;
+        const { activeLabelID } = this.state;
+        const [label] = jobInstance.task.labels.filter(
+            (_label: any): boolean => _label.id === activeLabelID,
+        );
+
+        if (!(e as CustomEvent).detail.isDone) {
+            return;
+        }
+
+        this.interactionIsDone = true;
+        try {
+            if (!isActivated) {
+                throw Error('Canvas raises event "canvas.interacted" when interaction with it is off');
+            }
+
+            const { points } = (e as CustomEvent).detail.shapes[0];
+            const state = new core.classes.ObjectState({
+                shapeType: ShapeType.RECTANGLE,
+                objectType: ObjectType.TRACK,
+                zOrder: curZOrder,
+                label,
+                points,
+                frame,
+                occluded: false,
+                source: 'auto',
+                attributes: {},
+            });
+
+            const [clientID] = await jobInstance.annotations.put([state]);
+
+            // update annotations on a canvas
+            fetchAnnotations();
+
+            const states = await jobInstance.annotations.get(frame);
+            const [objectState] = states
+                .filter((_state: any): boolean => _state.clientID === clientID);
+            await this.trackState(objectState);
+        } catch (err) {
+            notification.error({
+                description: err.toString(),
+                message: 'Tracking error occured',
+            });
+        }
+    };
+
+    private interactionListener = async (e: Event): Promise<void> => {
+        const { mode } = this.state;
+
+        if (mode === 'interaction') {
+            await this.onInteraction(e);
+        }
+
+        if (mode === 'tracking') {
+            await this.onTracking(e);
+        }
+    };
+
     private setActiveInteractor = (key: string): void => {
         const { interactors } = this.props;
         this.setState({
@@ -311,6 +396,72 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         });
     };
 
+    private setActiveTracker = (key: string): void => {
+        const { trackers } = this.props;
+        this.setState({
+            activeTracker: trackers.filter(
+                (tracker: Model) => tracker.id === key,
+            )[0],
+        });
+    };
+
+    public async trackState(state: any): Promise<void> {
+        const { jobInstance, frame } = this.props;
+        const { activeTracker, trackingFrames } = this.state;
+        const { clientID, points } = state;
+
+        const tracker = activeTracker as Model;
+        try {
+            this.setState({ trackingProgress: 0, fetching: true });
+            let response = await core.lambda.call(jobInstance.task, tracker, {
+                task: jobInstance.task,
+                frame,
+                shape: points,
+            });
+
+            for (const offset of range(1, trackingFrames + 1)) {
+                /* eslint-disable no-await-in-loop */
+                const states = await jobInstance.annotations.get(frame + offset);
+                const [objectState] = states
+                    .filter((_state: any): boolean => _state.clientID === clientID);
+                response = await core.lambda.call(jobInstance.task, tracker, {
+                    task: jobInstance.task,
+                    frame: frame + offset,
+                    shape: response.points,
+                    state: response.state,
+                });
+
+                const reduced = response.shape
+                    .reduce((acc: number[], value: number, index: number): number[] => {
+                        if (index % 2) { // y
+                            acc[1] = Math.min(acc[1], value);
+                            acc[3] = Math.max(acc[3], value);
+                        } else { // x
+                            acc[0] = Math.min(acc[0], value);
+                            acc[2] = Math.max(acc[2], value);
+                        }
+                        return acc;
+                    }, [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER,
+                        Number.MIN_SAFE_INTEGER, Number.MIN_SAFE_INTEGER,
+                    ]);
+
+                objectState.points = reduced;
+                await objectState.save();
+
+                this.setState({ trackingProgress: offset / trackingFrames });
+            }
+        } finally {
+            this.setState({ trackingProgress: null, fetching: false });
+        }
+    }
+
+    public trackingAvailable(): boolean {
+        const { activeTracker, trackingFrames } = this.state;
+        const { trackers } = this.props;
+
+        return !!trackingFrames && !!trackers.length && activeTracker !== null;
+    }
+
     private renderLabelBlock(): JSX.Element {
         const { labels } = this.props;
         const { activeLabelID } = this.state;
@@ -355,10 +506,119 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         );
     }
 
+    private renderTrackerBlock(): JSX.Element {
+        const {
+            trackers,
+            canvasInstance,
+            jobInstance,
+            frame,
+            onInteractionStart,
+        } = this.props;
+        const {
+            activeTracker,
+            activeLabelID,
+            fetching,
+            trackingFrames,
+        } = this.state;
+
+        if (!trackers.length) {
+            return (
+                <Row type='flex' justify='center' align='middle' style={{ marginTop: '5px' }}>
+                    <Col>
+                        <Text type='warning' className='cvat-text-color'>No available trackers found</Text>
+                    </Col>
+                </Row>
+            );
+        }
+
+        return (
+            <>
+                <Row type='flex' justify='start'>
+                    <Col>
+                        <Text className='cvat-text-color'>Tracker</Text>
+                    </Col>
+                </Row>
+                <Row type='flex' align='middle' justify='center'>
+                    <Col span={24}>
+                        <Select
+                            style={{ width: '100%' }}
+                            defaultValue={trackers[0].name}
+                            onChange={this.setActiveTracker}
+                        >
+                            {trackers.map((interactor: Model): JSX.Element => (
+                                <Select.Option title={interactor.description} key={interactor.id}>
+                                    {interactor.name}
+                                </Select.Option>
+                            ))}
+                        </Select>
+                    </Col>
+                </Row>
+                <Row type='flex' align='middle' justify='start' style={{ marginTop: '5px' }}>
+                    <Col>
+                        <Text>Tracking frames</Text>
+                    </Col>
+                    <Col offset={2}>
+                        <InputNumber
+                            value={trackingFrames}
+                            step={1}
+                            min={1}
+                            precision={0}
+                            max={jobInstance.stopFrame - frame}
+                            onChange={(value: number | undefined): void => {
+                                if (typeof (value) !== 'undefined') {
+                                    this.setState({
+                                        trackingFrames: value,
+                                    });
+                                }
+                            }}
+                        />
+                    </Col>
+                </Row>
+                <Row type='flex' align='middle' justify='end'>
+                    <Col>
+                        <Button
+                            type='primary'
+                            loading={fetching}
+                            className='cvat-tools-track-button'
+                            disabled={!activeTracker || fetching || frame === jobInstance.stopFrame}
+                            onClick={() => {
+                                this.setState({
+                                    mode: 'tracking',
+                                });
+
+                                if (activeTracker) {
+                                    canvasInstance.cancel();
+                                    canvasInstance.interact({
+                                        shapeType: 'rectangle',
+                                        enabled: true,
+                                    });
+
+                                    onInteractionStart(activeTracker, activeLabelID);
+                                }
+                            }}
+                        >
+                            Track
+                        </Button>
+                    </Col>
+                </Row>
+            </>
+        );
+    }
+
     private renderInteractorBlock(): JSX.Element {
         const { interactors, canvasInstance, onInteractionStart } = this.props;
         const { activeInteractor, activeLabelID, fetching } = this.state;
 
+        if (!interactors.length) {
+            return (
+                <Row type='flex' justify='center' align='middle' style={{ marginTop: '5px' }}>
+                    <Col>
+                        <Text type='warning' className='cvat-text-color'>No available interactors found</Text>
+                    </Col>
+                </Row>
+            );
+        }
+
         return (
             <>
                 <Row type='flex' justify='start'>
@@ -389,6 +649,10 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                             className='cvat-tools-interact-button'
                             disabled={!activeInteractor || fetching}
                             onClick={() => {
+                                this.setState({
+                                    mode: 'interaction',
+                                });
+
                                 if (activeInteractor) {
                                     canvasInstance.cancel();
                                     canvasInstance.interact({
@@ -413,10 +677,21 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
         const {
             jobInstance,
             detectors,
+            curZOrder,
             frame,
             fetchAnnotations,
         } = this.props;
 
+        if (!detectors.length) {
+            return (
+                <Row type='flex' justify='center' align='middle' style={{ marginTop: '5px' }}>
+                    <Col>
+                        <Text type='warning' className='cvat-text-color'>No available detectors found</Text>
+                    </Col>
+                </Row>
+            );
+        }
+
         return (
             <DetectorRunner
                 withCleanup={false}
@@ -424,6 +699,10 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                 task={jobInstance.task}
                 runInference={async (task: any, model: Model, body: object) => {
                     try {
+                        this.setState({
+                            mode: 'detection',
+                        });
+
                         this.setState({ fetching: true });
                         const result = await core.lambda.call(task, model, {
                             ...body,
@@ -444,7 +723,7 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                                     occluded: false,
                                     source: 'auto',
                                     attributes: {},
-                                    zOrder: 0, // TODO:  get current z order
+                                    zOrder: curZOrder,
                                 })
                             ));
 
@@ -471,7 +750,7 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                         <Text className='cvat-text-color' strong>AI Tools</Text>
                     </Col>
                 </Row>
-                <Tabs>
+                <Tabs type='card' tabBarGutter={8}>
                     <Tabs.TabPane key='interactors' tab='Interactors'>
                         { this.renderLabelBlock() }
                         { this.renderInteractorBlock() }
@@ -479,24 +758,34 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                     <Tabs.TabPane key='detectors' tab='Detectors'>
                         { this.renderDetectorBlock() }
                     </Tabs.TabPane>
+                    <Tabs.TabPane key='trackers' tab='Trackers'>
+                        { this.renderLabelBlock() }
+                        { this.renderTrackerBlock() }
+                    </Tabs.TabPane>
                 </Tabs>
             </div>
         );
     }
 
     public render(): JSX.Element | null {
-        const { interactors, isInteraction, canvasInstance } = this.props;
+        const {
-        const { fetching } = this.state;
+            interactors,
+            detectors,
+            trackers,
+            isActivated,
+            canvasInstance,
+        } = this.props;
+        const { fetching, trackingProgress } = this.state;
 
-        if (!interactors.length) return null;
+        if (![...interactors, ...detectors, ...trackers].length) return null;
 
-        const dynamcPopoverPros = isInteraction ? {
+        const dynamcPopoverPros = isActivated ? {
             overlayStyle: {
                 display: 'none',
             },
         } : {};
 
-        const dynamicIconProps = isInteraction ? {
+        const dynamicIconProps = isActivated ? {
             className: 'cvat-active-canvas-control cvat-tools-control',
             onClick: (): void => {
                 canvasInstance.interact({ enabled: false });
@@ -517,12 +806,15 @@ class ToolsControlComponent extends React.PureComponent<Props, State> {
                 >
                     <Text>Waiting for a server response..</Text>
                     <Icon style={{ marginLeft: '10px' }} type='loading' />
+                    { trackingProgress !== null && (
+                        <Progress percent={+(trackingProgress * 100).toFixed(0)} status='active' />
+                    )}
                 </Modal>
                 <Popover
                     {...dynamcPopoverPros}
                     placement='right'
                     overlayClassName='cvat-tools-control-popover'
-                    content={interactors.length && this.renderPopoverContent()}
+                    content={this.renderPopoverContent()}
                 >
                     <Icon {...dynamicIconProps} component={AIToolsIcon} />
                 </Popover>

cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx

@@ -41,6 +41,7 @@ interface Props {
     toBackground(): void;
     toForeground(): void;
     resetCuboidPerspective(): void;
+    activateTracking(): void;
 }
 
 function ItemTopComponent(props: Props): JSX.Element {
@@ -72,6 +73,7 @@ function ItemTopComponent(props: Props): JSX.Element {
         toBackground,
         toForeground,
         resetCuboidPerspective,
+        activateTracking,
     } = props;
 
     const [menuVisible, setMenuVisible] = useState(false);
@@ -150,6 +152,7 @@ function ItemTopComponent(props: Props): JSX.Element {
                         toForeground,
                         resetCuboidPerspective,
                         changeColorPickerVisible,
+                        activateTracking,
                     })}
                 >
                     <Icon type='more' />

cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx

@@ -33,16 +33,17 @@ interface Props {
     toBackgroundShortcut: string;
     toForegroundShortcut: string;
     removeShortcut: string;
-    changeColor: (value: string) => void;
+    changeColor(value: string): void;
-    copy: (() => void);
+    copy(): void;
-    remove: (() => void);
+    remove(): void;
-    propagate: (() => void);
+    propagate(): void;
-    createURL: (() => void);
+    createURL(): void;
-    switchOrientation: (() => void);
+    switchOrientation(): void;
-    toBackground: (() => void);
+    toBackground(): void;
-    toForeground: (() => void);
+    toForeground(): void;
-    resetCuboidPerspective: (() => void);
+    resetCuboidPerspective(): void;
-    changeColorPickerVisible: (visible: boolean) => void;
+    changeColorPickerVisible(visible: boolean): void;
+    activateTracking(): void;
 }
 
 export default function ItemMenu(props: Props): JSX.Element {
@@ -71,6 +72,7 @@ export default function ItemMenu(props: Props): JSX.Element {
         toForeground,
         resetCuboidPerspective,
         changeColorPickerVisible,
+        activateTracking,
     } = props;
 
     return (
@@ -94,6 +96,16 @@ export default function ItemMenu(props: Props): JSX.Element {
                     </Button>
                 </Tooltip>
             </Menu.Item>
+            {objectType === ObjectType.TRACK && shapeType === ShapeType.RECTANGLE && (
+                <Menu.Item>
+                    <Tooltip title='Run tracking with the active tracker' mouseLeaveDelay={0}>
+                        <Button type='link' onClick={activateTracking}>
+                            <Icon type='gateway' />
+                            Track
+                        </Button>
+                    </Tooltip>
+                </Menu.Item>
+            )}
             { [ShapeType.POLYGON, ShapeType.POLYLINE, ShapeType.CUBOID].includes(shapeType) && (
                 <Menu.Item>
                     <Button type='link' icon='retweet' onClick={switchOrientation}>

cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item.tsx

@@ -44,6 +44,7 @@ interface Props {
     changeColor(color: string): void;
     collapse(): void;
     resetCuboidPerspective(): void;
+    activateTracking(): void;
 }
 
 function objectItemsAreEqual(prevProps: Props, nextProps: Props): boolean {
@@ -94,6 +95,7 @@ function ObjectItemComponent(props: Props): JSX.Element {
         changeColor,
         collapse,
         resetCuboidPerspective,
+        activateTracking,
     } = props;
 
     const type = objectType === ObjectType.TAG ? ObjectType.TAG.toUpperCase()
@@ -142,6 +144,7 @@ function ObjectItemComponent(props: Props): JSX.Element {
                     toBackground={toBackground}
                     toForeground={toForeground}
                     resetCuboidPerspective={resetCuboidPerspective}
+                    activateTracking={activateTracking}
                 />
                 <ObjectButtonsContainer
                     clientID={clientID}

cvat-ui/src/components/annotation-page/standard-workspace/standard-workspace.tsx

@@ -13,6 +13,7 @@ import CanvasContextMenuContainer from 'containers/annotation-page/standard-work
 import ObjectSideBarComponent from 'components/annotation-page/standard-workspace/objects-side-bar/objects-side-bar';
 import CanvasPointContextMenuComponent from 'components/annotation-page/standard-workspace/canvas-point-context-menu';
 
+
 export default function StandardWorkspaceComponent(): JSX.Element {
     return (
         <Layout hasSider className='cvat-standard-workspace'>

cvat-ui/src/components/annotation-page/standard-workspace/styles.scss

@@ -92,6 +92,7 @@
     }
 }
 
+.cvat-tools-track-button,
 .cvat-tools-interact-button {
     width: 100%;
     margin-top: 10px;
@@ -102,7 +103,7 @@
 }
 
 .cvat-tools-control-popover-content {
-    width: 350px;
+    width: fit-content;
     padding: 10px;
     border-radius: 5px;
     background: $background-color-2;

cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: MIT
 
-import React from 'react';
+import React, { MutableRefObject } from 'react';
 import copy from 'copy-to-clipboard';
 import { connect } from 'react-redux';
 
@@ -26,6 +26,7 @@ import {
 } from 'actions/annotation-actions';
 
 import ObjectStateItemComponent from 'components/annotation-page/standard-workspace/objects-side-bar/object-item';
+import { ToolsControlComponent } from 'components/annotation-page/standard-workspace/controls-side-bar/tools-control';
 import { shift } from 'utils/math';
 
 interface OwnProps {
@@ -47,6 +48,7 @@ interface StateToProps {
     minZLayer: number;
     maxZLayer: number;
     normalizedKeyMap: Record<string, string>;
+    aiToolsRef: MutableRefObject<ToolsControlComponent>;
 }
 
 interface DispatchToProps {
@@ -86,6 +88,7 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps {
                 ready,
                 activeControl,
             },
+            aiToolsRef,
         },
         settings: {
             shapes: {
@@ -118,6 +121,7 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps {
         minZLayer,
         maxZLayer,
         normalizedKeyMap,
+        aiToolsRef,
     };
 }
 
@@ -259,6 +263,13 @@ class ObjectItemContainer extends React.PureComponent<Props> {
         collapseOrExpand([objectState], !collapsed);
     };
 
+    private activateTracking = (): void => {
+        const { objectState, aiToolsRef } = this.props;
+        if (aiToolsRef.current && aiToolsRef.current.trackingAvailable()) {
+            aiToolsRef.current.trackState(objectState);
+        }
+    };
+
     private changeColor = (color: string): void => {
         const {
             objectState,
@@ -402,6 +413,7 @@ class ObjectItemContainer extends React.PureComponent<Props> {
                 changeLabel={this.changeLabel}
                 changeAttribute={this.changeAttribute}
                 collapse={this.collapse}
+                activateTracking={this.activateTracking}
                 resetCuboidPerspective={() => this.resetCuboidPerspective()}
             />
         );

cvat-ui/src/reducers/annotation-reducer.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: MIT
 
+import React from 'react';
 import { AnyAction } from 'redux';
 
 import { Canvas, CanvasMode } from 'cvat-canvas-wrapper';
@@ -93,6 +94,7 @@ const defaultState: AnnotationState = {
         collecting: false,
         data: null,
     },
+    aiToolsRef: React.createRef(),
     colors: [],
     sidebarCollapsed: false,
     appearanceCollapsed: false,
@@ -1058,7 +1060,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
                 },
                 canvas: {
                     ...state.canvas,
-                    activeControl: ActiveControl.INTERACTION,
+                    activeControl: ActiveControl.AI_TOOLS,
                 },
             };
         }

cvat-ui/src/reducers/interfaces.ts

@@ -4,6 +4,7 @@
 
 import { ExtendedKeyMapOptions } from 'react-hotkeys';
 import { Canvas, RectDrawingMethod } from 'cvat-canvas-wrapper';
+import { MutableRefObject } from 'react';
 
 export type StringObject = {
     [index: string]: string;
@@ -275,7 +276,7 @@ export enum ActiveControl {
     GROUP = 'group',
     SPLIT = 'split',
     EDIT = 'edit',
-    INTERACTION = 'interaction',
+    AI_TOOLS = 'ai_tools',
 }
 
 export enum ShapeType {
@@ -394,6 +395,7 @@ export interface AnnotationState {
     appearanceCollapsed: boolean;
     tabContentHeight: number;
     workspace: Workspace;
+    aiToolsRef: MutableRefObject<any>;
 }
 
 export enum Workspace {

cvat-ui/src/utils/range.ts

@@ -0,0 +1,27 @@
+// Copyright (C) 2020 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+export default function range(x: number, y?: number): number[] {
+    if (typeof (x) !== 'undefined' && typeof (y) !== 'undefined') {
+        if (typeof (x) !== 'number' && typeof (y) !== 'number') {
+            throw new Error(`Range() expects number arguments. Got ${typeof (x)}, ${typeof (y)}`);
+        }
+
+        if (x >= y) {
+            throw new Error(`Range() expects the first argument less or equal than the second. Got ${x}, ${y}`);
+        }
+
+        return Array.from(Array(y - x), (_: number, i: number) => i + x);
+    }
+
+    if (typeof (x) !== 'undefined') {
+        if (typeof (x) !== 'number') {
+            throw new Error(`Range() expects number arguments. Got ${typeof (x)}`);
+        }
+
+        return [...Array(x).keys()];
+    }
+
+    return [];
+}