diff --git a/CHANGELOG.md b/CHANGELOG.md index 78445bd6..65c9ec5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- +- Support of context images for 2D image tasks () ### Changed -- +- Updated manifest format, added meta with related images () ### Deprecated @@ -111,7 +111,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed filters select overflow () - Fixed tasks in project auto annotation () - Cuboids are missed in annotations statistics () -- The list of files attached to the task is not displayed () - A couple of css-related issues (top bar disappear, wrong arrow position on collapse elements) () - Issue with point region doesn't work in Firefox () - Fixed cuboid perspective change () diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 5fecc130..2bdc1118 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.12.1", + "version": "3.13.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index a04727d5..e5e482fd 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.12.1", + "version": "3.13.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-core/src/frames.js b/cvat-core/src/frames.js index bd1009ef..478b6c65 100644 --- a/cvat-core/src/frames.js +++ b/cvat-core/src/frames.js @@ -19,7 +19,15 @@ */ class FrameData { constructor({ - width, height, name, taskID, frameNumber, startFrame, stopFrame, decodeForward, + width, + height, + name, + taskID, + frameNumber, + startFrame, + stopFrame, + decodeForward, + has_related_context: hasRelatedContext, }) { Object.defineProperties( this, @@ -72,6 +80,18 @@ value: frameNumber, writable: false, }, + /** + * True if some context images are associated with this frame + * @name hasRelatedContext + * @type {boolean} + * @memberof module:API.cvat.classes.FrameData + * @readonly + * @instance + */ + hasRelatedContext: { + value: hasRelatedContext, + writable: false, + }, startFrame: { value: startFrame, writable: false, diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index f3627d4e..cf31969a 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -756,11 +756,7 @@ }, ); } catch (errorData) { - const code = errorData.response ? errorData.response.status : errorData.code; - throw new ServerError( - `Could not get Image Context of the frame for the task ${tid} from the server`, - code, - ); + throw generateError(errorData); } return response.data; diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index ae3f43c1..c2a48c19 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.19.1", + "version": "1.20.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index ceda7cb0..f15cd393 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.19.1", + "version": "1.20.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 898306a0..0dd6eb55 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -196,6 +196,8 @@ export enum AnnotationActionTypes { GET_PREDICTIONS_SUCCESS = 'GET_PREDICTIONS_SUCCESS', HIDE_SHOW_CONTEXT_IMAGE = 'HIDE_SHOW_CONTEXT_IMAGE', GET_CONTEXT_IMAGE = 'GET_CONTEXT_IMAGE', + GET_CONTEXT_IMAGE_SUCCESS = 'GET_CONTEXT_IMAGE_SUCCESS', + GET_CONTEXT_IMAGE_FAILED = 'GET_CONTEXT_IMAGE_FAILED', } export function saveLogsAsync(): ThunkAction { @@ -715,6 +717,7 @@ export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameSte number: state.annotation.player.frame.number, data: state.annotation.player.frame.data, filename: state.annotation.player.frame.filename, + hasRelatedContext: state.annotation.player.frame.hasRelatedContext, delay: state.annotation.player.frame.delay, changeTime: state.annotation.player.frame.changeTime, states: state.annotation.annotations.states, @@ -766,6 +769,7 @@ export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameSte number: toFrame, data, filename: data.filename, + hasRelatedContext: data.hasRelatedContext, states, minZ, maxZ, @@ -1031,6 +1035,7 @@ export function getJobAsync(tid: number, jid: number, initialFrame: number, init states, frameNumber, frameFilename: frameData.filename, + frameHasRelatedContext: frameData.hasRelatedContext, frameData, colors, filters, @@ -1636,31 +1641,23 @@ export function getContextImage(): ThunkAction { return async (dispatch: ActionCreator): Promise => { const state: CombinedState = getStore().getState(); const { instance: job } = state.annotation.job; - const { frame, contextImage } = state.annotation.player; + const { number: frameNumber } = state.annotation.player.frame; try { - const context = await job.frames.contextImage(job.task.id, frame.number); - const loaded = true; - const contextImageHide = contextImage.hidden; dispatch({ type: AnnotationActionTypes.GET_CONTEXT_IMAGE, - payload: { - context, - loaded, - contextImageHide, - }, + payload: {}, + }); + + const contextImageData = await job.frames.contextImage(job.task.id, frameNumber); + dispatch({ + type: AnnotationActionTypes.GET_CONTEXT_IMAGE_SUCCESS, + payload: { contextImageData }, }); } catch (error) { - const context = ''; - const loaded = true; - const contextImageHide = contextImage.hidden; dispatch({ - type: AnnotationActionTypes.GET_CONTEXT_IMAGE, - payload: { - context, - loaded, - contextImageHide, - }, + type: AnnotationActionTypes.GET_CONTEXT_IMAGE_FAILED, + payload: { error }, }); } }; diff --git a/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx b/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx index 5909816a..595613d5 100644 --- a/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx @@ -18,6 +18,7 @@ import getCore from 'cvat-core-wrapper'; import consts from 'consts'; import CVATTooltip from 'components/common/cvat-tooltip'; import ImageSetupsContent from './image-setups-content'; +import ContextImage from '../standard-workspace/context-image/context-image'; const cvat = getCore(); @@ -773,12 +774,12 @@ export default class CanvasWrapperComponent extends React.PureComponent { maxZLayer, curZLayer, minZLayer, - onSwitchZLayer, - onAddZLayer, keyMap, switchableAutomaticBordering, automaticBordering, onSwitchAutomaticBordering, + onSwitchZLayer, + onAddZLayer, } = this.props; const preventDefault = (event: KeyboardEvent | undefined): void => { @@ -817,6 +818,8 @@ export default class CanvasWrapperComponent extends React.PureComponent { }} /> + + }> diff --git a/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx b/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx index 654cbb13..823a62fa 100644 --- a/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx @@ -14,20 +14,16 @@ import { Workspace } from 'reducers/interfaces'; import { CAMERA_ACTION, Canvas3d, MouseInteraction, ViewType, } from 'cvat-canvas3d-wrapper'; -import ContextImage from '../standard3D-workspace/context-image/context-image'; -import CVATTooltip from '../../common/cvat-tooltip'; +import ContextImage from 'components/annotation-page/standard-workspace/context-image/context-image'; +import CVATTooltip from 'components/common/cvat-tooltip'; interface Props { canvasInstance: Canvas3d; jobInstance: any; frameData: any; curZLayer: number; - contextImageHide: boolean; - loaded: boolean; - data: string; annotations: any[]; onSetupCanvas: () => void; - getContextImage(): void; onResetCanvas(): void; workspace: Workspace; animateID: any; @@ -119,9 +115,7 @@ const CanvasWrapperComponent = (props: Props): ReactElement => { const sideView = useRef(null); const frontView = useRef(null); - const { - frameData, contextImageHide, getContextImage, loaded, data, annotations, curZLayer, - } = props; + const { frameData, annotations, curZLayer } = props; const onCanvasSetup = (): void => { const { onSetupCanvas } = props; @@ -345,13 +339,7 @@ const CanvasWrapperComponent = (props: Props): ReactElement => { return ( - + state.annotation.player.frame); + const { data: contextImageData, hidden: contextImageHidden, fetching: contextImageFetching } = useSelector( + (state: CombinedState) => state.annotation.player.contextImage, + ); + const [requested, setRequested] = useState(false); + + useEffect(() => { + if (requested) { + setRequested(false); + } + }, [frame]); + + useEffect(() => { + if (hasRelatedContext && !contextImageHidden && !requested) { + dispatch(getContextImage()); + setRequested(true); + } + }, [contextImageHidden, requested, hasRelatedContext]); + + if (!hasRelatedContext) { + return null; + } + + return ( +
+
+ {contextImageFetching ? : null} + {contextImageHidden ? ( + + dispatch(hideShowContextImage(false))} + /> + + ) : ( + <> + dispatch(hideShowContextImage(true))} + /> + { + notification.error({ + message: 'Could not display context image', + description: `Source is ${ + contextImageData === null ? 'empty' : contextImageData.slice(0, 100) + }`, + }); + }} + className='cvat-context-image' + /> + + )} +
+ ); +} diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/control-visibility-observer.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/control-visibility-observer.tsx index c900c306..54e99d04 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/control-visibility-observer.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/control-visibility-observer.tsx @@ -41,7 +41,7 @@ export function ExtraControlsControl(): JSX.Element { > ); diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss index 81c60a53..86e819ff 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss +++ b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss @@ -8,6 +8,55 @@ height: 100%; } +.cvat-context-image-wrapper { + height: auto; + width: $grid-unit-size * 32; + position: absolute; + top: $grid-unit-size; + right: $grid-unit-size; + z-index: 100; + background: black; + display: flex; + flex-direction: column; + justify-content: space-between; + user-select: none; + + > .cvat-context-image-wrapper-header { + height: $grid-unit-size * 4; + width: 100%; + z-index: 101; + background: rgba(0, 0, 0, 0.2); + position: absolute; + top: 0; + left: 0; + } + + > .ant-image { + margin: $grid-unit-size / 2; + } + + > span { + position: absolute; + font-size: 18px; + top: 7px; + right: 7px; + z-index: 102; + color: white; + + &:hover { + > svg { + transform: scale(1.2); + } + } + } +} + +.cvat-context-image { + width: 100%; + height: auto; + display: block; +} + .cvat-objects-sidebar-sider { top: 0; right: 0; @@ -56,8 +105,7 @@ .cvat-issue-control, .cvat-tools-control, .cvat-extra-controls-control, -.cvat-opencv-control, -.cvat-context-image-control { +.cvat-opencv-control { border-radius: 3.3px; transform: scale(0.65); padding: 2px; @@ -76,7 +124,7 @@ } } -.cvat-extra-controls-control { +.cvat-antd-icon-control { > svg { width: 40px; height: 40px; diff --git a/cvat-ui/src/components/annotation-page/standard3D-workspace/context-image/context-image.tsx b/cvat-ui/src/components/annotation-page/standard3D-workspace/context-image/context-image.tsx deleted file mode 100644 index cdb114d7..00000000 --- a/cvat-ui/src/components/annotation-page/standard3D-workspace/context-image/context-image.tsx +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (C) 2021 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React, { useEffect } from 'react'; - -interface Props { - frame: number; - contextImageHide: boolean; - loaded: boolean; - data: string; - getContextImage(): void; -} - -export default function ContextImage(props: Props): JSX.Element { - const { - contextImageHide, loaded, data, getContextImage, - } = props; - - useEffect(() => { - if (!contextImageHide && !loaded) { - getContextImage(); - } - }, [contextImageHide, loaded]); - - if (!contextImageHide && data !== '') { - return ( -
- Context not available -
- ); - } - return null; -} diff --git a/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx index 304f8ebb..985bc669 100644 --- a/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx @@ -9,20 +9,15 @@ import { Canvas3d as Canvas } from 'cvat-canvas3d-wrapper'; import CursorControl from './cursor-control'; import MoveControl from './move-control'; import DrawCuboidControl from './draw-cuboid-control'; -import PhotoContextControl from './photo-context'; interface Props { canvasInstance: Canvas; activeControl: ActiveControl; normalizedKeyMap: Record; - contextImageHide: boolean; - hideShowContextImage: (hidden: boolean) => void; } export default function ControlsSideBarComponent(props: Props): JSX.Element { - const { - canvasInstance, activeControl, normalizedKeyMap, contextImageHide, hideShowContextImage, - } = props; + const { canvasInstance, activeControl, normalizedKeyMap } = props; return ( @@ -37,12 +32,6 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { canvasInstance={canvasInstance} isDrawing={activeControl === ActiveControl.DRAW_CUBOID} /> - ); } diff --git a/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/photo-context.tsx b/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/photo-context.tsx deleted file mode 100644 index 3a7f1c6c..00000000 --- a/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/photo-context.tsx +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (C) 2021 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import CameraIcon from '@ant-design/icons/CameraOutlined'; - -import CVATTooltip from 'components/common/cvat-tooltip'; -import { Canvas3d as Canvas } from 'cvat-canvas3d-wrapper'; -import { ActiveControl } from 'reducers/interfaces'; - -interface Props { - canvasInstance: Canvas; - activeControl: ActiveControl; - hideShowContextImage: (hidden: boolean) => void; - contextImageHide: boolean; -} - -function PhotoContextControl(props: Props): JSX.Element { - const { activeControl, contextImageHide, hideShowContextImage } = props; - - return ( - - { - hideShowContextImage(!contextImageHide); - }} - /> - - ); -} - -export default React.memo(PhotoContextControl); diff --git a/cvat-ui/src/components/annotation-page/standard3D-workspace/styles.scss b/cvat-ui/src/components/annotation-page/standard3D-workspace/styles.scss index ad321301..f58af1d2 100644 --- a/cvat-ui/src/components/annotation-page/standard3D-workspace/styles.scss +++ b/cvat-ui/src/components/annotation-page/standard3D-workspace/styles.scss @@ -4,173 +4,12 @@ @import 'base.scss'; -.cvat-standard-workspace.ant-layout { - height: 100%; -} - -.cvat-contextImage { - width: $grid-unit-size * 32; - position: absolute; - background: $border-color-3; - top: $grid-unit-size; - right: $grid-unit-size; - z-index: 100; - border-radius: $grid-unit-size; - border: 1px solid $border-color-3; - display: flex; - flex-direction: column; - justify-content: space-between; - padding: $grid-unit-size/2; -} - -.cvat-contextImage-show { - max-width: 100%; - max-height: 100%; -} - -.cvat-contextImage-loading { - text-align: center; -} - -.cvat-objects-sidebar-filter-input { - width: calc(100% - 35px); -} - -.cvat-objects-sidebar-sider { - top: 0; - right: 0; - left: auto; - background-color: $background-color-2; - border-left: 1px solid $border-color-1; - border-bottom: 1px solid $border-color-1; - border-radius: $grid-unit-size/2 0 0 $grid-unit-size/2; - z-index: 2; -} - -.cvat-objects-sidebar { - height: 100%; -} - -.cvat-rotate-canvas-controls-right > svg { - transform: scaleX(-1); -} - -.cvat-canvas-controls-sidebar { - background-color: $background-color-2; - border-right: 1px solid $border-color-1; - - > div { - > i { - border-radius: 3.3px; - transform: scale(0.65); - padding: $grid-unit-size/4; - - &:hover { - background: $header-color; - transform: scale(0.75); - } - - &:active { - transform: scale(0.65); - } - - > svg { - transform: scale(0.8); - } - } - } -} - -.cvat-active-canvas-control { - background: $header-color; - transform: scale(0.75); -} - -.cvat-rotate-canvas-controls-left, -.cvat-rotate-canvas-controls-right { - transform: scale(0.65); - border-radius: $grid-unit-size/2; - - &:hover { - transform: scale(0.75); - } - - &:active { - transform: scale(0.65); - } -} - -.cvat-rotate-canvas-controls > .ant-popover-content > .ant-popover-inner > div > .ant-popover-inner-content { - padding: 0; -} - -.cvat-draw-shape-popover, -.cvat-tools-control-popover { - > .ant-popover-content > .ant-popover-inner > div > .ant-popover-inner-content { - padding: 0; - } -} - -.cvat-tools-track-button, -.cvat-tools-interact-button { - width: 100%; - margin-top: $grid-unit-size; -} - -.cvat-draw-shape-popover-points-selector { - width: 100%; -} - -.cvat-tools-control-popover-content { - width: fit-content; - padding: $grid-unit-size; - border-radius: $grid-unit-size/2; - background: $background-color-2; -} - -.cvat-draw-shape-popover-content { - padding: $grid-unit-size; - border-radius: $grid-unit-size/2; - background: $background-color-2; - width: 270px; - - > div { - margin-top: $grid-unit-size/2; - } - - > div:nth-child(3) > div > div { - width: 100%; - } - - > div:last-child { - span { - width: 100%; - } - - button { - width: 100%; - - &:nth-child(1) { - border-radius: $grid-unit-size/2 0 0 $grid-unit-size/2; - } - - &:nth-child(2) { - border-radius: 0 $grid-unit-size/2 $grid-unit-size/2 0; - } - } - } -} - .cvat-canvas-container-overflow { overflow: hidden; width: 100%; height: 100%; } -.cvat-control-side-bar-icon-size { - font-size: $grid-unit-size * 5; -} - .cvat-canvas3d-perspective { height: 100%; width: 100%; diff --git a/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx b/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx index cae83c7c..d64c338a 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx @@ -57,7 +57,6 @@ interface Props { onUndoClick(): void; onRedoClick(): void; jobInstance: any; - hideShowContextImage(): any; } export default function AnnotationTopBarComponent(props: Props): JSX.Element { diff --git a/cvat-ui/src/containers/annotation-page/canvas/canvas-wrapper3D.tsx b/cvat-ui/src/containers/annotation-page/canvas/canvas-wrapper3D.tsx index f65bd054..bcb328ad 100644 --- a/cvat-ui/src/containers/annotation-page/canvas/canvas-wrapper3D.tsx +++ b/cvat-ui/src/containers/annotation-page/canvas/canvas-wrapper3D.tsx @@ -5,7 +5,7 @@ import { connect } from 'react-redux'; import CanvasWrapperComponent from 'components/annotation-page/canvas/canvas-wrapper3D'; -import { confirmCanvasReady, getContextImage, resetCanvas } from 'actions/annotation-actions'; +import { confirmCanvasReady, resetCanvas } from 'actions/annotation-actions'; import { CombinedState } from 'reducers/interfaces'; @@ -16,15 +16,11 @@ interface StateToProps { jobInstance: any; frameData: any; curZLayer: number; - contextImageHide: boolean; - loaded: boolean; - data: string; annotations: any[]; } interface DispatchToProps { onSetupCanvas(): void; - getContextImage(): void; onResetCanvas(): void; } @@ -35,7 +31,6 @@ function mapStateToProps(state: CombinedState): StateToProps { job: { instance: jobInstance }, player: { frame: { data: frameData }, - contextImage: { hidden: contextImageHide, data, loaded }, }, annotations: { states: annotations, @@ -49,9 +44,6 @@ function mapStateToProps(state: CombinedState): StateToProps { jobInstance, frameData, curZLayer, - contextImageHide, - loaded, - data, annotations, }; } @@ -61,9 +53,6 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { onSetupCanvas(): void { dispatch(confirmCanvasReady()); }, - getContextImage(): void { - dispatch(getContextImage()); - }, onResetCanvas(): void { dispatch(resetCanvas()); }, diff --git a/cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx index 59d6b7d2..a54c496a 100644 --- a/cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx @@ -2,11 +2,10 @@ // // SPDX-License-Identifier: MIT -import { KeyMap } from 'utils/mousetrap-react'; import { connect } from 'react-redux'; +import { KeyMap } from 'utils/mousetrap-react'; import { Canvas } from 'cvat-canvas-wrapper'; -import { hideShowContextImage } from 'actions/annotation-actions'; import ControlsSideBarComponent from 'components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar'; import { ActiveControl, CombinedState } from 'reducers/interfaces'; @@ -15,21 +14,13 @@ interface StateToProps { activeControl: ActiveControl; keyMap: KeyMap; normalizedKeyMap: Record; - contextImageHide: boolean; loaded: boolean; } -interface DispatchToProps { - hideShowContextImage(hidden: boolean): void; -} - function mapStateToProps(state: CombinedState): StateToProps { const { annotation: { canvas: { instance: canvasInstance, activeControl }, - player: { - contextImage: { hidden: contextImageHide, loaded }, - }, }, shortcuts: { keyMap, normalizedKeyMap }, } = state; @@ -39,17 +30,7 @@ function mapStateToProps(state: CombinedState): StateToProps { activeControl, normalizedKeyMap, keyMap, - contextImageHide, - loaded, - }; -} - -function dispatchToProps(dispatch: any): DispatchToProps { - return { - hideShowContextImage(hidden: boolean): void { - dispatch(hideShowContextImage(hidden)); - }, }; } -export default connect(mapStateToProps, dispatchToProps)(ControlsSideBarComponent); +export default connect(mapStateToProps)(ControlsSideBarComponent); diff --git a/cvat-ui/src/reducers/annotation-reducer.ts b/cvat-ui/src/reducers/annotation-reducer.ts index 22426c4f..927845db 100644 --- a/cvat-ui/src/reducers/annotation-reducer.ts +++ b/cvat-ui/src/reducers/annotation-reducer.ts @@ -51,6 +51,7 @@ const defaultState: AnnotationState = { number: 0, filename: '', data: null, + hasRelatedContext: false, fetching: false, delay: 0, changeTime: null, @@ -58,8 +59,8 @@ const defaultState: AnnotationState = { playing: false, frameAngles: [], contextImage: { - loaded: false, - data: '', + fetching: false, + data: null, hidden: false, }, }, @@ -145,6 +146,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { openTime, frameNumber: number, frameFilename: filename, + frameHasRelatedContext, colors, filters, frameData: data, @@ -189,6 +191,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { frame: { ...state.player.frame, filename, + hasRelatedContext: frameHasRelatedContext, number, data, }, @@ -226,11 +229,6 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { ...state.player.frame, fetching: false, }, - contextImage: { - loaded: false, - data: '', - hidden: state.player.contextImage.hidden, - }, }, }; } @@ -252,7 +250,16 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { } case AnnotationActionTypes.CHANGE_FRAME_SUCCESS: { const { - number, data, filename, states, minZ, maxZ, curZ, delay, changeTime, + number, + data, + filename, + hasRelatedContext, + states, + minZ, + maxZ, + curZ, + delay, + changeTime, } = action.payload; const activatedStateID = states @@ -268,6 +275,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { frame: { data, filename, + hasRelatedContext, number, fetching: false, changeTime, @@ -275,7 +283,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { }, contextImage: { ...state.player.contextImage, - loaded: false, + data: null, }, }, annotations: { @@ -1170,30 +1178,52 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { } case AnnotationActionTypes.HIDE_SHOW_CONTEXT_IMAGE: { const { hidden } = action.payload; - const { loaded, data } = state.player.contextImage; return { ...state, player: { ...state.player, contextImage: { - loaded, - data, + ...state.player.contextImage, hidden, }, }, }; } case AnnotationActionTypes.GET_CONTEXT_IMAGE: { - const { context, loaded } = action.payload; + return { + ...state, + player: { + ...state.player, + contextImage: { + ...state.player.contextImage, + fetching: true, + }, + }, + }; + } + case AnnotationActionTypes.GET_CONTEXT_IMAGE_SUCCESS: { + const { contextImageData } = action.payload; return { ...state, player: { ...state.player, contextImage: { - loaded, - data: context, - hidden: state.player.contextImage.hidden, + ...state.player.contextImage, + fetching: false, + data: contextImageData, + }, + }, + }; + } + case AnnotationActionTypes.GET_CONTEXT_IMAGE_FAILED: { + return { + ...state, + player: { + ...state.player, + contextImage: { + ...state.player.contextImage, + fetching: false, }, }, }; diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 4ddeb301..2f067eca 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -269,6 +269,7 @@ export interface NotificationsState { saving: null | ErrorState; jobFetching: null | ErrorState; frameFetching: null | ErrorState; + contextImageFetching: null | ErrorState; changingLabelColor: null | ErrorState; updating: null | ErrorState; creating: null | ErrorState; @@ -417,6 +418,7 @@ export interface AnnotationState { frame: { number: number; filename: string; + hasRelatedContext: boolean; data: any | null; fetching: boolean; delay: number; @@ -425,8 +427,8 @@ export interface AnnotationState { playing: boolean; frameAngles: number[]; contextImage: { - loaded: boolean; - data: string; + fetching: boolean; + data: string | null; hidden: boolean; }; }; diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index 497117be..ae424729 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -69,6 +69,7 @@ const defaultState: NotificationsState = { saving: null, jobFetching: null, frameFetching: null, + contextImageFetching: null, changingLabelColor: null, updating: null, creating: null, @@ -689,6 +690,21 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } + case AnnotationActionTypes.GET_CONTEXT_IMAGE_FAILED: { + return { + ...state, + errors: { + ...state.errors, + annotation: { + ...state.errors.annotation, + contextImageFetching: { + message: 'Could not fetch context image from the server', + reason: action.payload.error, + }, + }, + }, + }; + } case AnnotationActionTypes.SAVE_ANNOTATIONS_FAILED: { return { ...state, diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index dda3956e..ffa58deb 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -9,7 +9,6 @@ import zipfile import io import itertools import struct -import re from abc import ABC, abstractmethod from contextlib import closing @@ -112,6 +111,10 @@ class ImageListReader(IMediaReader): for i in range(self._start, self._stop, self._step): yield (self.get_image(i), self.get_path(i), i) + def filter(self, callback): + source_path = list(filter(callback, self._source_path)) + ImageListReader.__init__(self, source_path, step=self._step, start=self._start, stop=self._stop) + def get_path(self, i): return self._source_path[i] @@ -199,7 +202,7 @@ class ZipReader(ImageListReader): self._zip_source = zipfile.ZipFile(source_path[0], mode='a') self.extract_dir = source_path[1] if len(source_path) > 1 else None file_list = [f for f in self._zip_source.namelist() if files_to_ignore(f) and get_mime(f) == 'image'] - super().__init__(file_list, step, start, stop) + super().__init__(file_list, step=step, start=start, stop=stop) def __del__(self): self._zip_source.close() @@ -759,66 +762,6 @@ class ValidateDimension: self.image_files[file_name] = file_path return pcd_files - def validate_velodyne_points(self, *args): - root, actual_path, files = args - velodyne_files = self.process_files(root, actual_path, files) - related_path = os.path.split(os.path.split(root)[0])[0] - - path_list = [re.search(r'image_\d.*', path, re.IGNORECASE) for path in os.listdir(related_path) if - os.path.isdir(os.path.join(related_path, path))] - - for path_ in path_list: - if path_: - path = os.path.join(path_.group(), "data") - path = os.path.abspath(os.path.join(related_path, path)) - - files = [file for file in os.listdir(path) if - os.path.isfile(os.path.abspath(os.path.join(path, file)))] - for file in files: - - f_name = file.split(".")[0] - if velodyne_files.get(f_name, None): - self.related_files[velodyne_files[f_name]].append( - os.path.abspath(os.path.join(path, file))) - - def validate_pointcloud(self, *args): - root, actual_path, files = args - pointcloud_files = self.process_files(root, actual_path, files) - related_path = root.rsplit("/pointcloud", 1)[0] - related_images_path = os.path.join(related_path, "related_images") - - if os.path.isdir(related_images_path): - paths = [path for path in os.listdir(related_images_path) if - os.path.isdir(os.path.abspath(os.path.join(related_images_path, path)))] - - for k in pointcloud_files: - for path in paths: - - if k == path.rsplit("_", 1)[0]: - file_path = os.path.abspath(os.path.join(related_images_path, path)) - files = [file for file in os.listdir(file_path) if - os.path.isfile(os.path.join(file_path, file))] - for related_image in files: - self.related_files[pointcloud_files[k]].append(os.path.join(file_path, related_image)) - - def validate_default(self, *args): - root, actual_path, files = args - pcd_files = self.process_files(root, actual_path, files) - if len(list(pcd_files.keys())): - - for image in self.image_files.keys(): - if pcd_files.get(image, None): - self.related_files[pcd_files[image]].append(self.image_files[image]) - - current_directory_name = os.path.split(root) - - if len(pcd_files.keys()) == 1: - pcd_name = list(pcd_files.keys())[0].rsplit(".", 1)[0] - if current_directory_name[1] == pcd_name: - for related_image in self.image_files.values(): - if root == os.path.split(related_image)[0]: - self.related_files[pcd_files[pcd_name]].append(related_image) - def validate(self): """ Validate the directory structure for kitty and point cloud format. @@ -830,15 +773,7 @@ class ValidateDimension: if not files_to_ignore(root): continue - if root.endswith("data"): - if os.path.split(os.path.split(root)[0])[1] == "velodyne_points": - self.validate_velodyne_points(root, actual_path, files) - - elif os.path.split(root)[-1] == "pointcloud": - self.validate_pointcloud(root, actual_path, files) - - else: - self.validate_default(root, actual_path, files) + self.process_files(root, actual_path, files) if len(self.related_files.keys()): self.dimension = DimensionType.DIM_3D diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index dfbe4fa6..6e0cacc8 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -545,6 +545,7 @@ class FrameMetaSerializer(serializers.Serializer): width = serializers.IntegerField() height = serializers.IntegerField() name = serializers.CharField(max_length=1024) + has_related_context = serializers.BooleanField() class PluginsSerializer(serializers.Serializer): GIT_INTEGRATION = serializers.BooleanField() diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index e24865c7..656c3527 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -7,6 +7,7 @@ import itertools import os import sys import rq +import re import shutil from traceback import print_exception from urllib import parse as urlparse @@ -19,6 +20,7 @@ from cvat.apps.engine.utils import av_scan_paths from cvat.apps.engine.models import DimensionType from utils.dataset_manifest import ImageManifestManager, VideoManifestManager from utils.dataset_manifest.core import VideoManifestValidator +from utils.dataset_manifest.utils import detect_related_images import django_rq from django.conf import settings @@ -273,10 +275,14 @@ def _create_thread(tid, data): start=db_data.start_frame, stop=data['stop_frame'], dimension=DimensionType.DIM_3D, - ) extractor.add_files(validate_dimension.converted_files) + related_images = {} + if isinstance(extractor, MEDIA_TYPES['image']['extractor']): + extractor.filter(lambda x: not re.search(r'(^|{0})related_images{0}'.format(os.sep), x)) + related_images = detect_related_images(extractor.absolute_source_paths, upload_dir) + db_task.mode = task_mode db_data.compressed_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' and not data['use_zip_chunks'] else models.DataChoice.IMAGESET db_data.original_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' else models.DataChoice.IMAGESET @@ -394,13 +400,14 @@ def _create_thread(tid, data): base_msg = str(ex) if isinstance(ex, AssertionError) \ else "Uploaded video does not support a quick way of task creating." _update_status("{} The task will be created using the old method".format(base_msg)) - else:# images, archive, pdf + else: # images, archive, pdf db_data.size = len(extractor) manifest = ImageManifestManager(db_data.get_manifest_path()) if not manifest_file: if db_task.dimension == DimensionType.DIM_2D: meta_info = manifest.prepare_meta( sources=extractor.absolute_source_paths, + meta={ k: {'related_images': related_images[k] } for k in related_images }, data_dir=upload_dir ) content = meta_info.content @@ -410,6 +417,7 @@ def _create_thread(tid, data): name, ext = os.path.splitext(os.path.relpath(source, upload_dir)) content.append({ 'name': name, + 'meta': { 'related_images': related_images[''.join((name, ext))] }, 'extension': ext }) manifest.create(content) @@ -465,27 +473,15 @@ def _create_thread(tid, data): update_progress(progress) if db_task.mode == 'annotation': - if validate_dimension.dimension == DimensionType.DIM_2D: - models.Image.objects.bulk_create(db_images) - else: - related_file = [] - for image_data in db_images: - image_model = models.Image( - data=image_data.data, - path=image_data.path, - frame=image_data.frame, - width=image_data.width, - height=image_data.height - ) - - image_model.save() - image_data = models.Image.objects.get(id=image_model.id) - - if validate_dimension.related_files.get(image_data.path, None): - for related_image_file in validate_dimension.related_files[image_data.path]: - related_file.append( - RelatedFile(data=db_data, primary_image_id=image_data.id, path=related_image_file)) - RelatedFile.objects.bulk_create(related_file) + models.Image.objects.bulk_create(db_images) + created_images = models.Image.objects.filter(data_id=db_data.id) + + db_related_files = [ + RelatedFile(data=image.data, primary_image=image, path=os.path.join(upload_dir, related_file_path)) + for image in created_images + for related_file_path in related_images.get(image.path, []) + ] + RelatedFile.objects.bulk_create(db_related_files) db_images = [] else: models.Video.objects.create( diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 89751952..db79c232 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -42,7 +42,7 @@ from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer from cvat.apps.engine.frame_provider import FrameProvider from cvat.apps.engine.models import ( Job, StatusChoice, Task, Project, Review, Issue, - Comment, StorageMethodChoice, ReviewStatus, StorageChoice, DimensionType, Image + Comment, StorageMethodChoice, ReviewStatus, StorageChoice, Image ) from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, @@ -487,21 +487,17 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): return sendfile(request, frame_provider.get_preview()) elif data_type == 'context_image': - if db_task.dimension == DimensionType.DIM_3D: - data_id = int(data_id) - image = Image.objects.get(data_id=db_task.data_id, frame=data_id) - for i in image.related_files.all(): - path = os.path.realpath(str(i.path)) - image = cv2.imread(path) - success, result = cv2.imencode('.JPEG', image) - if not success: - raise Exception("Failed to encode image to '%s' format" % (".jpeg")) - return HttpResponse(io.BytesIO(result.tobytes()), content_type="image/jpeg") - return Response(data='No context image related to the frame', - status=status.HTTP_404_NOT_FOUND) - else: - return Response(data='Only 3D tasks support context images', - status=status.HTTP_400_BAD_REQUEST) + data_id = int(data_id) + image = Image.objects.get(data_id=db_data.id, frame=data_id) + for i in image.related_files.all(): + path = os.path.realpath(str(i.path)) + image = cv2.imread(path) + success, result = cv2.imencode('.JPEG', image) + if not success: + raise Exception('Failed to encode image to ".jpeg" format') + return HttpResponse(io.BytesIO(result.tobytes()), content_type='image/jpeg') + return Response(data='No context image related to the frame', + status=status.HTTP_404_NOT_FOUND) else: return Response(data='unknown data type {}.'.format(data_type), status=status.HTTP_400_BAD_REQUEST) except APIException as e: @@ -636,7 +632,7 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): @action(detail=True, methods=['GET'], serializer_class=DataMetaSerializer, url_path='data/meta') def data_info(request, pk): - db_task = models.Task.objects.prefetch_related('data__images').select_related('data__video').get(pk=pk) + db_task = models.Task.objects.prefetch_related('data__images__related_files').select_related('data__video').get(pk=pk) if hasattr(db_task.data, 'video'): media = [db_task.data.video] @@ -647,6 +643,7 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet): 'width': item.width, 'height': item.height, 'name': item.path, + 'has_related_context': hasattr(item, 'related_files') and bool(len(item.related_files.all())) } for item in media] db_data = db_task.data diff --git a/tests/cypress/integration/canvas3d_functionality/case_56_canvas3d_functionality_basic_actions.js b/tests/cypress/integration/canvas3d_functionality/case_56_canvas3d_functionality_basic_actions.js index ef058431..7bb27971 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_56_canvas3d_functionality_basic_actions.js +++ b/tests/cypress/integration/canvas3d_functionality/case_56_canvas3d_functionality_basic_actions.js @@ -57,10 +57,10 @@ context('Canvas 3D functionality. Basic actions.', () => { } function testContextImage() { - cy.get('.cvat-contextImage-show').should('exist').and('be.visible'); - cy.get('[data-icon="camera"]').click(); // Context image hide - cy.get('.cvat-contextImage-show').should('not.exist'); - cy.get('[data-icon="camera"]').click(); // Context image show + cy.get('.cvat-context-image-wrapper img').should('exist').and('be.visible'); + cy.get('.cvat-context-image-switcher').click(); // Context image hide + cy.get('.cvat-context-image-wrapper img').should('not.exist'); + cy.get('.cvat-context-image-switcher').click(); // Context image show } function testControlButtonTooltip(button, expectedTooltipText) { @@ -104,9 +104,11 @@ context('Canvas 3D functionality. Basic actions.', () => { cy.get('.cvat-canvas3d-topview').should('exist').and('be.visible'); cy.get('.cvat-canvas3d-sideview').should('exist').and('be.visible'); cy.get('.cvat-canvas3d-frontview').should('exist').and('be.visible'); - cy.get('.cvat-canvas-controls-sidebar').find('[role="img"]').then(($controlButtons) => { - expect($controlButtons.length).to.be.equal(4); - }); + cy.get('.cvat-canvas-controls-sidebar') + .find('[role="img"]') + .then(($controlButtons) => { + expect($controlButtons.length).to.be.equal(3); + }); cy.get('.cvat-canvas-controls-sidebar') .should('exist') .and('be.visible') @@ -114,12 +116,10 @@ context('Canvas 3D functionality. Basic actions.', () => { cy.get('.cvat-move-control').should('exist').and('be.visible'); cy.get('.cvat-cursor-control').should('exist').and('be.visible'); cy.get('.cvat-draw-cuboid-control').should('exist').and('be.visible'); - cy.get('.cvat-context-image-control').should('exist').and('be.visible'); }); [ ['.cvat-move-control', 'Move the image'], ['.cvat-cursor-control', 'Cursor [Esc]'], - ['.cvat-context-image-control', 'Photo context show/hide'] ].forEach(([button, tooltip]) => { testControlButtonTooltip(button, tooltip); }); diff --git a/tests/cypress/integration/canvas3d_functionality/case_63_canvas3d_functionality_control_button_mouse_interaction.js b/tests/cypress/integration/canvas3d_functionality/case_63_canvas3d_functionality_control_button_mouse_interaction.js index 34c6845d..34ccdde4 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_63_canvas3d_functionality_control_button_mouse_interaction.js +++ b/tests/cypress/integration/canvas3d_functionality/case_63_canvas3d_functionality_control_button_mouse_interaction.js @@ -34,7 +34,6 @@ context('Canvas 3D functionality. Control button. Mouse interaction.', () => { before(() => { cy.openTaskJob(taskName); - cy.get('.cvat-contextImage-show').should('be.visible'); }); describe(`Testing case "${caseId}"`, () => { diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index edb68fb2..7a82f8ea 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -145,8 +145,9 @@ class VideoStreamReader: class DatasetImagesReader: - def __init__(self, sources, is_sorted=True, use_image_hash=False, *args, **kwargs): + def __init__(self, sources, meta=None, is_sorted=True, use_image_hash=False, *args, **kwargs): self._sources = sources if is_sorted else sorted(sources) + self._meta = meta self._content = [] self._data_dir = kwargs.get('data_dir', None) self._use_image_hash = use_image_hash @@ -163,6 +164,8 @@ class DatasetImagesReader: 'width': img.width, 'height': img.height, } + if self._meta and img_name in self._meta: + image_properties['meta'] = self._meta[img_name] if self._use_image_hash: image_properties['checksum'] = md5_hash(img) yield image_properties @@ -177,7 +180,7 @@ class DatasetImagesReader: class _Manifest: FILE_NAME = 'manifest.jsonl' - VERSION = '1.0' + VERSION = '1.1' def __init__(self, path, is_created=False): assert path, 'A path to manifest file not found' diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index 680052f0..c54b1b5d 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -2,21 +2,12 @@ # # SPDX-License-Identifier: MIT import argparse -import mimetypes import os import sys +import re from glob import glob -def _define_data_type(media): - media_type, _ = mimetypes.guess_type(media) - if media_type: - return media_type.split('/')[0] - -def _is_video(media_file): - return _define_data_type(media_file) == 'video' - -def _is_image(media_file): - return _define_data_type(media_file) == 'image' +from utils import detect_related_images, is_image, is_video def get_args(): parser = argparse.ArgumentParser() @@ -33,7 +24,7 @@ def main(): manifest_directory = os.path.abspath(args.output_dir) os.makedirs(manifest_directory, exist_ok=True) - source = os.path.abspath(args.source) + source = os.path.abspath(os.path.expanduser(args.source)) sources = [] if not os.path.isfile(source): # directory/pattern with images @@ -41,7 +32,7 @@ def main(): if os.path.isdir(source): data_dir = source for root, _, files in os.walk(source): - sources.extend([os.path.join(root, f) for f in files if _is_image(f)]) + sources.extend([os.path.join(root, f) for f in files if is_image(f)]) else: items = source.lstrip('/').split('/') position = 0 @@ -56,18 +47,28 @@ def main(): data_dir = source.split(items[position])[0] except Exception as ex: sys.exit(str(ex)) - sources = list(filter(_is_image, glob(source, recursive=True))) + sources = list(filter(is_image, glob(source, recursive=True))) + + sources = list(filter(lambda x: 'related_images{}'.format(os.sep) not in x, sources)) + + # If the source is a glob expression, we need additional processing + abs_root = source + while abs_root and re.search('[*?\[\]]', abs_root): + abs_root = os.path.split(abs_root)[0] + + related_images = detect_related_images(sources, abs_root) + meta = { k: {'related_images': related_images[k] } for k in related_images } try: assert len(sources), 'A images was not found' manifest = ImageManifestManager(manifest_path=manifest_directory) - meta_info = manifest.prepare_meta(sources=sources, is_sorted=False, + meta_info = manifest.prepare_meta(sources=sources, meta=meta, is_sorted=False, use_image_hash=True, data_dir=data_dir) manifest.create(meta_info) except Exception as ex: sys.exit(str(ex)) else: # video try: - assert _is_video(source), 'You can specify a video path or a directory/pattern with images' + assert is_video(source), 'You can specify a video path or a directory/pattern with images' manifest = VideoManifestManager(manifest_path=manifest_directory) try: meta_info = manifest.prepare_meta(media_file=source, force=args.force) diff --git a/utils/dataset_manifest/utils.py b/utils/dataset_manifest/utils.py index c5e9feea..09690add 100644 --- a/utils/dataset_manifest/utils.py +++ b/utils/dataset_manifest/utils.py @@ -1,7 +1,10 @@ # Copyright (C) 2021 Intel Corporation # # SPDX-License-Identifier: MIT +import os +import re import hashlib +import mimetypes import cv2 as cv from av import VideoFrame @@ -21,4 +24,163 @@ def rotate_image(image, angle): def md5_hash(frame): if isinstance(frame, VideoFrame): frame = frame.to_image() - return hashlib.md5(frame.tobytes()).hexdigest() # nosec \ No newline at end of file + return hashlib.md5(frame.tobytes()).hexdigest() # nosec + +def _define_data_type(media): + return mimetypes.guess_type(media)[0] + +def is_video(media_file): + data_type = _define_data_type(media_file) + return data_type is not None and data_type.startswith('video') + +def is_image(media_file): + data_type = _define_data_type(media_file) + return data_type is not None and data_type.startswith('image') and \ + not data_type.startswith('image/svg') + + +def _list_and_join(root): + files = os.listdir(root) + for f in files: + yield os.path.join(root, f) + +def _prepare_context_list(files, base_dir): + return sorted(map(lambda x: os.path.relpath(x, base_dir), filter(is_image, files))) + +# Expected 2D format is: +# data/ +# 00001.png +# related_images/ +# 00001_png/ +# context_image_1.jpeg +# context_image_2.png +def _detect_related_images_2D(image_paths, root_path): + related_images = {} + latest_dirname = '' + related_images_exist = False + + for image_path in sorted(image_paths): + rel_image_path = os.path.relpath(image_path, root_path) + dirname = os.path.dirname(image_path) + related_images_dirname = os.path.join(dirname, 'related_images') + related_images[rel_image_path] = [] + + if latest_dirname == dirname and not related_images_exist: + continue + elif latest_dirname != dirname: + # Update some data applicable for a subset of paths (within the current dirname) + latest_dirname = dirname + related_images_exist = os.path.isdir(related_images_dirname) + + if related_images_exist: + related_images_dirname = os.path.join( + related_images_dirname, '_'.join(os.path.basename(image_path).rsplit('.', 1)) + ) + + if os.path.isdir(related_images_dirname): + related_images[rel_image_path] = _prepare_context_list(_list_and_join(related_images_dirname), root_path) + return related_images + +# Possible 3D formats are: +# velodyne_points/ +# data/ +# image_01.bin +# IMAGE_00 # any number? +# data/ +# image_01.png + +# pointcloud/ +# 00001.pcd +# related_images/ +# 00001_pcd/ +# image_01.png # or other image + +# Default formats +# Option 1 +# data/ +# image.pcd +# image.png + +# Option 2 +# data/ +# image_1/ +# image_1.pcd +# context_1.png +# context_2.jpg +def _detect_related_images_3D(image_paths, root_path): + related_images = {} + latest_dirname = '' + dirname_files = [] + related_images_exist = False + velodyne_context_images_dirs = [] + + for image_path in sorted(image_paths): + rel_image_path = os.path.relpath(image_path, root_path) + name = os.path.splitext(os.path.basename(image_path))[0] + dirname = os.path.dirname(image_path) + related_images_dirname = os.path.normpath(os.path.join(dirname, '..', 'related_images')) + related_images[rel_image_path] = [] + + if latest_dirname != dirname: + # Update some data applicable for a subset of paths (within the current dirname) + latest_dirname = dirname + related_images_exist = os.path.isdir(related_images_dirname) + dirname_files = list(filter(lambda x: x != image_path, _list_and_join(dirname))) + velodyne_context_images_dirs = [directory for directory + in _list_and_join(os.path.normpath(os.path.join(dirname, '..', '..'))) + if os.path.isdir(os.path.join(directory, 'data')) and re.search(r'image_\d.*', directory, re.IGNORECASE) + ] + + if os.path.basename(dirname) == name: + # default format (option 2) + related_images[rel_image_path].extend(_prepare_context_list(dirname_files, root_path)) + + filtered_dirname_files = list(filter(lambda x: os.path.splitext(os.path.basename(x))[0] == name, dirname_files)) + if len(filtered_dirname_files): + # default format (option 1) + related_images[rel_image_path].extend(_prepare_context_list(filtered_dirname_files, root_path)) + + if related_images_exist: + related_images_dirname = os.path.join( + related_images_dirname, '_'.join(os.path.basename(image_path).rsplit('.', 1)) + ) + if os.path.isdir(related_images_dirname): + related_images[rel_image_path].extend( + _prepare_context_list(_list_and_join(related_images_dirname), root_path) + ) + + if dirname.endswith(os.path.join('velodyne_points', 'data')): + # velodynepoints format + for context_images_dir in velodyne_context_images_dirs: + context_files = _list_and_join(os.path.join(context_images_dir, 'data')) + context_files = list( + filter(lambda x: os.path.splitext(os.path.basename(x))[0] == name, context_files) + ) + related_images[rel_image_path].extend( + _prepare_context_list(context_files, root_path) + ) + + related_images[rel_image_path].sort() + return related_images + +# This function is expected to be called only for images tasks +# image_path is expected to be a list of absolute path to images +# root_path is expected to be a string (dataset root) +def detect_related_images(image_paths, root_path): + data_are_2d = False + data_are_3d = False + + # First of all need to define data type we are working with + for image_path in image_paths: + # .bin files are expected to be converted to .pcd before this code + if os.path.splitext(image_path)[1].lower() == '.pcd': + data_are_3d = True + else: + data_are_2d = True + assert not (data_are_3d and data_are_2d), 'Combined data types 2D and 3D are not supported' + + if data_are_2d: + return _detect_related_images_2D(image_paths, root_path) + elif data_are_3d: + return _detect_related_images_3D(image_paths, root_path) + return {}