CVAT-3D: support lidar data on the server side (#2534)

* CVAT-3D Updated the Mime Types with Bin Support, added dependency of open3D
* CVAT-3D Added additional column as Dimension for engine_task table and created a relatedfiles table for PCD to Image mapping.
* Added Support for 3D file Upload in BIN and PCD.
* Added Dimension attribute defaulting to 2D for importer and exporter.
* Added props passing for dimension attribute, filtering of import, Migration Scripts and Dimension attribute for MpegChunk Writers

Co-authored-by: cdp <cdp123>

Dockerfile

@@ -73,6 +73,8 @@ RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \
         apache2 \
         libapache2-mod-xsendfile \
+        libgomp1 \
+        libgl1 \
         supervisor \
         libldap-2.4-2 \
         libsasl2-2 \

cvat-core/src/annotation-formats.js

@@ -15,6 +15,7 @@
                 format: initialData.ext,
                 version: initialData.version,
                 enabled: initialData.enabled,
+                dimension: initialData.dimension,
             };
 
             Object.defineProperties(this, {
@@ -58,6 +59,16 @@
                      */
                     get: () => data.enabled,
                 },
+                dimension: {
+                    /**
+                     * @name dimension
+                     * @type {string}
+                     * @memberof module:API.cvat.enums.DimensionType
+                     * @readonly
+                     * @instance
+                     */
+                    get: () => data.dimension,
+                },
             });
         }
     }
@@ -74,6 +85,7 @@
                 format: initialData.ext,
                 version: initialData.version,
                 enabled: initialData.enabled,
+                dimension: initialData.dimension,
             };
 
             Object.defineProperties(this, {
@@ -117,6 +129,16 @@
                      */
                     get: () => data.enabled,
                 },
+                dimension: {
+                    /**
+                     * @name dimension
+                     * @type {string}
+                     * @memberof module:API.cvat.enums.DimensionType
+                     * @readonly
+                     * @instance
+                     */
+                    get: () => data.dimension,
+                },
             });
         }
     }

cvat-core/src/api-implementation.js

@@ -10,7 +10,7 @@
         isBoolean, isInteger, isEnum, isString, checkFilter,
     } = require('./common');
 
-    const { TaskStatus, TaskMode } = require('./enums');
+    const { TaskStatus, TaskMode, DimensionType } = require('./enums');
 
     const User = require('./user');
     const { AnnotationFormats } = require('./annotation-formats');
@@ -176,6 +176,7 @@
                 search: isString,
                 status: isEnum.bind(TaskStatus),
                 mode: isEnum.bind(TaskMode),
+                dimension: isEnum.bind(DimensionType),
             });
 
             if ('search' in filter && Object.keys(filter).length > 1) {
@@ -198,7 +199,7 @@
             }
 
             const searchParams = new URLSearchParams();
-            for (const field of ['name', 'owner', 'assignee', 'search', 'status', 'mode', 'id', 'page', 'projectId']) {
+            for (const field of ['name', 'owner', 'assignee', 'search', 'status', 'mode', 'id', 'page', 'projectId', 'dimension']) {
                 if (Object.prototype.hasOwnProperty.call(filter, field)) {
                     searchParams.set(field, filter[field]);
                 }

cvat-core/src/enums.js

@@ -33,6 +33,20 @@
         COMPLETED: 'completed',
     });
 
+    /**
+     * Task dimension
+     * @enum
+     * @name DimensionType
+     * @memberof module:API.cvat.enums
+     * @property {string} DIMENSION_2D '2d'
+     * @property {string} DIMENSION_3D '3d'
+     * @readonly
+     */
+    const DimensionType = Object.freeze({
+        DIMENSION_2D: '2d',
+        DIMENSION_3D: '3d',
+    });
+
     /**
      * Review statuses
      * @enum {string}
@@ -333,5 +347,6 @@
         RQStatus,
         colors,
         Source,
+        DimensionType,
     };
 })();

cvat-core/src/session.js

@@ -974,6 +974,7 @@
                 use_zip_chunks: undefined,
                 use_cache: undefined,
                 copy_data: undefined,
+                dimension: undefined,
             };
 
             let updatedFields = {
@@ -1452,6 +1453,16 @@
                             updatedFields = fields;
                         },
                     },
+                    dimension: {
+                        /**
+                         * @name enabled
+                         * @type {string}
+                         * @memberof module:API.cvat.enums.DimensionType
+                         * @readonly
+                         * @instance
+                         */
+                        get: () => data.dimension,
+                    },
                 }),
             );
 

cvat-ui/src/components/actions-menu/actions-menu.tsx

@@ -11,6 +11,7 @@ import { MenuInfo } from 'rc-menu/lib/interface';
 import DumpSubmenu from './dump-submenu';
 import LoadSubmenu from './load-submenu';
 import ExportSubmenu from './export-submenu';
+import { DimensionType } from '../../reducers/interfaces';
 
 interface Props {
     taskID: number;
@@ -22,7 +23,7 @@ interface Props {
     dumpActivities: string[] | null;
     exportActivities: string[] | null;
     inferenceIsActive: boolean;
-
+    taskDimension: DimensionType;
     onClickMenu: (params: MenuInfo, file?: File) => void;
 }
 
@@ -47,6 +48,7 @@ export default function ActionsMenuComponent(props: Props): JSX.Element {
         dumpActivities,
         exportActivities,
         loadActivity,
+        taskDimension,
     } = props;
 
     let latestParams: MenuInfo | null = null;
@@ -102,6 +104,7 @@ export default function ActionsMenuComponent(props: Props): JSX.Element {
                 dumpers,
                 dumpActivities,
                 menuKey: Actions.DUMP_TASK_ANNO,
+                taskDimension,
             })}
             {LoadSubmenu({
                 loaders,
@@ -110,11 +113,13 @@ export default function ActionsMenuComponent(props: Props): JSX.Element {
                     onClickMenuWrapper(null, file);
                 },
                 menuKey: Actions.LOAD_TASK_ANNO,
+                taskDimension,
             })}
             {ExportSubmenu({
                 exporters: dumpers,
                 exportActivities,
                 menuKey: Actions.EXPORT_TASK_DATASET,
+                taskDimension,
             })}
             {!!bugTracker && <Menu.Item key={Actions.OPEN_BUG_TRACKER}>Open bug tracker</Menu.Item>}
             <Menu.Item disabled={inferenceIsActive} key={Actions.RUN_AUTO_ANNOTATION}>

cvat-ui/src/components/actions-menu/dump-submenu.tsx

@@ -6,6 +6,7 @@ import React from 'react';
 import Menu from 'antd/lib/menu';
 import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons';
 import Text from 'antd/lib/typography/Text';
+import { DimensionType } from '../../reducers/interfaces';
 
 function isDefaultFormat(dumperName: string, taskMode: string): boolean {
     return (
@@ -19,15 +20,19 @@ interface Props {
     menuKey: string;
     dumpers: any[];
     dumpActivities: string[] | null;
+    taskDimension: DimensionType;
 }
 
 export default function DumpSubmenu(props: Props): JSX.Element {
-    const { taskMode, menuKey, dumpers, dumpActivities } = props;
+    const {
+        taskMode, menuKey, dumpers, dumpActivities, taskDimension,
+    } = props;
 
     return (
         <Menu.SubMenu key={menuKey} title='Dump annotations'>
             {dumpers
                 .sort((a: any, b: any) => a.name.localeCompare(b.name))
+                .filter((dumper: any): boolean => dumper.dimension === taskDimension)
                 .map(
                     (dumper: any): JSX.Element => {
                         const pending = (dumpActivities || []).includes(dumper.name);

cvat-ui/src/components/actions-menu/export-submenu.tsx

@@ -6,20 +6,25 @@ import React from 'react';
 import Menu from 'antd/lib/menu';
 import Text from 'antd/lib/typography/Text';
 import { ExportOutlined, LoadingOutlined } from '@ant-design/icons';
+import { DimensionType } from '../../reducers/interfaces';
 
 interface Props {
     menuKey: string;
     exporters: any[];
     exportActivities: string[] | null;
+    taskDimension: DimensionType;
 }
 
 export default function ExportSubmenu(props: Props): JSX.Element {
-    const { menuKey, exporters, exportActivities } = props;
+    const {
+        menuKey, exporters, exportActivities, taskDimension,
+    } = props;
 
     return (
         <Menu.SubMenu key={menuKey} title='Export as a dataset'>
             {exporters
                 .sort((a: any, b: any) => a.name.localeCompare(b.name))
+                .filter((exporter: any): boolean => exporter.dimension === taskDimension)
                 .map(
                     (exporter: any): JSX.Element => {
                         const pending = (exportActivities || []).includes(exporter.name);

cvat-ui/src/components/actions-menu/load-submenu.tsx

@@ -8,23 +8,26 @@ import Upload from 'antd/lib/upload';
 import Button from 'antd/lib/button';
 import Text from 'antd/lib/typography/Text';
 import { UploadOutlined, LoadingOutlined } from '@ant-design/icons';
+import { DimensionType } from '../../reducers/interfaces';
 
 interface Props {
     menuKey: string;
     loaders: any[];
     loadActivity: string | null;
     onFileUpload(file: File): void;
+    taskDimension: DimensionType;
 }
 
 export default function LoadSubmenu(props: Props): JSX.Element {
     const {
-        menuKey, loaders, loadActivity, onFileUpload,
+        menuKey, loaders, loadActivity, onFileUpload, taskDimension,
     } = props;
 
     return (
         <Menu.SubMenu key={menuKey} title='Upload annotations'>
             {loaders
                 .sort((a: any, b: any) => a.name.localeCompare(b.name))
+                .filter((loader: any): boolean => loader.dimension === taskDimension)
                 .map(
                     (loader: any): JSX.Element => {
                         const accept = loader.format

cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx

@@ -164,6 +164,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element {
                 dumpers,
                 dumpActivities,
                 menuKey: Actions.DUMP_TASK_ANNO,
+                taskDimension: jobInstance.task.dimension,
             })}
             {LoadSubmenu({
                 loaders,
@@ -172,11 +173,13 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element {
                     onClickMenuWrapper(null, file);
                 },
                 menuKey: Actions.LOAD_JOB_ANNO,
+                taskDimension: jobInstance.task.dimension,
             })}
             {ExportSubmenu({
                 exporters: dumpers,
                 exportActivities,
                 menuKey: Actions.EXPORT_TASK_DATASET,
+                taskDimension: jobInstance.task.dimension,
             })}
 
             <Menu.Item key={Actions.REMOVE_ANNO}>Remove annotations</Menu.Item>

cvat-ui/src/components/model-runner-modal/detector-runner.tsx

@@ -21,6 +21,7 @@ import { Model, StringObject } from 'reducers/interfaces';
 
 import { clamp } from 'utils/math';
 import consts from 'consts';
+import { DimensionType } from '../../reducers/interfaces';
 
 interface Props {
     withCleanup: boolean;
@@ -127,7 +128,8 @@ function DetectorRunner(props: Props): JSX.Element {
                 <Col span={4}>Model:</Col>
                 <Col span={20}>
                     <Select
-                        placeholder='Select a model'
+                        placeholder={task.dimension === DimensionType.DIM_2D ? 'Select a model' : 'No models available'}
+                        disabled={task.dimension !== DimensionType.DIM_2D}
                         style={{ width: '100%' }}
                         onChange={(_modelID: string): void => {
                             const newmodel = models.filter((_model): boolean => _model.id === _modelID)[0];

cvat-ui/src/containers/actions-menu/actions-menu.tsx

@@ -137,6 +137,7 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps):
             exportActivities={exportActivities}
             inferenceIsActive={inferenceIsActive}
             onClickMenu={onClickMenu}
+            taskDimension={taskInstance.dimension}
         />
     );
 }

cvat-ui/src/reducers/interfaces.ts

@@ -568,3 +568,8 @@ export interface CombinedState {
     shortcuts: ShortcutsState;
     review: ReviewState;
 }
+
+export enum DimensionType {
+    DIM_3D = '3d',
+    DIM_2D = '2d',
+}

cvat/apps/dataset_manager/formats/registry.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: MIT
 
 from datumaro.components.project import Environment
+from cvat.apps.engine.models import DimensionType
 
 
 dm_env = Environment()
@@ -23,7 +24,7 @@ class Importer(_Format):
     def __call__(self, src_file, task_data, **options):
         raise NotImplementedError()
 
-def _wrap_format(f_or_cls, klass, name, version, ext, display_name, enabled):
+def _wrap_format(f_or_cls, klass, name, version, ext, display_name, enabled, dimension=DimensionType.DIM_2D):
     import inspect
     assert inspect.isclass(f_or_cls) or inspect.isfunction(f_or_cls)
     if inspect.isclass(f_or_cls):
@@ -45,17 +46,18 @@ def _wrap_format(f_or_cls, klass, name, version, ext, display_name, enabled):
     target.DISPLAY_NAME = (display_name or klass.DISPLAY_NAME).format(
         NAME=name, VERSION=version, EXT=ext)
     assert all([target.NAME, target.VERSION, target.EXT, target.DISPLAY_NAME])
+    target.DIMENSION = dimension
     target.ENABLED = enabled
 
     return target
 
 EXPORT_FORMATS = {}
-def exporter(name, version, ext, display_name=None, enabled=True):
+def exporter(name, version, ext, display_name=None, enabled=True, dimension=DimensionType.DIM_2D):
     assert name not in EXPORT_FORMATS, "Export format '%s' already registered" % name
     def wrap_with_params(f_or_cls):
         t = _wrap_format(f_or_cls, Exporter,
             name=name, ext=ext, version=version, display_name=display_name,
-            enabled=enabled)
+            enabled=enabled, dimension=dimension)
         key = t.DISPLAY_NAME
         assert key not in EXPORT_FORMATS, "Export format '%s' already registered" % name
         EXPORT_FORMATS[key] = t
@@ -63,11 +65,11 @@ def exporter(name, version, ext, display_name=None, enabled=True):
     return wrap_with_params
 
 IMPORT_FORMATS = {}
-def importer(name, version, ext, display_name=None, enabled=True):
+def importer(name, version, ext, display_name=None, enabled=True, dimension=DimensionType.DIM_2D):
     def wrap_with_params(f_or_cls):
         t = _wrap_format(f_or_cls, Importer,
             name=name, ext=ext, version=version, display_name=display_name,
-            enabled=enabled)
+            enabled=enabled, dimension=dimension)
         key = t.DISPLAY_NAME
         assert key not in IMPORT_FORMATS, "Import format '%s' already registered" % name
         IMPORT_FORMATS[key] = t
@@ -92,4 +94,4 @@ import cvat.apps.dataset_manager.formats.pascal_voc
 import cvat.apps.dataset_manager.formats.tfrecord
 import cvat.apps.dataset_manager.formats.yolo
 import cvat.apps.dataset_manager.formats.imagenet
-import cvat.apps.dataset_manager.formats.camvid
+import cvat.apps.dataset_manager.formats.camvid

cvat/apps/dataset_manager/serializers.py

@@ -10,7 +10,8 @@ class DatasetFormatSerializer(serializers.Serializer):
     ext = serializers.CharField(max_length=64, source='EXT')
     version = serializers.CharField(max_length=64, source='VERSION')
     enabled = serializers.BooleanField(source='ENABLED')
+    dimension = serializers.CharField(max_length=2, source='DIMENSION')
 
 class DatasetFormatsSerializer(serializers.Serializer):
     importers = DatasetFormatSerializer(many=True)
-    exporters = DatasetFormatSerializer(many=True)
+    exporters = DatasetFormatSerializer(many=True)

cvat/apps/engine/cache.py

@@ -12,10 +12,12 @@ from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter,
     Mpeg4CompressedChunkWriter, ZipChunkWriter, ZipCompressedChunkWriter)
 from cvat.apps.engine.models import DataChoice, StorageChoice
 from cvat.apps.engine.prepare import PrepareInfo
+from cvat.apps.engine.models import DimensionType
 
 class CacheInteraction:
-    def __init__(self):
+    def __init__(self, dimension=DimensionType.DIM_2D):
         self._cache = Cache(settings.CACHE_ROOT)
+        self._dimension = dimension
 
     def __del__(self):
         self._cache.close()
@@ -38,7 +40,10 @@ class CacheInteraction:
         image_quality = 100 if writer_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality
         mime_type = 'video/mp4' if writer_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip'
 
-        writer = writer_classes[quality](image_quality)
+        kwargs = {}
+        if self._dimension == DimensionType.DIM_3D:
+            kwargs["dimension"] = DimensionType.DIM_3D
+        writer = writer_classes[quality](image_quality, **kwargs)
 
         images = []
         buff = BytesIO()

cvat/apps/engine/frame_provider.py

@@ -13,7 +13,7 @@ from PIL import Image
 from cvat.apps.engine.cache import CacheInteraction
 from cvat.apps.engine.media_extractors import VideoReader, ZipReader
 from cvat.apps.engine.mime_types import mimetypes
-from cvat.apps.engine.models import DataChoice, StorageMethodChoice
+from cvat.apps.engine.models import DataChoice, StorageMethodChoice, DimensionType
 
 
 class RandomAccessIterator:
@@ -83,7 +83,7 @@ class FrameProvider:
                     self.reader_class([self.get_chunk_path(chunk_id, self.quality, self.db_data)[0]]))
             return self.chunk_reader
 
-    def __init__(self, db_data):
+    def __init__(self, db_data, dimension=DimensionType.DIM_2D):
         self._db_data = db_data
         self._loaders = {}
 
@@ -93,7 +93,7 @@ class FrameProvider:
         }
 
         if db_data.storage_method == StorageMethodChoice.CACHE:
-            cache = CacheInteraction()
+            cache = CacheInteraction(dimension=dimension)
 
             self._loaders[self.Quality.COMPRESSED] = self.BuffChunkLoader(
                 reader_class[db_data.compressed_chunk_type],

cvat/apps/engine/media.mimetypes

@@ -106,7 +106,8 @@ image/x-jng	jng
 image/jp2	jp2
 image/x-portable-bitmap	pbm
 image/x-eps	epsf
-image/x-photo-cd	pcd
+image/x.point-cloud-data pcd
+image/x.kitti-velodyne bin
 image/jpeg	jpe
 image/jp2	jpf
 image/jpeg	jpg

cvat/apps/engine/media_extractors.py

@@ -8,13 +8,17 @@ import shutil
 import zipfile
 import io
 import itertools
+import struct
+import re
 from abc import ABC, abstractmethod
 
 import av
 import numpy as np
 from pyunpack import Archive
 from PIL import Image, ImageFile
+import open3d as o3d
 from cvat.apps.engine.utils import rotate_image
+from cvat.apps.engine.models import DimensionType
 
 # fixes: "OSError:broken data stream" when executing line 72 while loading images downloaded from the web
 # see: https://stackoverflow.com/questions/42462431/oserror-broken-data-stream-when-reading-image-file
@@ -179,7 +183,8 @@ class PdfReader(ImageListReader):
 
 class ZipReader(ImageListReader):
     def __init__(self, source_path, step=1, start=0, stop=None):
-        self._zip_source = zipfile.ZipFile(source_path[0], mode='r')
+        self._dimension = DimensionType.DIM_2D
+        self._zip_source = zipfile.ZipFile(source_path[0], mode='a')
         self.extract_dir = source_path[1] if len(source_path) > 1 else None
         file_list = [f for f in self._zip_source.namelist() if get_mime(f) == 'image']
         super().__init__(file_list, step, start, stop)
@@ -188,18 +193,42 @@ class ZipReader(ImageListReader):
         self._zip_source.close()
 
     def get_preview(self):
+        if self._dimension == DimensionType.DIM_3D:
+            fp = open(os.path.join(os.path.dirname(__file__), 'assets/3d_preview.jpeg'), "rb")
+            return self._get_preview(fp)
         io_image = io.BytesIO(self._zip_source.read(self._source_path[0]))
         return self._get_preview(io_image)
 
     def get_image_size(self, i):
+        if self._dimension == DimensionType.DIM_3D:
+            with self._zip_source.open(self._source_path[i], "r") as file:
+                properties = ValidateDimension.get_pcd_properties(file)
+                return int(properties["WIDTH"]),  int(properties["HEIGHT"])
         img = Image.open(io.BytesIO(self._zip_source.read(self._source_path[i])))
         return img.width, img.height
 
     def get_image(self, i):
         return io.BytesIO(self._zip_source.read(self._source_path[i]))
 
+    def add_files(self, source_path):
+        root_path = os.path.split(self._zip_source.filename)[0]
+        for path in source_path:
+            self._zip_source.write(path, path.replace(root_path, ""))
+
+    def get_zip_filename(self):
+        return self._zip_source.filename
+
+    def reconcile(self, source_files, step=1, start=0, stop=None, dimension=DimensionType.DIM_2D):
+        self._dimension = dimension
+        super().__init__(
+            source_path=source_files,
+            step=step,
+            start=start,
+            stop=stop
+        )
+
     def get_path(self, i):
-        if  self._zip_source.filename:
+        if self._zip_source.filename:
             return os.path.join(os.path.dirname(self._zip_source.filename), self._source_path[i]) \
                 if not self.extract_dir else os.path.join(self.extract_dir, self._source_path[i])
         else: # necessary for mime_type definition
@@ -283,8 +312,9 @@ class VideoReader(IMediaReader):
         return image.width, image.height
 
 class IChunkWriter(ABC):
-    def __init__(self, quality):
+    def __init__(self, quality, dimension=DimensionType.DIM_2D):
         self._image_quality = quality
+        self._dimension = dimension
 
     @staticmethod
     def _compress_image(image_path, quality):
@@ -326,12 +356,20 @@ class ZipCompressedChunkWriter(IChunkWriter):
     def save_as_chunk(self, images, chunk_path):
         image_sizes = []
         with zipfile.ZipFile(chunk_path, 'x') as zip_chunk:
-            for idx, (image, _ , _) in enumerate(images):
+            for idx, (image, _, _) in enumerate(images):
-                w, h, image_buf = self._compress_image(image, self._image_quality)
+                if self._dimension == DimensionType.DIM_2D:
+                    w, h, image_buf = self._compress_image(image, self._image_quality)
+                    extension = "jpeg"
+                else:
+                    image_buf = open(image, "rb") if isinstance(image, str) else image
+                    properties = ValidateDimension.get_pcd_properties(image_buf)
+                    w, h = int(properties["WIDTH"]), int(properties["HEIGHT"])
+                    extension = "pcd"
+                    image_buf.seek(0, 0)
+                    image_buf = io.BytesIO(image_buf.read())
                 image_sizes.append((w, h))
-                arcname = '{:06d}.jpeg'.format(idx)
+                arcname = '{:06d}.{}'.format(idx, extension)
                 zip_chunk.writestr(arcname, image_buf.getvalue())
-
         return image_sizes
 
 class Mpeg4ChunkWriter(IChunkWriter):
@@ -511,3 +549,173 @@ MEDIA_TYPES = {
         'unique': True,
     }
 }
+
+
+class ValidateDimension:
+
+    def __init__(self, path=None):
+        self.dimension = DimensionType.DIM_2D
+        self.path = path
+        self.related_files = {}
+        self.image_files = {}
+        self.converted_files = []
+
+    @staticmethod
+    def get_pcd_properties(fp, verify_version=False):
+        kv = {}
+        pcd_version = ["0.7", "0.6", "0.5", "0.4", "0.3", "0.2", "0.1",
+                       ".7", ".6", ".5", ".4", ".3", ".2", ".1"]
+        try:
+            for line in fp:
+                line = line.decode("utf-8")
+                if line.startswith("#"):
+                    continue
+                k, v = line.split(" ", maxsplit=1)
+                kv[k] = v.strip()
+                if "DATA" in line:
+                    break
+            if verify_version:
+                if "VERSION" in kv and kv["VERSION"] in pcd_version:
+                    return True
+                return None
+            return kv
+        except AttributeError:
+            return None
+
+    @staticmethod
+    def convert_bin_to_pcd(path, delete_source=True):
+        list_pcd = []
+        with open(path, "rb") as f:
+            size_float = 4
+            byte = f.read(size_float * 4)
+            while byte:
+                x, y, z, _ = struct.unpack("ffff", byte)
+                list_pcd.append([x, y, z])
+                byte = f.read(size_float * 4)
+        np_pcd = np.asarray(list_pcd)
+        pcd = o3d.geometry.PointCloud()
+        pcd.points = o3d.utility.Vector3dVector(np_pcd)
+        pcd_filename = path.replace(".bin", ".pcd")
+        o3d.io.write_point_cloud(pcd_filename, pcd)
+        if delete_source:
+            os.remove(path)
+        return pcd_filename
+
+    def set_path(self, path):
+        self.path = path
+
+    def bin_operation(self, file_path, actual_path):
+        pcd_path = ValidateDimension.convert_bin_to_pcd(file_path)
+        self.converted_files.append(pcd_path)
+        return pcd_path.split(actual_path)[-1][1:]
+
+    @staticmethod
+    def pcd_operation(file_path, actual_path):
+        with open(file_path, "rb") as file:
+            is_pcd = ValidateDimension.get_pcd_properties(file, verify_version=True)
+        return file_path.split(actual_path)[-1][1:] if is_pcd else file_path
+
+    def process_files(self, root, actual_path, files):
+        pcd_files = {}
+
+        for file in files:
+            file_name, file_extension = file.rsplit('.', maxsplit=1)
+            file_path = os.path.abspath(os.path.join(root, file))
+
+            if file_extension == "bin":
+                path = self.bin_operation(file_path, actual_path)
+                pcd_files[file_name] = path
+                self.related_files[path] = []
+
+            elif file_extension == "pcd":
+                path = ValidateDimension.pcd_operation(file_path, actual_path)
+                if path == file_path:
+                    self.image_files[file_name] = file_path
+                else:
+                    pcd_files[file_name] = path
+                    self.related_files[path] = []
+            else:
+                self.image_files[file_name] = file_path
+        return pcd_files
+
+    def validate_velodyne_points(self, *args):
+        root, actual_path, files = args
+        velodyne_files = self.process_files(root, actual_path, files)
+        related_path = os.path.split(os.path.split(root)[0])[0]
+
+        path_list = [re.search(r'image_\d.*', path, re.IGNORECASE) for path in os.listdir(related_path) if
+                     os.path.isdir(os.path.join(related_path, path))]
+
+        for path_ in path_list:
+            if path_:
+                path = os.path.join(path_.group(), "data")
+                path = os.path.abspath(os.path.join(related_path, path))
+
+                files = [file for file in os.listdir(path) if
+                         os.path.isfile(os.path.abspath(os.path.join(path, file)))]
+                for file in files:
+
+                    f_name = file.split(".")[0]
+                    if velodyne_files.get(f_name, None):
+                        self.related_files[velodyne_files[f_name]].append(
+                            os.path.abspath(os.path.join(path, file)))
+
+    def validate_pointcloud(self, *args):
+        root, actual_path, files = args
+        pointcloud_files = self.process_files(root, actual_path, files)
+        related_path = root.split("pointcloud")[0]
+        related_images_path = os.path.join(related_path, "related_images")
+
+        if os.path.isdir(related_images_path):
+            paths = [path for path in os.listdir(related_images_path) if
+                     os.path.isdir(os.path.abspath(os.path.join(related_images_path, path)))]
+
+            for k in pointcloud_files:
+                for path in paths:
+
+                    if k == path.split("_")[0]:
+                        file_path = os.path.abspath(os.path.join(related_images_path, path))
+                        files = [file for file in os.listdir(file_path) if
+                                 os.path.isfile(os.path.join(file_path, file))]
+                        for related_image in files:
+                            self.related_files[pointcloud_files[k]].append(os.path.join(file_path, related_image))
+
+    def validate_default(self, *args):
+        root, actual_path, files = args
+        pcd_files = self.process_files(root, actual_path, files)
+        if len(list(pcd_files.keys())):
+
+            for image in self.image_files.keys():
+                if pcd_files.get(image, None):
+                    self.related_files[pcd_files[image]].append(self.image_files[image])
+
+            current_directory_name = os.path.split(root)
+
+            if len(pcd_files.keys()) == 1:
+                pcd_name = list(pcd_files.keys())[0].split(".")[0]
+                if current_directory_name[1] == pcd_name:
+                    for related_image in self.image_files.values():
+                        if root == os.path.split(related_image)[0]:
+                            self.related_files[pcd_files[pcd_name]].append(related_image)
+
+    def validate(self):
+        """
+            Validate the directory structure for kitty and point cloud format.
+        """
+        if not self.path:
+            return
+        actual_path = self.path
+        for root, _, files in os.walk(actual_path):
+
+            if root.endswith("data"):
+                if os.path.split(os.path.split(root)[0])[1] == "velodyne_points":
+                    self.validate_velodyne_points(root, actual_path, files)
+
+            elif os.path.split(root)[-1] == "pointcloud":
+                self.validate_pointcloud(root, actual_path, files)
+
+            else:
+                self.validate_default(root, actual_path, files)
+
+        if len(self.related_files.keys()):
+            self.dimension = DimensionType.DIM_3D

cvat/apps/engine/migrations/0036_auto_20201216_0943.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.1.1 on 2020-12-16 09:43
+
+import cvat.apps.engine.models
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('engine', '0035_data_storage'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='dimension',
+            field=models.CharField(choices=[('3d', 'DIM_3D'), ('2d', 'DIM_2D')], default=cvat.apps.engine.models.DimensionType['DIM_2D'], max_length=2),
+        ),
+        migrations.CreateModel(
+            name='RelatedFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('path', models.FileField(max_length=1024, storage=cvat.apps.engine.models.MyFileSystemStorage(), upload_to=cvat.apps.engine.models.upload_path_handler)),
+                ('data', models.ForeignKey(default=1, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='related_files', to='engine.data')),
+                ('primary_image', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='related_files', to='engine.image')),
+            ],
+            options={
+                'default_permissions': (),
+                'unique_together': {('data', 'path')},
+            },
+        ),
+    ]

cvat/apps/engine/models.py

@@ -19,6 +19,17 @@ class SafeCharField(models.CharField):
             return value[:self.max_length]
         return value
 
+class DimensionType(str, Enum):
+    DIM_3D = '3d'
+    DIM_2D = '2d'
+
+    @classmethod
+    def choices(cls):
+        return tuple((x.value, x.name) for x in cls)
+
+    def __str__(self):
+        return self.value
+
 class StatusChoice(str, Enum):
     ANNOTATION = 'annotation'
     VALIDATION = 'validation'
@@ -202,6 +213,7 @@ class Task(models.Model):
     status = models.CharField(max_length=32, choices=StatusChoice.choices(),
         default=StatusChoice.ANNOTATION)
     data = models.ForeignKey(Data, on_delete=models.CASCADE, null=True, related_name="tasks")
+    dimension = models.CharField(max_length=2, choices=DimensionType.choices(), default=DimensionType.DIM_2D)
 
     # Extend default permission model
     class Meta:
@@ -265,6 +277,17 @@ class RemoteFile(models.Model):
     class Meta:
         default_permissions = ()
 
+
+class RelatedFile(models.Model):
+    data = models.ForeignKey(Data, on_delete=models.CASCADE, related_name="related_files", default=1, null=True)
+    path = models.FileField(upload_to=upload_path_handler,
+                            max_length=1024, storage=MyFileSystemStorage())
+    primary_image = models.ForeignKey(Image, on_delete=models.CASCADE, related_name="related_files", null=True)
+
+    class Meta:
+        default_permissions = ()
+        unique_together = ("data", "path")
+
 class Segment(models.Model):
     task = models.ForeignKey(Task, on_delete=models.CASCADE)
     start_frame = models.IntegerField()

cvat/apps/engine/serializers.py

@@ -329,13 +329,15 @@ class TaskSerializer(WriteOnceMixin, serializers.ModelSerializer):
     assignee = BasicUserSerializer(allow_null=True, required=False)
     assignee_id = serializers.IntegerField(write_only=True, allow_null=True, required=False)
     project_id = serializers.IntegerField(required=False)
+    dimension = serializers.CharField(allow_blank=True, required=False)
 
     class Meta:
         model = models.Task
         fields = ('url', 'id', 'name', 'project_id', 'mode', 'owner', 'assignee', 'owner_id', 'assignee_id',
             'bug_tracker', 'created_date', 'updated_date', 'overlap',
             'segment_size', 'status', 'labels', 'segments',
-            'data_chunk_size', 'data_compressed_chunk_type', 'data_original_chunk_type', 'size', 'image_quality', 'data')
+            'data_chunk_size', 'data_compressed_chunk_type', 'data_original_chunk_type', 'size', 'image_quality',
+            'data', 'dimension')
         read_only_fields = ('mode', 'created_date', 'updated_date', 'status', 'data_chunk_size', 'owner', 'assignee',
             'data_compressed_chunk_type', 'data_original_chunk_type', 'size', 'image_quality', 'data')
         write_once_fields = ('overlap', 'segment_size', 'project_id')

cvat/apps/engine/task.py

@@ -14,10 +14,11 @@ from urllib import error as urlerror
 from urllib import parse as urlparse
 from urllib import request as urlrequest
 
-from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter
+from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter, ValidateDimension
-from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice
+from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice, RelatedFile
 from cvat.apps.engine.utils import av_scan_paths
 from cvat.apps.engine.prepare import prepare_meta
+from cvat.apps.engine.models import DimensionType
 
 import django_rq
 from django.conf import settings
@@ -259,8 +260,25 @@ def _create_thread(tid, data):
                 start=db_data.start_frame,
                 stop=data['stop_frame'],
             )
+
+    validate_dimension = ValidateDimension()
     if extractor.__class__ == MEDIA_TYPES['zip']['extractor']:
         extractor.extract()
+        validate_dimension.set_path(os.path.split(extractor.get_zip_filename())[0])
+        validate_dimension.validate()
+        if validate_dimension.dimension == DimensionType.DIM_3D:
+            db_task.dimension = DimensionType.DIM_3D
+
+            extractor.reconcile(
+                source_files=list(validate_dimension.related_files.keys()),
+                step=db_data.get_frame_step(),
+                start=db_data.start_frame,
+                stop=data['stop_frame'],
+                dimension=DimensionType.DIM_3D,
+
+            )
+            extractor.add_files(validate_dimension.converted_files)
+
     db_task.mode = task_mode
     db_data.compressed_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' and not data['use_zip_chunks'] else models.DataChoice.IMAGESET
     db_data.original_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' else models.DataChoice.IMAGESET
@@ -282,7 +300,10 @@ def _create_thread(tid, data):
     compressed_chunk_writer_class = Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter
     original_chunk_writer_class = Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter
 
-    compressed_chunk_writer = compressed_chunk_writer_class(db_data.image_quality)
+    kwargs = {}
+    if validate_dimension.dimension == DimensionType.DIM_3D:
+        kwargs["dimension"] = validate_dimension.dimension
+    compressed_chunk_writer = compressed_chunk_writer_class(db_data.image_quality, **kwargs)
     original_chunk_writer = original_chunk_writer_class(100)
 
     # calculate chunk size if it isn't specified
@@ -399,7 +420,27 @@ def _create_thread(tid, data):
             update_progress(progress)
 
     if db_task.mode == 'annotation':
-        models.Image.objects.bulk_create(db_images)
+        if validate_dimension.dimension == DimensionType.DIM_2D:
+            models.Image.objects.bulk_create(db_images)
+        else:
+            related_file = []
+            for image_data in db_images:
+                image_model = models.Image(
+                    data=image_data.data,
+                    path=image_data.path,
+                    frame=image_data.frame,
+                    width=image_data.width,
+                    height=image_data.height
+                )
+
+                image_model.save()
+                image_data = models.Image.objects.get(id=image_model.id)
+
+                if validate_dimension.related_files.get(image_data.path, None):
+                    for related_image_file in validate_dimension.related_files[image_data.path]:
+                        related_file.append(
+                            RelatedFile(data=db_data, primary_image_id=image_data.id, path=related_image_file))
+            RelatedFile.objects.bulk_create(related_file)
         db_images = []
     else:
         models.Video.objects.create(
@@ -414,4 +455,4 @@ def _create_thread(tid, data):
     preview.save(db_data.get_preview_path())
 
     slogger.glob.info("Found frames {} for Data #{}".format(db_data.size, db_data.id))
-    _save_task_to_db(db_task)
+    _save_task_to_db(db_task)

cvat/apps/engine/tests/test_rest_api.py

@@ -16,6 +16,8 @@ from enum import Enum
 from glob import glob
 from io import BytesIO
 from unittest import mock
+import open3d as o3d
+import struct
 
 import av
 import numpy as np
@@ -31,6 +33,8 @@ from rest_framework.test import APIClient, APITestCase
 from cvat.apps.engine.models import (AttributeType, Data, Job, Project,
     Segment, StatusChoice, Task, Label, StorageMethodChoice, StorageChoice)
 from cvat.apps.engine.prepare import prepare_meta, prepare_meta_for_upload
+from cvat.apps.engine.media_extractors import ValidateDimension
+from cvat.apps.engine.models import DimensionType
 
 def create_db_users(cls):
     (group_admin, _) = Group.objects.get_or_create(name="admin")
@@ -1840,6 +1844,47 @@ class TaskDataAPITestCase(APITestCase):
             zip_archive.write(data.read())
         cls._image_sizes[filename] = img_sizes
 
+        filename = "test_pointcloud_pcd.zip"
+        path = os.path.join(os.path.dirname(__file__), 'assets', filename)
+        image_sizes = []
+        # container = av.open(path, 'r')
+        zip_file = zipfile.ZipFile(path)
+        for info in zip_file.namelist():
+            if info.rsplit(".", maxsplit=1)[-1] == "pcd":
+                with zip_file.open(info, "r") as file:
+                    data = ValidateDimension.get_pcd_properties(file)
+                    image_sizes.append((int(data["WIDTH"]), int(data["HEIGHT"])))
+        cls._image_sizes[filename] = image_sizes
+
+        filename = "test_velodyne_points.zip"
+        path = os.path.join(os.path.dirname(__file__), 'assets', filename)
+        image_sizes = []
+        # create zip instance
+
+        zip_file = zipfile.ZipFile(path, mode='a')
+
+        source_path = []
+        root_path = os.path.abspath(os.path.split(path)[0])
+
+        for info in zip_file.namelist():
+            if os.path.splitext(info)[1][1:] == "bin":
+                zip_file.extract(info, root_path)
+                bin_path = os.path.abspath(os.path.join(root_path, info))
+                source_path.append(ValidateDimension.convert_bin_to_pcd(bin_path))
+
+        for path in source_path:
+            zip_file.write(path, os.path.abspath(path.replace(root_path, "")))
+
+        for info in zip_file.namelist():
+            if os.path.splitext(info)[1][1:] == "pcd":
+                with zip_file.open(info, "r") as file:
+                    data = ValidateDimension.get_pcd_properties(file)
+                    image_sizes.append((int(data["WIDTH"]), int(data["HEIGHT"])))
+        root_path = os.path.abspath(os.path.join(root_path, filename.split(".")[0]))
+
+        shutil.rmtree(root_path)
+        cls._image_sizes[filename] = image_sizes
+
     @classmethod
     def tearDownClass(cls):
         super().tearDownClass()
@@ -1905,8 +1950,10 @@ class TaskDataAPITestCase(APITestCase):
         return self._run_api_v1_task_id_data_get(tid, user, "frame", "original", number)
 
     @staticmethod
-    def _extract_zip_chunk(chunk_buffer):
+    def _extract_zip_chunk(chunk_buffer, dimension=DimensionType.DIM_2D):
         chunk = zipfile.ZipFile(chunk_buffer, mode='r')
+        if dimension == DimensionType.DIM_3D:
+            return [BytesIO(chunk.read(f)) for f in sorted(chunk.namelist()) if f.rsplit(".", maxsplit=1)[-1] == "pcd"]
         return [Image.open(BytesIO(chunk.read(f))) for f in sorted(chunk.namelist())]
 
     @staticmethod
@@ -1917,7 +1964,7 @@ class TaskDataAPITestCase(APITestCase):
 
     def _test_api_v1_tasks_id_data_spec(self, user, spec, data, expected_compressed_type, expected_original_type, image_sizes,
                                         expected_storage_method=StorageMethodChoice.FILE_SYSTEM,
-                                        expected_uploaded_data_location=StorageChoice.LOCAL):
+                                        expected_uploaded_data_location=StorageChoice.LOCAL, dimension=DimensionType.DIM_2D):
         # create task
         response = self._create_task(user, spec)
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
@@ -1953,8 +2000,9 @@ class TaskDataAPITestCase(APITestCase):
         response = self._get_preview(task_id, user)
         self.assertEqual(response.status_code, expected_status_code)
         if expected_status_code == status.HTTP_200_OK:
-            preview = Image.open(io.BytesIO(b"".join(response.streaming_content)))
+            if dimension == DimensionType.DIM_2D:
-            self.assertLessEqual(preview.size, image_sizes[0])
+                preview = Image.open(io.BytesIO(b"".join(response.streaming_content)))
+                self.assertLessEqual(preview.size, image_sizes[0])
 
         # check compressed chunk
         response = self._get_compressed_chunk(task_id, user, 0)
@@ -1965,14 +2013,18 @@ class TaskDataAPITestCase(APITestCase):
             else:
                 compressed_chunk = io.BytesIO(b"".join(response.streaming_content))
             if task["data_compressed_chunk_type"] == self.ChunkType.IMAGESET:
-                images = self._extract_zip_chunk(compressed_chunk)
+                images = self._extract_zip_chunk(compressed_chunk, dimension=dimension)
             else:
                 images = self._extract_video_chunk(compressed_chunk)
 
             self.assertEqual(len(images), min(task["data_chunk_size"], len(image_sizes)))
 
             for image_idx, image in enumerate(images):
-                self.assertEqual(image.size, image_sizes[image_idx])
+                if dimension == DimensionType.DIM_3D:
+                    properties = ValidateDimension.get_pcd_properties(image)
+                    self.assertEqual((int(properties["WIDTH"]),int(properties["HEIGHT"])), image_sizes[image_idx])
+                else:
+                    self.assertEqual(image.size, image_sizes[image_idx])
 
         # check original chunk
         response = self._get_original_chunk(task_id, user, 0)
@@ -1983,12 +2035,16 @@ class TaskDataAPITestCase(APITestCase):
             else:
                 original_chunk  = io.BytesIO(b"".join(response.streaming_content))
             if task["data_original_chunk_type"] == self.ChunkType.IMAGESET:
-                images = self._extract_zip_chunk(original_chunk)
+                images = self._extract_zip_chunk(original_chunk, dimension=dimension)
             else:
                 images = self._extract_video_chunk(original_chunk)
 
             for image_idx, image in enumerate(images):
-                self.assertEqual(image.size, image_sizes[image_idx])
+                if dimension == DimensionType.DIM_3D:
+                    properties = ValidateDimension.get_pcd_properties(image)
+                    self.assertEqual((int(properties["WIDTH"]), int(properties["HEIGHT"])), image_sizes[image_idx])
+                else:
+                    self.assertEqual(image.size, image_sizes[image_idx])
 
             self.assertEqual(len(images), min(task["data_chunk_size"], len(image_sizes)))
 
@@ -2004,7 +2060,7 @@ class TaskDataAPITestCase(APITestCase):
                 source_images = []
                 for f in source_files:
                     if zipfile.is_zipfile(f):
-                        source_images.extend(self._extract_zip_chunk(f))
+                        source_images.extend(self._extract_zip_chunk(f, dimension=dimension))
                     elif isinstance(f, io.BytesIO) and \
                             str(getattr(f, 'name', None)).endswith('.pdf'):
                         source_images.extend(convert_from_bytes(f.getvalue(),
@@ -2013,9 +2069,14 @@ class TaskDataAPITestCase(APITestCase):
                         source_images.append(Image.open(f))
 
                 for img_idx, image in enumerate(images):
-                    server_image = np.array(image)
+                    if dimension == DimensionType.DIM_3D:
-                    source_image = np.array(source_images[img_idx])
+                        server_image = np.array(image.getbuffer())
-                    self.assertTrue(np.array_equal(source_image, server_image))
+                        source_image = np.array(source_images[img_idx].getbuffer())
+                        self.assertTrue(np.array_equal(source_image, server_image))
+                    else:
+                        server_image = np.array(image)
+                        source_image = np.array(source_images[img_idx])
+                        self.assertTrue(np.array_equal(source_image, server_image))
 
     def _test_api_v1_tasks_id_data(self, user):
         task_spec = {
@@ -2415,6 +2476,45 @@ class TaskDataAPITestCase(APITestCase):
 
         self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.VIDEO, self.ChunkType.VIDEO, image_sizes)
 
+        task_spec = {
+            "name": "my archive task #24",
+            "overlap": 0,
+            "segment_size": 0,
+            "labels": [
+                {"name": "car"},
+                {"name": "person"},
+            ]
+        }
+
+        task_data = {
+            "client_files[0]": open(os.path.join(os.path.dirname(__file__), 'assets', 'test_pointcloud_pcd.zip'), 'rb'),
+            "image_quality": 100,
+        }
+        image_sizes = self._image_sizes["test_pointcloud_pcd.zip"]
+        self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET,
+                                             self.ChunkType.IMAGESET,
+                                             image_sizes, dimension=DimensionType.DIM_3D)
+
+        task_spec = {
+            "name": "my archive task #25",
+            "overlap": 0,
+            "segment_size": 0,
+            "labels": [
+                {"name": "car"},
+                {"name": "person"},
+            ]
+        }
+
+        task_data = {
+            "client_files[0]": open(os.path.join(os.path.dirname(__file__), 'assets', 'test_velodyne_points.zip'),
+                                    'rb'),
+            "image_quality": 100,
+        }
+        image_sizes = self._image_sizes["test_velodyne_points.zip"]
+        self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET,
+                                             self.ChunkType.IMAGESET,
+                                             image_sizes, dimension=DimensionType.DIM_3D)
+
     def test_api_v1_tasks_id_data_admin(self):
         self._test_api_v1_tasks_id_data(self.admin)
 

cvat/apps/engine/views.py

@@ -447,7 +447,7 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet):
                 if not db_data:
                     raise NotFound(detail='Cannot find requested data for the task')
 
-                frame_provider = FrameProvider(db_task.data)
+                frame_provider = FrameProvider(db_task.data, db_task.dimension)
 
                 if data_type == 'chunk':
                     data_id = int(data_id)

cvat/requirements/base.txt

@@ -44,4 +44,5 @@ tensorflow==2.4.0 # Optional requirement of Datumaro
 # archives. Don't use as a python module because it has GPL license.
 patool==1.12
 diskcache==5.0.2
+open3d==0.11.2
 git+https://github.com/openvinotoolkit/datumaro@v0.1.4