Add dataset export facility (#813)

* Add datumaro django application
* Add cvat task datumaro bindings
* Add REST api for task export
* Add scheduler service
* Updated CHANGELOG.md

.travis.yml

@@ -13,4 +13,5 @@ before_script:
 
 script:
   - docker exec -it cvat /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
+  - docker exec -it cvat /bin/bash -c 'python3 manage.py test datumaro/'
   - docker exec -it cvat /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls'

.vscode/launch.json

@@ -71,6 +71,22 @@
             "env": {},
             "console": "internalConsole"
         },
+        {
+            "name": "server: RQ - scheduler",
+            "type": "python",
+            "request": "launch",
+            "stopOnEntry": false,
+            "justMyCode": false,
+            "pythonPath": "${config:python.pythonPath}",
+            "program": "${workspaceRoot}/manage.py",
+            "args": [
+                "rqscheduler",
+            ],
+            "django": true,
+            "cwd": "${workspaceFolder}",
+            "env": {},
+            "console": "internalConsole"
+        },
         {
             "name": "server: RQ - low",
             "type": "python",
@@ -177,6 +193,7 @@
                 "server: django",
                 "server: RQ - default",
                 "server: RQ - low",
+                "server: RQ - scheduler",
                 "server: git",
             ]
         }

CHANGELOG.md

@@ -15,6 +15,8 @@ https://github.com/opencv/cvat/issues/750).
 - Auto segmentation using Mask_RCNN component (Keras+Tensorflow Mask R-CNN Segmentation)
 - Added MOT CSV format support
 - Ability to dump/load annotations in LabelMe format from UI
+- REST API to export an annotation task (images + annotations)
+- Datumaro is an experimental framework to build, analyze, debug and visualize datasets for DL algorithms
 
 ### Changed
 -

Dockerfile

@@ -154,6 +154,10 @@ COPY utils ${HOME}/utils
 COPY cvat/ ${HOME}/cvat
 COPY cvat-core/ ${HOME}/cvat-core
 COPY tests ${HOME}/tests
+COPY datumaro/ ${HOME}/datumaro
+
+RUN sed -r "s/^(.*)#.*$/\1/g" ${HOME}/datumaro/requirements.txt | xargs -n 1 -L 1 pip3 install --no-cache-dir
+
 # Binary option is necessary to correctly apply the patch on Windows platform.
 # https://unix.stackexchange.com/questions/239364/how-to-fix-hunk-1-failed-at-1-different-line-endings-message
 RUN patch --binary -p1 < ${HOME}/cvat/apps/engine/static/engine/js/3rdparty.patch

cvat/apps/dataset_manager/bindings.py

@@ -0,0 +1,176 @@
+from collections import OrderedDict
+import os
+import os.path as osp
+
+from django.db import transaction
+
+from cvat.apps.annotation.annotation import Annotation
+from cvat.apps.engine.annotation import TaskAnnotation
+from cvat.apps.engine.models import Task, ShapeType
+
+import datumaro.components.extractor as datumaro
+from datumaro.util.image import lazy_image
+
+
+class CvatImagesDirExtractor(datumaro.Extractor):
+    _SUPPORTED_FORMATS = ['.png', '.jpg']
+
+    def __init__(self, url):
+        super().__init__()
+
+        items = []
+        for (dirpath, _, filenames) in os.walk(url):
+            for name in filenames:
+                path = osp.join(dirpath, name)
+                if self._is_image(path):
+                    item_id = Task.get_image_frame(path)
+                    item = datumaro.DatasetItem(
+                        id=item_id, image=lazy_image(path))
+                    items.append((item.id, item))
+
+        items = sorted(items, key=lambda e: e[0])
+        items = OrderedDict(items)
+        self._items = items
+
+        self._subsets = None
+
+    def __iter__(self):
+        for item in self._items.values():
+            yield item
+
+    def __len__(self):
+        return len(self._items)
+
+    def subsets(self):
+        return self._subsets
+
+    def get(self, item_id, subset=None, path=None):
+        if path or subset:
+            raise KeyError()
+        return self._items[item_id]
+
+    def _is_image(self, path):
+        for ext in self._SUPPORTED_FORMATS:
+            if osp.isfile(path) and path.endswith(ext):
+                return True
+        return False
+
+
+class CvatTaskExtractor(datumaro.Extractor):
+    def __init__(self, url, db_task, user):
+        self._db_task = db_task
+        self._categories = self._load_categories()
+
+        cvat_annotations = TaskAnnotation(db_task.id, user)
+        with transaction.atomic():
+            cvat_annotations.init_from_db()
+        cvat_annotations = Annotation(cvat_annotations.ir_data, db_task)
+
+        dm_annotations = []
+
+        for cvat_anno in cvat_annotations.group_by_frame():
+            dm_anno = self._read_cvat_anno(cvat_anno)
+            dm_item = datumaro.DatasetItem(
+                id=cvat_anno.frame, annotations=dm_anno)
+            dm_annotations.append((dm_item.id, dm_item))
+
+        dm_annotations = sorted(dm_annotations, key=lambda e: e[0])
+        self._items = OrderedDict(dm_annotations)
+
+        self._subsets = None
+
+    def __iter__(self):
+        for item in self._items.values():
+            yield item
+
+    def __len__(self):
+        return len(self._items)
+
+    def subsets(self):
+        return self._subsets
+
+    def get(self, item_id, subset=None, path=None):
+        if path or subset:
+            raise KeyError()
+        return self._items[item_id]
+
+    def _load_categories(self):
+        categories = {}
+        label_categories = datumaro.LabelCategories()
+
+        db_labels = self._db_task.label_set.all()
+        for db_label in db_labels:
+            db_attributes = db_label.attributespec_set.all()
+            label_categories.add(db_label.name)
+
+            for db_attr in db_attributes:
+                label_categories.attributes.add(db_attr.name)
+
+        categories[datumaro.AnnotationType.label] = label_categories
+
+        return categories
+
+    def categories(self):
+        return self._categories
+
+    def _read_cvat_anno(self, cvat_anno):
+        item_anno = []
+
+        categories = self.categories()
+        label_cat = categories[datumaro.AnnotationType.label]
+
+        label_map = {}
+        label_attrs = {}
+        db_labels = self._db_task.label_set.all()
+        for db_label in db_labels:
+            label_map[db_label.name] = label_cat.find(db_label.name)[0]
+
+            attrs = {}
+            db_attributes = db_label.attributespec_set.all()
+            for db_attr in db_attributes:
+                attrs[db_attr.name] = db_attr.default_value
+            label_attrs[db_label.name] = attrs
+        map_label = lambda label_db_name: label_map[label_db_name]
+
+        for tag_obj in cvat_anno.tags:
+            anno_group = tag_obj.group
+            if isinstance(anno_group, int):
+                anno_group = anno_group
+            anno_label = map_label(tag_obj.label)
+            anno_attr = dict(label_attrs[tag_obj.label])
+            for attr in tag_obj.attributes:
+                anno_attr[attr.name] = attr.value
+
+            anno = datumaro.LabelObject(label=anno_label,
+                attributes=anno_attr, group=anno_group)
+            item_anno.append(anno)
+
+        for shape_obj in cvat_anno.labeled_shapes:
+            anno_group = shape_obj.group
+            if isinstance(anno_group, int):
+                anno_group = anno_group
+            anno_label = map_label(shape_obj.label)
+            anno_attr = dict(label_attrs[shape_obj.label])
+            for attr in shape_obj.attributes:
+                anno_attr[attr.name] = attr.value
+
+            anno_points = shape_obj.points
+            if shape_obj.type == ShapeType.POINTS:
+                anno = datumaro.PointsObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj.type == ShapeType.POLYLINE:
+                anno = datumaro.PolyLineObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj.type == ShapeType.POLYGON:
+                anno = datumaro.PolygonObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj.type == ShapeType.RECTANGLE:
+                x0, y0, x1, y1 = anno_points
+                anno = datumaro.BboxObject(x0, y0, x1 - x0, y1 - y0,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            else:
+                raise Exception("Unknown shape type '%s'" % (shape_obj.type))
+
+            item_anno.append(anno)
+
+        return item_anno

cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py

@@ -0,0 +1,120 @@
+from collections import OrderedDict
+import getpass
+import json
+import os, os.path as osp
+import requests
+
+from datumaro.components.config import (Config,
+    SchemaBuilder as _SchemaBuilder,
+)
+import datumaro.components.extractor as datumaro
+from datumaro.util.image import lazy_image, load_image
+
+from cvat.utils.cli.core import CLI as CVAT_CLI, CVAT_API_V1
+
+
+CONFIG_SCHEMA = _SchemaBuilder() \
+    .add('task_id', int) \
+    .add('server_host', str) \
+    .add('server_port', int) \
+    .build()
+
+class cvat_rest_api_task_images(datumaro.Extractor):
+    def _image_local_path(self, item_id):
+        task_id = self._config.task_id
+        return osp.join(self._cache_dir,
+            'task_{}_frame_{:06d}.jpg'.format(task_id, item_id))
+
+    def _make_image_loader(self, item_id):
+        return lazy_image(item_id,
+            lambda item_id: self._image_loader(item_id, self))
+
+    def _is_image_cached(self, item_id):
+        return osp.isfile(self._image_local_path(item_id))
+
+    def _download_image(self, item_id):
+        self._connect()
+        os.makedirs(self._cache_dir, exist_ok=True)
+        self._cvat_cli.tasks_frame(task_id=self._config.task_id,
+            frame_ids=[item_id], outdir=self._cache_dir)
+
+    def _connect(self):
+        if self._session is not None:
+            return
+
+        session = None
+        try:
+            print("Enter credentials for '%s:%s':" % \
+                (self._config.server_host, self._config.server_port))
+            username = input('User: ')
+            password = getpass.getpass()
+
+            session = requests.Session()
+            session.auth = (username, password)
+
+            api = CVAT_API_V1(self._config.server_host,
+                self._config.server_port)
+            cli = CVAT_CLI(session, api)
+
+            self._session = session
+            self._cvat_cli = cli
+        except Exception:
+            if session is not None:
+                session.close()
+
+    def __del__(self):
+        if hasattr(self, '_session'):
+            if self._session is not None:
+                self._session.close()
+
+    @staticmethod
+    def _image_loader(item_id, extractor):
+        if not extractor._is_image_cached(item_id):
+            extractor._download_image(item_id)
+        local_path = extractor._image_local_path(item_id)
+        return load_image(local_path)
+
+    def __init__(self, url):
+        super().__init__()
+
+        local_dir = url
+        self._local_dir = local_dir
+        self._cache_dir = osp.join(local_dir, 'images')
+
+        with open(osp.join(url, 'config.json'), 'r') as config_file:
+            config = json.load(config_file)
+            config = Config(config, schema=CONFIG_SCHEMA)
+        self._config = config
+
+        with open(osp.join(url, 'images_meta.json'), 'r') as images_file:
+            images_meta = json.load(images_file)
+            image_list = images_meta['images']
+
+        items = []
+        for entry in image_list:
+            item_id = entry['id']
+            item = datumaro.DatasetItem(
+                id=item_id, image=self._make_image_loader(item_id))
+            items.append((item.id, item))
+
+        items = sorted(items, key=lambda e: e[0])
+        items = OrderedDict(items)
+        self._items = items
+
+        self._cvat_cli = None
+        self._session = None
+
+    def __iter__(self):
+        for item in self._items.values():
+            yield item
+
+    def __len__(self):
+        return len(self._items)
+
+    def subsets(self):
+        return None
+
+    def get(self, item_id, subset=None, path=None):
+        if path or subset:
+            raise KeyError()
+        return self._items[item_id]

cvat/apps/dataset_manager/task.py

@@ -0,0 +1,351 @@
+from datetime import timedelta
+import json
+import os
+import os.path as osp
+import shutil
+import sys
+import tempfile
+from urllib.parse import urlsplit
+
+from django.utils import timezone
+import django_rq
+
+from cvat.apps.engine.log import slogger
+from cvat.apps.engine.models import Task, ShapeType
+from .util import current_function_name, make_zip_archive
+
+_DATUMARO_REPO_PATH = osp.join(__file__[:__file__.rfind('cvat/')], 'datumaro')
+sys.path.append(_DATUMARO_REPO_PATH)
+from datumaro.components.project import Project
+import datumaro.components.extractor as datumaro
+from .bindings import CvatImagesDirExtractor, CvatTaskExtractor
+
+
+_MODULE_NAME = __package__ + '.' + osp.splitext(osp.basename(__file__))[0]
+def log_exception(logger=None, exc_info=True):
+    if logger is None:
+        logger = slogger
+    logger.exception("[%s @ %s]: exception occurred" % \
+            (_MODULE_NAME, current_function_name(2)),
+        exc_info=exc_info)
+
+_TASK_IMAGES_EXTRACTOR = '_cvat_task_images'
+_TASK_ANNO_EXTRACTOR = '_cvat_task_anno'
+_TASK_IMAGES_REMOTE_EXTRACTOR = 'cvat_rest_api_task_images'
+
+def get_export_cache_dir(db_task):
+    return osp.join(db_task.get_task_dirname(), 'export_cache')
+
+class TaskProject:
+    @staticmethod
+    def _get_datumaro_project_dir(db_task):
+        return osp.join(db_task.get_task_dirname(), 'datumaro')
+
+    @staticmethod
+    def create(db_task):
+        task_project = TaskProject(db_task)
+        task_project._create()
+        return task_project
+
+    @staticmethod
+    def load(db_task):
+        task_project = TaskProject(db_task)
+        task_project._load()
+        task_project._init_dataset()
+        return task_project
+
+    @staticmethod
+    def from_task(db_task, user):
+        task_project = TaskProject(db_task)
+        task_project._import_from_task(user)
+        return task_project
+
+    def __init__(self, db_task):
+        self._db_task = db_task
+        self._project_dir = self._get_datumaro_project_dir(db_task)
+        self._project = None
+        self._dataset = None
+
+    def _create(self):
+        self._project = Project.generate(self._project_dir)
+        self._project.add_source('task_%s' % self._db_task.id, {
+            'url': self._db_task.get_data_dirname(),
+            'format': _TASK_IMAGES_EXTRACTOR,
+        })
+        self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR,
+            CvatImagesDirExtractor)
+
+        self._init_dataset()
+        self._dataset.define_categories(self._generate_categories())
+
+        self.save()
+
+    def _load(self):
+        self._project = Project.load(self._project_dir)
+        self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR,
+            CvatImagesDirExtractor)
+
+    def _import_from_task(self, user):
+        self._project = Project.generate(self._project_dir)
+
+        self._project.add_source('task_%s_images' % self._db_task.id, {
+            'url': self._db_task.get_data_dirname(),
+            'format': _TASK_IMAGES_EXTRACTOR,
+        })
+        self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR,
+            CvatImagesDirExtractor)
+
+        self._project.add_source('task_%s_anno' % self._db_task.id, {
+            'format': _TASK_ANNO_EXTRACTOR,
+        })
+        self._project.env.extractors.register(_TASK_ANNO_EXTRACTOR,
+            lambda url: CvatTaskExtractor(url,
+                db_task=self._db_task, user=user))
+
+        self._init_dataset()
+
+    def _init_dataset(self):
+        self._dataset = self._project.make_dataset()
+
+    def _generate_categories(self):
+        categories = {}
+        label_categories = datumaro.LabelCategories()
+
+        db_labels = self._db_task.label_set.all()
+        for db_label in db_labels:
+            db_attributes = db_label.attributespec_set.all()
+            label_categories.add(db_label.name)
+
+            for db_attr in db_attributes:
+                label_categories.attributes.add(db_attr.name)
+
+        categories[datumaro.AnnotationType.label] = label_categories
+
+        return categories
+
+    def put_annotations(self, annotations):
+        patch = {}
+
+        categories = self._dataset.categories()
+        label_cat = categories[datumaro.AnnotationType.label]
+
+        label_map = {}
+        attr_map = {}
+        db_labels = self._db_task.label_set.all()
+        for db_label in db_labels:
+            label_map[db_label.id] = label_cat.find(db_label.name)
+
+            db_attributes = db_label.attributespec_set.all()
+            for db_attr in db_attributes:
+                attr_map[(db_label.id, db_attr.id)] = db_attr.name
+        map_label = lambda label_db_id: label_map[label_db_id]
+        map_attr = lambda label_db_id, attr_db_id: \
+            attr_map[(label_db_id, attr_db_id)]
+
+        for tag_obj in annotations['tags']:
+            item_id = str(tag_obj['frame'])
+            item_anno = patch.get(item_id, [])
+
+            anno_group = tag_obj['group']
+            if isinstance(anno_group, int):
+                anno_group = [anno_group]
+            anno_label = map_label(tag_obj['label_id'])
+            anno_attr = {}
+            for attr in tag_obj['attributes']:
+                attr_name = map_attr(tag_obj['label_id'], attr['id'])
+                anno_attr[attr_name] = attr['value']
+
+            anno = datumaro.LabelObject(label=anno_label,
+                attributes=anno_attr, group=anno_group)
+            item_anno.append(anno)
+
+            patch[item_id] = item_anno
+
+        for shape_obj in annotations['shapes']:
+            item_id = str(shape_obj['frame'])
+            item_anno = patch.get(item_id, [])
+
+            anno_group = shape_obj['group']
+            if isinstance(anno_group, int):
+                anno_group = [anno_group]
+            anno_label = map_label(shape_obj['label_id'])
+            anno_attr = {}
+            for attr in shape_obj['attributes']:
+                attr_name = map_attr(shape_obj['label_id'], attr['id'])
+                anno_attr[attr_name] = attr['value']
+
+            anno_points = shape_obj['points']
+            if shape_obj['type'] == ShapeType.POINTS:
+                anno = datumaro.PointsObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj['type'] == ShapeType.POLYLINE:
+                anno = datumaro.PolyLineObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj['type'] == ShapeType.POLYGON:
+                anno = datumaro.PolygonObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj['type'] == ShapeType.RECTANGLE:
+                x0, y0, x1, y1 = anno_points
+                anno = datumaro.BboxObject(x0, y0, x1 - x0, y1 - y0,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            else:
+                raise Exception("Unknown shape type '%s'" % (shape_obj['type']))
+
+            item_anno.append(anno)
+
+            patch[item_id] = item_anno
+
+        # TODO: support track annotations
+
+        patch = [datumaro.DatasetItem(id=id_, annotations=anno) \
+            for id_, ann in patch.items()]
+
+        self._dataset.update(patch)
+
+    def save(self, save_dir=None, save_images=False):
+        if self._dataset is not None:
+            self._dataset.save(save_dir=save_dir, save_images=save_images)
+        else:
+            self._project.save(save_dir=save_dir)
+
+    def export(self, dst_format, save_dir, save_images=False, server_url=None):
+        if self._dataset is None:
+            self._init_dataset()
+        if dst_format == DEFAULT_FORMAT:
+            self._dataset.save(save_dir=save_dir, save_images=save_images)
+        elif dst_format == DEFAULT_FORMAT_REMOTE:
+            self._remote_export(save_dir=save_dir, server_url=server_url)
+        else:
+            self._dataset.export(output_format=dst_format,
+                save_dir=save_dir, save_images=save_images)
+
+    def _remote_image_converter(self, save_dir, server_url=None):
+        os.makedirs(save_dir, exist_ok=True)
+
+        db_task = self._db_task
+        items = []
+        config = {
+            'server_host': 'localhost',
+            'server_port': '',
+            'task_id': db_task.id,
+        }
+        if server_url:
+            parsed_url = urlsplit(server_url)
+            config['server_host'] = parsed_url.netloc
+            port = 80
+            if parsed_url.port:
+                port = parsed_url.port
+            config['server_port'] = int(port)
+
+        images_meta = {
+            'images': items,
+        }
+        for db_image in self._db_task.image_set.all():
+            frame_info = {
+                'id': db_image.frame,
+                'width': db_image.width,
+                'height': db_image.height,
+            }
+            items.append(frame_info)
+
+        with open(osp.join(save_dir, 'config.json'), 'w') as config_file:
+            json.dump(config, config_file)
+        with open(osp.join(save_dir, 'images_meta.json'), 'w') as images_file:
+            json.dump(images_meta, images_file)
+
+    def _remote_export(self, save_dir, server_url=None):
+        if self._dataset is None:
+            self._init_dataset()
+
+        os.makedirs(save_dir, exist_ok=True)
+        self._dataset.save(save_dir=save_dir, save_images=False, merge=True)
+
+        exported_project = Project.load(save_dir)
+        source_name = 'task_%s_images' % self._db_task.id
+        exported_project.add_source(source_name, {
+            'format': _TASK_IMAGES_REMOTE_EXTRACTOR,
+        })
+        self._remote_image_converter(
+            osp.join(save_dir, exported_project.local_source_dir(source_name)),
+            server_url=server_url)
+        exported_project.save()
+
+        templates_dir = osp.join(osp.dirname(__file__),
+            'export_templates', 'extractors')
+        target_dir = osp.join(
+            exported_project.config.project_dir,
+            exported_project.config.env_dir,
+            exported_project.env.config.extractors_dir)
+        os.makedirs(target_dir, exist_ok=True)
+        shutil.copyfile(
+            osp.join(templates_dir, _TASK_IMAGES_REMOTE_EXTRACTOR + '.py'),
+            osp.join(target_dir, _TASK_IMAGES_REMOTE_EXTRACTOR + '.py'))
+
+        # NOTE: put datumaro component to the archive so that
+        # it was available to the user
+        shutil.copytree(_DATUMARO_REPO_PATH, osp.join(save_dir, 'datumaro'),
+            ignore=lambda src, names: ['__pycache__'] + [
+                n for n in names
+                if sum([int(n.endswith(ext)) for ext in
+                        ['.pyx', '.pyo', '.pyd', '.pyc']])
+            ])
+
+
+DEFAULT_FORMAT = "datumaro_project"
+DEFAULT_FORMAT_REMOTE = "datumaro_project_remote"
+DEFAULT_CACHE_TTL = timedelta(hours=10)
+CACHE_TTL = DEFAULT_CACHE_TTL
+
+def export_project(task_id, user, dst_format=None, server_url=None):
+    try:
+        db_task = Task.objects.get(pk=task_id)
+
+        if not dst_format:
+            dst_format = DEFAULT_FORMAT
+
+        cache_dir = get_export_cache_dir(db_task)
+        save_dir = osp.join(cache_dir, dst_format)
+        archive_path = osp.normpath(save_dir) + '.zip'
+
+        task_time = timezone.localtime(db_task.updated_date).timestamp()
+        if not (osp.exists(archive_path) and \
+                task_time <= osp.getmtime(archive_path)):
+            os.makedirs(cache_dir, exist_ok=True)
+            with tempfile.TemporaryDirectory(
+                    dir=cache_dir, prefix=dst_format + '_') as temp_dir:
+                project = TaskProject.from_task(db_task, user)
+                project.export(dst_format, save_dir=temp_dir, save_images=True,
+                    server_url=server_url)
+
+                os.makedirs(cache_dir, exist_ok=True)
+                make_zip_archive(temp_dir, archive_path)
+
+            archive_ctime = osp.getctime(archive_path)
+            scheduler = django_rq.get_scheduler()
+            cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL,
+                func=clear_export_cache,
+                task_id=task_id,
+                file_path=archive_path, file_ctime=archive_ctime)
+            slogger.task[task_id].info(
+                "The task '{}' is exported as '{}' "
+                "and available for downloading for next '{}'. "
+                "Export cache cleaning job is enqueued, "
+                "id '{}', start in '{}'".format(
+                    db_task.name, dst_format, CACHE_TTL,
+                    cleaning_job.id, CACHE_TTL))
+
+        return archive_path
+    except Exception:
+        log_exception(slogger.task[task_id])
+        raise
+
+def clear_export_cache(task_id, file_path, file_ctime):
+    try:
+        if osp.exists(file_path) and osp.getctime(file_path) == file_ctime:
+            os.remove(file_path)
+            slogger.task[task_id].info(
+                "Export cache file '{}' successfully removed" \
+                .format(file_path))
+    except Exception:
+        log_exception(slogger.task[task_id])
+        raise

cvat/apps/dataset_manager/util.py

@@ -0,0 +1,15 @@
+import inspect
+import os, os.path as osp
+import zipfile
+
+
+def current_function_name(depth=1):
+    return inspect.getouterframes(inspect.currentframe())[depth].function
+
+
+def make_zip_archive(src_path, dst_path):
+    with zipfile.ZipFile(dst_path, 'w') as archive:
+        for (dirpath, _, filenames) in os.walk(src_path):
+            for name in filenames:
+                path = osp.join(dirpath, name)
+                archive.write(path, osp.relpath(path, src_path))

cvat/apps/engine/models.py

@@ -86,6 +86,17 @@ class Task(models.Model):
 
         return path
 
+    @staticmethod
+    def get_image_frame(image_path):
+        assert image_path.endswith('.jpg')
+        index = os.path.splitext(os.path.basename(image_path))[0]
+
+        path = os.path.dirname(image_path)
+        d2 = os.path.basename(path)
+        d1 = os.path.basename(os.path.dirname(path))
+
+        return int(d1) * 10000 + int(d2) * 100 + int(index)
+
     def get_frame_step(self):
         match = re.search("step\s*=\s*([1-9]\d*)", self.frame_filter)
         return int(match.group(1)) if match else 1

cvat/apps/engine/views.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: MIT
 
 import os
+import os.path as osp
 import re
 import traceback
 from ast import literal_eval
@@ -25,6 +26,7 @@ from rest_framework import mixins
 from django_filters import rest_framework as filters
 import django_rq
 from django.db import IntegrityError
+from django.utils import timezone
 
 
 from . import annotation, task, models
@@ -45,6 +47,7 @@ from cvat.apps.authentication import auth
 from rest_framework.permissions import SAFE_METHODS
 from cvat.apps.annotation.models import AnnotationDumper, AnnotationLoader
 from cvat.apps.annotation.format import get_annotation_formats
+import cvat.apps.dataset_manager.task as DatumaroTask
 
 # Server REST API
 @login_required
@@ -443,6 +446,66 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet):
                 "cannot get frame #{}".format(frame), exc_info=True)
             return HttpResponseBadRequest(str(e))
 
+    @action(detail=True, methods=['GET'], serializer_class=None,
+        url_path='export/')
+    def dataset_export(self, request, pk):
+        """Export task as a dataset in a specific format"""
+
+        db_task = self.get_object()
+
+        action = request.query_params.get("action", "")
+        action = action.lower()
+        if action not in ["", "download"]:
+            raise serializers.ValidationError(
+                "Unexpected parameter 'action' specified for the request")
+
+        dst_format = request.query_params.get("format", "")
+        if not dst_format:
+            dst_format = DatumaroTask.DEFAULT_FORMAT
+        dst_format = dst_format.lower()
+        if 100 < len(dst_format) or not re.fullmatch(r"^[\w_-]+$", dst_format):
+            raise serializers.ValidationError(
+                "Unexpected parameter 'format' specified for the request")
+
+        rq_id = "task_dataset_export.{}.{}".format(pk, dst_format)
+        queue = django_rq.get_queue("default")
+
+        rq_job = queue.fetch_job(rq_id)
+        if rq_job:
+            task_time = timezone.localtime(db_task.updated_date)
+            request_time = timezone.localtime(rq_job.meta.get('request_time', datetime.min))
+            if request_time < task_time:
+                rq_job.cancel()
+                rq_job.delete()
+            else:
+                if rq_job.is_finished:
+                    file_path = rq_job.return_value
+                    if action == "download" and osp.exists(file_path):
+                        rq_job.delete()
+
+                        timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
+                        filename = "task_{}-{}-{}.zip".format(
+                            db_task.name, timestamp, dst_format)
+                        return sendfile(request, file_path, attachment=True,
+                            attachment_filename=filename.lower())
+                    else:
+                        if osp.exists(file_path):
+                            return Response(status=status.HTTP_201_CREATED)
+                elif rq_job.is_failed:
+                    exc_info = str(rq_job.exc_info)
+                    rq_job.delete()
+                    return Response(exc_info,
+                        status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+                else:
+                    return Response(status=status.HTTP_202_ACCEPTED)
+
+        ttl = DatumaroTask.CACHE_TTL.total_seconds()
+        queue.enqueue_call(func=DatumaroTask.export_project,
+            args=(pk, request.user, dst_format), job_id=rq_id,
+            meta={ 'request_time': timezone.localtime() },
+            result_ttl=ttl, failure_ttl=ttl)
+        return Response(status=status.HTTP_201_CREATED)
+
 class JobViewSet(viewsets.GenericViewSet,
     mixins.RetrieveModelMixin, mixins.UpdateModelMixin):
     queryset = Job.objects.all().order_by('id')

cvat/requirements/base.txt

@@ -18,6 +18,7 @@ redis==3.2.0
 requests==2.20.0
 rjsmin==1.0.12
 rq==1.0.0
+rq-scheduler==0.9.1
 scipy==1.2.1
 sqlparse==0.2.4
 django-sendfile==0.3.11

cvat/settings/base.py

@@ -95,6 +95,7 @@ INSTALLED_APPS = [
     'cvat.apps.authentication',
     'cvat.apps.documentation',
     'cvat.apps.git',
+    'cvat.apps.dataset_manager',
     'cvat.apps.annotation',
     'django_rq',
     'compressor',

datumaro/README.md

@@ -0,0 +1,36 @@
+# Dataset framework
+
+A framework to prepare, manage, build, analyze datasets
+
+## Documentation
+
+-[Quick start guide](docs/quickstart.md)
+
+## Installation
+
+Python3.5+ is required.
+
+To install into a virtual environment do:
+
+``` bash
+python -m pip install virtualenv
+python -m virtualenv venv
+. venv/bin/activate
+pip install -r requirements.txt
+```
+
+## Execution
+
+The tool can be executed both as a script and as a module.
+
+``` bash
+PYTHONPATH="..."
+python -m datumaro <command>
+python path/to/datum.py
+```
+
+## Testing
+
+``` bash
+python -m unittest discover -s tests
+```

datumaro/datum.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+import sys
+
+from datumaro import main
+
+
+if __name__ == '__main__':
+    sys.exit(main())

datumaro/datumaro/__init__.py

@@ -0,0 +1,89 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import logging as log
+import sys
+
+from .cli import (
+    project as project_module,
+    source as source_module,
+    item as item_module,
+    model as model_module,
+    # inference as inference_module,
+
+    create_command as create_command_module,
+    add_command as add_command_module,
+    remove_command as remove_command_module,
+    export_command as export_command_module,
+    # diff_command as diff_command_module,
+    # build_command as build_command_module,
+    stats_command as stats_command_module,
+    explain_command as explain_command_module,
+)
+from .components.config import VERSION
+
+
+KNOWN_COMMANDS = {
+    # contexts
+    'project': project_module.main,
+    'source': source_module.main,
+    'item': item_module.main,
+    'model': model_module.main,
+    # 'inference': inference_module.main,
+
+    # shortcuts
+    'create': create_command_module.main,
+    'add': add_command_module.main,
+    'remove': remove_command_module.main,
+    'export': export_command_module.main,
+    # 'diff': diff_command_module.main,
+    # 'build': build_command_module.main,
+    'stats': stats_command_module.main,
+    'explain': explain_command_module.main,
+}
+
+def get_command(name, args=None):
+    return KNOWN_COMMANDS[name]
+
+def loglevel(name):
+    numeric = getattr(log, name.upper(), None)
+    if not isinstance(numeric, int):
+        raise ValueError('Invalid log level: %s' % name)
+    return numeric
+
+def parse_command(input_args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('command', choices=KNOWN_COMMANDS.keys(),
+        help='A command to execute')
+    parser.add_argument('args', nargs=argparse.REMAINDER)
+    parser.add_argument('--version', action='version', version=VERSION)
+    parser.add_argument('--loglevel', type=loglevel, default='info',
+        help="Logging level (default: %(default)s)")
+
+    general_args = parser.parse_args(input_args)
+    command_name = general_args.command
+    command_args = general_args.args
+    return general_args, command_name, command_args
+
+def set_up_logger(general_args):
+    loglevel = general_args.loglevel
+    log.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
+        level=loglevel)
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    general_args, command_name, command_args = parse_command(args)
+
+    set_up_logger(general_args)
+
+    command = get_command(command_name, general_args)
+    return command(command_args)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

datumaro/datumaro/__main__.py

@@ -0,0 +1,12 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import sys
+from . import main
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+

datumaro/datumaro/cli/__init__.py

@@ -0,0 +1,5 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+

datumaro/datumaro/cli/add_command.py

@@ -0,0 +1,21 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+
+from . import source as source_module
+
+
+def build_parser(parser=argparse.ArgumentParser()):
+    source_module.build_add_parser(parser). \
+        set_defaults(command=source_module.add_command)
+
+    return parser
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+
+    return args.command(args)

datumaro/datumaro/cli/create_command.py

@@ -0,0 +1,21 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+
+from . import project as project_module
+
+
+def build_parser(parser=argparse.ArgumentParser()):
+    project_module.build_create_parser(parser) \
+        .set_defaults(command=project_module.create_command)
+
+    return parser
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+
+    return args.command(args)

datumaro/datumaro/cli/explain_command.py

@@ -0,0 +1,192 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import cv2
+import logging as log
+import os
+import os.path as osp
+
+from datumaro.components.project import Project
+from datumaro.components.algorithms.rise import RISE
+from datumaro.util.command_targets import (TargetKinds, target_selector,
+    ProjectTarget, SourceTarget, ImageTarget, is_project_path)
+from datumaro.util.image import load_image
+from .util.project import load_project
+
+
+def build_parser(parser=argparse.ArgumentParser()):
+    parser.add_argument('-m', '--model', required=True,
+        help="Model to use for inference")
+    parser.add_argument('-t', '--target', default=None,
+        help="Inference target - image, source, project "
+             "(default: current dir)")
+    parser.add_argument('-d', '--save-dir', default=None,
+        help="Directory to save output (default: display only)")
+
+    method_sp = parser.add_subparsers(dest='algorithm')
+
+    rise_parser = method_sp.add_parser('rise')
+    rise_parser.add_argument('-s', '--max-samples', default=None, type=int,
+        help="Number of algorithm iterations (default: mask size ^ 2)")
+    rise_parser.add_argument('--mw', '--mask-width',
+        dest='mask_width', default=7, type=int,
+        help="Mask width (default: %(default)s)")
+    rise_parser.add_argument('--mh', '--mask-height',
+        dest='mask_height', default=7, type=int,
+        help="Mask height (default: %(default)s)")
+    rise_parser.add_argument('--prob', default=0.5, type=float,
+        help="Mask pixel inclusion probability (default: %(default)s)")
+    rise_parser.add_argument('--iou', '--iou-thresh',
+        dest='iou_thresh', default=0.9, type=float,
+        help="IoU match threshold for detections (default: %(default)s)")
+    rise_parser.add_argument('--nms', '--nms-iou-thresh',
+        dest='nms_iou_thresh', default=0.0, type=float,
+        help="IoU match threshold in Non-maxima suppression (default: no NMS)")
+    rise_parser.add_argument('--conf', '--det-conf-thresh',
+        dest='det_conf_thresh', default=0.0, type=float,
+        help="Confidence threshold for detections (default: do not filter)")
+    rise_parser.add_argument('-b', '--batch-size', default=1, type=int,
+        help="Inference batch size (default: %(default)s)")
+    rise_parser.add_argument('--progressive', action='store_true',
+        help="Visualize results during computations")
+
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    parser.set_defaults(command=explain_command)
+
+    return parser
+
+def explain_command(args):
+    from matplotlib import cm
+
+    project = load_project(args.project_dir)
+
+    model = project.make_executable_model(args.model)
+
+    if str(args.algorithm).lower() != 'rise':
+        raise NotImplementedError()
+
+    rise = RISE(model,
+        max_samples=args.max_samples,
+        mask_width=args.mask_width,
+        mask_height=args.mask_height,
+        prob=args.prob,
+        iou_thresh=args.iou_thresh,
+        nms_thresh=args.nms_iou_thresh,
+        det_conf_thresh=args.det_conf_thresh,
+        batch_size=args.batch_size)
+
+    if args.target[0] == TargetKinds.image:
+        image_path = args.target[1]
+        image = load_image(image_path)
+        if model.preferred_input_size() is not None:
+            h, w = model.preferred_input_size()
+            image = cv2.resize(image, (w, h))
+
+        log.info("Running inference explanation for '%s'" % image_path)
+        heatmap_iter = rise.apply(image, progressive=args.progressive)
+
+        image = image / 255.0
+        file_name = osp.splitext(osp.basename(image_path))[0]
+        if args.progressive:
+            for i, heatmaps in enumerate(heatmap_iter):
+                for j, heatmap in enumerate(heatmaps):
+                    hm_painted = cm.jet(heatmap)[:, :, 2::-1]
+                    disp = (image + hm_painted) / 2
+                    cv2.imshow('heatmap-%s' % j, hm_painted)
+                    cv2.imshow(file_name + '-heatmap-%s' % j, disp)
+                cv2.waitKey(10)
+                print("Iter", i, "of", args.max_samples, end='\r')
+        else:
+            heatmaps = next(heatmap_iter)
+
+        if args.save_dir is not None:
+            log.info("Saving inference heatmaps at '%s'" % args.save_dir)
+            os.makedirs(args.save_dir, exist_ok=True)
+
+            for j, heatmap in enumerate(heatmaps):
+                save_path = osp.join(args.save_dir,
+                    file_name + '-heatmap-%s.png' % j)
+                cv2.imwrite(save_path, heatmap * 255.0)
+        else:
+            for j, heatmap in enumerate(heatmaps):
+                disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2
+                cv2.imshow(file_name + '-heatmap-%s' % j, disp)
+            cv2.waitKey(0)
+    elif args.target[0] == TargetKinds.source or \
+         args.target[0] == TargetKinds.project:
+        if args.target[0] == TargetKinds.source:
+            source_name = args.target[1]
+            dataset = project.make_source_project(source_name).make_dataset()
+            log.info("Running inference explanation for '%s'" % source_name)
+        else:
+            project_name = project.config.project_name
+            dataset = project.make_dataset()
+            log.info("Running inference explanation for '%s'" % project_name)
+
+        for item in dataset:
+            image = item.image
+            if image is None:
+                log.warn(
+                    "Dataset item %s does not have image data. Skipping." % \
+                    (item.id))
+                continue
+
+            if model.preferred_input_size() is not None:
+                h, w = model.preferred_input_size()
+                image = cv2.resize(image, (w, h))
+            heatmap_iter = rise.apply(image)
+
+            image = image / 255.0
+            file_name = osp.splitext(osp.basename(image_path))[0]
+            heatmaps = next(heatmap_iter)
+
+            if args.save_dir is not None:
+                log.info("Saving inference heatmaps at '%s'" % args.save_dir)
+                os.makedirs(args.save_dir, exist_ok=True)
+
+                for j, heatmap in enumerate(heatmaps):
+                    save_path = osp.join(args.save_dir,
+                        file_name + '-heatmap-%s.png' % j)
+                    cv2.imwrite(save_path, heatmap * 255.0)
+
+            if args.progressive:
+                for j, heatmap in enumerate(heatmaps):
+                    disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2
+                    cv2.imshow(file_name + '-heatmap-%s' % j, disp)
+                cv2.waitKey(0)
+    else:
+        raise NotImplementedError()
+
+    return 0
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+    if 'command' not in args:
+        parser.print_help()
+        return 1
+
+    project_path = args.project_dir
+    if is_project_path(project_path):
+        project = Project.load(project_path)
+    else:
+        project = None
+    try:
+        args.target = target_selector(
+            ProjectTarget(is_default=True, project=project),
+            SourceTarget(project=project),
+            ImageTarget()
+        )(args.target)
+        if args.target[0] == TargetKinds.project:
+            if is_project_path(args.target[1]):
+                args.project_dir = osp.dirname(osp.abspath(args.target[1]))
+    except argparse.ArgumentTypeError as e:
+        print(e)
+        parser.print_help()
+        return 1
+
+    return args.command(args)

datumaro/datumaro/cli/export_command.py

@@ -0,0 +1,69 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import os.path as osp
+
+from datumaro.components.project import Project
+from datumaro.util.command_targets import (TargetKinds, target_selector,
+    ProjectTarget, SourceTarget, ImageTarget, ExternalDatasetTarget,
+    is_project_path
+)
+
+from . import project as project_module
+from . import source as source_module
+from . import item as item_module
+
+
+def export_external_dataset(target, params):
+    raise NotImplementedError()
+
+def build_parser(parser=argparse.ArgumentParser()):
+    parser.add_argument('target', nargs='?', default=None)
+    parser.add_argument('params', nargs=argparse.REMAINDER)
+
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+
+    return parser
+
+def process_command(target, params, args):
+    project_dir = args.project_dir
+    target_kind, target_value = target
+    if target_kind == TargetKinds.project:
+        return project_module.main(['export', '-p', target_value] + params)
+    elif target_kind == TargetKinds.source:
+        return source_module.main(['export', '-p', project_dir, '-n', target_value] + params)
+    elif target_kind == TargetKinds.item:
+        return item_module.main(['export', '-p', project_dir, target_value] + params)
+    elif target_kind == TargetKinds.external_dataset:
+        return export_external_dataset(target_value, params)
+    return 1
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+
+    project_path = args.project_dir
+    if is_project_path(project_path):
+        project = Project.load(project_path)
+    else:
+        project = None
+    try:
+        args.target = target_selector(
+            ProjectTarget(is_default=True, project=project),
+            SourceTarget(project=project),
+            ExternalDatasetTarget(),
+            ImageTarget()
+        )(args.target)
+        if args.target[0] == TargetKinds.project:
+            if is_project_path(args.target[1]):
+                args.project_dir = osp.dirname(osp.abspath(args.target[1]))
+    except argparse.ArgumentTypeError as e:
+        print(e)
+        parser.print_help()
+        return 1
+
+    return process_command(args.target, args.params, args)

datumaro/datumaro/cli/inference/__init__.py

@@ -0,0 +1,33 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+
+
+def run_command(args):
+    return 0
+
+def build_run_parser(parser):
+    return parser
+
+def build_parser(parser=argparse.ArgumentParser()):
+    command_parsers = parser.add_subparsers(dest='command')
+
+    build_run_parser(command_parsers.add_parser('run')). \
+        set_defaults(command=run_command)
+
+    return parser
+
+def process_command(command, args):
+    return 0
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+    if 'command' not in args:
+        parser.print_help()
+        return 1
+
+    return args.command(args)

datumaro/datumaro/cli/item/__init__.py

@@ -0,0 +1,38 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+
+
+def build_export_parser(parser):
+    return parser
+
+def build_stats_parser(parser):
+    return parser
+
+def build_diff_parser(parser):
+    return parser
+
+def build_edit_parser(parser):
+    return parser
+
+def build_parser(parser=argparse.ArgumentParser()):
+    command_parsers = parser.add_subparsers(dest='command_name')
+
+    build_export_parser(command_parsers.add_parser('export'))
+    build_stats_parser(command_parsers.add_parser('stats'))
+    build_diff_parser(command_parsers.add_parser('diff'))
+    build_edit_parser(command_parsers.add_parser('edit'))
+
+    return parser
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+    if 'command' not in args:
+        parser.print_help()
+        return 1
+
+    return args.command(args)

datumaro/datumaro/cli/model/__init__.py

@@ -0,0 +1,127 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import logging as log
+import os
+import os.path as osp
+import shutil
+
+from ..util.project import load_project
+
+
+def add_command(args):
+    project = load_project(args.project_dir)
+
+    log.info("Adding '%s' model to '%s' project" % \
+        (args.launcher, project.config.project_name))
+
+    options = args.launcher_args_extractor(args)
+
+    if args.launcher == 'openvino' and args.copy:
+        config = project.config
+        env_config = project.env.config
+
+        model_dir_rel = osp.join(
+            config.env_dir, env_config.models_dir, args.name)
+        model_dir = osp.join(
+            config.project_dir, model_dir_rel)
+
+        os.makedirs(model_dir, exist_ok=True)
+
+        shutil.copy(options.description,
+            osp.join(model_dir, osp.basename(options.description)))
+        options.description = \
+            osp.join(model_dir_rel, osp.basename(options.description))
+
+        shutil.copy(options.weights,
+            osp.join(model_dir, osp.basename(options.weights)))
+        options.weights = \
+            osp.join(model_dir_rel, osp.basename(options.weights))
+
+        shutil.copy(options.interpretation_script,
+            osp.join(model_dir, osp.basename(options.interpretation_script)))
+        options.interpretation_script = \
+            osp.join(model_dir_rel, osp.basename(options.interpretation_script))
+
+    project.add_model(args.name, {
+        'launcher': args.launcher,
+        'options': vars(options),
+    })
+
+    project.save()
+
+    return 0
+
+def build_openvino_add_parser(parser):
+    parser.add_argument('-d', '--description', required=True,
+        help="Path to the model description file (.xml)")
+    parser.add_argument('-w', '--weights', required=True,
+        help="Path to the model weights file (.bin)")
+    parser.add_argument('-i', '--interpretation-script', required=True,
+        help="Path to the network output interpretation script (.py)")
+    parser.add_argument('--plugins-path', default=None,
+        help="Path to the custom Inference Engine plugins directory")
+    parser.add_argument('--copy', action='store_true',
+        help="Copy the model data to the project")
+    return parser
+
+def openvino_args_extractor(args):
+    my_args = argparse.Namespace()
+    my_args.description = args.description
+    my_args.weights = args.weights
+    my_args.interpretation_script = args.interpretation_script
+    my_args.plugins_path = args.plugins_path
+    return my_args
+
+def build_add_parser(parser):
+    parser.add_argument('name',
+        help="Name of the model to be added")
+    launchers_sp = parser.add_subparsers(dest='launcher')
+
+    build_openvino_add_parser(launchers_sp.add_parser('openvino')) \
+        .set_defaults(launcher_args_extractor=openvino_args_extractor)
+
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    return parser
+
+
+def remove_command(args):
+    project = load_project(args.project_dir)
+
+    project.remove_model(args.name)
+    project.save()
+
+    return 0
+
+def build_remove_parser(parser):
+    parser.add_argument('name',
+        help="Name of the model to be removed")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+
+    return parser
+
+
+def build_parser(parser=argparse.ArgumentParser()):
+    command_parsers = parser.add_subparsers(dest='command_name')
+
+    build_add_parser(command_parsers.add_parser('add')) \
+        .set_defaults(command=add_command)
+
+    build_remove_parser(command_parsers.add_parser('remove')) \
+        .set_defaults(command=remove_command)
+
+    return parser
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+    if 'command' not in args:
+        parser.print_help()
+        return 1
+
+    return args.command(args)

datumaro/datumaro/cli/project/__init__.py

@@ -0,0 +1,283 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import logging as log
+import os
+import os.path as osp
+
+from datumaro.components.project import Project
+from datumaro.components.comparator import Comparator
+from .diff import DiffVisualizer
+from ..util.project import make_project_path, load_project
+
+
+def build_create_parser(parser):
+    parser.add_argument('-d', '--dest', default='.', dest='dst_dir',
+        help="Save directory for the new project (default: current dir")
+    parser.add_argument('-n', '--name', default=None,
+        help="Name of the new project (default: same as project dir)")
+    parser.add_argument('--overwrite', action='store_true',
+        help="Overwrite existing files in the save directory")
+    return parser
+
+def create_command(args):
+    project_dir = osp.abspath(args.dst_dir)
+    project_path = make_project_path(project_dir)
+    if not args.overwrite and osp.isfile(project_path):
+        log.error("Project file '%s' already exists" % (project_path))
+        return 1
+
+    project_name = args.name
+    if project_name is None:
+        project_name = osp.basename(project_dir)
+
+    log.info("Creating project at '%s'" % (project_dir))
+
+    Project.generate(project_dir, {
+        'project_name': project_name,
+    })
+
+    log.info("Project has been created at '%s'" % (project_dir))
+
+    return 0
+
+def build_import_parser(parser):
+    import datumaro.components.importers as importers_module
+    importers_list = [name for name, cls in importers_module.items]
+
+    parser.add_argument('source_path',
+        help="Path to import a project from")
+    parser.add_argument('-f', '--format', required=True,
+        help="Source project format (options: %s)" % (', '.join(importers_list)))
+    parser.add_argument('-d', '--dest', default='.', dest='dst_dir',
+        help="Directory to save the new project to (default: current dir)")
+    parser.add_argument('extra_args', nargs=argparse.REMAINDER,
+        help="Additional arguments for importer")
+    parser.add_argument('-n', '--name', default=None,
+        help="Name of the new project (default: same as project dir)")
+    parser.add_argument('--overwrite', action='store_true',
+        help="Overwrite existing files in the save directory")
+    return parser
+
+def import_command(args):
+    project_dir = osp.abspath(args.dst_dir)
+    project_path = make_project_path(project_dir)
+    if not args.overwrite and osp.isfile(project_path):
+        log.error("Project file '%s' already exists" % (project_path))
+        return 1
+
+    project_name = args.name
+    if project_name is None:
+        project_name = osp.basename(project_dir)
+
+    log.info("Importing project from '%s' as '%s'" % \
+        (args.source_path, args.format))
+
+    source_path = osp.abspath(args.source_path)
+    project = Project.import_from(source_path, args.format)
+    project.config.project_name = project_name
+    project.config.project_dir = project_dir
+    project = project.make_dataset()
+    project.save(merge=True, save_images=False)
+
+    log.info("Project has been created at '%s'" % (project_dir))
+
+    return 0
+
+def build_build_parser(parser):
+    return parser
+
+def build_export_parser(parser):
+    parser.add_argument('-e', '--filter', default=None,
+        help="Filter expression for dataset items. Examples: "
+             "extract images with width < height: "
+             "'/item[image/width < image/height]'; "
+             "extract images with large-area bboxes: "
+             "'/item[annotation/type=\"bbox\" and annotation/area>2000]'"
+        )
+    parser.add_argument('-d', '--dest', dest='dst_dir', required=True,
+        help="Directory to save output")
+    parser.add_argument('-f', '--output-format', required=True,
+        help="Output format")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    parser.add_argument('--save-images', action='store_true',
+        help="Save images")
+    return parser
+
+def export_command(args):
+    project = load_project(args.project_dir)
+
+    dst_dir = osp.abspath(args.dst_dir)
+    os.makedirs(dst_dir, exist_ok=False)
+
+    project.make_dataset().export(
+        save_dir=dst_dir,
+        output_format=args.output_format,
+        filter_expr=args.filter,
+        save_images=args.save_images)
+    log.info("Project exported to '%s' as '%s'" % \
+        (dst_dir, args.output_format))
+
+    return 0
+
+def build_stats_parser(parser):
+    parser.add_argument('name')
+    return parser
+
+def build_docs_parser(parser):
+    return parser
+
+def build_extract_parser(parser):
+    parser.add_argument('-e', '--filter', default=None,
+        help="Filter expression for dataset items. Examples: "
+             "extract images with width < height: "
+             "'/item[image/width < image/height]'; "
+             "extract images with large-area bboxes: "
+             "'/item[annotation/type=\"bbox\" and annotation/area>2000]'"
+        )
+    parser.add_argument('-d', '--dest', dest='dst_dir', required=True,
+        help="Output directory")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    return parser
+
+def extract_command(args):
+    project = load_project(args.project_dir)
+
+    dst_dir = osp.abspath(args.dst_dir)
+    os.makedirs(dst_dir, exist_ok=False)
+
+    project.make_dataset().extract(filter_expr=args.filter, save_dir=dst_dir)
+    log.info("Subproject extracted to '%s'" % (dst_dir))
+
+    return 0
+
+def build_merge_parser(parser):
+    parser.add_argument('other_project_dir',
+        help="Directory of the project to get data updates from")
+    parser.add_argument('-d', '--dest', dest='dst_dir', default=None,
+        help="Output directory (default: current project's dir)")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    return parser
+
+def merge_command(args):
+    first_project = load_project(args.project_dir)
+    second_project = load_project(args.other_project_dir)
+
+    first_dataset = first_project.make_dataset()
+    first_dataset.update(second_project.make_dataset())
+
+    dst_dir = args.dst_dir
+    first_dataset.save(save_dir=dst_dir)
+
+    if dst_dir is None:
+        dst_dir = first_project.config.project_dir
+    dst_dir = osp.abspath(dst_dir)
+    log.info("Merge result saved to '%s'" % (dst_dir))
+
+    return 0
+
+def build_diff_parser(parser):
+    parser.add_argument('other_project_dir',
+        help="Directory of the second project to be compared")
+    parser.add_argument('-d', '--dest', default=None, dest='dst_dir',
+        help="Directory to save comparison results (default: do not save)")
+    parser.add_argument('-f', '--output-format',
+        default=DiffVisualizer.DEFAULT_FORMAT,
+        choices=[f.name for f in DiffVisualizer.Format],
+        help="Output format (default: %(default)s)")
+    parser.add_argument('--iou-thresh', default=0.5, type=float,
+        help="IoU match threshold for detections (default: %(default)s)")
+    parser.add_argument('--conf-thresh', default=0.5, type=float,
+        help="Confidence threshold for detections (default: %(default)s)")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the first project to be compared (default: current dir)")
+    return parser
+
+def diff_command(args):
+    first_project = load_project(args.project_dir)
+    second_project = load_project(args.other_project_dir)
+
+    comparator = Comparator(
+        iou_threshold=args.iou_thresh,
+        conf_threshold=args.conf_thresh)
+
+    save_dir = args.dst_dir
+    if save_dir is not None:
+        log.info("Saving diff to '%s'" % save_dir)
+        os.makedirs(osp.abspath(save_dir))
+    visualizer = DiffVisualizer(save_dir=save_dir, comparator=comparator,
+        output_format=args.output_format)
+    visualizer.save_dataset_diff(
+        first_project.make_dataset(),
+        second_project.make_dataset())
+
+    return 0
+
+def build_transform_parser(parser):
+    parser.add_argument('-d', '--dest', dest='dst_dir', required=True,
+        help="Directory to save output")
+    parser.add_argument('-m', '--model', dest='model_name', required=True,
+        help="Model to apply to the project")
+    parser.add_argument('-f', '--output-format', required=True,
+        help="Output format")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    return parser
+
+def transform_command(args):
+    project = load_project(args.project_dir)
+
+    dst_dir = osp.abspath(args.dst_dir)
+    os.makedirs(dst_dir, exist_ok=False)
+    project.make_dataset().transform(
+        save_dir=dst_dir,
+        model_name=args.model_name)
+
+    log.info("Transform results saved to '%s'" % (dst_dir))
+
+    return 0
+
+
+def build_parser(parser=argparse.ArgumentParser()):
+    command_parsers = parser.add_subparsers(dest='command_name')
+
+    build_create_parser(command_parsers.add_parser('create')) \
+        .set_defaults(command=create_command)
+
+    build_import_parser(command_parsers.add_parser('import')) \
+        .set_defaults(command=import_command)
+
+    build_export_parser(command_parsers.add_parser('export')) \
+        .set_defaults(command=export_command)
+
+    build_extract_parser(command_parsers.add_parser('extract')) \
+        .set_defaults(command=extract_command)
+
+    build_merge_parser(command_parsers.add_parser('merge')) \
+        .set_defaults(command=merge_command)
+
+    build_build_parser(command_parsers.add_parser('build'))
+    build_stats_parser(command_parsers.add_parser('stats'))
+    build_docs_parser(command_parsers.add_parser('docs'))
+    build_diff_parser(command_parsers.add_parser('diff')) \
+        .set_defaults(command=diff_command)
+
+    build_transform_parser(command_parsers.add_parser('transform')) \
+        .set_defaults(command=transform_command)
+
+    return parser
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+    if 'command' not in args:
+        parser.print_help()
+        return 1
+
+    return args.command(args)

datumaro/datumaro/cli/project/diff.py

@@ -0,0 +1,274 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from collections import Counter
+import cv2
+from enum import Enum
+import numpy as np
+import os
+import os.path as osp
+
+_formats = ['simple']
+
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    import tensorboardX as tb
+    _formats.append('tensorboard')
+
+from datumaro.components.extractor import AnnotationType
+
+
+Format = Enum('Formats', _formats)
+
+class DiffVisualizer:
+    Format = Format
+    DEFAULT_FORMAT = Format.simple
+
+    _UNMATCHED_LABEL = -1
+
+
+    def __init__(self, comparator, save_dir, output_format=DEFAULT_FORMAT):
+        self.comparator = comparator
+
+        if isinstance(output_format, str):
+            output_format = Format[output_format]
+        assert output_format in Format
+        self.output_format = output_format
+
+        self.save_dir = save_dir
+        if output_format is Format.tensorboard:
+            logdir = osp.join(self.save_dir, 'logs', 'diff')
+            self.file_writer = tb.SummaryWriter(logdir)
+        if output_format is Format.simple:
+            self.label_diff_writer = None
+
+        self.categories = {}
+
+        self.label_confusion_matrix = Counter()
+        self.bbox_confusion_matrix = Counter()
+
+    def save_dataset_diff(self, extractor_a, extractor_b):
+        if self.save_dir:
+            os.makedirs(self.save_dir, exist_ok=True)
+
+        if len(extractor_a) != len(extractor_b):
+            print("Datasets have different lengths: %s vs %s" % \
+                (len(extractor_a), len(extractor_b)))
+
+        self.categories = {}
+
+        label_mismatch = self.comparator. \
+            compare_dataset_labels(extractor_a, extractor_b)
+        if label_mismatch is None:
+            print("Datasets have no label information")
+        elif len(label_mismatch) != 0:
+            print("Datasets have mismatching labels:")
+            for a_label, b_label in label_mismatch:
+                if a_label is None:
+                    print("  > %s" % b_label.name)
+                elif b_label is None:
+                    print("  < %s" % a_label.name)
+                else:
+                    print("  %s != %s" % (a_label.name, b_label.name))
+        else:
+            self.categories.update(extractor_a.categories())
+            self.categories.update(extractor_b.categories())
+
+        self.label_confusion_matrix = Counter()
+        self.bbox_confusion_matrix = Counter()
+
+        if self.output_format is Format.tensorboard:
+            self.file_writer.reopen()
+
+        for i, (item_a, item_b) in enumerate(zip(extractor_a, extractor_b)):
+            if item_a.id != item_b.id or not item_a.id or not item_b.id:
+                print("Dataset items #%s '%s' '%s' do not match" % \
+                    (i + 1, item_a.id, item_b.id))
+                continue
+
+            label_diff = self.comparator.compare_item_labels(item_a, item_b)
+            self.update_label_confusion(label_diff)
+
+            bbox_diff = self.comparator.compare_item_bboxes(item_a, item_b)
+            self.update_bbox_confusion(bbox_diff)
+
+            self.save_item_label_diff(item_a, item_b, label_diff)
+            self.save_item_bbox_diff(item_a, item_b, bbox_diff)
+
+        if len(self.label_confusion_matrix) != 0:
+            self.save_conf_matrix(self.label_confusion_matrix,
+                'labels_confusion.png')
+        if len(self.bbox_confusion_matrix) != 0:
+            self.save_conf_matrix(self.bbox_confusion_matrix,
+                'bbox_confusion.png')
+
+        if self.output_format is Format.tensorboard:
+            self.file_writer.flush()
+            self.file_writer.close()
+        elif self.output_format is Format.simple:
+            if self.label_diff_writer:
+                self.label_diff_writer.flush()
+                self.label_diff_writer.close()
+
+    def update_label_confusion(self, label_diff):
+        matches, a_unmatched, b_unmatched = label_diff
+        for label in matches:
+            self.label_confusion_matrix[(label, label)] += 1
+        for a_label in a_unmatched:
+            self.label_confusion_matrix[(a_label, self._UNMATCHED_LABEL)] += 1
+        for b_label in b_unmatched:
+            self.label_confusion_matrix[(self._UNMATCHED_LABEL, b_label)] += 1
+
+    def update_bbox_confusion(self, bbox_diff):
+        matches, mispred, a_unmatched, b_unmatched = bbox_diff
+        for a_bbox, b_bbox in matches:
+            self.bbox_confusion_matrix[(a_bbox.label, b_bbox.label)] += 1
+        for a_bbox, b_bbox in mispred:
+            self.bbox_confusion_matrix[(a_bbox.label, b_bbox.label)] += 1
+        for a_bbox in a_unmatched:
+            self.bbox_confusion_matrix[(a_bbox.label, self._UNMATCHED_LABEL)] += 1
+        for b_bbox in b_unmatched:
+            self.bbox_confusion_matrix[(self._UNMATCHED_LABEL, b_bbox.label)] += 1
+
+    @classmethod
+    def draw_text_with_background(cls, frame, text, origin,
+            font=cv2.FONT_HERSHEY_SIMPLEX, scale=1.0,
+            color=(0, 0, 0), thickness=1, bgcolor=(1, 1, 1)):
+        text_size, baseline = cv2.getTextSize(text, font, scale, thickness)
+        cv2.rectangle(frame,
+            tuple((origin + (0, baseline)).astype(int)),
+            tuple((origin + (text_size[0], -text_size[1])).astype(int)),
+            bgcolor, cv2.FILLED)
+        cv2.putText(frame, text,
+            tuple(origin.astype(int)),
+            font, scale, color, thickness)
+        return text_size, baseline
+
+    def draw_detection_roi(self, frame, x, y, w, h, label, conf, color):
+        cv2.rectangle(frame, (x, y), (x + w, y + h), color, 2)
+
+        text = '%s %.2f%%' % (label, 100.0 * conf)
+        text_scale = 0.5
+        font = cv2.FONT_HERSHEY_SIMPLEX
+        text_size = cv2.getTextSize(text, font, text_scale, 1)
+        line_height = np.array([0, text_size[0][1]])
+        self.draw_text_with_background(frame, text,
+            np.array([x, y]) - line_height * 0.5,
+            font, scale=text_scale, color=[255 - c for c in color])
+
+    def get_label(self, label_id):
+        cat = self.categories.get(AnnotationType.label)
+        if cat is None:
+            return str(label_id)
+        return cat.items[label_id].name
+
+    def draw_bbox(self, img, shape, color):
+        x, y, w, h = shape.get_bbox()
+        self.draw_detection_roi(img, int(x), int(y), int(w), int(h),
+            self.get_label(shape.label), shape.attributes.get('score', 1),
+            color)
+
+    def get_label_diff_file(self):
+        if self.label_diff_writer is None:
+            self.label_diff_writer = \
+                open(osp.join(self.save_dir, 'label_diff.txt'), 'w')
+        return self.label_diff_writer
+
+    def save_item_label_diff(self, item_a, item_b, diff):
+        _, a_unmatched, b_unmatched = diff
+
+        if 0 < len(a_unmatched) + len(b_unmatched):
+            if self.output_format is Format.simple:
+                f = self.get_label_diff_file()
+                f.write(item_a.id + '\n')
+                for a_label in a_unmatched:
+                    f.write('  >%s\n' % self.get_label(a_label))
+                for b_label in b_unmatched:
+                    f.write('  <%s\n' % self.get_label(b_label))
+            elif self.output_format is Format.tensorboard:
+                tag = item_a.id
+                for a_label in a_unmatched:
+                    self.file_writer.add_text(tag,
+                        '>%s\n' % self.get_label(a_label))
+                for b_label in b_unmatched:
+                    self.file_writer.add_text(tag,
+                        '<%s\n' % self.get_label(b_label))
+
+    def save_item_bbox_diff(self, item_a, item_b, diff):
+        _, mispred, a_unmatched, b_unmatched = diff
+
+        if 0 < len(a_unmatched) + len(b_unmatched) + len(mispred):
+            img_a = item_a.image.copy()
+            img_b = img_a.copy()
+            for a_bbox, b_bbox in mispred:
+                self.draw_bbox(img_a, a_bbox, (0, 255, 0))
+                self.draw_bbox(img_b, b_bbox, (0, 0, 255))
+            for a_bbox in a_unmatched:
+                self.draw_bbox(img_a, a_bbox, (255, 255, 0))
+            for b_bbox in b_unmatched:
+                self.draw_bbox(img_b, b_bbox, (255, 255, 0))
+
+            img = np.hstack([img_a, img_b])
+
+            path = osp.join(self.save_dir, 'diff_%s' % item_a.id)
+
+            if self.output_format is Format.simple:
+                cv2.imwrite(path + '.png', img)
+            elif self.output_format is Format.tensorboard:
+                self.save_as_tensorboard(img, path)
+
+    def save_as_tensorboard(self, img, name):
+        img = img[:, :, ::-1] # to RGB
+        img = np.transpose(img, (2, 0, 1)) # to (C, H, W)
+        img = img.astype(dtype=np.uint8)
+        self.file_writer.add_image(name, img)
+
+    def save_conf_matrix(self, conf_matrix, filename):
+        import matplotlib.pyplot as plt
+
+        classes = None
+        label_categories = self.categories.get(AnnotationType.label)
+        if label_categories is not None:
+            classes = { id: c.name for id, c in enumerate(label_categories.items) }
+        if classes is None:
+            classes = { c: 'label_%s' % c for c, _ in conf_matrix }
+        classes[self._UNMATCHED_LABEL] = 'unmatched'
+
+        class_idx = { id: i for i, id in enumerate(classes.keys()) }
+        matrix = np.zeros((len(classes), len(classes)), dtype=int)
+        for idx_pair in conf_matrix:
+            index = (class_idx[idx_pair[0]], class_idx[idx_pair[1]])
+            matrix[index] = conf_matrix[idx_pair]
+
+        labels = [label for id, label in classes.items()]
+
+        fig = plt.figure()
+        fig.add_subplot(111)
+        table = plt.table(
+            cellText=matrix,
+            colLabels=labels,
+            rowLabels=labels,
+            loc ='center')
+        table.auto_set_font_size(False)
+        table.set_fontsize(8)
+        table.scale(3, 3)
+        # Removing ticks and spines enables you to get the figure only with table
+        plt.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False)
+        plt.tick_params(axis='y', which='both', right=False, left=False, labelleft=False)
+        for pos in ['right','top','bottom','left']:
+            plt.gca().spines[pos].set_visible(False)
+
+        for idx_pair in conf_matrix:
+            i = class_idx[idx_pair[0]]
+            j = class_idx[idx_pair[1]]
+            if conf_matrix[idx_pair] != 0:
+                if i != j:
+                    table._cells[(i + 1, j)].set_facecolor('#FF0000')
+                else:
+                    table._cells[(i + 1, j)].set_facecolor('#00FF00')
+
+        plt.savefig(osp.join(self.save_dir, filename),
+            bbox_inches='tight', pad_inches=0.05)

datumaro/datumaro/cli/remove_command.py

@@ -0,0 +1,21 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+
+from . import source as source_module
+
+
+def build_parser(parser=argparse.ArgumentParser()):
+    source_module.build_add_parser(parser). \
+        set_defaults(command=source_module.remove_command)
+
+    return parser
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+
+    return args.command(args)

datumaro/datumaro/cli/source/__init__.py

@@ -0,0 +1,219 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import logging as log
+import os
+import os.path as osp
+import shutil
+
+from ..util.project import load_project
+
+
+def build_create_parser(parser):
+    parser.add_argument('-n', '--name', required=True,
+        help="Name of the source to be created")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    return parser
+
+def create_command(args):
+    project = load_project(args.project_dir)
+    config = project.config
+
+    name = args.name
+
+    if project.env.git.has_submodule(name):
+        log.fatal("Source '%s' already exists" % (name))
+        return 1
+
+    try:
+        project.get_source(name)
+        log.fatal("Source '%s' already exists" % (name))
+        return 1
+    except KeyError:
+        pass
+
+    dst_dir = osp.join(config.project_dir, config.sources_dir, name)
+    project.env.git.init(dst_dir)
+
+    project.add_source(name, { 'url': name })
+    project.save()
+
+    log.info("Source '%s' has been added to the project, location: '%s'" \
+        % (name, dst_dir))
+
+    return 0
+
+def build_import_parser(parser):
+    sp = parser.add_subparsers(dest='source_type')
+
+    repo_parser = sp.add_parser('repo')
+    repo_parser.add_argument('url',
+        help="URL of the source git repository")
+    repo_parser.add_argument('-b', '--branch', default='master',
+        help="Branch of the source repository (default: %(default)s)")
+    repo_parser.add_argument('--checkout', action='store_true',
+        help="Do branch checkout")
+
+    dir_parser = sp.add_parser('dir')
+    dir_parser.add_argument('url',
+        help="Path to the source directory")
+    dir_parser.add_argument('--copy', action='store_true',
+        help="Copy data to the project")
+
+    parser.add_argument('-f', '--format', default=None,
+        help="Name of the source dataset format (default: 'project')")
+    parser.add_argument('-n', '--name', default=None,
+        help="Name of the source to be imported")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    return parser
+
+def import_command(args):
+    project = load_project(args.project_dir)
+
+    if args.source_type == 'repo':
+        name = args.name
+        if name is None:
+            name = osp.splitext(osp.basename(args.url))[0]
+
+        if project.env.git.has_submodule(name):
+            log.fatal("Submodule '%s' already exists" % (name))
+            return 1
+
+        try:
+            project.get_source(name)
+            log.fatal("Source '%s' already exists" % (name))
+            return 1
+        except KeyError:
+            pass
+
+        dst_dir = project.local_source_dir(name)
+        project.env.git.create_submodule(name, dst_dir,
+            url=args.url, branch=args.branch, no_checkout=not args.checkout)
+
+        source = { 'url': args.url }
+        if args.format:
+            source['format'] = args.format
+        project.add_source(name, source)
+        project.save()
+
+        log.info("Source '%s' has been added to the project, location: '%s'" \
+            % (name, dst_dir))
+    elif args.source_type == 'dir':
+        url = osp.abspath(args.url)
+        if not osp.exists(url):
+            log.fatal("Source path '%s' does not exist" % url)
+            return 1
+
+        name = args.name
+        if name is None:
+            name = osp.splitext(osp.basename(url))[0]
+
+        try:
+            project.get_source(name)
+            log.fatal("Source '%s' already exists" % (name))
+            return 1
+        except KeyError:
+            pass
+
+        dst_dir = url
+        if args.copy:
+            dst_dir = project.local_source_dir(name)
+            log.info("Copying from '%s' to '%s'" % (url, dst_dir))
+            shutil.copytree(url, dst_dir)
+            url = name
+
+        source = { 'url': url }
+        if args.format:
+            source['format'] = args.format
+        project.add_source(name, source)
+        project.save()
+
+        log.info("Source '%s' has been added to the project, location: '%s'" \
+            % (name, dst_dir))
+
+    return 0
+
+def build_remove_parser(parser):
+    parser.add_argument('-n', '--name', required=True,
+        help="Name of the source to be removed")
+    parser.add_argument('--force', action='store_true',
+        help="Ignore possible errors during removal")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    return parser
+
+def remove_command(args):
+    project = load_project(args.project_dir)
+
+    name = args.name
+    if name is None:
+        log.fatal("Expected source name")
+        return
+
+    if project.env.git.has_submodule(name):
+        if args.force:
+            log.warning("Forcefully removing the '%s' source..." % (name))
+
+        project.env.git.remove_submodule(name, force=args.force)
+
+    project.remove_source(name)
+    project.save()
+
+    log.info("Source '%s' has been removed from the project" % (name))
+
+    return 0
+
+def build_export_parser(parser):
+    parser.add_argument('-n', '--name', required=True,
+        help="Source dataset to be extracted")
+    parser.add_argument('-d', '--dest', dest='dst_dir', required=True,
+        help="Directory to save output")
+    parser.add_argument('-f', '--output-format', required=True,
+        help="Output format (default: %(default)s)")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    return parser
+
+def export_command(args):
+    project = load_project(args.project_dir)
+
+    dst_dir = osp.abspath(args.dst_dir)
+    os.makedirs(dst_dir, exist_ok=False)
+
+    source_project = project.make_source_project(args.name)
+    source_project.make_dataset().export(
+        save_dir=args.dst_dir,
+        output_format=args.output_format)
+    log.info("Source '%s' exported to '%s' as '%s'" % \
+        (args.name, dst_dir, args.output_format))
+
+    return 0
+
+def build_parser(parser=argparse.ArgumentParser()):
+    command_parsers = parser.add_subparsers(dest='command_name')
+
+    build_create_parser(command_parsers.add_parser('create')) \
+        .set_defaults(command=create_command)
+    build_import_parser(command_parsers.add_parser('import')) \
+        .set_defaults(command=import_command)
+    build_remove_parser(command_parsers.add_parser('remove')) \
+        .set_defaults(command=remove_command)
+    build_export_parser(command_parsers.add_parser('export')) \
+        .set_defaults(command=export_command)
+
+    return parser
+
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+    if 'command' not in args:
+        parser.print_help()
+        return 1
+
+    return args.command(args)

datumaro/datumaro/cli/stats_command.py

@@ -0,0 +1,69 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import os.path as osp
+
+from datumaro.components.project import Project
+from datumaro.util.command_targets import (TargetKinds, target_selector,
+    ProjectTarget, SourceTarget, ExternalDatasetTarget, ImageTarget,
+    is_project_path
+)
+
+from . import project as project_module
+from . import source as source_module
+from . import item as item_module
+
+
+def compute_external_dataset_stats(target, params):
+    raise NotImplementedError()
+
+def build_parser(parser=argparse.ArgumentParser()):
+    parser.add_argument('target', nargs='?', default=None)
+    parser.add_argument('params', nargs=argparse.REMAINDER)
+
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+
+    return parser
+
+def process_command(target, params, args):
+    project_dir = args.project_dir
+    target_kind, target_value = target
+    if target_kind == TargetKinds.project:
+        return project_module.main(['stats', '-p', target_value] + params)
+    elif target_kind == TargetKinds.source:
+        return source_module.main(['stats', '-p', project_dir, target_value] + params)
+    elif target_kind == TargetKinds.item:
+        return item_module.main(['stats', '-p', project_dir, target_value] + params)
+    elif target_kind == TargetKinds.external_dataset:
+        return compute_external_dataset_stats(target_value, params)
+    return 1
+
+def main(args=None):
+    parser = build_parser()
+    args = parser.parse_args(args)
+
+    project_path = args.project_dir
+    if is_project_path(project_path):
+        project = Project.load(project_path)
+    else:
+        project = None
+    try:
+        args.target = target_selector(
+            ProjectTarget(is_default=True, project=project),
+            SourceTarget(project=project),
+            ExternalDatasetTarget(),
+            ImageTarget()
+        )(args.target)
+        if args.target[0] == TargetKinds.project:
+            if is_project_path(args.target[1]):
+                args.project_dir = osp.dirname(osp.abspath(args.target[1]))
+    except argparse.ArgumentTypeError as e:
+        print(e)
+        parser.print_help()
+        return 1
+
+    return process_command(args.target, args.params, args)

datumaro/datumaro/cli/util/__init__.py

@@ -0,0 +1,5 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+

datumaro/datumaro/cli/util/project.py

@@ -0,0 +1,20 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import os.path as osp
+
+from datumaro.components.project import Project, \
+    PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG
+
+
+def make_project_path(project_dir, project_filename=None):
+    if project_filename is None:
+        project_filename = DEFAULT_CONFIG.project_filename
+    return osp.join(project_dir, project_filename)
+
+def load_project(project_dir, project_filename=None):
+    if project_filename:
+        project_dir = osp.join(project_dir, project_filename)
+    return Project.load(project_dir)

datumaro/datumaro/components/__init__.py

@@ -0,0 +1,5 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+

datumaro/datumaro/components/algorithms/__init__.py

@@ -0,0 +1,5 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+

datumaro/datumaro/components/algorithms/rise.py

@@ -0,0 +1,219 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-variable
+
+import cv2
+import numpy as np
+from math import ceil
+
+from datumaro.components.extractor import *
+
+
+def flatmatvec(mat):
+    return np.reshape(mat, (len(mat), -1))
+
+def expand(array, axis=None):
+    if axis is None:
+        axis = len(array.shape)
+    return np.expand_dims(array, axis=axis)
+
+class RISE:
+    """
+    Implements RISE: Randomized Input Sampling for
+    Explanation of Black-box Models algorithm
+    See explanations at: https://arxiv.org/pdf/1806.07421.pdf
+    """
+
+    def __init__(self, model,
+            max_samples=None, mask_width=7, mask_height=7, prob=0.5,
+            iou_thresh=0.9, nms_thresh=0.0, det_conf_thresh=0.0,
+            batch_size=1):
+        self.model = model
+        self.max_samples = max_samples
+        self.mask_height = mask_height
+        self.mask_width = mask_width
+        self.prob = prob
+        self.iou_thresh = iou_thresh
+        self.nms_thresh = nms_thresh
+        self.det_conf_thresh = det_conf_thresh
+        self.batch_size = batch_size
+
+    @staticmethod
+    def split_outputs(annotations):
+        labels = []
+        bboxes = []
+        for r in annotations:
+            if r.type is AnnotationType.label:
+                labels.append(r)
+            elif r.type is AnnotationType.bbox:
+                bboxes.append(r)
+        return labels, bboxes
+
+    @staticmethod
+    def nms(boxes, iou_thresh=0.5):
+        indices = np.argsort([b.attributes['score'] for b in boxes])
+        ious = np.array([[a.iou(b) for b in boxes] for a in boxes])
+
+        predictions = []
+        while len(indices) != 0:
+            i = len(indices) - 1
+            pred_idx = indices[i]
+            to_remove = [i]
+            predictions.append(boxes[pred_idx])
+            for i, box_idx in enumerate(indices[:i]):
+                if iou_thresh < ious[pred_idx, box_idx]:
+                    to_remove.append(i)
+            indices = np.delete(indices, to_remove)
+
+        return predictions
+
+    def normalize_hmaps(self, heatmaps, counts):
+        eps = np.finfo(heatmaps.dtype).eps
+        mhmaps = flatmatvec(heatmaps)
+        mhmaps /= expand(counts * self.prob + eps)
+        mhmaps -= expand(np.min(mhmaps, axis=1))
+        mhmaps /= expand(np.max(mhmaps, axis=1) + eps)
+        return np.reshape(mhmaps, heatmaps.shape)
+
+    def apply(self, image, progressive=False):
+        assert len(image.shape) == 3, \
+            "Expected an input image in (H, W, C) format"
+        assert image.shape[2] in [3, 4], \
+            "Expected BGR or BGRA input"
+        image = image[:, :, :3].astype(np.float32)
+
+        model = self.model
+        iou_thresh = self.iou_thresh
+
+        image_size = np.array((image.shape[:2]))
+        mask_size = np.array((self.mask_height, self.mask_width))
+        cell_size = np.ceil(image_size / mask_size)
+        upsampled_size = np.ceil((mask_size + 1) * cell_size)
+
+        rng = lambda shape=None: np.random.rand(*shape)
+        samples = np.prod(image_size)
+        if self.max_samples is not None:
+            samples = min(self.max_samples, samples)
+        batch_size = self.batch_size
+
+        result = next(iter(model.launch(expand(image, 0))))
+        result_labels, result_bboxes = self.split_outputs(result)
+        if 0 < self.det_conf_thresh:
+            result_bboxes = [b for b in result_bboxes \
+                if self.det_conf_thresh <= b.attributes['score']]
+        if 0 < self.nms_thresh:
+            result_bboxes = self.nms(result_bboxes, self.nms_thresh)
+
+        predicted_labels = set()
+        if len(result_labels) != 0:
+            predicted_label = max(result_labels,
+                key=lambda r: r.attributes['score']).label
+            predicted_labels.add(predicted_label)
+        if len(result_bboxes) != 0:
+            for bbox in result_bboxes:
+                predicted_labels.add(bbox.label)
+        predicted_labels = { label: idx \
+            for idx, label in enumerate(predicted_labels) }
+
+        predicted_bboxes = result_bboxes
+
+        heatmaps_count = len(predicted_labels) + len(predicted_bboxes)
+        heatmaps = np.zeros((heatmaps_count, *image_size), dtype=np.float32)
+        total_counts = np.zeros(heatmaps_count, dtype=np.int32)
+        confs = np.zeros(heatmaps_count, dtype=np.float32)
+
+        heatmap_id = 0
+
+        label_heatmaps = None
+        label_total_counts = None
+        label_confs = None
+        if len(predicted_labels) != 0:
+            step = len(predicted_labels)
+            label_heatmaps = heatmaps[heatmap_id : heatmap_id + step]
+            label_total_counts = total_counts[heatmap_id : heatmap_id + step]
+            label_confs = confs[heatmap_id : heatmap_id + step]
+            heatmap_id += step
+
+        bbox_heatmaps = None
+        bbox_total_counts = None
+        bbox_confs = None
+        if len(predicted_bboxes) != 0:
+            step = len(predicted_bboxes)
+            bbox_heatmaps = heatmaps[heatmap_id : heatmap_id + step]
+            bbox_total_counts = total_counts[heatmap_id : heatmap_id + step]
+            bbox_confs = confs[heatmap_id : heatmap_id + step]
+            heatmap_id += step
+
+        ups_mask = np.empty(upsampled_size.astype(int), dtype=np.float32)
+        masks = np.empty((batch_size, *image_size), dtype=np.float32)
+
+        full_batch_inputs = np.empty((batch_size, *image.shape), dtype=np.float32)
+        current_heatmaps = np.empty_like(heatmaps)
+        for b in range(ceil(samples / batch_size)):
+            batch_pos = b * batch_size
+            current_batch_size = min(samples - batch_pos, batch_size)
+
+            batch_masks = masks[: current_batch_size]
+            for i in range(current_batch_size):
+                mask = (rng(mask_size) < self.prob).astype(np.float32)
+                cv2.resize(mask, (int(upsampled_size[1]), int(upsampled_size[0])),
+                    ups_mask)
+
+                offsets = np.round(rng((2,)) * cell_size)
+                mask = ups_mask[
+                    int(offsets[0]):int(image_size[0] + offsets[0]),
+                    int(offsets[1]):int(image_size[1] + offsets[1]) ]
+                batch_masks[i] = mask
+
+            batch_inputs = full_batch_inputs[:current_batch_size]
+            np.multiply(expand(batch_masks), expand(image, 0), out=batch_inputs)
+
+            results = model.launch(batch_inputs)
+            for mask, result in zip(batch_masks, results):
+                result_labels, result_bboxes = self.split_outputs(result)
+
+                confs.fill(0)
+                if len(predicted_labels) != 0:
+                    for r in result_labels:
+                        idx = predicted_labels.get(r.label, None)
+                        if idx is not None:
+                            label_total_counts[idx] += 1
+                            label_confs[idx] += r.attributes['score']
+                    for r in result_bboxes:
+                        idx = predicted_labels.get(r.label, None)
+                        if idx is not None:
+                            label_total_counts[idx] += 1
+                            label_confs[idx] += r.attributes['score']
+
+                if len(predicted_bboxes) != 0 and len(result_bboxes) != 0:
+                    if 0 < self.det_conf_thresh:
+                        result_bboxes = [b for b in result_bboxes \
+                            if self.det_conf_thresh <= b.attributes['score']]
+                    if 0 < self.nms_thresh:
+                        result_bboxes = self.nms(result_bboxes, self.nms_thresh)
+
+                    for detection in result_bboxes:
+                        for pred_idx, pred in enumerate(predicted_bboxes):
+                            if pred.label != detection.label:
+                                continue
+
+                            iou = pred.iou(detection)
+                            assert 0 <= iou and iou <= 1
+                            if iou < iou_thresh:
+                                continue
+
+                            bbox_total_counts[pred_idx] += 1
+
+                            conf = detection.attributes['score']
+                            bbox_confs[pred_idx] += conf
+
+                np.multiply.outer(confs, mask, out=current_heatmaps)
+                heatmaps += current_heatmaps
+
+                if progressive:
+                    yield self.normalize_hmaps(heatmaps.copy(), total_counts)
+
+        yield self.normalize_hmaps(heatmaps, total_counts)

datumaro/datumaro/components/comparator.py

@@ -0,0 +1,113 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from itertools import zip_longest
+import numpy as np
+
+from datumaro.components.extractor import AnnotationType, LabelCategories
+
+
+class Comparator:
+    def __init__(self,
+            iou_threshold=0.5, conf_threshold=0.9):
+        self.iou_threshold = iou_threshold
+        self.conf_threshold = conf_threshold
+
+    @staticmethod
+    def iou(box_a, box_b):
+        return box_a.iou(box_b)
+
+    # pylint: disable=no-self-use
+    def compare_dataset_labels(self, extractor_a, extractor_b):
+        a_label_cat = extractor_a.categories().get(AnnotationType.label)
+        b_label_cat = extractor_b.categories().get(AnnotationType.label)
+        if not a_label_cat and not b_label_cat:
+            return None
+        if not a_label_cat:
+            a_label_cat = LabelCategories()
+        if not b_label_cat:
+            b_label_cat = LabelCategories()
+
+        mismatches = []
+        for a_label, b_label in zip_longest(a_label_cat.items, b_label_cat.items):
+            if a_label != b_label:
+                mismatches.append((a_label, b_label))
+        return mismatches
+    # pylint: enable=no-self-use
+
+    def compare_item_labels(self, item_a, item_b):
+        conf_threshold = self.conf_threshold
+
+        a_labels = set([ann.label for ann in item_a.annotations \
+            if ann.type is AnnotationType.label and \
+               conf_threshold < ann.attributes.get('score', 1)])
+        b_labels = set([ann.label for ann in item_b.annotations \
+            if ann.type is AnnotationType.label and \
+               conf_threshold < ann.attributes.get('score', 1)])
+
+        a_unmatched = a_labels - b_labels
+        b_unmatched = b_labels - a_labels
+        matches = a_labels & b_labels
+
+        return matches, a_unmatched, b_unmatched
+
+    def compare_item_bboxes(self, item_a, item_b):
+        iou_threshold = self.iou_threshold
+        conf_threshold = self.conf_threshold
+
+        a_boxes = [ann for ann in item_a.annotations \
+            if ann.type is AnnotationType.bbox and \
+               conf_threshold < ann.attributes.get('score', 1)]
+        b_boxes = [ann for ann in item_b.annotations \
+            if ann.type is AnnotationType.bbox and \
+               conf_threshold < ann.attributes.get('score', 1)]
+        a_boxes.sort(key=lambda ann: 1 - ann.attributes.get('score', 1))
+        b_boxes.sort(key=lambda ann: 1 - ann.attributes.get('score', 1))
+
+        # a_matches: indices of b_boxes matched to a bboxes
+        # b_matches: indices of a_boxes matched to b bboxes
+        a_matches = -np.ones(len(a_boxes), dtype=int)
+        b_matches = -np.ones(len(b_boxes), dtype=int)
+
+        iou_matrix = np.array([
+            [self.iou(a, b) for b in b_boxes] for a in a_boxes
+        ])
+
+        # matches: boxes we succeeded to match completely
+        # mispred: boxes we succeeded to match, having label mismatch
+        matches = []
+        mispred = []
+
+        for a_idx, a_bbox in enumerate(a_boxes):
+            if len(b_boxes) == 0:
+                break
+            matched_b = a_matches[a_idx]
+            iou_max = max(iou_matrix[a_idx, matched_b], iou_threshold)
+            for b_idx, b_bbox in enumerate(b_boxes):
+                if 0 <= b_matches[b_idx]: # assign a_bbox with max conf
+                    continue
+                iou = iou_matrix[a_idx, b_idx]
+                if iou < iou_max:
+                    continue
+                iou_max = iou
+                matched_b = b_idx
+
+            if matched_b < 0:
+                continue
+            a_matches[a_idx] = matched_b
+            b_matches[matched_b] = a_idx
+
+            b_bbox = b_boxes[matched_b]
+
+            if a_bbox.label == b_bbox.label:
+                matches.append( (a_bbox, b_bbox) )
+            else:
+                mispred.append( (a_bbox, b_bbox) )
+
+        # *_umatched: boxes of (*) we failed to match
+        a_unmatched = [a_boxes[i] for i, m in enumerate(a_matches) if m < 0]
+        b_unmatched = [b_boxes[i] for i, m in enumerate(b_matches) if m < 0]
+
+        return matches, mispred, a_unmatched, b_unmatched

datumaro/datumaro/components/config.py

@@ -0,0 +1,238 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import yaml
+
+
+class Schema:
+    class Item:
+        def __init__(self, ctor, internal=False):
+            self.ctor = ctor
+            self.internal = internal
+
+        def __call__(self, *args, **kwargs):
+            return self.ctor(*args, **kwargs)
+
+    def __init__(self, items=None, fallback=None):
+        self._items = {}
+        if items is not None:
+            self._items.update(items)
+        self._fallback = fallback
+
+    def _get_items(self, allow_fallback=True):
+        all_items = {}
+
+        if allow_fallback and self._fallback is not None:
+            all_items.update(self._fallback)
+        all_items.update(self._items)
+
+        return all_items
+
+    def items(self, allow_fallback=True):
+        return self._get_items(allow_fallback=allow_fallback).items()
+
+    def keys(self, allow_fallback=True):
+        return self._get_items(allow_fallback=allow_fallback).keys()
+
+    def values(self, allow_fallback=True):
+        return self._get_items(allow_fallback=allow_fallback).values()
+
+    def __contains__(self, key):
+        return key in self.keys()
+
+    def __len__(self):
+        return len(self._get_items())
+
+    def __iter__(self):
+        return iter(self._get_items())
+
+    def __getitem__(self, key):
+        default = object()
+        value = self.get(key, default=default)
+        if value is default:
+            raise KeyError('Key "%s" does not exist' % (key))
+        return value
+
+    def get(self, key, default=None):
+        found = self._items.get(key, default)
+        if found is not default:
+            return found
+
+        if self._fallback is not None:
+            return self._fallback.get(key, default)
+
+class SchemaBuilder:
+    def __init__(self):
+        self._items = {}
+
+    def add(self, name, ctor=str, internal=False):
+        if name in self._items:
+            raise KeyError('Key "%s" already exists' % (name))
+
+        self._items[name] = Schema.Item(ctor, internal=internal)
+        return self
+
+    def build(self):
+        return Schema(self._items)
+
+class Config:
+    def __init__(self, config=None, fallback=None, schema=None, mutable=True):
+        # schema should be established first
+        self.__dict__['_schema'] = schema
+        self.__dict__['_mutable'] = True
+
+        self.__dict__['_config'] = {}
+        if fallback is not None:
+            for k, v in fallback.items(allow_fallback=False):
+                self.set(k, v)
+        if config is not None:
+            self.update(config)
+
+        self.__dict__['_mutable'] = mutable
+
+    def _items(self, allow_fallback=True, allow_internal=True):
+        all_config = {}
+        if allow_fallback and self._schema is not None:
+            for key, item in self._schema.items():
+                all_config[key] = item()
+        all_config.update(self._config)
+
+        if not allow_internal and self._schema is not None:
+            for key, item in self._schema.items():
+                if item.internal:
+                    all_config.pop(key)
+        return all_config
+
+    def items(self, allow_fallback=True, allow_internal=True):
+        return self._items(
+                allow_fallback=allow_fallback,
+                allow_internal=allow_internal
+            ).items()
+
+    def keys(self, allow_fallback=True, allow_internal=True):
+        return self._items(
+                allow_fallback=allow_fallback,
+                allow_internal=allow_internal
+            ).keys()
+
+    def values(self, allow_fallback=True, allow_internal=True):
+        return self._items(
+                allow_fallback=allow_fallback,
+                allow_internal=allow_internal
+            ).values()
+
+    def __contains__(self, key):
+        return key in self.keys()
+
+    def __len__(self):
+        return len(self.items())
+
+    def __iter__(self):
+        return iter(zip(self.keys(), self.values()))
+
+    def __getitem__(self, key):
+        default = object()
+        value = self.get(key, default=default)
+        if value is default:
+            raise KeyError('Key "%s" does not exist' % (key))
+        return value
+
+    def __setitem__(self, key, value):
+        return self.set(key, value)
+
+    def __getattr__(self, key):
+        return self.get(key)
+
+    def __setattr__(self, key, value):
+        return self.set(key, value)
+
+    def __eq__(self, other):
+        try:
+            for k, my_v in self.items(allow_internal=False):
+                other_v = other[k]
+                if my_v != other_v:
+                    return False
+            return True
+        except Exception:
+            return False
+
+    def update(self, other):
+        for k, v in other.items():
+            self.set(k, v)
+
+    def remove(self, key):
+        if not self._mutable:
+            raise Exception("Cannot set value of immutable object")
+
+        self._config.pop(key, None)
+
+    def get(self, key, default=None):
+        found = self._config.get(key, default)
+        if found is not default:
+            return found
+
+        if self._schema is not None:
+            found = self._schema.get(key, default)
+            if found is not default:
+                # ignore mutability
+                found = found()
+                self._config[key] = found
+                return found
+
+        return found
+
+    def set(self, key, value):
+        if not self._mutable:
+            raise Exception("Cannot set value of immutable object")
+
+        if self._schema is not None:
+            if key not in self._schema:
+                raise Exception("Can not set key '%s' - schema mismatch" % (key))
+
+            schema_entry = self._schema[key]
+            schema_entry_instance = schema_entry()
+            if not isinstance(value, type(schema_entry_instance)):
+                if isinstance(value, dict) and \
+                        isinstance(schema_entry_instance, Config):
+                    schema_entry_instance.update(value)
+                    value = schema_entry_instance
+                else:
+                    raise Exception("Can not set key '%s' - schema mismatch" % (key))
+
+        self._config[key] = value
+        return value
+
+    @staticmethod
+    def parse(path):
+        with open(path, 'r') as f:
+            return Config(yaml.safe_load(f))
+
+    @staticmethod
+    def yaml_representer(dumper, value):
+        return dumper.represent_data(
+            value._items(allow_internal=False, allow_fallback=False))
+
+    def dump(self, path):
+        with open(path, 'w+') as f:
+            yaml.dump(self, f)
+
+yaml.add_multi_representer(Config, Config.yaml_representer)
+
+
+class DefaultConfig(Config):
+    def __init__(self, default=None):
+        super().__init__()
+        self.__dict__['_default'] = default
+
+    def set(self, key, value):
+        if key not in self.keys(allow_fallback=False):
+            value = self._default(value)
+            return super().set(key, value)
+        else:
+            return super().set(key, value)
+
+
+VERSION = '0.1.0'
+DEFAULT_FORMAT = 'datumaro'

datumaro/datumaro/components/config_model.py

@@ -0,0 +1,83 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from datumaro.components.config import Config, \
+    DefaultConfig as _DefaultConfig, \
+    SchemaBuilder as _SchemaBuilder
+
+
+SOURCE_SCHEMA = _SchemaBuilder() \
+    .add('url', str) \
+    .add('format', str) \
+    .add('options', str) \
+    .build()
+
+class Source(Config):
+    def __init__(self, config=None):
+        super().__init__(config, schema=SOURCE_SCHEMA)
+
+
+MODEL_SCHEMA = _SchemaBuilder() \
+    .add('launcher', str) \
+    .add('model_dir', str, internal=True) \
+    .add('options', dict) \
+    .build()
+
+class Model(Config):
+    def __init__(self, config=None):
+        super().__init__(config, schema=MODEL_SCHEMA)
+
+
+ENV_SCHEMA = _SchemaBuilder() \
+    .add('models_dir', str) \
+    .add('importers_dir', str) \
+    .add('launchers_dir', str) \
+    .add('converters_dir', str) \
+    .add('extractors_dir', str) \
+    \
+    .add('models', lambda: _DefaultConfig(
+        lambda v=None: Model(v))) \
+    .build()
+
+ENV_DEFAULT_CONFIG = Config({
+    'models_dir': 'models',
+    'importers_dir': 'importers',
+    'launchers_dir': 'launchers',
+    'converters_dir': 'converters',
+    'extractors_dir': 'extractors',
+}, mutable=False, schema=ENV_SCHEMA)
+
+
+PROJECT_SCHEMA = _SchemaBuilder() \
+    .add('project_name', str) \
+    .add('format_version', int) \
+    \
+    .add('sources_dir', str) \
+    .add('dataset_dir', str) \
+    .add('build_dir', str) \
+    .add('subsets', list) \
+    .add('sources', lambda: _DefaultConfig(
+        lambda v=None: Source(v))) \
+    .add('filter', str) \
+    \
+    .add('project_filename', str, internal=True) \
+    .add('project_dir', str, internal=True) \
+    .add('env_filename', str, internal=True) \
+    .add('env_dir', str, internal=True) \
+    .build()
+
+PROJECT_DEFAULT_CONFIG = Config({
+    'project_name': 'undefined',
+    'format_version': 1,
+
+    'sources_dir': 'sources',
+    'dataset_dir': 'dataset',
+    'build_dir': 'build',
+
+    'project_filename': 'config.yaml',
+    'project_dir': '',
+    'env_filename': 'datumaro.yaml',
+    'env_dir': '.datumaro',
+}, mutable=False, schema=PROJECT_SCHEMA)

datumaro/datumaro/components/converter.py

@@ -0,0 +1,8 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+class Converter:
+    def __call__(self, extractor, save_dir):
+        raise NotImplementedError()

datumaro/datumaro/components/converters/__init__.py

@@ -0,0 +1,43 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from datumaro.components.converters.datumaro import DatumaroConverter
+
+from datumaro.components.converters.ms_coco import (
+    CocoConverter,
+    CocoImageInfoConverter,
+    CocoCaptionsConverter,
+    CocoInstancesConverter,
+    CocoPersonKeypointsConverter,
+    CocoLabelsConverter,
+)
+
+from datumaro.components.converters.voc import (
+    VocConverter,
+    VocClassificationConverter,
+    VocDetectionConverter,
+    VocLayoutConverter,
+    VocActionConverter,
+    VocSegmentationConverter,
+)
+
+
+items = [
+    ('datumaro', DatumaroConverter),
+
+    ('coco', CocoConverter),
+    ('coco_images', CocoImageInfoConverter),
+    ('coco_captions', CocoCaptionsConverter),
+    ('coco_instances', CocoInstancesConverter),
+    ('coco_person_kp', CocoPersonKeypointsConverter),
+    ('coco_labels', CocoLabelsConverter),
+
+    ('voc', VocConverter),
+    ('voc_cls', VocClassificationConverter),
+    ('voc_det', VocDetectionConverter),
+    ('voc_segm', VocSegmentationConverter),
+    ('voc_action', VocActionConverter),
+    ('voc_layout', VocLayoutConverter),
+]

datumaro/datumaro/components/converters/datumaro.py

@@ -0,0 +1,294 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=no-self-use
+
+import cv2
+import json
+import os
+import os.path as osp
+
+from datumaro.components.converter import Converter
+from datumaro.components.extractor import (
+    DEFAULT_SUBSET_NAME,
+    AnnotationType, Annotation,
+    LabelObject, MaskObject, PointsObject, PolygonObject,
+    PolyLineObject, BboxObject, CaptionObject,
+    LabelCategories, MaskCategories, PointsCategories
+)
+from datumaro.components.formats.datumaro import DatumaroPath
+from datumaro.util.mask_tools import apply_colormap
+
+
+def _cast(value, type_conv, default=None):
+    if value is None:
+        return default
+    try:
+        return type_conv(value)
+    except Exception:
+        return default
+
+class _SubsetWriter:
+    def __init__(self, name, converter):
+        self._name = name
+        self._converter = converter
+
+        self._data = {
+            'info': {},
+            'categories': {},
+            'items': [],
+        }
+
+        self._next_mask_id = 1
+
+    @property
+    def categories(self):
+        return self._data['categories']
+
+    @property
+    def items(self):
+        return self._data['items']
+
+    def write_item(self, item):
+        annotations = []
+        self.items.append({
+            'id': item.id,
+            'path': item.path,
+            'annotations': annotations,
+        })
+
+        for ann in item.annotations:
+            if isinstance(ann, LabelObject):
+                converted_ann = self._convert_label_object(ann)
+            elif isinstance(ann, MaskObject):
+                converted_ann = self._convert_mask_object(ann)
+            elif isinstance(ann, PointsObject):
+                converted_ann = self._convert_points_object(ann)
+            elif isinstance(ann, PolyLineObject):
+                converted_ann = self._convert_polyline_object(ann)
+            elif isinstance(ann, PolygonObject):
+                converted_ann = self._convert_polygon_object(ann)
+            elif isinstance(ann, BboxObject):
+                converted_ann = self._convert_bbox_object(ann)
+            elif isinstance(ann, CaptionObject):
+                converted_ann = self._convert_caption_object(ann)
+            else:
+                raise NotImplementedError()
+            annotations.append(converted_ann)
+
+    def write_categories(self, categories):
+        for ann_type, desc in categories.items():
+            if isinstance(desc, LabelCategories):
+                converted_desc = self._convert_label_categories(desc)
+            elif isinstance(desc, MaskCategories):
+                converted_desc = self._convert_mask_categories(desc)
+            elif isinstance(desc, PointsCategories):
+                converted_desc = self._convert_points_categories(desc)
+            else:
+                raise NotImplementedError()
+            self.categories[ann_type.name] = converted_desc
+
+    def write(self, save_dir):
+        with open(osp.join(save_dir, '%s.json' % (self._name)), 'w') as f:
+            json.dump(self._data, f)
+
+    def _convert_annotation(self, obj):
+        assert isinstance(obj, Annotation)
+
+        ann_json = {
+            'id': _cast(obj.id, int),
+            'type': _cast(obj.type.name, str),
+            'attributes': obj.attributes,
+            'group': _cast(obj.group, int, None),
+        }
+        return ann_json
+
+    def _convert_label_object(self, obj):
+        converted = self._convert_annotation(obj)
+
+        converted.update({
+            'label_id': _cast(obj.label, int),
+        })
+        return converted
+
+    def _save_mask(self, mask):
+        mask_id = None
+        if mask is None:
+            return mask_id
+
+        if self._converter._apply_colormap:
+            categories = self._converter._extractor.categories()
+            categories = categories[AnnotationType.mask]
+            colormap = categories.colormap
+
+            mask = apply_colormap(mask, colormap)
+
+        mask_id = self._next_mask_id
+        self._next_mask_id += 1
+
+        filename = '%d%s' % (mask_id, DatumaroPath.MASK_EXT)
+        masks_dir = osp.join(self._converter._annotations_dir,
+            DatumaroPath.MASKS_DIR)
+        os.makedirs(masks_dir, exist_ok=True)
+        path = osp.join(masks_dir, filename)
+        cv2.imwrite(path, mask)
+        return mask_id
+
+    def _convert_mask_object(self, obj):
+        converted = self._convert_annotation(obj)
+
+        mask = obj.image
+        mask_id = None
+        if mask is not None:
+            mask_id = self._save_mask(mask)
+
+        converted.update({
+            'label_id': _cast(obj.label, int),
+            'mask_id': _cast(mask_id, int),
+        })
+        return converted
+
+    def _convert_polyline_object(self, obj):
+        converted = self._convert_annotation(obj)
+
+        converted.update({
+            'label_id': _cast(obj.label, int),
+            'points': [float(p) for p in obj.get_points()],
+        })
+        return converted
+
+    def _convert_polygon_object(self, obj):
+        converted = self._convert_annotation(obj)
+
+        converted.update({
+            'label_id': _cast(obj.label, int),
+            'points': [float(p) for p in obj.get_points()],
+        })
+        return converted
+
+    def _convert_bbox_object(self, obj):
+        converted = self._convert_annotation(obj)
+
+        converted.update({
+            'label_id': _cast(obj.label, int),
+            'bbox': [float(p) for p in obj.get_bbox()],
+        })
+        return converted
+
+    def _convert_points_object(self, obj):
+        converted = self._convert_annotation(obj)
+
+        converted.update({
+            'label_id': _cast(obj.label, int),
+            'points': [float(p) for p in obj.points],
+            'visibility': [int(v.value) for v in obj.visibility],
+        })
+        return converted
+
+    def _convert_caption_object(self, obj):
+        converted = self._convert_annotation(obj)
+
+        converted.update({
+            'caption': _cast(obj.caption, str),
+        })
+        return converted
+
+    def _convert_label_categories(self, obj):
+        converted = {
+            'labels': [],
+        }
+        for label in obj.items:
+            converted['labels'].append({
+                'name': _cast(label.name, str),
+                'parent': _cast(label.parent, str),
+            })
+        return converted
+
+    def _convert_mask_categories(self, obj):
+        converted = {
+            'colormap': [],
+        }
+        for label_id, color in obj.colormap.items():
+            converted['colormap'].append({
+                'label_id': int(label_id),
+                'r': int(color[0]),
+                'g': int(color[1]),
+                'b': int(color[2]),
+            })
+        return converted
+
+    def _convert_points_categories(self, obj):
+        converted = {
+            'items': [],
+        }
+        for label_id, item in obj.items.items():
+            converted['items'].append({
+                'label_id': int(label_id),
+                'labels': [_cast(label, str) for label in item.labels],
+                'adjacent': [int(v) for v in item.adjacent],
+            })
+        return converted
+
+class _Converter:
+    def __init__(self, extractor, save_dir,
+            save_images=False, apply_colormap=False):
+        self._extractor = extractor
+        self._save_dir = save_dir
+        self._save_images = save_images
+        self._apply_colormap = apply_colormap
+
+    def convert(self):
+        os.makedirs(self._save_dir, exist_ok=True)
+
+        images_dir = osp.join(self._save_dir, DatumaroPath.IMAGES_DIR)
+        os.makedirs(images_dir, exist_ok=True)
+        self._images_dir = images_dir
+
+        annotations_dir = osp.join(self._save_dir, DatumaroPath.ANNOTATIONS_DIR)
+        os.makedirs(annotations_dir, exist_ok=True)
+        self._annotations_dir = annotations_dir
+
+        subsets = self._extractor.subsets()
+        if len(subsets) == 0:
+            subsets = [ None ]
+        subsets = [n if n else DEFAULT_SUBSET_NAME for n in subsets]
+        subsets = { name: _SubsetWriter(name, self) for name in subsets }
+
+        for subset, writer in subsets.items():
+            writer.write_categories(self._extractor.categories())
+
+        for item in self._extractor:
+            subset = item.subset
+            if not subset:
+                subset = DEFAULT_SUBSET_NAME
+            writer = subsets[subset]
+
+            if self._save_images:
+                self._save_image(item)
+            writer.write_item(item)
+
+        for subset, writer in subsets.items():
+            writer.write(annotations_dir)
+
+    def _save_image(self, item):
+        image = item.image
+        if image is None:
+            return
+
+        image_path = osp.join(self._images_dir,
+            str(item.id) + DatumaroPath.IMAGE_EXT)
+        cv2.imwrite(image_path, image)
+
+class DatumaroConverter(Converter):
+    def __init__(self, save_images=False, apply_colormap=False):
+        super().__init__()
+        self._save_images = save_images
+        self._apply_colormap = apply_colormap
+
+    def __call__(self, extractor, save_dir):
+        converter = _Converter(extractor, save_dir,
+            apply_colormap=self._apply_colormap,
+            save_images=self._save_images)
+        converter.convert()

datumaro/datumaro/components/converters/ms_coco.py

@@ -0,0 +1,386 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import cv2
+import json
+import numpy as np
+import os
+import os.path as osp
+
+import pycocotools.mask as mask_utils
+
+from datumaro.components.converter import Converter
+from datumaro.components.extractor import (
+    DEFAULT_SUBSET_NAME, AnnotationType, PointsObject, BboxObject
+)
+from datumaro.components.formats.ms_coco import CocoAnnotationType, CocoPath
+from datumaro.util import find
+import datumaro.util.mask_tools as mask_tools
+
+
+def _cast(value, type_conv, default=None):
+    if value is None:
+        return default
+    try:
+        return type_conv(value)
+    except Exception:
+        return default
+
+class _TaskConverter:
+    def __init__(self):
+        self._min_ann_id = 1
+
+        data = {
+            'licenses': [],
+            'info': {},
+            'categories': [],
+            'images': [],
+            'annotations': []
+            }
+
+        data['licenses'].append({
+            'name': '',
+            'id': 0,
+            'url': ''
+        })
+
+        data['info'] = {
+            'contributor': '',
+            'date_created': '',
+            'description': '',
+            'url': '',
+            'version': '',
+            'year': ''
+        }
+        self._data = data
+
+    def is_empty(self):
+        return len(self._data['annotations']) == 0
+
+    def save_image_info(self, item, filename):
+        if item.has_image:
+            h, w, _ = item.image.shape
+        else:
+            h = 0
+            w = 0
+
+        self._data['images'].append({
+            'id': _cast(item.id, int, 0),
+            'width': int(w),
+            'height': int(h),
+            'file_name': filename,
+            'license': 0,
+            'flickr_url': '',
+            'coco_url': '',
+            'date_captured': 0,
+        })
+
+    def save_categories(self, dataset):
+        raise NotImplementedError()
+
+    def save_annotations(self, item):
+        raise NotImplementedError()
+
+    def write(self, path):
+        next_id = self._min_ann_id
+        for ann in self.annotations:
+            if ann['id'] is None:
+                ann['id'] = next_id
+                next_id += 1
+
+        with open(path, 'w') as outfile:
+            json.dump(self._data, outfile)
+
+    @property
+    def annotations(self):
+        return self._data['annotations']
+
+    @property
+    def categories(self):
+        return self._data['categories']
+
+    def _get_ann_id(self, annotation):
+        ann_id = annotation.id
+        if ann_id:
+            self._min_ann_id = max(ann_id, self._min_ann_id)
+        return ann_id
+
+class _InstancesConverter(_TaskConverter):
+    def save_categories(self, dataset):
+        label_categories = dataset.categories().get(AnnotationType.label)
+        if label_categories is None:
+            return
+
+        for idx, cat in enumerate(label_categories.items):
+            self.categories.append({
+                'id': 1 + idx,
+                'name': cat.name,
+                'supercategory': cat.parent,
+            })
+
+    def save_annotations(self, item):
+        for ann in item.annotations:
+            if ann.type != AnnotationType.bbox:
+                continue
+
+            is_crowd = ann.attributes.get('is_crowd', False)
+            segmentation = None
+            if ann.group is not None:
+                if is_crowd:
+                    segmentation = find(item.annotations, lambda x: \
+                        x.group == ann.group and x.type == AnnotationType.mask)
+                    if segmentation is not None:
+                        binary_mask = np.array(segmentation.image, dtype=np.bool)
+                        binary_mask = np.asfortranarray(binary_mask, dtype=np.uint8)
+                        segmentation = mask_utils.encode(binary_mask)
+                        area = mask_utils.area(segmentation)
+                        segmentation = mask_tools.convert_mask_to_rle(binary_mask)
+                else:
+                    segmentation = find(item.annotations, lambda x: \
+                        x.group == ann.group and x.type == AnnotationType.polygon)
+                    if segmentation is not None:
+                        area = ann.area()
+                        segmentation = [segmentation.get_points()]
+            if segmentation is None:
+                is_crowd = False
+                segmentation = [ann.get_polygon()]
+                area = ann.area()
+
+            elem = {
+                'id': self._get_ann_id(ann),
+                'image_id': _cast(item.id, int, 0),
+                'category_id': _cast(ann.label, int, -1) + 1,
+                'segmentation': segmentation,
+                'area': float(area),
+                'bbox': ann.get_bbox(),
+                'iscrowd': int(is_crowd),
+            }
+            if 'score' in ann.attributes:
+                elem['score'] = float(ann.attributes['score'])
+
+            self.annotations.append(elem)
+
+class _ImageInfoConverter(_TaskConverter):
+    def is_empty(self):
+        return len(self._data['images']) == 0
+
+    def save_categories(self, dataset):
+        pass
+
+    def save_annotations(self, item):
+        pass
+
+class _CaptionsConverter(_TaskConverter):
+    def save_categories(self, dataset):
+        pass
+
+    def save_annotations(self, item):
+        for ann in item.annotations:
+            if ann.type != AnnotationType.caption:
+                continue
+
+            elem = {
+                'id': self._get_ann_id(ann),
+                'image_id': _cast(item.id, int, 0),
+                'category_id': 0, # NOTE: workaround for a bug in cocoapi
+                'caption': ann.caption,
+            }
+            if 'score' in ann.attributes:
+                elem['score'] = float(ann.attributes['score'])
+
+            self.annotations.append(elem)
+
+class _KeypointsConverter(_TaskConverter):
+    def save_categories(self, dataset):
+        label_categories = dataset.categories().get(AnnotationType.label)
+        if label_categories is None:
+            return
+        points_categories = dataset.categories().get(AnnotationType.points)
+        if points_categories is None:
+            return
+
+        for idx, kp_cat in points_categories.items.items():
+            label_cat = label_categories.items[idx]
+
+            cat = {
+                'id': 1 + idx,
+                'name': label_cat.name,
+                'supercategory': label_cat.parent,
+                'keypoints': [str(l) for l in kp_cat.labels],
+                'skeleton': [int(i) for i in kp_cat.adjacent],
+            }
+            self.categories.append(cat)
+
+    def save_annotations(self, item):
+        for ann in item.annotations:
+            if ann.type != AnnotationType.points:
+                continue
+
+            elem = {
+                'id': self._get_ann_id(ann),
+                'image_id': _cast(item.id, int, 0),
+                'category_id': _cast(ann.label, int, -1) + 1,
+            }
+            if 'score' in ann.attributes:
+                elem['score'] = float(ann.attributes['score'])
+
+            keypoints = []
+            points = ann.get_points()
+            visibility = ann.visibility
+            for index in range(0, len(points), 2):
+                kp = points[index : index + 2]
+                state = visibility[index // 2].value
+                keypoints.extend([*kp, state])
+
+            num_visible = len([v for v in visibility \
+                if v == PointsObject.Visibility.visible])
+
+            bbox = find(item.annotations, lambda x: \
+                x.group == ann.group and \
+                x.type == AnnotationType.bbox and
+                x.label == ann.label)
+            if bbox is None:
+                bbox = BboxObject(*ann.get_bbox())
+            elem.update({
+                'segmentation': bbox.get_polygon(),
+                'area': bbox.area(),
+                'bbox': bbox.get_bbox(),
+                'iscrowd': 0,
+                'keypoints': keypoints,
+                'num_keypoints': num_visible,
+            })
+
+            self.annotations.append(elem)
+
+class _LabelsConverter(_TaskConverter):
+    def save_categories(self, dataset):
+        label_categories = dataset.categories().get(AnnotationType.label)
+        if label_categories is None:
+            return
+
+        for idx, cat in enumerate(label_categories.items):
+            self.categories.append({
+                'id': 1 + idx,
+                'name': cat.name,
+                'supercategory': cat.parent,
+            })
+
+    def save_annotations(self, item):
+        for ann in item.annotations:
+            if ann.type != AnnotationType.label:
+                continue
+
+            elem = {
+                'id': self._get_ann_id(ann),
+                'image_id': _cast(item.id, int, 0),
+                'category_id': int(ann.label) + 1,
+            }
+            if 'score' in ann.attributes:
+                elem['score'] = float(ann.attributes['score'])
+
+            self.annotations.append(elem)
+
+class _Converter:
+    _TASK_CONVERTER = {
+        CocoAnnotationType.image_info: _ImageInfoConverter,
+        CocoAnnotationType.instances: _InstancesConverter,
+        CocoAnnotationType.person_keypoints: _KeypointsConverter,
+        CocoAnnotationType.captions: _CaptionsConverter,
+        CocoAnnotationType.labels: _LabelsConverter,
+    }
+
+    def __init__(self, extractor, save_dir, save_images=False, task=None):
+        if not task:
+            task = list(self._TASK_CONVERTER.keys())
+        elif task in CocoAnnotationType:
+            task = [task]
+        self._task = task
+        self._extractor = extractor
+        self._save_dir = save_dir
+        self._save_images = save_images
+
+    def make_dirs(self):
+        self._images_dir = osp.join(self._save_dir, CocoPath.IMAGES_DIR)
+        os.makedirs(self._images_dir, exist_ok=True)
+
+        self._ann_dir = osp.join(self._save_dir, CocoPath.ANNOTATIONS_DIR)
+        os.makedirs(self._ann_dir, exist_ok=True)
+
+    def make_task_converter(self, task):
+        return self._TASK_CONVERTER[task]()
+
+    def make_task_converters(self):
+        return {
+            task: self.make_task_converter(task) for task in self._task
+        }
+
+    def save_image(self, item, subset_name, filename):
+        path = osp.join(self._images_dir, subset_name, filename)
+        cv2.imwrite(path, item.image)
+
+        return path
+
+    def convert(self):
+        self.make_dirs()
+
+        subsets = self._extractor.subsets()
+        if len(subsets) == 0:
+            subsets = [ None ]
+
+        for subset_name in subsets:
+            if subset_name:
+                subset = self._extractor.get_subset(subset_name)
+            else:
+                subset_name = DEFAULT_SUBSET_NAME
+                subset = self._extractor
+
+            task_converters = self.make_task_converters()
+            for task_conv in task_converters.values():
+                task_conv.save_categories(subset)
+            for item in subset:
+                filename = ''
+                if item.has_image:
+                    filename = str(item.id) + CocoPath.IMAGE_EXT
+                    if self._save_images:
+                        self.save_image(item, subset_name, filename)
+                for task_conv in task_converters.values():
+                    task_conv.save_image_info(item, filename)
+                    task_conv.save_annotations(item)
+
+            for task, task_conv in task_converters.items():
+                if not task_conv.is_empty():
+                    task_conv.write(osp.join(self._ann_dir,
+                        '%s_%s.json' % (task.name, subset_name)))
+
+class CocoConverter(Converter):
+    def __init__(self, task=None, save_images=False):
+        super().__init__()
+        self._task = task
+        self._save_images = save_images
+
+    def __call__(self, extractor, save_dir):
+        converter = _Converter(extractor, save_dir,
+            save_images=self._save_images, task=self._task)
+        converter.convert()
+
+def CocoInstancesConverter(save_images=False):
+    return CocoConverter(CocoAnnotationType.instances,
+        save_images=save_images)
+
+def CocoImageInfoConverter(save_images=False):
+    return CocoConverter(CocoAnnotationType.image_info,
+        save_images=save_images)
+
+def CocoPersonKeypointsConverter(save_images=False):
+    return CocoConverter(CocoAnnotationType.person_keypoints,
+        save_images=save_images)
+
+def CocoCaptionsConverter(save_images=False):
+    return CocoConverter(CocoAnnotationType.captions,
+        save_images=save_images)
+
+def CocoLabelsConverter(save_images=False):
+    return CocoConverter(CocoAnnotationType.labels,
+        save_images=save_images)

datumaro/datumaro/components/converters/voc.py

@@ -0,0 +1,370 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import cv2
+from collections import OrderedDict, defaultdict
+import os
+import os.path as osp
+from lxml import etree as ET
+
+from datumaro.components.converter import Converter
+from datumaro.components.extractor import DEFAULT_SUBSET_NAME, AnnotationType
+from datumaro.components.formats.voc import VocLabel, VocAction, \
+    VocBodyPart, VocPose, VocTask, VocPath, VocColormap, VocInstColormap
+from datumaro.util import find
+from datumaro.util.mask_tools import apply_colormap
+
+
+def _write_xml_bbox(bbox, parent_elem):
+    x, y, w, h = bbox
+    bbox_elem = ET.SubElement(parent_elem, 'bndbox')
+    ET.SubElement(bbox_elem, 'xmin').text = str(x)
+    ET.SubElement(bbox_elem, 'ymin').text = str(y)
+    ET.SubElement(bbox_elem, 'xmax').text = str(x + w)
+    ET.SubElement(bbox_elem, 'ymax').text = str(y + h)
+    return bbox_elem
+
+class _Converter:
+    _LABELS = set([entry.name for entry in VocLabel])
+    _BODY_PARTS = set([entry.name for entry in VocBodyPart])
+    _ACTIONS = set([entry.name for entry in VocAction])
+
+    def __init__(self, task, extractor, save_dir,
+            apply_colormap=True, save_images=False):
+
+        assert not task or task in VocTask
+        self._task = task
+        self._extractor = extractor
+        self._save_dir = save_dir
+        self._apply_colormap = apply_colormap
+        self._save_images = save_images
+
+        self._label_categories = extractor.categories() \
+            .get(AnnotationType.label)
+        self._mask_categories = extractor.categories() \
+            .get(AnnotationType.mask)
+
+    def convert(self):
+        self.init_dirs()
+        self.save_subsets()
+
+    def init_dirs(self):
+        save_dir = self._save_dir
+        subsets_dir = osp.join(save_dir, VocPath.SUBSETS_DIR)
+        cls_subsets_dir = osp.join(subsets_dir,
+            VocPath.TASK_DIR[VocTask.classification])
+        action_subsets_dir = osp.join(subsets_dir,
+            VocPath.TASK_DIR[VocTask.action_classification])
+        layout_subsets_dir = osp.join(subsets_dir,
+            VocPath.TASK_DIR[VocTask.person_layout])
+        segm_subsets_dir = osp.join(subsets_dir,
+            VocPath.TASK_DIR[VocTask.segmentation])
+        ann_dir = osp.join(save_dir, VocPath.ANNOTATIONS_DIR)
+        img_dir = osp.join(save_dir, VocPath.IMAGES_DIR)
+        segm_dir = osp.join(save_dir, VocPath.SEGMENTATION_DIR)
+        inst_dir = osp.join(save_dir, VocPath.INSTANCES_DIR)
+        images_dir = osp.join(save_dir, VocPath.IMAGES_DIR)
+
+        os.makedirs(subsets_dir, exist_ok=True)
+        os.makedirs(ann_dir, exist_ok=True)
+        os.makedirs(img_dir, exist_ok=True)
+        os.makedirs(segm_dir, exist_ok=True)
+        os.makedirs(inst_dir, exist_ok=True)
+        os.makedirs(images_dir, exist_ok=True)
+
+        self._subsets_dir = subsets_dir
+        self._cls_subsets_dir = cls_subsets_dir
+        self._action_subsets_dir = action_subsets_dir
+        self._layout_subsets_dir = layout_subsets_dir
+        self._segm_subsets_dir = segm_subsets_dir
+        self._ann_dir = ann_dir
+        self._img_dir = img_dir
+        self._segm_dir = segm_dir
+        self._inst_dir = inst_dir
+        self._images_dir = images_dir
+
+    def get_label(self, label_id):
+        return self._label_categories.items[label_id].name
+
+    def save_subsets(self):
+        subsets = self._extractor.subsets()
+        if len(subsets) == 0:
+            subsets = [ None ]
+
+        for subset_name in subsets:
+            if subset_name:
+                subset = self._extractor.get_subset(subset_name)
+            else:
+                subset_name = DEFAULT_SUBSET_NAME
+                subset = self._extractor
+
+            class_lists = OrderedDict()
+            clsdet_list = OrderedDict()
+            action_list = OrderedDict()
+            layout_list = OrderedDict()
+            segm_list = OrderedDict()
+
+            for item in subset:
+                item_id = str(item.id)
+                if self._save_images:
+                    data = item.image
+                    if data is not None:
+                        cv2.imwrite(osp.join(self._images_dir,
+                                str(item_id) + VocPath.IMAGE_EXT),
+                            data)
+
+                labels = []
+                bboxes = []
+                masks = []
+                for a in item.annotations:
+                    if a.type == AnnotationType.label:
+                        labels.append(a)
+                    elif a.type == AnnotationType.bbox:
+                        bboxes.append(a)
+                    elif a.type == AnnotationType.mask:
+                        masks.append(a)
+
+                if len(bboxes) != 0:
+                    root_elem = ET.Element('annotation')
+                    if '_' in item_id:
+                        folder = item_id[ : item_id.find('_')]
+                    else:
+                        folder = ''
+                    ET.SubElement(root_elem, 'folder').text = folder
+                    ET.SubElement(root_elem, 'filename').text = \
+                        item_id + VocPath.IMAGE_EXT
+
+                    if item.has_image:
+                        h, w, c = item.image.shape
+                        size_elem = ET.SubElement(root_elem, 'size')
+                        ET.SubElement(size_elem, 'width').text = str(w)
+                        ET.SubElement(size_elem, 'height').text = str(h)
+                        ET.SubElement(size_elem, 'depth').text = str(c)
+
+                    item_segmented = 0 < len(masks)
+                    if item_segmented:
+                        ET.SubElement(root_elem, 'segmented').text = '1'
+
+                    objects_with_parts = []
+                    objects_with_actions = defaultdict(dict)
+
+                    main_bboxes = []
+                    layout_bboxes = []
+                    for bbox in bboxes:
+                        label = self.get_label(bbox.label)
+                        if label in self._LABELS:
+                            main_bboxes.append(bbox)
+                        elif label in self._BODY_PARTS:
+                            layout_bboxes.append(bbox)
+
+                    for new_obj_id, obj in enumerate(main_bboxes):
+                        attr = obj.attributes
+
+                        obj_elem = ET.SubElement(root_elem, 'object')
+                        ET.SubElement(obj_elem, 'name').text = self.get_label(obj.label)
+
+                        pose = attr.get('pose')
+                        if pose is not None:
+                            ET.SubElement(obj_elem, 'pose').text = VocPose[pose].name
+
+                        truncated = attr.get('truncated')
+                        if truncated is not None:
+                            ET.SubElement(obj_elem, 'truncated').text = '%d' % truncated
+
+                        difficult = attr.get('difficult')
+                        if difficult is not None:
+                            ET.SubElement(obj_elem, 'difficult').text = '%d' % difficult
+
+                        bbox = obj.get_bbox()
+                        if bbox is not None:
+                            _write_xml_bbox(bbox, obj_elem)
+
+                        for part in VocBodyPart:
+                            part_bbox = find(layout_bboxes, lambda x: \
+                                obj.id == x.group and \
+                                self.get_label(x.label) == part.name)
+                            if part_bbox is not None:
+                                part_elem = ET.SubElement(obj_elem, 'part')
+                                ET.SubElement(part_elem, 'name').text = part.name
+                                _write_xml_bbox(part_bbox.get_bbox(), part_elem)
+
+                                objects_with_parts.append(new_obj_id)
+
+                        actions = [x for x in labels
+                            if obj.id == x.group and \
+                               self.get_label(x.label) in self._ACTIONS]
+                        if len(actions) != 0:
+                            actions_elem = ET.SubElement(obj_elem, 'actions')
+                            for action in VocAction:
+                                presented = find(actions, lambda x: \
+                                    self.get_label(x.label) == action.name) is not None
+                                ET.SubElement(actions_elem, action.name).text = \
+                                    '%d' % presented
+
+                                objects_with_actions[new_obj_id][action] = presented
+
+                    if self._task in [None,
+                            VocTask.detection,
+                            VocTask.person_layout,
+                            VocTask.action_classification]:
+                        with open(osp.join(self._ann_dir, item_id + '.xml'), 'w') as f:
+                            f.write(ET.tostring(root_elem,
+                                encoding='unicode', pretty_print=True))
+
+                    clsdet_list[item_id] = True
+                    layout_list[item_id] = objects_with_parts
+                    action_list[item_id] = objects_with_actions
+
+                for label_obj in labels:
+                    label = self.get_label(label_obj.label)
+                    if label not in self._LABELS:
+                        continue
+                    class_list = class_lists.get(item_id, set())
+                    class_list.add(label_obj.label)
+                    class_lists[item_id] = class_list
+
+                    clsdet_list[item_id] = True
+
+                for mask_obj in masks:
+                    if mask_obj.attributes.get('class') == True:
+                        self.save_segm(osp.join(self._segm_dir,
+                                item_id + VocPath.SEGM_EXT),
+                            mask_obj, self._mask_categories.colormap)
+                    if mask_obj.attributes.get('instances') == True:
+                        self.save_segm(osp.join(self._inst_dir,
+                                item_id + VocPath.SEGM_EXT),
+                            mask_obj, VocInstColormap)
+
+                    segm_list[item_id] = True
+
+                if len(item.annotations) == 0:
+                    clsdet_list[item_id] = None
+                    layout_list[item_id] = None
+                    action_list[item_id] = None
+                    segm_list[item_id] = None
+
+                if self._task in [None,
+                        VocTask.classification,
+                        VocTask.detection,
+                        VocTask.action_classification,
+                        VocTask.person_layout]:
+                    self.save_clsdet_lists(subset_name, clsdet_list)
+                    if self._task in [None, VocTask.classification]:
+                        self.save_class_lists(subset_name, class_lists)
+                if self._task in [None, VocTask.action_classification]:
+                    self.save_action_lists(subset_name, action_list)
+                if self._task in [None, VocTask.person_layout]:
+                    self.save_layout_lists(subset_name, layout_list)
+                if self._task in [None, VocTask.segmentation]:
+                    self.save_segm_lists(subset_name, segm_list)
+
+    def save_action_lists(self, subset_name, action_list):
+        os.makedirs(self._action_subsets_dir, exist_ok=True)
+
+        ann_file = osp.join(self._action_subsets_dir, subset_name + '.txt')
+        with open(ann_file, 'w') as f:
+            for item in action_list:
+                f.write('%s\n' % item)
+
+        if len(action_list) == 0:
+            return
+
+        for action in VocAction:
+            ann_file = osp.join(self._action_subsets_dir,
+                '%s_%s.txt' % (action.name, subset_name))
+            with open(ann_file, 'w') as f:
+                for item, objs in action_list.items():
+                    if not objs:
+                        continue
+                    for obj_id, obj_actions in objs.items():
+                        presented = obj_actions[action]
+                        f.write('%s %s % d\n' % \
+                            (item, 1 + obj_id, 1 if presented else -1))
+
+    def save_class_lists(self, subset_name, class_lists):
+        os.makedirs(self._cls_subsets_dir, exist_ok=True)
+
+        if len(class_lists) == 0:
+            return
+
+        for label in VocLabel:
+            ann_file = osp.join(self._cls_subsets_dir,
+                '%s_%s.txt' % (label.name, subset_name))
+            with open(ann_file, 'w') as f:
+                for item, item_labels in class_lists.items():
+                    if not item_labels:
+                        continue
+                    presented = label.value in item_labels
+                    f.write('%s % d\n' % \
+                        (item, 1 if presented else -1))
+
+    def save_clsdet_lists(self, subset_name, clsdet_list):
+        os.makedirs(self._cls_subsets_dir, exist_ok=True)
+
+        ann_file = osp.join(self._cls_subsets_dir, subset_name + '.txt')
+        with open(ann_file, 'w') as f:
+            for item in clsdet_list:
+                f.write('%s\n' % item)
+
+    def save_segm_lists(self, subset_name, segm_list):
+        os.makedirs(self._segm_subsets_dir, exist_ok=True)
+
+        ann_file = osp.join(self._segm_subsets_dir, subset_name + '.txt')
+        with open(ann_file, 'w') as f:
+            for item in segm_list:
+                f.write('%s\n' % item)
+
+    def save_layout_lists(self, subset_name, layout_list):
+        os.makedirs(self._layout_subsets_dir, exist_ok=True)
+
+        ann_file = osp.join(self._layout_subsets_dir, subset_name + '.txt')
+        with open(ann_file, 'w') as f:
+            for item, item_layouts in layout_list.items():
+                if item_layouts:
+                    for obj_id in item_layouts:
+                        f.write('%s % d\n' % (item, 1 + obj_id))
+                else:
+                    f.write('%s\n' % (item))
+
+    def save_segm(self, path, annotation, colormap):
+        data = annotation.image
+        if self._apply_colormap:
+            if colormap is None:
+                colormap = VocColormap
+            data = apply_colormap(data, colormap)
+        cv2.imwrite(path, data)
+
+class VocConverter(Converter):
+    def __init__(self, task=None, save_images=False, apply_colormap=False):
+        super().__init__()
+        self._task = task
+        self._save_images = save_images
+        self._apply_colormap = apply_colormap
+
+    def __call__(self, extractor, save_dir):
+        converter = _Converter(self._task, extractor, save_dir,
+            apply_colormap=self._apply_colormap,
+            save_images=self._save_images)
+        converter.convert()
+
+def VocClassificationConverter(save_images=False):
+    return VocConverter(VocTask.classification,
+        save_images=save_images)
+
+def VocDetectionConverter(save_images=False):
+    return VocConverter(VocTask.detection,
+        save_images=save_images)
+
+def VocLayoutConverter(save_images=False):
+    return VocConverter(VocTask.person_layout,
+        save_images=save_images)
+
+def VocActionConverter(save_images=False):
+    return VocConverter(VocTask.action_classification,
+        save_images=save_images)
+
+def VocSegmentationConverter(save_images=False, apply_colormap=True):
+    return VocConverter(VocTask.segmentation,
+        save_images=save_images, apply_colormap=apply_colormap)

datumaro/datumaro/components/dataset_filter.py

@@ -0,0 +1,193 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from lxml import etree as ET
+from datumaro.components.extractor import (DatasetItem, Annotation,
+    LabelObject, MaskObject, PointsObject, PolygonObject,
+    PolyLineObject, BboxObject, CaptionObject,
+)
+
+
+def _cast(value, type_conv, default=None):
+    if value is None:
+        return default
+    try:
+        return type_conv(value)
+    except Exception:
+        return default
+
+class DatasetItemEncoder:
+    def encode_item(self, item):
+        item_elem = ET.Element('item')
+        ET.SubElement(item_elem, 'id').text = str(item.id)
+        ET.SubElement(item_elem, 'subset').text = str(item.subset)
+
+        # Dataset wrapper-specific
+        ET.SubElement(item_elem, 'source').text = \
+            str(getattr(item, 'source', None))
+        ET.SubElement(item_elem, 'extractor').text = \
+            str(getattr(item, 'extractor', None))
+
+        image = item.image
+        if image is not None:
+            item_elem.append(self.encode_image(image))
+
+        for ann in item.annotations:
+            item_elem.append(self.encode_object(ann))
+
+        return item_elem
+
+    @classmethod
+    def encode_image(cls, image):
+        image_elem = ET.Element('image')
+
+        h, w, c = image.shape
+        ET.SubElement(image_elem, 'width').text = str(w)
+        ET.SubElement(image_elem, 'height').text = str(h)
+        ET.SubElement(image_elem, 'depth').text = str(c)
+
+        return image_elem
+
+    @classmethod
+    def encode_annotation(cls, annotation):
+        assert isinstance(annotation, Annotation)
+        ann_elem = ET.Element('annotation')
+        ET.SubElement(ann_elem, 'id').text = str(annotation.id)
+        ET.SubElement(ann_elem, 'type').text = str(annotation.type.name)
+
+        for k, v in annotation.attributes.items():
+            ET.SubElement(ann_elem, k).text = str(v)
+
+        ET.SubElement(ann_elem, 'group').text = str(annotation.group)
+
+        return ann_elem
+
+    @classmethod
+    def encode_label_object(cls, obj):
+        ann_elem = cls.encode_annotation(obj)
+
+        ET.SubElement(ann_elem, 'label_id').text = str(obj.label)
+
+        return ann_elem
+
+    @classmethod
+    def encode_mask_object(cls, obj):
+        ann_elem = cls.encode_annotation(obj)
+
+        ET.SubElement(ann_elem, 'label_id').text = str(obj.label)
+
+        mask = obj.image
+        if mask is not None:
+            ann_elem.append(cls.encode_image(mask))
+
+        return ann_elem
+
+    @classmethod
+    def encode_bbox_object(cls, obj):
+        ann_elem = cls.encode_annotation(obj)
+
+        ET.SubElement(ann_elem, 'label_id').text = str(obj.label)
+        ET.SubElement(ann_elem, 'x').text = str(obj.x)
+        ET.SubElement(ann_elem, 'y').text = str(obj.y)
+        ET.SubElement(ann_elem, 'w').text = str(obj.w)
+        ET.SubElement(ann_elem, 'h').text = str(obj.h)
+        ET.SubElement(ann_elem, 'area').text = str(obj.area())
+
+        return ann_elem
+
+    @classmethod
+    def encode_points_object(cls, obj):
+        ann_elem = cls.encode_annotation(obj)
+
+        ET.SubElement(ann_elem, 'label_id').text = str(obj.label)
+
+        x, y, w, h = obj.get_bbox()
+        area = w * h
+        bbox_elem = ET.SubElement(ann_elem, 'bbox')
+        ET.SubElement(bbox_elem, 'x').text = str(x)
+        ET.SubElement(bbox_elem, 'y').text = str(y)
+        ET.SubElement(bbox_elem, 'w').text = str(w)
+        ET.SubElement(bbox_elem, 'h').text = str(h)
+        ET.SubElement(bbox_elem, 'area').text = str(area)
+
+        points = ann_elem.points
+        for i in range(0, len(points), 2):
+            point_elem = ET.SubElement(ann_elem, 'point')
+            ET.SubElement(point_elem, 'x').text = str(points[i * 2])
+            ET.SubElement(point_elem, 'y').text = str(points[i * 2 + 1])
+            ET.SubElement(point_elem, 'visible').text = \
+                str(ann_elem.visibility[i // 2].name)
+
+        return ann_elem
+
+    @classmethod
+    def encode_polyline_object(cls, obj):
+        ann_elem = cls.encode_annotation(obj)
+
+        ET.SubElement(ann_elem, 'label_id').text = str(obj.label)
+
+        x, y, w, h = obj.get_bbox()
+        area = w * h
+        bbox_elem = ET.SubElement(ann_elem, 'bbox')
+        ET.SubElement(bbox_elem, 'x').text = str(x)
+        ET.SubElement(bbox_elem, 'y').text = str(y)
+        ET.SubElement(bbox_elem, 'w').text = str(w)
+        ET.SubElement(bbox_elem, 'h').text = str(h)
+        ET.SubElement(bbox_elem, 'area').text = str(area)
+
+        points = ann_elem.points
+        for i in range(0, len(points), 2):
+            point_elem = ET.SubElement(ann_elem, 'point')
+            ET.SubElement(point_elem, 'x').text = str(points[i * 2])
+            ET.SubElement(point_elem, 'y').text = str(points[i * 2 + 1])
+
+        return ann_elem
+
+    @classmethod
+    def encode_caption_object(cls, obj):
+        ann_elem = cls.encode_annotation(obj)
+
+        ET.SubElement(ann_elem, 'caption').text = str(obj.caption)
+
+        return ann_elem
+
+    def encode_object(self, o):
+        if isinstance(o, LabelObject):
+            return self.encode_label_object(o)
+        if isinstance(o, MaskObject):
+            return self.encode_mask_object(o)
+        if isinstance(o, BboxObject):
+            return self.encode_bbox_object(o)
+        if isinstance(o, PointsObject):
+            return self.encode_points_object(o)
+        if isinstance(o, PolyLineObject):
+            return self.encode_polyline_object(o)
+        if isinstance(o, PolygonObject):
+            return self.encode_polygon_object(o)
+        if isinstance(o, CaptionObject):
+            return self.encode_caption_object(o)
+        if isinstance(o, Annotation): # keep after derived classes
+            return self.encode_annotation(o)
+
+        if isinstance(o, DatasetItem):
+            return self.encode_item(o)
+
+        return None
+
+class XPathDatasetFilter:
+    def __init__(self, filter_text=None):
+        self._filter = None
+        if filter_text is not None:
+            self._filter = ET.XPath(filter_text)
+        self._encoder = DatasetItemEncoder()
+
+    def __call__(self, item):
+        encoded_item = self._serialize_item(item)
+        if self._filter is None:
+            return True
+        return bool(self._filter(encoded_item))
+
+    def _serialize_item(self, item):
+        return self._encoder.encode_item(item)

datumaro/datumaro/components/extractor.py

@@ -0,0 +1,549 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from collections import namedtuple
+from enum import Enum
+import numpy as np
+
+
+AnnotationType = Enum('AnnotationType',
+    [
+        'label',
+        'mask',
+        'points',
+        'polygon',
+        'polyline',
+        'bbox',
+        'caption',
+    ])
+
+class Annotation:
+    # pylint: disable=redefined-builtin
+    def __init__(self, id=None, type=None, attributes=None, group=None):
+        if id is not None:
+            id = int(id)
+        self.id = id
+
+        assert type in AnnotationType
+        self.type = type
+
+        if attributes is None:
+            attributes = {}
+        else:
+            attributes = dict(attributes)
+        self.attributes = attributes
+
+        if group is not None:
+            group = int(group)
+        self.group = group
+    # pylint: enable=redefined-builtin
+
+    def __eq__(self, other):
+        if not isinstance(other, Annotation):
+            return False
+        return \
+            (self.id == other.id) and \
+            (self.type == other.type) and \
+            (self.attributes == other.attributes) and \
+            (self.group == other.group)
+
+class Categories:
+    def __init__(self, attributes=None):
+        if attributes is None:
+            attributes = set()
+        self.attributes = attributes
+
+    def __eq__(self, other):
+        if not isinstance(other, Categories):
+            return False
+        return \
+            (self.attributes == other.attributes)
+
+class LabelCategories(Categories):
+    Category = namedtuple('Category', ['name', 'parent'])
+
+    def __init__(self, items=None, attributes=None):
+        super().__init__(attributes=attributes)
+
+        if items is None:
+            items = []
+        self.items = items
+
+        self._indices = {}
+        self._reindex()
+
+    def _reindex(self):
+        indices = {}
+        for index, item in enumerate(self.items):
+            assert item.name not in self._indices
+            indices[item.name] = index
+        self._indices = indices
+
+    def add(self, name, parent=None):
+        assert name not in self._indices
+
+        index = len(self.items)
+        self.items.append(self.Category(name, parent))
+        self._indices[name] = index
+
+    def find(self, name):
+        index = self._indices.get(name)
+        if index:
+            return index, self.items[index]
+        return index, None
+
+    def __eq__(self, other):
+        if not super().__eq__(other):
+            return False
+        return \
+            (self.items == other.items)
+
+class LabelObject(Annotation):
+    # pylint: disable=redefined-builtin
+    def __init__(self, label=None,
+            id=None, attributes=None, group=None):
+        super().__init__(id=id, type=AnnotationType.label,
+            attributes=attributes, group=group)
+        self.label = label
+    # pylint: enable=redefined-builtin
+
+    def __eq__(self, other):
+        if not super().__eq__(other):
+            return False
+        return \
+            (self.label == other.label)
+
+class MaskCategories(Categories):
+    def __init__(self, colormap=None, inverse_colormap=None, attributes=None):
+        super().__init__(attributes=attributes)
+
+        # colormap: label id -> color
+        if colormap is None:
+            colormap = {}
+        self.colormap = colormap
+        self._inverse_colormap = inverse_colormap
+
+    @property
+    def inverse_colormap(self):
+        from datumaro.util.mask_tools import invert_colormap
+        if self._inverse_colormap is None:
+            if self.colormap is not None:
+                try:
+                    self._inverse_colormap = invert_colormap(self.colormap)
+                except Exception:
+                    pass
+        return self._inverse_colormap
+
+    def __eq__(self, other):
+        if not super().__eq__(other):
+            return False
+        for label_id, my_color in self.colormap.items():
+            other_color = other.colormap.get(label_id)
+            if not np.array_equal(my_color, other_color):
+                return False
+        return True
+
+class MaskObject(Annotation):
+    # pylint: disable=redefined-builtin
+    def __init__(self, image=None, label=None,
+            id=None, attributes=None, group=None):
+        super().__init__(id=id, type=AnnotationType.mask,
+            attributes=attributes, group=group)
+        self._image = image
+        self._label = label
+    # pylint: enable=redefined-builtin
+
+    @property
+    def label(self):
+        return self._label
+
+    @property
+    def image(self):
+        if callable(self._image):
+            return self._image()
+        return self._image
+
+    def painted_data(self, colormap):
+        raise NotImplementedError()
+
+    def area(self):
+        raise NotImplementedError()
+
+    def extract(self, class_id):
+        raise NotImplementedError()
+
+    def bbox(self):
+        raise NotImplementedError()
+
+    def __eq__(self, other):
+        if not super().__eq__(other):
+            return False
+        return \
+            (self.label == other.label) and \
+            (np.all(self.image == other.image))
+
+def compute_iou(bbox_a, bbox_b):
+    aX, aY, aW, aH = bbox_a
+    bX, bY, bW, bH = bbox_b
+    in_right = min(aX + aW, bX + bW)
+    in_left = max(aX, bX)
+    in_top = max(aY, bY)
+    in_bottom = min(aY + aH, bY + bH)
+
+    in_w = max(0, in_right - in_left)
+    in_h = max(0, in_bottom - in_top)
+    intersection = in_w * in_h
+
+    a_area = aW * aH
+    b_area = bW * bH
+    union = a_area + b_area - intersection
+
+    return intersection / max(1.0, union)
+
+class ShapeObject(Annotation):
+    # pylint: disable=redefined-builtin
+    def __init__(self, type, points=None, label=None,
+            id=None, attributes=None, group=None):
+        super().__init__(id=id, type=type,
+            attributes=attributes, group=group)
+        self.points = points
+        self.label = label
+    # pylint: enable=redefined-builtin
+
+    def area(self):
+        raise NotImplementedError()
+
+    def get_polygon(self):
+        raise NotImplementedError()
+
+    def get_bbox(self):
+        points = self.get_points()
+        if not self.points:
+            return None
+
+        xs = [p for p in points[0::2]]
+        ys = [p for p in points[1::2]]
+        x0 = min(xs)
+        x1 = max(xs)
+        y0 = min(ys)
+        y1 = max(ys)
+        return [x0, y0, x1 - x0, y1 - y0]
+
+    def get_points(self):
+        return self.points
+
+    def get_mask(self):
+        raise NotImplementedError()
+
+    def __eq__(self, other):
+        if not super().__eq__(other):
+            return False
+        return \
+            (self.points == other.points) and \
+            (self.label == other.label)
+
+class PolyLineObject(ShapeObject):
+    # pylint: disable=redefined-builtin
+    def __init__(self, points=None,
+            label=None, id=None, attributes=None, group=None):
+        super().__init__(type=AnnotationType.polyline,
+            points=points, label=label,
+            id=id, attributes=attributes, group=group)
+    # pylint: enable=redefined-builtin
+
+    def get_polygon(self):
+        return self.get_points()
+
+    def area(self):
+        return 0
+
+class PolygonObject(ShapeObject):
+    # pylint: disable=redefined-builtin
+    def __init__(self, points=None,
+            label=None, id=None, attributes=None, group=None):
+        super().__init__(type=AnnotationType.polygon,
+            points=points, label=label,
+            id=id, attributes=attributes, group=group)
+    # pylint: enable=redefined-builtin
+
+    def get_polygon(self):
+        return self.get_points()
+
+class BboxObject(ShapeObject):
+    # pylint: disable=redefined-builtin
+    def __init__(self, x=0, y=0, w=0, h=0,
+            label=None, id=None, attributes=None, group=None):
+        super().__init__(type=AnnotationType.bbox,
+            points=[x, y, x + w, y + h], label=label,
+            id=id, attributes=attributes, group=group)
+    # pylint: enable=redefined-builtin
+
+    @property
+    def x(self):
+        return self.points[0]
+
+    @property
+    def y(self):
+        return self.points[1]
+
+    @property
+    def w(self):
+        return self.points[2] - self.points[0]
+
+    @property
+    def h(self):
+        return self.points[3] - self.points[1]
+
+    def area(self):
+        return self.w * self.h
+
+    def get_bbox(self):
+        return [self.x, self.y, self.w, self.h]
+
+    def get_polygon(self):
+        x, y, w, h = self.get_bbox()
+        return [
+            x, y,
+            x + w, y,
+            x + w, y + h,
+            x, y + h
+        ]
+
+    def iou(self, other):
+        return compute_iou(self.get_bbox(), other.get_bbox())
+
+class PointsCategories(Categories):
+    Category = namedtuple('Category', ['labels', 'adjacent'])
+
+    def __init__(self, items=None, attributes=None):
+        super().__init__(attributes=attributes)
+
+        if items is None:
+            items = {}
+        self.items = items
+
+    def add(self, label_id, labels=None, adjacent=None):
+        if labels is None:
+            labels = []
+        if adjacent is None:
+            adjacent = []
+        self.items[label_id] = self.Category(labels, set(adjacent))
+
+    def __eq__(self, other):
+        if not super().__eq__(other):
+            return False
+        return \
+            (self.items == other.items)
+
+class PointsObject(ShapeObject):
+    Visibility = Enum('Visibility', [
+        ('absent', 0),
+        ('hidden', 1),
+        ('visible', 2),
+    ])
+
+    # pylint: disable=redefined-builtin
+    def __init__(self, points=None, visibility=None, label=None,
+            id=None, attributes=None, group=None):
+        if points is not None:
+            assert len(points) % 2 == 0
+
+            if visibility is not None:
+                assert len(visibility) == len(points) // 2
+                for i, v in enumerate(visibility):
+                    if not isinstance(v, self.Visibility):
+                        visibility[i] = self.Visibility(v)
+            else:
+                visibility = []
+                for _ in range(len(points) // 2):
+                    visibility.append(self.Visibility.absent)
+
+        super().__init__(type=AnnotationType.points,
+            points=points, label=label,
+            id=id, attributes=attributes, group=group)
+
+        self.visibility = visibility
+    # pylint: enable=redefined-builtin
+
+    def area(self):
+        return 0
+
+    def __eq__(self, other):
+        if not super().__eq__(other):
+            return False
+        return \
+            (self.visibility == other.visibility)
+
+class CaptionObject(Annotation):
+    # pylint: disable=redefined-builtin
+    def __init__(self, caption=None,
+            id=None, attributes=None, group=None):
+        super().__init__(id=id, type=AnnotationType.caption,
+            attributes=attributes, group=group)
+
+        if caption is None:
+            caption = ''
+        self.caption = caption
+    # pylint: enable=redefined-builtin
+
+    def __eq__(self, other):
+        if not super().__eq__(other):
+            return False
+        return \
+            (self.caption == other.caption)
+
+class DatasetItem:
+    # pylint: disable=redefined-builtin
+    def __init__(self, id, annotations=None,
+            subset=None, path=None, image=None):
+        assert id is not None
+        if not isinstance(id, str):
+            id = str(id)
+        assert len(id) != 0
+        self._id = id
+
+        if subset is None:
+            subset = ''
+        assert isinstance(subset, str)
+        self._subset = subset
+
+        if path is None:
+            path = []
+        self._path = path
+
+        if annotations is None:
+            annotations = []
+        self._annotations = annotations
+
+        self._image = image
+    # pylint: enable=redefined-builtin
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def subset(self):
+        return self._subset
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def annotations(self):
+        return self._annotations
+
+    @property
+    def image(self):
+        if callable(self._image):
+            return self._image()
+        return self._image
+
+    @property
+    def has_image(self):
+        return self._image is not None
+
+    def __eq__(self, other):
+        if not isinstance(other, __class__):
+            return False
+        return \
+            (self.id == other.id) and \
+            (self.subset == other.subset) and \
+            (self.annotations == other.annotations) and \
+            (self.image == other.image)
+
+class IExtractor:
+    def __iter__(self):
+        raise NotImplementedError()
+
+    def __len__(self):
+        raise NotImplementedError()
+
+    def subsets(self):
+        raise NotImplementedError()
+
+    def get_subset(self, name):
+        raise NotImplementedError()
+
+    def categories(self):
+        raise NotImplementedError()
+
+    def select(self, pred):
+        raise NotImplementedError()
+
+    def get(self, item_id, subset=None, path=None):
+        raise NotImplementedError()
+
+class _DatasetFilter:
+    def __init__(self, iterable, predicate):
+        self.iterable = iterable
+        self.predicate = predicate
+
+    def __iter__(self):
+        return filter(self.predicate, self.iterable)
+
+class _ExtractorBase(IExtractor):
+    def __init__(self, length=None):
+        self._length = length
+        self._subsets = None
+
+    def _init_cache(self):
+        subsets = set()
+        length = -1
+        for length, item in enumerate(self):
+            subsets.add(item.subset)
+        length += 1
+
+        if self._length is None:
+            self._length = length
+        if self._subsets is None:
+            self._subsets = subsets
+
+    def __len__(self):
+        if self._length is None:
+            self._init_cache()
+        return self._length
+
+    def subsets(self):
+        if self._subsets is None:
+            self._init_cache()
+        return list(self._subsets)
+
+    def get_subset(self, name):
+        if name in self.subsets():
+            return self.select(lambda item: item.subset == name)
+        else:
+            raise Exception("Unknown subset '%s' requested" % name)
+
+class DatasetIteratorWrapper(_ExtractorBase):
+    def __init__(self, iterable, categories):
+        super().__init__(length=None)
+        self._iterable = iterable
+        self._categories = categories
+
+    def __iter__(self):
+        return iter(self._iterable)
+
+    def categories(self):
+        return self._categories
+
+    def select(self, pred):
+        return DatasetIteratorWrapper(
+            _DatasetFilter(self, pred), self.categories())
+
+class Extractor(_ExtractorBase):
+    def __init__(self, length=None):
+        super().__init__(length=None)
+
+    def categories(self):
+        return {}
+
+    def select(self, pred):
+        return DatasetIteratorWrapper(
+            _DatasetFilter(self, pred), self.categories())
+
+
+DEFAULT_SUBSET_NAME = 'default'

datumaro/datumaro/components/extractors/__init__.py

@@ -0,0 +1,50 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from datumaro.components.extractors.datumaro import DatumaroExtractor
+
+from datumaro.components.extractors.ms_coco import (
+    CocoImageInfoExtractor,
+    CocoCaptionsExtractor,
+    CocoInstancesExtractor,
+    CocoLabelsExtractor,
+    CocoPersonKeypointsExtractor,
+)
+
+from datumaro.components.extractors.voc import (
+    VocClassificationExtractor,
+    VocDetectionExtractor,
+    VocSegmentationExtractor,
+    VocLayoutExtractor,
+    VocActionExtractor,
+    VocComp_1_2_Extractor,
+    VocComp_3_4_Extractor,
+    VocComp_5_6_Extractor,
+    VocComp_7_8_Extractor,
+    VocComp_9_10_Extractor,
+)
+
+
+items = [
+    ('datumaro', DatumaroExtractor),
+
+    ('coco_images', CocoImageInfoExtractor),
+    ('coco_captions', CocoCaptionsExtractor),
+    ('coco_instances', CocoInstancesExtractor),
+    ('coco_person_kp', CocoPersonKeypointsExtractor),
+    ('coco_labels', CocoLabelsExtractor),
+
+    ('voc_cls', VocClassificationExtractor),
+    ('voc_det', VocDetectionExtractor),
+    ('voc_segm', VocSegmentationExtractor),
+    ('voc_layout', VocLayoutExtractor),
+    ('voc_action', VocActionExtractor),
+
+    ('voc_comp_1_2', VocComp_1_2_Extractor),
+    ('voc_comp_3_4', VocComp_3_4_Extractor),
+    ('voc_comp_5_6', VocComp_5_6_Extractor),
+    ('voc_comp_7_8', VocComp_7_8_Extractor),
+    ('voc_comp_9_10', VocComp_9_10_Extractor),
+]

datumaro/datumaro/components/extractors/datumaro.py

@@ -0,0 +1,214 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from collections import defaultdict
+import json
+import os.path as osp
+
+from datumaro.components.extractor import (Extractor, DatasetItem,
+    DEFAULT_SUBSET_NAME,
+    AnnotationType,
+    LabelObject, MaskObject, PointsObject, PolygonObject,
+    PolyLineObject, BboxObject, CaptionObject,
+    LabelCategories, MaskCategories, PointsCategories
+)
+from datumaro.components.formats.datumaro import DatumaroPath
+from datumaro.util import dir_items
+from datumaro.util.image import lazy_image
+from datumaro.util.mask_tools import lazy_mask
+
+
+class DatumaroExtractor(Extractor):
+    class Subset(Extractor):
+        def __init__(self, name, parent):
+            super().__init__()
+            self._parent = parent
+            self._name = name
+            self.items = []
+
+        def __iter__(self):
+            for item in self.items:
+                yield self._parent._get(item, self._name)
+
+        def __len__(self):
+            return len(self.items)
+
+        def categories(self):
+            return self._parent.categories()
+
+    def __init__(self, path):
+        super().__init__()
+
+        assert osp.isdir(path)
+        self._path = path
+
+        annotations = defaultdict(list)
+        found_subsets = self._find_subsets(path)
+        parsed_anns = None
+        subsets = {}
+        for subset_name, subset_path in found_subsets.items():
+            if subset_name == DEFAULT_SUBSET_NAME:
+                subset_name = None
+            subset = self.Subset(subset_name, self)
+            with open(subset_path, 'r') as f:
+                parsed_anns = json.load(f)
+
+            for index, _ in enumerate(parsed_anns['items']):
+                subset.items.append(index)
+
+            annotations[subset_name] = parsed_anns
+            subsets[subset_name] = subset
+        self._annotations = dict(annotations)
+        self._subsets = subsets
+
+        self._categories = {}
+        if parsed_anns is not None:
+            self._categories = self._load_categories(parsed_anns)
+
+    @staticmethod
+    def _load_categories(parsed):
+        categories = {}
+
+        parsed_label_cat = parsed['categories'].get(AnnotationType.label.name)
+        if parsed_label_cat:
+            label_categories = LabelCategories()
+            for item in parsed_label_cat['labels']:
+                label_categories.add(item['name'], parent=item['parent'])
+
+            categories[AnnotationType.label] = label_categories
+
+        parsed_mask_cat = parsed['categories'].get(AnnotationType.mask.name)
+        if parsed_mask_cat:
+            colormap = {}
+            for item in parsed_mask_cat['colormap']:
+                colormap[int(item['label_id'])] = \
+                    (item['r'], item['g'], item['b'])
+
+            mask_categories = MaskCategories(colormap=colormap)
+            categories[AnnotationType.mask] = mask_categories
+
+        parsed_points_cat = parsed['categories'].get(AnnotationType.points.name)
+        if parsed_points_cat:
+            point_categories = PointsCategories()
+            for item in parsed_points_cat['items']:
+                point_categories.add(int(item['label_id']),
+                    item['labels'], adjacent=item['adjacent'])
+
+            categories[AnnotationType.points] = point_categories
+
+        return categories
+
+    def _get(self, index, subset_name):
+        item = self._annotations[subset_name]['items'][index]
+
+        item_id = item.get('id')
+
+        image_path = osp.join(self._path, DatumaroPath.IMAGES_DIR,
+            item_id + DatumaroPath.IMAGE_EXT)
+        image = None
+        if osp.isfile(image_path):
+            image = lazy_image(image_path)
+
+        annotations = self._load_annotations(item)
+
+        return DatasetItem(id=item_id, subset=subset_name,
+            annotations=annotations, image=image)
+
+    def _load_annotations(self, item):
+        parsed = item['annotations']
+        loaded = []
+
+        for ann in parsed:
+            ann_id = ann.get('id')
+            ann_type = AnnotationType[ann['type']]
+            attributes = ann.get('attributes')
+            group = ann.get('group')
+
+            if ann_type == AnnotationType.label:
+                label_id = ann.get('label_id')
+                loaded.append(LabelObject(label=label_id,
+                    id=ann_id, attributes=attributes, group=group))
+
+            elif ann_type == AnnotationType.mask:
+                label_id = ann.get('label_id')
+                mask_id = str(ann.get('mask_id'))
+
+                mask_path = osp.join(self._path, DatumaroPath.ANNOTATIONS_DIR,
+                    DatumaroPath.MASKS_DIR, mask_id + DatumaroPath.MASK_EXT)
+                mask = None
+
+                if osp.isfile(mask_path):
+                    mask_cat = self._categories.get(AnnotationType.mask)
+                    if mask_cat is not None:
+                        mask = lazy_mask(mask_path, mask_cat.inverse_colormap)
+                    else:
+                        mask = lazy_image(mask_path)
+
+                loaded.append(MaskObject(label=label_id, image=mask,
+                    id=ann_id, attributes=attributes, group=group))
+
+            elif ann_type == AnnotationType.polyline:
+                label_id = ann.get('label_id')
+                points = ann.get('points')
+                loaded.append(PolyLineObject(points, label=label_id,
+                    id=ann_id, attributes=attributes, group=group))
+
+            elif ann_type == AnnotationType.polygon:
+                label_id = ann.get('label_id')
+                points = ann.get('points')
+                loaded.append(PolygonObject(points, label=label_id,
+                    id=ann_id, attributes=attributes, group=group))
+
+            elif ann_type == AnnotationType.bbox:
+                label_id = ann.get('label_id')
+                x, y, w, h = ann.get('bbox')
+                loaded.append(BboxObject(x, y, w, h, label=label_id,
+                    id=ann_id, attributes=attributes, group=group))
+
+            elif ann_type == AnnotationType.points:
+                label_id = ann.get('label_id')
+                points = ann.get('points')
+                loaded.append(PointsObject(points, label=label_id,
+                    id=ann_id, attributes=attributes, group=group))
+
+            elif ann_type == AnnotationType.caption:
+                caption = ann.get('caption')
+                loaded.append(CaptionObject(caption,
+                    id=ann_id, attributes=attributes, group=group))
+
+            else:
+                raise NotImplementedError()
+
+        return loaded
+
+    def categories(self):
+        return self._categories
+
+    def __iter__(self):
+        for subset_name, subset in self._subsets.items():
+            for index in subset.items:
+                yield self._get(index, subset_name)
+
+    def __len__(self):
+        length = 0
+        for subset in self._subsets.values():
+            length += len(subset)
+        return length
+
+    def subsets(self):
+        return list(self._subsets)
+
+    def get_subset(self, name):
+        return self._subsets[name]
+
+    @staticmethod
+    def _find_subsets(path):
+        anno_dir = osp.join(path, DatumaroPath.ANNOTATIONS_DIR)
+        if not osp.isdir(anno_dir):
+            raise Exception('Datumaro dataset not found at "%s"' % path)
+
+        return { name: osp.join(anno_dir, name + '.json')
+            for name in dir_items(anno_dir, '.json', truncate_ext=True)
+        }

datumaro/datumaro/components/extractors/ms_coco.py

@@ -0,0 +1,297 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import numpy as np
+import os.path as osp
+
+from pycocotools.coco import COCO
+import pycocotools.mask as mask_utils
+
+from datumaro.components.extractor import (Extractor, DatasetItem,
+    AnnotationType,
+    LabelObject, MaskObject, PointsObject, PolygonObject,
+    BboxObject, CaptionObject,
+    LabelCategories, PointsCategories
+)
+from datumaro.components.formats.ms_coco import CocoAnnotationType, CocoPath
+from datumaro.util.image import lazy_image
+
+
+class RleMask(MaskObject):
+    # pylint: disable=redefined-builtin
+    def __init__(self, rle=None, label=None,
+            id=None, attributes=None, group=None):
+        lazy_decode = lambda: mask_utils.decode(rle).astype(np.bool)
+        super().__init__(image=lazy_decode, label=label,
+            id=id, attributes=attributes, group=group)
+
+        self._rle = rle
+    # pylint: enable=redefined-builtin
+
+    def area(self):
+        return mask_utils.area(self._rle)
+
+    def bbox(self):
+        return mask_utils.toBbox(self._rle)
+
+    def __eq__(self, other):
+        if not isinstance(other, __class__):
+            return super().__eq__(other)
+        return self._rle == other._rle
+
+
+class CocoExtractor(Extractor):
+    class Subset(Extractor):
+        def __init__(self, name, parent):
+            super().__init__()
+            self._name = name
+            self._parent = parent
+            self.loaders = {}
+            self.items = set()
+
+        def __iter__(self):
+            for img_id in self.items:
+                yield self._parent._get(img_id, self._name)
+
+        def __len__(self):
+            return len(self.items)
+
+        def categories(self):
+            return self._parent.categories()
+
+    def __init__(self, path, task):
+        super().__init__()
+
+        rootpath = path.rsplit(CocoPath.ANNOTATIONS_DIR, maxsplit=1)[0]
+        self._path = rootpath
+        self._task = task
+        self._subsets = {}
+
+        subset_name = osp.splitext(osp.basename(path))[0] \
+            .rsplit('_', maxsplit=1)[1]
+        subset = CocoExtractor.Subset(subset_name, self)
+        loader = self._make_subset_loader(path)
+        subset.loaders[task] = loader
+        for img_id in loader.getImgIds():
+            subset.items.add(img_id)
+        self._subsets[subset_name] = subset
+
+        self._load_categories()
+
+    @staticmethod
+    def _make_subset_loader(path):
+        # COCO API has an 'unclosed file' warning
+        coco_api = COCO()
+        with open(path, 'r') as f:
+            import json
+            dataset = json.load(f)
+
+        coco_api.dataset = dataset
+        coco_api.createIndex()
+        return coco_api
+
+    def _load_categories(self):
+        loaders = {}
+
+        for subset in self._subsets.values():
+            loaders.update(subset.loaders)
+
+        self._categories = {}
+
+        label_loader = loaders.get(CocoAnnotationType.labels)
+        instances_loader = loaders.get(CocoAnnotationType.instances)
+        person_kp_loader = loaders.get(CocoAnnotationType.person_keypoints)
+
+        if label_loader is None and instances_loader is not None:
+            label_loader = instances_loader
+        if label_loader is None and person_kp_loader is not None:
+            label_loader = person_kp_loader
+        if label_loader is not None:
+            label_categories, label_map = \
+                self._load_label_categories(label_loader)
+            self._categories[AnnotationType.label] = label_categories
+            self._label_map = label_map
+
+        if person_kp_loader is not None:
+            person_kp_categories = \
+                self._load_person_kp_categories(person_kp_loader)
+            self._categories[AnnotationType.points] = person_kp_categories
+
+    # pylint: disable=no-self-use
+    def _load_label_categories(self, loader):
+        catIds = loader.getCatIds()
+        cats = loader.loadCats(catIds)
+
+        categories = LabelCategories()
+        label_map = {}
+        for idx, cat in enumerate(cats):
+            label_map[cat['id']] = idx
+            categories.add(name=cat['name'], parent=cat['supercategory'])
+
+        return categories, label_map
+    # pylint: enable=no-self-use
+
+    def _load_person_kp_categories(self, loader):
+        catIds = loader.getCatIds()
+        cats = loader.loadCats(catIds)
+
+        categories = PointsCategories()
+        for cat in cats:
+            label_id, _ = self._categories[AnnotationType.label].find(cat['name'])
+            categories.add(label_id=label_id,
+                labels=cat['keypoints'], adjacent=cat['skeleton'])
+
+        return categories
+
+    def categories(self):
+        return self._categories
+
+    def __iter__(self):
+        for subset_name, subset in self._subsets.items():
+            for img_id in subset.items:
+                yield self._get(img_id, subset_name)
+
+    def __len__(self):
+        length = 0
+        for subset in self._subsets.values():
+            length += len(subset)
+        return length
+
+    def subsets(self):
+        return list(self._subsets)
+
+    def get_subset(self, name):
+        return self._subsets[name]
+
+    def _get(self, img_id, subset):
+        file_name = None
+        image_info = None
+        image = None
+        annotations = []
+        for ann_type, loader in self._subsets[subset].loaders.items():
+            if image is None:
+                image_info = loader.loadImgs(img_id)[0]
+                file_name = image_info['file_name']
+                if file_name != '':
+                    file_path = osp.join(
+                        self._path, CocoPath.IMAGES_DIR, subset, file_name)
+                    if osp.exists(file_path):
+                        image = lazy_image(file_path)
+
+            annIds = loader.getAnnIds(imgIds=img_id)
+            anns = loader.loadAnns(annIds)
+
+            for ann in anns:
+                self._parse_annotation(ann, ann_type, annotations, image_info)
+        return DatasetItem(id=img_id, subset=subset,
+            image=image, annotations=annotations)
+
+    def _parse_label(self, ann):
+        cat_id = ann.get('category_id')
+        if cat_id in [0, None]:
+            return None
+        return self._label_map[cat_id]
+
+    def _parse_annotation(self, ann, ann_type, parsed_annotations,
+            image_info=None):
+        ann_id = ann.get('id')
+        attributes = {}
+        if 'score' in ann:
+            attributes['score'] = ann['score']
+
+        if ann_type is CocoAnnotationType.instances:
+            x, y, w, h = ann['bbox']
+            label_id = self._parse_label(ann)
+            group = None
+
+            is_crowd = bool(ann['iscrowd'])
+            attributes['is_crowd'] = is_crowd
+
+            segmentation = ann.get('segmentation')
+            if segmentation is not None:
+                group = ann_id
+
+                if isinstance(segmentation, list):
+                    # polygon -- a single object might consist of multiple parts
+                    for polygon_points in segmentation:
+                        parsed_annotations.append(PolygonObject(
+                            points=polygon_points, label=label_id,
+                            group=group
+                        ))
+
+                    # we merge all parts into one mask RLE code
+                    img_h = image_info['height']
+                    img_w = image_info['width']
+                    rles = mask_utils.frPyObjects(segmentation, img_h, img_w)
+                    rle = mask_utils.merge(rles)
+                elif isinstance(segmentation['counts'], list):
+                    # uncompressed RLE
+                    img_h, img_w = segmentation['size']
+                    rle = mask_utils.frPyObjects([segmentation], img_h, img_w)[0]
+                else:
+                    # compressed RLE
+                    rle = segmentation
+
+                parsed_annotations.append(RleMask(rle=rle, label=label_id,
+                    group=group
+                ))
+
+            parsed_annotations.append(
+                BboxObject(x, y, w, h, label=label_id,
+                    id=ann_id, attributes=attributes, group=group)
+            )
+        elif ann_type is CocoAnnotationType.labels:
+            label_id = self._parse_label(ann)
+            parsed_annotations.append(
+                LabelObject(label=label_id,
+                    id=ann_id, attributes=attributes)
+            )
+        elif ann_type is CocoAnnotationType.person_keypoints:
+            keypoints = ann['keypoints']
+            points = [p for i, p in enumerate(keypoints) if i % 3 != 2]
+            visibility = keypoints[2::3]
+            bbox = ann.get('bbox')
+            label_id = self._parse_label(ann)
+            group = None
+            if bbox is not None:
+                group = ann_id
+            parsed_annotations.append(
+                PointsObject(points, visibility, label=label_id,
+                    id=ann_id, attributes=attributes, group=group)
+            )
+            if bbox is not None:
+                parsed_annotations.append(
+                    BboxObject(*bbox, label=label_id, group=group)
+                )
+        elif ann_type is CocoAnnotationType.captions:
+            caption = ann['caption']
+            parsed_annotations.append(
+                CaptionObject(caption,
+                    id=ann_id, attributes=attributes)
+            )
+        else:
+            raise NotImplementedError()
+
+        return parsed_annotations
+
+class CocoImageInfoExtractor(CocoExtractor):
+    def __init__(self, path):
+        super().__init__(path, task=CocoAnnotationType.image_info)
+
+class CocoCaptionsExtractor(CocoExtractor):
+    def __init__(self, path):
+        super().__init__(path, task=CocoAnnotationType.captions)
+
+class CocoInstancesExtractor(CocoExtractor):
+    def __init__(self, path):
+        super().__init__(path, task=CocoAnnotationType.instances)
+
+class CocoPersonKeypointsExtractor(CocoExtractor):
+    def __init__(self, path):
+        super().__init__(path, task=CocoAnnotationType.person_keypoints)
+
+class CocoLabelsExtractor(CocoExtractor):
+    def __init__(self, path):
+        super().__init__(path, task=CocoAnnotationType.labels)

datumaro/datumaro/components/extractors/voc.py

@@ -0,0 +1,705 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from collections import defaultdict
+from itertools import chain
+import os
+import os.path as osp
+from xml.etree import ElementTree as ET
+
+from datumaro.components.extractor import (Extractor, DatasetItem,
+    AnnotationType, LabelObject, MaskObject, BboxObject,
+    LabelCategories, MaskCategories
+)
+from datumaro.components.formats.voc import VocLabel, VocAction, \
+    VocBodyPart, VocTask, VocPath, VocColormap, VocInstColormap
+from datumaro.util import dir_items
+from datumaro.util.image import lazy_image
+from datumaro.util.mask_tools import lazy_mask, invert_colormap
+
+
+_inverse_inst_colormap = invert_colormap(VocInstColormap)
+
+# pylint: disable=pointless-statement
+def _make_voc_categories():
+    categories = {}
+
+    label_categories = LabelCategories()
+    for label in chain(VocLabel, VocAction, VocBodyPart):
+        label_categories.add(label.name)
+    categories[AnnotationType.label] = label_categories
+
+    def label_id(class_index):
+        class_label = VocLabel(class_index).name
+        label_id, _ = label_categories.find(class_label)
+        return label_id
+    colormap = { label_id(idx): tuple(color) \
+        for idx, color in VocColormap.items() }
+    mask_categories = MaskCategories(colormap)
+    mask_categories.inverse_colormap # init inverse colormap
+    categories[AnnotationType.mask] = mask_categories
+
+    return categories
+# pylint: enable=pointless-statement
+
+class VocExtractor(Extractor):
+    class Subset(Extractor):
+        def __init__(self, name, parent):
+            super().__init__()
+            self._parent = parent
+            self._name = name
+            self.items = []
+
+        def __iter__(self):
+            for item in self.items:
+                yield self._parent._get(item, self._name)
+
+        def __len__(self):
+            return len(self.items)
+
+        def categories(self):
+            return self._parent.categories()
+
+    def _load_subsets(self, subsets_dir):
+        dir_files = dir_items(subsets_dir, '.txt', truncate_ext=True)
+        subset_names = [s for s in dir_files if '_' not in s]
+
+        subsets = {}
+        for subset_name in subset_names:
+            subset = __class__.Subset(subset_name, self)
+
+            with open(osp.join(subsets_dir, subset_name + '.txt'), 'r') as f:
+                subset.items = [line.split()[0] for line in f]
+
+            subsets[subset_name] = subset
+        return subsets
+
+    def _load_cls_annotations(self, subsets_dir, subset_names):
+        dir_files = dir_items(subsets_dir, '.txt', truncate_ext=True)
+
+        label_annotations = defaultdict(list)
+        label_anno_files = [s for s in dir_files \
+            if '_' in s and s[s.rfind('_') + 1:] in subset_names]
+        for ann_file in label_anno_files:
+            with open(osp.join(subsets_dir, ann_file + '.txt'), 'r') as f:
+                label = ann_file[:ann_file.rfind('_')]
+                label_id = VocLabel[label].value
+                for line in f:
+                    item, present = line.split()
+                    if present == '1':
+                        label_annotations[item].append(label_id)
+
+        self._annotations[VocTask.classification] = dict(label_annotations)
+
+    def _load_det_annotations(self):
+        det_anno_dir = osp.join(self._path, VocPath.ANNOTATIONS_DIR)
+        det_anno_items = dir_items(det_anno_dir, '.xml', truncate_ext=True)
+        det_annotations = dict()
+        for ann_item in det_anno_items:
+            with open(osp.join(det_anno_dir, ann_item + '.xml'), 'r') as f:
+                ann_file_data = f.read()
+                ann_file_root = ET.fromstring(ann_file_data)
+                item = ann_file_root.find('filename').text
+                item = osp.splitext(item)[0]
+                det_annotations[item] = ann_file_data
+
+        self._annotations[VocTask.detection] = det_annotations
+
+    def _load_categories(self):
+        self._categories = _make_voc_categories()
+
+    def __init__(self, path, task):
+        super().__init__()
+
+        self._path = path
+        self._subsets = {}
+        self._categories = {}
+        self._annotations = {}
+        self._task = task
+
+        self._load_categories()
+
+    def __len__(self):
+        length = 0
+        for subset in self._subsets.values():
+            length += len(subset)
+        return length
+
+    def subsets(self):
+        return list(self._subsets)
+
+    def get_subset(self, name):
+        return self._subsets[name]
+
+    def categories(self):
+        return self._categories
+
+    def __iter__(self):
+        for subset_name, subset in self._subsets.items():
+            for item in subset.items:
+                yield self._get(item, subset_name)
+
+    def _get(self, item, subset_name):
+        image = None
+        image_path = osp.join(self._path, VocPath.IMAGES_DIR,
+            item + VocPath.IMAGE_EXT)
+        if osp.isfile(image_path):
+            image = lazy_image(image_path)
+
+        annotations = self._get_annotations(item)
+
+        return DatasetItem(annotations=annotations,
+            id=item, subset=subset_name, image=image)
+
+    def _get_label_id(self, label):
+        label_id, _ = self._categories[AnnotationType.label].find(label)
+        assert label_id is not None
+        return label_id
+
+    def _get_annotations(self, item):
+        item_annotations = []
+
+        if self._task is VocTask.segmentation:
+            segm_path = osp.join(self._path, VocPath.SEGMENTATION_DIR,
+                item + VocPath.SEGM_EXT)
+            if osp.isfile(segm_path):
+                inverse_cls_colormap = \
+                    self._categories[AnnotationType.mask].inverse_colormap
+                item_annotations.append(MaskObject(
+                    image=lazy_mask(segm_path, inverse_cls_colormap),
+                    attributes={ 'class': True }
+                ))
+
+            inst_path = osp.join(self._path, VocPath.INSTANCES_DIR,
+                item + VocPath.SEGM_EXT)
+            if osp.isfile(inst_path):
+                item_annotations.append(MaskObject(
+                    image=lazy_mask(inst_path, _inverse_inst_colormap),
+                    attributes={ 'instances': True }
+                ))
+
+        cls_annotations = self._annotations.get(VocTask.classification)
+        if cls_annotations is not None and \
+           self._task is VocTask.classification:
+            item_labels = cls_annotations.get(item)
+            if item_labels is not None:
+                for label in item_labels:
+                    label_id = self._get_label_id(VocLabel(label).name)
+                    item_annotations.append(LabelObject(label_id))
+
+        det_annotations = self._annotations.get(VocTask.detection)
+        if det_annotations is not None:
+            det_annotations = det_annotations.get(item)
+        if det_annotations is not None:
+            root_elem = ET.fromstring(det_annotations)
+
+            for obj_id, object_elem in enumerate(root_elem.findall('object')):
+                attributes = {}
+                group = None
+
+                obj_label_id = None
+                label_elem = object_elem.find('name')
+                if label_elem is not None:
+                    obj_label_id = self._get_label_id(label_elem.text)
+
+                obj_bbox = self._parse_bbox(object_elem)
+
+                if obj_label_id is None or obj_bbox is None:
+                    continue
+
+                difficult_elem = object_elem.find('difficult')
+                if difficult_elem is not None:
+                    attributes['difficult'] = (difficult_elem.text == '1')
+
+                truncated_elem = object_elem.find('truncated')
+                if truncated_elem is not None:
+                    attributes['truncated'] = (truncated_elem.text == '1')
+
+                occluded_elem = object_elem.find('occluded')
+                if occluded_elem is not None:
+                    attributes['occluded'] = (occluded_elem.text == '1')
+
+                pose_elem = object_elem.find('pose')
+                if pose_elem is not None:
+                    attributes['pose'] = pose_elem.text
+
+                point_elem = object_elem.find('point')
+                if point_elem is not None:
+                    point_x = point_elem.find('x')
+                    point_y = point_elem.find('y')
+                    point = [float(point_x.text), float(point_y.text)]
+                    attributes['point'] = point
+
+                actions_elem = object_elem.find('actions')
+                if actions_elem is not None and \
+                   self._task is VocTask.action_classification:
+                    for action in VocAction:
+                        action_elem = actions_elem.find(action.name)
+                        if action_elem is None or action_elem.text != '1':
+                            continue
+
+                        act_label_id = self._get_label_id(action.name)
+                        assert group in [None, obj_id]
+                        group = obj_id
+                        item_annotations.append(LabelObject(act_label_id,
+                            group=obj_id))
+
+                if self._task is VocTask.person_layout:
+                    for part_elem in object_elem.findall('part'):
+                        part = part_elem.find('name').text
+                        part_label_id = self._get_label_id(part)
+                        bbox = self._parse_bbox(part_elem)
+                        group = obj_id
+                        item_annotations.append(BboxObject(
+                            *bbox, label=part_label_id,
+                            group=obj_id))
+
+                if self._task in [VocTask.action_classification, VocTask.person_layout]:
+                    if group is None:
+                        continue
+
+                item_annotations.append(BboxObject(*obj_bbox, label=obj_label_id,
+                    attributes=attributes, id=obj_id, group=group))
+
+        return item_annotations
+
+    @staticmethod
+    def _parse_bbox(object_elem):
+        try:
+            bbox_elem = object_elem.find('bndbox')
+            xmin = int(bbox_elem.find('xmin').text)
+            xmax = int(bbox_elem.find('xmax').text)
+            ymin = int(bbox_elem.find('ymin').text)
+            ymax = int(bbox_elem.find('ymax').text)
+            return [xmin, ymin, xmax - xmin, ymax - ymin]
+        except Exception:
+            return None
+
+class VocClassificationExtractor(VocExtractor):
+    _ANNO_DIR = 'Main'
+
+    def __init__(self, path):
+        super().__init__(path, task=VocTask.classification)
+
+        subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR)
+        subsets = self._load_subsets(subsets_dir)
+        self._subsets = subsets
+
+        self._load_cls_annotations(subsets_dir, subsets)
+
+class VocDetectionExtractor(VocExtractor):
+    _ANNO_DIR = 'Main'
+
+    def __init__(self, path):
+        super().__init__(path, task=VocTask.detection)
+
+        subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR)
+        subsets = self._load_subsets(subsets_dir)
+        self._subsets = subsets
+
+        self._load_det_annotations()
+
+class VocSegmentationExtractor(VocExtractor):
+    _ANNO_DIR = 'Segmentation'
+
+    def __init__(self, path):
+        super().__init__(path, task=VocTask.segmentation)
+
+        subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR)
+        subsets = self._load_subsets(subsets_dir)
+        self._subsets = subsets
+
+class VocLayoutExtractor(VocExtractor):
+    _ANNO_DIR = 'Layout'
+
+    def __init__(self, path):
+        super().__init__(path, task=VocTask.person_layout)
+
+        subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR)
+        subsets = self._load_subsets(subsets_dir)
+        self._subsets = subsets
+
+        self._load_det_annotations()
+
+class VocActionExtractor(VocExtractor):
+    _ANNO_DIR = 'Action'
+
+    def __init__(self, path):
+        super().__init__(path, task=VocTask.action_classification)
+
+        subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR)
+        subsets = self._load_subsets(subsets_dir)
+        self._subsets = subsets
+
+        self._load_det_annotations()
+
+
+class VocResultsExtractor(Extractor):
+    class Subset(Extractor):
+        def __init__(self, name, parent):
+            super().__init__()
+            self._parent = parent
+            self._name = name
+            self.items = []
+
+        def __iter__(self):
+            for item in self.items:
+                yield self._parent._get(item, self._name)
+
+        def __len__(self):
+            return len(self.items)
+
+        def categories(self):
+            return self._parent.categories()
+
+    _SUPPORTED_TASKS = {
+        VocTask.classification: {
+            'dir': 'Main',
+            'mark': 'cls',
+            'ext': '.txt',
+            'path' : ['%(comp)s_cls_%(subset)s_%(label)s.txt'],
+            'comp': ['comp1', 'comp2'],
+        },
+        VocTask.detection: {
+            'dir': 'Main',
+            'mark': 'det',
+            'ext': '.txt',
+            'path': ['%(comp)s_det_%(subset)s_%(label)s.txt'],
+            'comp': ['comp3', 'comp4'],
+        },
+        VocTask.segmentation: {
+            'dir': 'Segmentation',
+            'mark': ['cls', 'inst'],
+            'ext': '.png',
+            'path': ['%(comp)s_%(subset)s_cls', '%(item)s.png'],
+            'comp': ['comp5', 'comp6'],
+        },
+        VocTask.person_layout: {
+            'dir': 'Layout',
+            'mark': 'layout',
+            'ext': '.xml',
+            'path': ['%(comp)s_layout_%(subset)s.xml'],
+            'comp': ['comp7', 'comp8'],
+        },
+        VocTask.action_classification: {
+            'dir': 'Action',
+            'mark': 'action',
+            'ext': '.txt',
+            'path': ['%(comp)s_action_%(subset)s_%(label)s.txt'],
+            'comp': ['comp9', 'comp10'],
+        },
+    }
+
+    def _parse_txt_ann(self, path, subsets, annotations, task):
+        task_desc = self._SUPPORTED_TASKS[task]
+        task_dir = osp.join(path, task_desc['dir'])
+        ann_ext = task_desc['ext']
+        if not osp.isdir(task_dir):
+            return
+
+        ann_files = dir_items(task_dir, ann_ext, truncate_ext=True)
+
+        for ann_file in ann_files:
+            ann_parts = filter(None, ann_file.strip().split('_'))
+            if len(ann_parts) != 4:
+                continue
+            _, mark, subset_name, label = ann_parts
+            if mark != task_desc['mark']:
+                continue
+
+            label_id = VocLabel[label].value
+            anns = defaultdict(list)
+            with open(osp.join(task_dir, ann_file + ann_ext), 'r') as f:
+                for line in f:
+                    line_parts = line.split()
+                    item = line_parts[0]
+                    anns[item].append((label_id, *line_parts[1:]))
+
+            subset = VocResultsExtractor.Subset(subset_name, self)
+            subset.items = list(anns)
+
+            subsets[subset_name] = subset
+            annotations[subset_name] = dict(anns)
+
+    def _parse_classification(self, path, subsets, annotations):
+        self._parse_txt_ann(path, subsets, annotations,
+            VocTask.classification)
+
+    def _parse_detection(self, path, subsets, annotations):
+        self._parse_txt_ann(path, subsets, annotations,
+            VocTask.detection)
+
+    def _parse_action(self, path, subsets, annotations):
+        self._parse_txt_ann(path, subsets, annotations,
+            VocTask.action_classification)
+
+    def _load_categories(self):
+        self._categories = _make_voc_categories()
+
+    def _get_label_id(self, label):
+        label_id = self._categories[AnnotationType.label].find(label)
+        assert label_id is not None
+        return label_id
+
+    def __init__(self, path):
+        super().__init__()
+
+        self._path = path
+        self._subsets = {}
+        self._annotations = {}
+
+        self._load_categories()
+
+    def __len__(self):
+        length = 0
+        for subset in self._subsets.values():
+            length += len(subset)
+        return length
+
+    def subsets(self):
+        return list(self._subsets)
+
+    def get_subset(self, name):
+        return self._subsets[name]
+
+    def categories(self):
+        return self._categories
+
+    def __iter__(self):
+        for subset_name, subset in self._subsets.items():
+            for item in subset.items:
+                yield self._get(item, subset_name)
+
+    def _get(self, item, subset_name):
+        image = None
+        image_path = osp.join(self._path, VocPath.IMAGES_DIR,
+            item + VocPath.IMAGE_EXT)
+        if osp.isfile(image_path):
+            image = lazy_image(image_path)
+
+        annotations = self._get_annotations(item, subset_name)
+
+        return DatasetItem(annotations=annotations,
+            id=item, subset=subset_name, image=image)
+
+    def _get_annotations(self, item, subset_name):
+        raise NotImplementedError()
+
+class VocComp_1_2_Extractor(VocResultsExtractor):
+    def __init__(self, path):
+        super().__init__(path)
+
+        subsets = {}
+        annotations = defaultdict(dict)
+
+        self._parse_classification(path, subsets, annotations)
+
+        self._subsets = subsets
+        self._annotations = dict(annotations)
+
+    def _get_annotations(self, item, subset_name):
+        annotations = []
+
+        cls_ann = self._annotations[subset_name].get(item)
+        if cls_ann is not None:
+            for desc in cls_ann:
+                label_id, conf = desc
+                label_id = self._get_label_id(VocLabel(int(label_id)).name)
+                annotations.append(LabelObject(
+                    label_id,
+                    attributes={ 'score': float(conf) }
+                ))
+
+        return annotations
+
+class VocComp_3_4_Extractor(VocResultsExtractor):
+    def __init__(self, path):
+        super().__init__(path)
+
+        subsets = {}
+        annotations = defaultdict(dict)
+
+        self._parse_detection(path, subsets, annotations)
+
+        self._subsets = subsets
+        self._annotations = dict(annotations)
+
+    def _get_annotations(self, item, subset_name):
+        annotations = []
+
+        det_ann = self._annotations[subset_name].get(item)
+        if det_ann is not None:
+            for desc in det_ann:
+                label_id, conf, left, top, right, bottom = desc
+                label_id = self._get_label_id(VocLabel(int(label_id)).name)
+                annotations.append(BboxObject(
+                    x=float(left), y=float(top),
+                    w=float(right) - float(left), h=float(bottom) - float(top),
+                    label=label_id,
+                    attributes={ 'score': float(conf) }
+                ))
+
+        return annotations
+
+class VocComp_5_6_Extractor(VocResultsExtractor):
+    def __init__(self, path):
+        super().__init__(path)
+
+        subsets = {}
+        annotations = defaultdict(dict)
+
+        task_dir = osp.join(path, 'Segmentation')
+        if not osp.isdir(task_dir):
+            return
+
+        ann_files = os.listdir(task_dir)
+
+        for ann_dir in ann_files:
+            ann_parts = filter(None, ann_dir.strip().split('_'))
+            if len(ann_parts) != 4:
+                continue
+            _, subset_name, mark = ann_parts
+            if mark not in ['cls', 'inst']:
+                continue
+
+            item_dir = osp.join(task_dir, ann_dir)
+            items = dir_items(item_dir, '.png', truncate_ext=True)
+            items = { name: osp.join(item_dir, item + '.png') \
+                for name, item in items }
+
+            subset = VocResultsExtractor.Subset(subset_name, self)
+            subset.items = list(items)
+
+            subsets[subset_name] = subset
+            annotations[subset_name][mark] = items
+
+        self._subsets = subsets
+        self._annotations = dict(annotations)
+
+    def _get_annotations(self, item, subset_name):
+        annotations = []
+
+        segm_ann = self._annotations[subset_name]
+        cls_image_path = segm_ann.get(item)
+        if cls_image_path and osp.isfile(cls_image_path):
+            inverse_cls_colormap = \
+                self._categories[AnnotationType.mask].inverse_colormap
+            annotations.append(MaskObject(
+                image=lazy_mask(cls_image_path, inverse_cls_colormap),
+                attributes={ 'class': True }
+            ))
+
+        inst_ann = self._annotations[subset_name]
+        inst_image_path = inst_ann.get(item)
+        if inst_image_path and osp.isfile(inst_image_path):
+            annotations.append(MaskObject(
+                image=lazy_mask(inst_image_path, _inverse_inst_colormap),
+                attributes={ 'instances': True }
+            ))
+
+        return annotations
+
+class VocComp_7_8_Extractor(VocResultsExtractor):
+    def __init__(self, path):
+        super().__init__(path)
+
+        subsets = {}
+        annotations = defaultdict(dict)
+
+        task = VocTask.person_layout
+        task_desc = self._SUPPORTED_TASKS[task]
+        task_dir = osp.join(path, task_desc['dir'])
+        if not osp.isdir(task_dir):
+            return
+
+        ann_ext = task_desc['ext']
+        ann_files = dir_items(task_dir, ann_ext, truncate_ext=True)
+
+        for ann_file in ann_files:
+            ann_parts = filter(None, ann_file.strip().split('_'))
+            if len(ann_parts) != 4:
+                continue
+            _, mark, subset_name, _ = ann_parts
+            if mark != task_desc['mark']:
+                continue
+
+            layouts = {}
+            root = ET.parse(osp.join(task_dir, ann_file + ann_ext))
+            root_elem = root.getroot()
+            for layout_elem in root_elem.findall('layout'):
+                item = layout_elem.find('image').text
+                obj_id = int(layout_elem.find('object').text)
+                conf = float(layout_elem.find('confidence').text)
+                parts = []
+                for part_elem in layout_elem.findall('part'):
+                    label_id = VocBodyPart[part_elem.find('class').text].value
+                    bbox_elem = part_elem.find('bndbox')
+                    xmin = float(bbox_elem.find('xmin').text)
+                    xmax = float(bbox_elem.find('xmax').text)
+                    ymin = float(bbox_elem.find('ymin').text)
+                    ymax = float(bbox_elem.find('ymax').text)
+                    bbox = [xmin, ymin, xmax - xmin, ymax - ymin]
+                    parts.append((label_id, bbox))
+                layouts[item] = [obj_id, conf, parts]
+
+            subset = VocResultsExtractor.Subset(subset_name, self)
+            subset.items = list(layouts)
+
+            subsets[subset_name] = subset
+            annotations[subset_name] = layouts
+
+        self._subsets = subsets
+        self._annotations = dict(annotations)
+
+    def _get_annotations(self, item, subset_name):
+        annotations = []
+
+        layout_ann = self._annotations[subset_name].get(item)
+        if layout_ann is not None:
+            for desc in layout_ann:
+                obj_id, conf, parts = desc
+                attributes = {
+                    'score': conf,
+                    'object_id': obj_id,
+                }
+
+                for part in parts:
+                    part_id, bbox = part
+                    label_id = self._get_label_id(VocBodyPart(part_id).name)
+                    annotations.append(BboxObject(
+                        *bbox, label=label_id,
+                        attributes=attributes))
+
+        return annotations
+
+class VocComp_9_10_Extractor(VocResultsExtractor):
+    def __init__(self, path):
+        super().__init__(path)
+
+        subsets = {}
+        annotations = defaultdict(dict)
+
+        self._parse_action(path, subsets, annotations)
+
+        self._subsets = subsets
+        self._annotations = dict(annotations)
+
+    def _get_annotations(self, item, subset_name):
+        annotations = []
+
+        action_ann = self._annotations[subset_name].get(item)
+        if action_ann is not None:
+            for desc in action_ann:
+                action_id, obj_id, conf = desc
+                label_id = self._get_label_id(VocAction(int(action_id)).name)
+                annotations.append(LabelObject(
+                    label_id,
+                    attributes={
+                        'score': conf,
+                        'object_id': int(obj_id),
+                    }
+                ))
+
+        return annotations

datumaro/datumaro/components/formats/__init__.py

@@ -0,0 +1,5 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+

datumaro/datumaro/components/formats/datumaro.py

@@ -0,0 +1,12 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+class DatumaroPath:
+    IMAGES_DIR = 'images'
+    ANNOTATIONS_DIR = 'annotations'
+    MASKS_DIR = 'masks'
+
+    IMAGE_EXT = '.jpg'
+    MASK_EXT = '.png'

datumaro/datumaro/components/formats/ms_coco.py

@@ -0,0 +1,23 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from enum import Enum
+
+
+CocoAnnotationType = Enum('CocoAnnotationType', [
+    'instances',
+    'person_keypoints',
+    'captions',
+    'labels', # extension, does not exist in original COCO format
+    'image_info',
+    'panoptic',
+    'stuff',
+])
+
+class CocoPath:
+    IMAGES_DIR = 'images'
+    ANNOTATIONS_DIR = 'annotations'
+
+    IMAGE_EXT = '.jpg'

datumaro/datumaro/components/formats/voc.py

@@ -0,0 +1,103 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from enum import Enum
+import numpy as np
+
+
+VocTask = Enum('VocTask', [
+    'classification',
+    'detection',
+    'segmentation',
+    'action_classification',
+    'person_layout',
+])
+
+VocLabel = Enum('VocLabel', [
+    ('aeroplane', 0),
+    ('bicycle', 1),
+    ('bird', 2),
+    ('boat', 3),
+    ('bottle', 4),
+    ('bus', 5),
+    ('car', 6),
+    ('cat', 7),
+    ('chair', 8),
+    ('cow', 9),
+    ('diningtable', 10),
+    ('dog', 11),
+    ('horse', 12),
+    ('motorbike', 13),
+    ('person', 14),
+    ('pottedplant', 15),
+    ('sheep', 16),
+    ('sofa', 17),
+    ('train', 18),
+    ('tvmonitor', 19),
+])
+
+VocPose = Enum('VocPose', [
+    'Unspecified',
+    'Left',
+    'Right',
+    'Frontal',
+    'Rear',
+])
+
+VocBodyPart = Enum('VocBodyPart', [
+    'head',
+    'hand',
+    'foot',
+])
+
+VocAction = Enum('VocAction', [
+    'other',
+    'jumping',
+    'phoning',
+    'playinginstrument',
+    'reading',
+    'ridingbike',
+    'ridinghorse',
+    'running',
+    'takingphoto',
+    'usingcomputer',
+    'walking',
+])
+
+def generate_colormap(length=256):
+    def get_bit(number, index):
+        return (number >> index) & 1
+
+    colormap = np.zeros((length, 3), dtype=int)
+    indices = np.arange(length, dtype=int)
+
+    for j in range(7, -1, -1):
+        for c in range(3):
+            colormap[:, c] |= get_bit(indices, c) << j
+        indices >>= 3
+
+    return {
+        id: tuple(color) for id, color in enumerate(colormap)
+    }
+
+VocColormap = generate_colormap(len(VocLabel))
+VocInstColormap = generate_colormap(256)
+
+class VocPath:
+    IMAGES_DIR = 'JPEGImages'
+    ANNOTATIONS_DIR = 'Annotations'
+    SEGMENTATION_DIR = 'SegmentationClass'
+    INSTANCES_DIR = 'SegmentationObject'
+    SUBSETS_DIR = 'ImageSets'
+    IMAGE_EXT = '.jpg'
+    SEGM_EXT = '.png'
+
+    TASK_DIR = {
+        VocTask.classification: 'Main',
+        VocTask.detection: 'Main',
+        VocTask.segmentation: 'Segmentation',
+        VocTask.action_classification: 'Action',
+        VocTask.person_layout: 'Layout',
+    }

datumaro/datumaro/components/importers/__init__.py

@@ -0,0 +1,24 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from datumaro.components.importers.datumaro import DatumaroImporter
+
+from datumaro.components.importers.ms_coco import (
+    CocoImporter,
+)
+
+from datumaro.components.importers.voc import (
+    VocImporter,
+    VocResultsImporter,
+)
+
+items = [
+    ('datumaro', DatumaroImporter),
+
+    ('ms_coco', CocoImporter),
+
+    ('voc', VocImporter),
+    ('voc_results', VocResultsImporter),
+]

datumaro/datumaro/components/importers/datumaro.py

@@ -0,0 +1,25 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import os.path as osp
+
+
+class DatumaroImporter:
+    EXTRACTOR_NAME = 'datumaro'
+
+    def __call__(self, path):
+        from datumaro.components.project import Project # cyclic import
+        project = Project()
+
+        if not osp.exists(path):
+            raise Exception("Failed to find 'datumaro' dataset at '%s'" % path)
+
+        source_name = osp.splitext(osp.basename(path))[0]
+        project.add_source(source_name, {
+            'url': path,
+            'format': self.EXTRACTOR_NAME,
+        })
+
+        return project

datumaro/datumaro/components/importers/ms_coco.py

@@ -0,0 +1,69 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from collections import defaultdict
+import os
+import os.path as osp
+
+from datumaro.components.formats.ms_coco import CocoAnnotationType, CocoPath
+
+
+class CocoImporter:
+    _COCO_EXTRACTORS = {
+        CocoAnnotationType.instances: 'coco_instances',
+        CocoAnnotationType.person_keypoints: 'coco_person_kp',
+        CocoAnnotationType.captions: 'coco_captions',
+        CocoAnnotationType.labels: 'coco_labels',
+        CocoAnnotationType.image_info: 'coco_images',
+    }
+
+    def __init__(self, task_filter=None):
+        self._task_filter = task_filter
+
+    def __call__(self, path):
+        from datumaro.components.project import Project # cyclic import
+        project = Project()
+
+        subsets = self.find_subsets(path)
+
+        if len(subsets) == 0:
+            raise Exception("Failed to find 'coco' dataset at '%s'" % path)
+
+        for ann_files in subsets.values():
+            for ann_type, ann_file in ann_files.items():
+                source_name = osp.splitext(osp.basename(ann_file))[0]
+                project.add_source(source_name, {
+                    'url': ann_file,
+                    'format': self._COCO_EXTRACTORS[ann_type],
+                })
+
+        return project
+
+    @staticmethod
+    def find_subsets(dataset_dir):
+        ann_dir = os.path.join(dataset_dir, CocoPath.ANNOTATIONS_DIR)
+        if not osp.isdir(ann_dir):
+            raise NotADirectoryError(
+                'COCO annotations directory not found at "%s"' % ann_dir)
+
+        subsets = defaultdict(dict)
+        for ann_file in os.listdir(ann_dir):
+            subset_path = osp.join(ann_dir, ann_file)
+            if not subset_path.endswith('.json'):
+                continue
+
+            name_parts = osp.splitext(ann_file)[0].rsplit('_', maxsplit=1)
+            ann_type = name_parts[0]
+            try:
+                ann_type = CocoAnnotationType[ann_type]
+            except KeyError:
+                raise Exception(
+                    'Unknown subset type %s, only known are: %s' % \
+                    (ann_type,
+                     ', '.join([e.name for e in CocoAnnotationType])
+                    ))
+            subset_name = name_parts[1]
+            subsets[subset_name][ann_type] = subset_path
+        return dict(subsets)

datumaro/datumaro/components/importers/voc.py

@@ -0,0 +1,77 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import os
+import os.path as osp
+
+from datumaro.components.formats.voc import VocTask, VocPath
+from datumaro.util import find
+
+
+class VocImporter:
+    _TASKS = [
+        (VocTask.classification, 'voc_cls', 'Main'),
+        (VocTask.detection, 'voc_det', 'Main'),
+        (VocTask.segmentation, 'voc_segm', 'Segmentation'),
+        (VocTask.person_layout, 'voc_layout', 'Layout'),
+        (VocTask.action_classification, 'voc_action', 'Action'),
+    ]
+
+    def __call__(self, path):
+        from datumaro.components.project import Project # cyclic import
+        project = Project()
+
+        for task, extractor_type, task_dir in self._TASKS:
+            task_dir = osp.join(path, VocPath.SUBSETS_DIR, task_dir)
+            if not osp.isdir(task_dir):
+                continue
+
+            project.add_source(task.name, {
+                'url': path,
+                'format': extractor_type,
+            })
+
+        if len(project.config.sources) == 0:
+            raise Exception("Failed to find 'voc' dataset at '%s'" % path)
+
+        return project
+
+
+class VocResultsImporter:
+    _TASKS = [
+        ('comp1', 'voc_comp_1_2', 'Main'),
+        ('comp2', 'voc_comp_1_2', 'Main'),
+        ('comp3', 'voc_comp_3_4', 'Main'),
+        ('comp4', 'voc_comp_3_4', 'Main'),
+        ('comp5', 'voc_comp_5_6', 'Segmentation'),
+        ('comp6', 'voc_comp_5_6', 'Segmentation'),
+        ('comp7', 'voc_comp_7_8', 'Layout'),
+        ('comp8', 'voc_comp_7_8', 'Layout'),
+        ('comp9', 'voc_comp_9_10', 'Action'),
+        ('comp10', 'voc_comp_9_10', 'Action'),
+    ]
+
+    def __call__(self, path):
+        from datumaro.components.project import Project # cyclic import
+        project = Project()
+
+        for task_name, extractor_type, task_dir in self._TASKS:
+            task_dir = osp.join(path, task_dir)
+            if not osp.isdir(task_dir):
+                continue
+            dir_items = os.listdir(task_dir)
+            if not find(dir_items, lambda x: x == task_name):
+                continue
+
+            project.add_source(task_name, {
+                'url': task_dir,
+                'format': extractor_type,
+            })
+
+        if len(project.config.sources) == 0:
+            raise Exception("Failed to find 'voc_results' dataset at '%s'" % \
+                path)
+
+        return project

datumaro/datumaro/components/launcher.py

@@ -0,0 +1,95 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import numpy as np
+
+from datumaro.components.extractor import DatasetItem, Extractor
+
+
+# pylint: disable=no-self-use
+class Launcher:
+    def __init__(self):
+        pass
+
+    def launch(self, inputs):
+        raise NotImplementedError()
+
+    def preferred_input_size(self):
+        return None
+
+    def get_categories(self):
+        return None
+# pylint: enable=no-self-use
+
+class InferenceWrapper(Extractor):
+    class ItemWrapper(DatasetItem):
+        def __init__(self, item, annotations, path=None):
+            super().__init__(id=item.id)
+            self._annotations = annotations
+            self._item = item
+            self._path = path
+
+        @DatasetItem.id.getter
+        def id(self):
+            return self._item.id
+
+        @DatasetItem.subset.getter
+        def subset(self):
+            return self._item.subset
+
+        @DatasetItem.path.getter
+        def path(self):
+            return self._path
+
+        @DatasetItem.annotations.getter
+        def annotations(self):
+            return self._annotations
+
+        @DatasetItem.image.getter
+        def image(self):
+            return self._item.image
+
+    def __init__(self, extractor, launcher, batch_size=1):
+        super().__init__()
+        self._extractor = extractor
+        self._launcher = launcher
+        self._batch_size = batch_size
+
+    def __iter__(self):
+        stop = False
+        data_iter = iter(self._extractor)
+        while not stop:
+            batch_items = []
+            try:
+                for _ in range(self._batch_size):
+                    item = next(data_iter)
+                    batch_items.append(item)
+            except StopIteration:
+                stop = True
+                if len(batch_items) == 0:
+                    break
+
+            inputs = np.array([item.image for item in batch_items])
+            inference = self._launcher.launch(inputs)
+
+            for item, annotations in zip(batch_items, inference):
+                yield self.ItemWrapper(item, annotations)
+
+    def __len__(self):
+        return len(self._extractor)
+
+    def subsets(self):
+        return self._extractor.subsets()
+
+    def get_subset(self, name):
+        subset = self._extractor.get_subset(name)
+        return InferenceWrapper(subset,
+            self._launcher, self._batch_size)
+
+    def categories(self):
+        launcher_override = self._launcher.get_categories()
+        if launcher_override is not None:
+            return launcher_override
+        return self._extractor.categories()

datumaro/datumaro/components/launchers/__init__.py

@@ -0,0 +1,13 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+items = [
+]
+
+try:
+    from datumaro.components.launchers.openvino import OpenVinoLauncher
+    items.append(('openvino', OpenVinoLauncher))
+except ImportError:
+    pass

datumaro/datumaro/components/launchers/openvino.py

@@ -0,0 +1,189 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=exec-used
+
+import cv2
+import os
+import os.path as osp
+import numpy as np
+import subprocess
+import platform
+
+from openvino.inference_engine import IENetwork, IEPlugin
+
+from datumaro.components.launcher import Launcher
+
+
+class InterpreterScript:
+    def __init__(self, path):
+        with open(path, 'r') as f:
+            script = f.read()
+
+        context = {}
+        exec(script, context, context)
+
+        process_outputs = context['process_outputs']
+        assert callable(process_outputs)
+        self.__dict__['process_outputs'] = process_outputs
+
+        get_categories = context.get('get_categories')
+        assert callable(get_categories) or get_categories is None
+        self.__dict__['get_categories'] = get_categories
+
+    @staticmethod
+    def get_categories():
+        return None
+
+    @staticmethod
+    def process_outputs(inputs, outputs):
+        return []
+
+class OpenVinoLauncher(Launcher):
+    _DEFAULT_IE_PLUGINS_PATH = "/opt/intel/openvino_2019.1.144/deployment_tools/inference_engine/lib/intel64"
+    _IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH", _DEFAULT_IE_PLUGINS_PATH)
+
+    @staticmethod
+    def _check_instruction_set(instruction):
+        return instruction == str.strip(
+            subprocess.check_output(
+                'lscpu | grep -o "{}" | head -1'.format(instruction), shell=True
+            ).decode('utf-8')
+        )
+
+    @staticmethod
+    def make_plugin(device='cpu', plugins_path=_IE_PLUGINS_PATH):
+        if plugins_path is None or not osp.isdir(plugins_path):
+            raise Exception('Inference engine plugins directory "%s" not found' % \
+                (plugins_path))
+
+        plugin = IEPlugin(device='CPU', plugin_dirs=[plugins_path])
+        if (OpenVinoLauncher._check_instruction_set('avx2')):
+            plugin.add_cpu_extension(os.path.join(plugins_path,
+                'libcpu_extension_avx2.so'))
+        elif (OpenVinoLauncher._check_instruction_set('sse4')):
+            plugin.add_cpu_extension(os.path.join(plugins_path,
+                'libcpu_extension_sse4.so'))
+        elif platform.system() == 'Darwin':
+            plugin.add_cpu_extension(os.path.join(plugins_path,
+                'libcpu_extension.dylib'))
+        else:
+            raise Exception('Inference engine requires support of avx2 or sse4')
+
+        return plugin
+
+    @staticmethod
+    def make_network(model, weights):
+        return IENetwork.from_ir(model=model, weights=weights)
+
+    def __init__(self, description, weights, interpretation_script,
+            plugins_path=None, model_dir=None, **kwargs):
+        if model_dir is None:
+            model_dir = ''
+        if not osp.isfile(description):
+            description = osp.join(model_dir, description)
+        if not osp.isfile(description):
+            raise Exception('Failed to open model description file "%s"' % \
+                (description))
+
+        if not osp.isfile(weights):
+            weights = osp.join(model_dir, weights)
+        if not osp.isfile(weights):
+            raise Exception('Failed to open model weights file "%s"' % \
+                (weights))
+
+        if not osp.isfile(interpretation_script):
+            interpretation_script = \
+                osp.join(model_dir, interpretation_script)
+        if not osp.isfile(interpretation_script):
+            raise Exception('Failed to open model interpretation script file "%s"' % \
+                (interpretation_script))
+
+        self._interpreter_script = InterpreterScript(interpretation_script)
+
+        if plugins_path is None:
+            plugins_path = OpenVinoLauncher._IE_PLUGINS_PATH
+
+        plugin = OpenVinoLauncher.make_plugin(plugins_path=plugins_path)
+        network = OpenVinoLauncher.make_network(description, weights)
+        self._network = network
+        self._plugin = plugin
+        self._load_executable_net()
+
+    def _load_executable_net(self, batch_size=1):
+        network = self._network
+        plugin = self._plugin
+
+        supported_layers = plugin.get_supported_layers(network)
+        not_supported_layers = [l for l in network.layers.keys() if l not in supported_layers]
+        if len(not_supported_layers) != 0:
+            raise Exception('Following layers are not supported by the plugin'
+                ' for the specified device {}:\n {}'. format( \
+                plugin.device, ", ".join(not_supported_layers)))
+
+        iter_inputs = iter(network.inputs)
+        self._input_blob_name = next(iter_inputs)
+        self._output_blob_name = next(iter(network.outputs))
+
+        # NOTE: handling for the inclusion of `image_info` in OpenVino2019
+        self._require_image_info = 'image_info' in network.inputs
+        if self._input_blob_name == 'image_info':
+            self._input_blob_name = next(iter_inputs)
+
+        input_type = network.inputs[self._input_blob_name]
+        self._input_layout = input_type if isinstance(input_type, list) else input_type.shape
+
+        self._input_layout[0] = batch_size
+        network.reshape({self._input_blob_name: self._input_layout})
+        self._batch_size = batch_size
+
+        self._net = plugin.load(network=network, num_requests=1)
+
+    def infer(self, inputs):
+        assert len(inputs.shape) == 4, \
+            "Expected an input image in (N, H, W, C) format, got %s" % \
+                (inputs.shape)
+        assert inputs.shape[3] == 3, \
+            "Expected BGR input"
+
+        n, c, h, w = self._input_layout
+        if inputs.shape[1:3] != (h, w):
+            resized_inputs = np.empty((n, h, w, c), dtype=inputs.dtype)
+            for inp, resized_input in zip(inputs, resized_inputs):
+                cv2.resize(inp, (w, h), resized_input)
+            inputs = resized_inputs
+        inputs = inputs.transpose((0, 3, 1, 2)) # NHWC to NCHW
+        inputs = {self._input_blob_name: inputs}
+        if self._require_image_info:
+            info = np.zeros([1, 3])
+            info[0, 0] = h
+            info[0, 1] = w
+            info[0, 2] = 1.0 # scale
+            inputs['image_info'] = info
+
+        results = self._net.infer(inputs)
+        if len(results) == 1:
+            return results[self._output_blob_name]
+        else:
+            return results
+
+    def launch(self, inputs):
+        batch_size = len(inputs)
+        if self._batch_size < batch_size:
+            self._load_executable_net(batch_size)
+
+        outputs = self.infer(inputs)
+        results = self.process_outputs(inputs, outputs)
+        return results
+
+    def get_categories(self):
+        return self._interpreter_script.get_categories()
+
+    def process_outputs(self, inputs, outputs):
+        return self._interpreter_script.process_outputs(inputs, outputs)
+
+    def preferred_input_size(self):
+        _, _, h, w = self._input_layout
+        return (h, w)

datumaro/datumaro/components/project.py

@@ -0,0 +1,712 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from collections import OrderedDict, defaultdict
+import git
+import importlib
+from functools import reduce
+import logging as log
+import os
+import os.path as osp
+import sys
+
+from datumaro.components.config import Config, DEFAULT_FORMAT
+from datumaro.components.config_model import *
+from datumaro.components.extractor import *
+from datumaro.components.launcher import *
+from datumaro.components.dataset_filter import XPathDatasetFilter
+
+
+def import_foreign_module(name, path):
+    module = None
+    default_path = sys.path.copy()
+    try:
+        sys.path = [ osp.abspath(path), ] + default_path
+        sys.modules.pop(name, None) # remove from cache
+        module = importlib.import_module(name)
+        sys.modules.pop(name) # remove from cache
+    except ImportError as e:
+        log.warn("Failed to import module '%s': %s" % (name, e))
+    finally:
+        sys.path = default_path
+    return module
+
+
+class Registry:
+    def __init__(self, config=None, item_type=None):
+        self.item_type = item_type
+
+        self.items = {}
+
+        if config is not None:
+            self.load(config)
+
+    def load(self, config):
+        pass
+
+    def register(self, name, value):
+        if self.item_type:
+            value = self.item_type(value)
+        self.items[name] = value
+        return value
+
+    def unregister(self, name):
+        return self.items.pop(name, None)
+
+    def get(self, key):
+        return self.items[key] # returns a class / ctor
+
+
+class ModelRegistry(Registry):
+    def __init__(self, config=None):
+        super().__init__(config, item_type=Model)
+
+    def load(self, config):
+        # TODO: list default dir, insert values
+        if 'models' in config:
+            for name, model in config.models.items():
+                self.register(name, model)
+
+
+class SourceRegistry(Registry):
+    def __init__(self, config=None):
+        super().__init__(config, item_type=Source)
+
+    def load(self, config):
+        # TODO: list default dir, insert values
+        if 'sources' in config:
+            for name, source in config.sources.items():
+                self.register(name, source)
+
+
+class ModuleRegistry(Registry):
+    def __init__(self, config=None, builtin=None, local=None):
+        super().__init__(config)
+
+        if builtin is not None:
+            for k, v in builtin:
+                self.register(k, v)
+        if local is not None:
+            for k, v in local:
+                self.register(k, v)
+
+
+class GitWrapper:
+    def __init__(self, config=None):
+        self.repo = None
+
+        if config is not None:
+            self.init(config.project_dir)
+
+    @staticmethod
+    def _git_dir(base_path):
+        return osp.join(base_path, '.git')
+
+    def init(self, path):
+        spawn = not osp.isdir(GitWrapper._git_dir(path))
+        self.repo = git.Repo.init(path=path)
+        if spawn:
+            author = git.Actor("Nobody", "nobody@example.com")
+            self.repo.index.commit('Initial commit', author=author)
+        return self.repo
+
+    def get_repo(self):
+        return self.repo
+
+    def is_initialized(self):
+        return self.repo is not None
+
+    def create_submodule(self, name, dst_dir, **kwargs):
+        self.repo.create_submodule(name, dst_dir, **kwargs)
+
+    def has_submodule(self, name):
+        return name in [submodule.name for submodule in self.repo.submodules]
+
+    def remove_submodule(self, name, **kwargs):
+        return self.repo.submodule(name).remove(**kwargs)
+
+def load_project_as_dataset(url):
+    # symbol forward declaration
+    raise NotImplementedError()
+
+class Environment:
+    PROJECT_EXTRACTOR_NAME = 'project'
+
+    def __init__(self, config=None):
+        config = Config(config,
+            fallback=PROJECT_DEFAULT_CONFIG, schema=PROJECT_SCHEMA)
+
+        env_dir = osp.join(config.project_dir, config.env_dir)
+        env_config_path = osp.join(env_dir, config.env_filename)
+        env_config = Config(fallback=ENV_DEFAULT_CONFIG, schema=ENV_SCHEMA)
+        if osp.isfile(env_config_path):
+            env_config.update(Config.parse(env_config_path))
+
+        self.config = env_config
+
+        self.models = ModelRegistry(env_config)
+        self.sources = SourceRegistry(config)
+
+        import datumaro.components.importers as builtin_importers
+        builtin_importers = builtin_importers.items
+        custom_importers = self._get_custom_module_items(
+            env_dir, env_config.importers_dir)
+        self.importers = ModuleRegistry(config,
+            builtin=builtin_importers, local=custom_importers)
+
+        import datumaro.components.extractors as builtin_extractors
+        builtin_extractors = builtin_extractors.items
+        custom_extractors = self._get_custom_module_items(
+            env_dir, env_config.extractors_dir)
+        self.extractors = ModuleRegistry(config,
+            builtin=builtin_extractors, local=custom_extractors)
+        self.extractors.register(self.PROJECT_EXTRACTOR_NAME,
+            load_project_as_dataset)
+
+        import datumaro.components.launchers as builtin_launchers
+        builtin_launchers = builtin_launchers.items
+        custom_launchers = self._get_custom_module_items(
+            env_dir, env_config.launchers_dir)
+        self.launchers = ModuleRegistry(config,
+            builtin=builtin_launchers, local=custom_launchers)
+
+        import datumaro.components.converters as builtin_converters
+        builtin_converters = builtin_converters.items
+        custom_converters = self._get_custom_module_items(
+            env_dir, env_config.converters_dir)
+        if custom_converters is not None:
+            custom_converters = custom_converters.items
+        self.converters = ModuleRegistry(config,
+            builtin=builtin_converters, local=custom_converters)
+
+        self.statistics = ModuleRegistry(config)
+        self.visualizers = ModuleRegistry(config)
+        self.git = GitWrapper(config)
+
+    def _get_custom_module_items(self, module_dir, module_name):
+        items = None
+
+        module = None
+        if osp.exists(osp.join(module_dir, module_name)):
+            module = import_foreign_module(module_name, module_dir)
+        if module is not None:
+            if hasattr(module, 'items'):
+                items = module.items
+            else:
+                items = self._find_custom_module_items(
+                    osp.join(module_dir, module_name))
+
+        return items
+
+    @staticmethod
+    def _find_custom_module_items(module_dir):
+        files = [p for p in os.listdir(module_dir)
+            if p.endswith('.py') and p != '__init__.py']
+
+        all_items = []
+        for f in files:
+            name = osp.splitext(f)[0]
+            module = import_foreign_module(name, module_dir)
+
+            items = []
+            if hasattr(module, 'items'):
+                items = module.items
+            else:
+                if hasattr(module, name):
+                    items = [ (name, getattr(module, name)) ]
+                else:
+                    log.warn("Failed to import custom module '%s'."
+                        " Custom module is expected to provide 'items' "
+                        "list or have an item matching its file name."
+                        " Skipping this module." % \
+                        (module_dir + '.' + name))
+
+            all_items.extend(items)
+
+        return all_items
+
+    def save(self, path):
+        self.config.dump(path)
+
+    def make_extractor(self, name, *args, **kwargs):
+        return self.extractors.get(name)(*args, **kwargs)
+
+    def make_importer(self, name, *args, **kwargs):
+        return self.importers.get(name)(*args, **kwargs)
+
+    def make_launcher(self, name, *args, **kwargs):
+        return self.launchers.get(name)(*args, **kwargs)
+
+    def make_converter(self, name, *args, **kwargs):
+        return self.converters.get(name)(*args, **kwargs)
+
+    def register_model(self, name, model):
+        self.config.models[name] = model
+        self.models.register(name, model)
+
+    def unregister_model(self, name):
+        self.config.models.remove(name)
+        self.models.unregister(name)
+
+
+class Subset(Extractor):
+    def __init__(self, parent):
+        self._parent = parent
+        self.items = OrderedDict()
+
+    def __iter__(self):
+        for item in self.items.values():
+            yield item
+
+    def __len__(self):
+        return len(self.items)
+
+    def categories(self):
+        return self._parent.categories()
+
+class DatasetItemWrapper(DatasetItem):
+    def __init__(self, item, path, annotations, image=None):
+        self._item = item
+        self._path = path
+        self._annotations = annotations
+        self._image = image
+
+    @DatasetItem.id.getter
+    def id(self):
+        return self._item.id
+
+    @DatasetItem.subset.getter
+    def subset(self):
+        return self._item.subset
+
+    @DatasetItem.path.getter
+    def path(self):
+        return self._path
+
+    @DatasetItem.annotations.getter
+    def annotations(self):
+        return self._annotations
+
+    @DatasetItem.has_image.getter
+    def has_image(self):
+        if self._image is not None:
+            return True
+        return self._item.has_image
+
+    @DatasetItem.image.getter
+    def image(self):
+        if self._image is not None:
+            if callable(self._image):
+                return self._image()
+            return self._image
+        return self._item.image
+
+class ProjectDataset(Extractor):
+    def __init__(self, project):
+        super().__init__()
+
+        self._project = project
+        config = self.config
+        env = self.env
+
+        dataset_filter = None
+        if config.filter:
+            dataset_filter = XPathDatasetFilter(config.filter)
+        self._filter = dataset_filter
+
+        sources = {}
+        for s_name, source in config.sources.items():
+            s_format = source.format
+            if not s_format:
+                s_format = env.PROJECT_EXTRACTOR_NAME
+            options = {}
+            options.update(source.options)
+
+            url = source.url
+            if not source.url:
+                url = osp.join(config.project_dir, config.sources_dir, s_name)
+            sources[s_name] = env.make_extractor(s_format,
+                url, **options)
+        self._sources = sources
+
+        own_source = None
+        own_source_dir = osp.join(config.project_dir, config.dataset_dir)
+        if osp.isdir(own_source_dir):
+            own_source = env.make_extractor(DEFAULT_FORMAT, own_source_dir)
+
+        # merge categories
+        # TODO: implement properly with merging and annotations remapping
+        categories = {}
+        for source in self._sources.values():
+            categories.update(source.categories())
+        for source in self._sources.values():
+            for cat_type, source_cat in source.categories().items():
+                assert categories[cat_type] == source_cat
+        if own_source is not None and len(own_source) != 0:
+            categories.update(own_source.categories())
+        self._categories = categories
+
+        # merge items
+        subsets = defaultdict(lambda: Subset(self))
+        for source_name, source in self._sources.items():
+            for item in source:
+                if dataset_filter and not dataset_filter(item):
+                    continue
+
+                existing_item = subsets[item.subset].items.get(item.id)
+                if existing_item is not None:
+                    image = None
+                    if existing_item.has_image:
+                        # TODO: think of image comparison
+                        image = lambda: existing_item.image
+
+                    path = existing_item.path
+                    if item.path != path:
+                        path = None
+                    item = DatasetItemWrapper(item=item, path=path,
+                        image=image, annotations=self._merge_anno(
+                            existing_item.annotations, item.annotations))
+                else:
+                    s_config = config.sources[source_name]
+                    if s_config and \
+                            s_config.format != self.env.PROJECT_EXTRACTOR_NAME:
+                        # NOTE: consider imported sources as our own dataset
+                        path = None
+                    else:
+                        path = item.path
+                        if path is None:
+                            path = []
+                        path = [source_name] + path
+                    item = DatasetItemWrapper(item=item, path=path,
+                        annotations=item.annotations)
+
+                subsets[item.subset].items[item.id] = item
+
+        # override with our items, fallback to existing images
+        if own_source is not None:
+            for item in own_source:
+                if dataset_filter and not dataset_filter(item):
+                    continue
+
+                if not item.has_image:
+                    existing_item = subsets[item.subset].items.get(item.id)
+                    if existing_item is not None:
+                        image = None
+                        if existing_item.has_image:
+                            # TODO: think of image comparison
+                            image = lambda: existing_item.image
+                        item = DatasetItemWrapper(item=item, path=None,
+                            annotations=item.annotations, image=image)
+
+                subsets[item.subset].items[item.id] = item
+
+        # TODO: implement subset remapping when needed
+        subsets_filter = config.subsets
+        if len(subsets_filter) != 0:
+            subsets = { k: v for k, v in subsets.items() if k in subsets_filter}
+        self._subsets = dict(subsets)
+
+        self._length = None
+
+    @staticmethod
+    def _merge_anno(a, b):
+        from itertools import chain
+        merged = []
+        for item in chain(a, b):
+            found = False
+            for elem in merged:
+                if elem == item:
+                    found = True
+                    break
+            if not found:
+                merged.append(item)
+
+        return merged
+
+    def iterate_own(self):
+        return self.select(lambda item: not item.path)
+
+    def __iter__(self):
+        for subset in self._subsets.values():
+            for item in subset:
+                if self._filter and not self._filter(item):
+                    continue
+                yield item
+
+    def __len__(self):
+        if self._length is None:
+            self._length = reduce(lambda s, x: s + len(x),
+                self._subsets.values(), 0)
+        return self._length
+
+    def get_subset(self, name):
+        return self._subsets[name]
+
+    def subsets(self):
+        return list(self._subsets)
+
+    def categories(self):
+        return self._categories
+
+    def define_categories(self, categories):
+        assert not self._categories
+        self._categories = categories
+
+    def get(self, item_id, subset=None, path=None):
+        if path:
+            source = path[0]
+            rest_path = path[1:]
+            return self._sources[source].get(
+                item_id=item_id, subset=subset, path=rest_path)
+        return self._subsets[subset].items[item_id]
+
+    def put(self, item, item_id=None, subset=None, path=None):
+        if path is None:
+            path = item.path
+        if path:
+            source = path[0]
+            rest_path = path[1:]
+            # TODO: reverse remapping
+            self._sources[source].put(item,
+                item_id=item_id, subset=subset, path=rest_path)
+
+        if item_id is None:
+            item_id = item.id
+        if subset is None:
+            subset = item.subset
+
+        item = DatasetItemWrapper(item=item, path=path,
+            annotations=item.annotations)
+        if item.subset not in self._subsets:
+            self._subsets[item.subset] = Subset(self)
+        self._subsets[subset].items[item_id] = item
+        self._length = None
+
+        return item
+
+    def build(self, tasks=None):
+        pass
+
+    def docs(self):
+        pass
+
+    def transform(self, model_name, save_dir=None):
+        project = Project(self.config)
+        project.config.remove('sources')
+
+        if save_dir is None:
+            save_dir = self.config.project_dir
+        project.config.project_dir = save_dir
+
+        dataset = project.make_dataset()
+        launcher = self._project.make_executable_model(model_name)
+        inference = InferenceWrapper(self, launcher)
+        dataset.update(inference)
+
+        dataset.save(merge=True)
+
+    def export(self, save_dir, output_format,
+            filter_expr=None, **converter_kwargs):
+        save_dir = osp.abspath(save_dir)
+        os.makedirs(save_dir, exist_ok=True)
+
+        dataset = self
+        if filter_expr:
+            dataset_filter = XPathDatasetFilter(filter_expr)
+            dataset = dataset.select(dataset_filter)
+
+        converter = self.env.make_converter(output_format, **converter_kwargs)
+        converter(dataset, save_dir)
+
+    def extract(self, save_dir, filter_expr=None):
+        project = Project(self.config)
+        if filter_expr:
+            XPathDatasetFilter(filter_expr)
+            project.set_filter(filter_expr)
+        project.save(save_dir)
+
+    def update(self, items):
+        for item in items:
+            if self._filter and not self._filter(item):
+                continue
+            self.put(item)
+        return self
+
+    def save(self, save_dir=None, merge=False, recursive=True,
+            save_images=False, apply_colormap=True):
+        if save_dir is None:
+            assert self.config.project_dir
+            save_dir = self.config.project_dir
+            project = self._project
+        else:
+            merge = True
+
+        if merge:
+            project = Project(Config(self.config))
+            project.config.remove('sources')
+
+        save_dir = osp.abspath(save_dir)
+        os.makedirs(save_dir, exist_ok=True)
+
+        dataset_save_dir = osp.join(save_dir, project.config.dataset_dir)
+        os.makedirs(dataset_save_dir, exist_ok=True)
+
+        converter_kwargs = {
+            'save_images': save_images,
+            'apply_colormap': apply_colormap,
+        }
+
+        if merge:
+            # merge and save the resulting dataset
+            converter = self.env.make_converter(
+                DEFAULT_FORMAT, **converter_kwargs)
+            converter(self, dataset_save_dir)
+        else:
+            if recursive:
+                # children items should already be updated
+                # so we just save them recursively
+                for source in self._sources.values():
+                    if isinstance(source, ProjectDataset):
+                        source.save(**converter_kwargs)
+
+            converter = self.env.make_converter(
+                DEFAULT_FORMAT, **converter_kwargs)
+            converter(self.iterate_own(), dataset_save_dir)
+
+        project.save(save_dir)
+
+    @property
+    def env(self):
+        return self._project.env
+
+    @property
+    def config(self):
+        return self._project.config
+
+    @property
+    def sources(self):
+        return self._sources
+
+class Project:
+    @staticmethod
+    def load(path):
+        path = osp.abspath(path)
+        if osp.isdir(path):
+            path = osp.join(path, PROJECT_DEFAULT_CONFIG.project_filename)
+        config = Config.parse(path)
+        config.project_dir = osp.dirname(path)
+        config.project_filename = osp.basename(path)
+        return Project(config)
+
+    def save(self, save_dir=None):
+        config = self.config
+        if save_dir is None:
+            assert config.project_dir
+            save_dir = osp.abspath(config.project_dir)
+        config_path = osp.join(save_dir, config.project_filename)
+
+        env_dir = osp.join(save_dir, config.env_dir)
+        os.makedirs(env_dir, exist_ok=True)
+        self.env.save(osp.join(env_dir, config.env_filename))
+
+        config.dump(config_path)
+
+    @staticmethod
+    def generate(save_dir, config=None):
+        project = Project(config)
+        project.save(save_dir)
+        project.config.project_dir = save_dir
+        return project
+
+    @staticmethod
+    def import_from(path, dataset_format, env=None, **kwargs):
+        if env is None:
+            env = Environment()
+        importer = env.make_importer(dataset_format)
+        return importer(path, **kwargs)
+
+    def __init__(self, config=None):
+        self.config = Config(config,
+            fallback=PROJECT_DEFAULT_CONFIG, schema=PROJECT_SCHEMA)
+        self.env = Environment(self.config)
+
+    def make_dataset(self):
+        return ProjectDataset(self)
+
+    def add_source(self, name, value=Source()):
+        if isinstance(value, (dict, Config)):
+            value = Source(value)
+        self.config.sources[name] = value
+        self.env.sources.register(name, value)
+
+    def remove_source(self, name):
+        self.config.sources.remove(name)
+        self.env.sources.unregister(name)
+
+    def get_source(self, name):
+        return self.config.sources[name]
+
+    def get_subsets(self):
+        return self.config.subsets
+
+    def set_subsets(self, value):
+        if not value:
+            self.config.remove('subsets')
+        else:
+            self.config.subsets = value
+
+    def add_model(self, name, value=Model()):
+        if isinstance(value, (dict, Config)):
+            value = Model(value)
+        self.env.register_model(name, value)
+
+    def get_model(self, name):
+        return self.env.models.get(name)
+
+    def remove_model(self, name):
+        self.env.unregister_model(name)
+
+    def make_executable_model(self, name):
+        model = self.get_model(name)
+        model.model_dir = self.local_model_dir(name)
+        return self.env.make_launcher(model.launcher,
+            **model.options, model_dir=model.model_dir)
+
+    def make_source_project(self, name):
+        source = self.get_source(name)
+
+        config = Config(self.config)
+        config.remove('sources')
+        config.remove('subsets')
+        config.remove('filter')
+        project = Project(config)
+        project.add_source(name, source)
+        return project
+
+    def get_filter(self):
+        if 'filter' in self.config:
+            return self.config.filter
+        return ''
+
+    def set_filter(self, value=None):
+        if not value:
+            self.config.remove('filter')
+        else:
+            # check filter
+            XPathDatasetFilter(value)
+            self.config.filter = value
+
+    def local_model_dir(self, model_name):
+        return osp.join(
+            self.config.env_dir, self.env.config.models_dir, model_name)
+
+    def local_source_dir(self, source_name):
+        return osp.join(self.config.sources_dir, source_name)
+
+# pylint: disable=function-redefined
+def load_project_as_dataset(url):
+    # implement the function declared above
+    return Project.load(url).make_dataset()
+# pylint: enable=function-redefined

datumaro/datumaro/util/__init__.py

@@ -0,0 +1,20 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import os
+
+
+def find(iterable, pred=lambda x: True, default=None):
+    return next((x for x in iterable if pred(x)), default)
+
+def dir_items(path, ext, truncate_ext=False):
+    items = []
+    for f in os.listdir(path):
+        ext_pos = f.rfind(ext)
+        if ext_pos != -1:
+            if truncate_ext:
+                f = f[:ext_pos]
+            items.append(f)
+    return items

datumaro/datumaro/util/command_targets.py

@@ -0,0 +1,110 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import cv2
+from enum import Enum
+
+from datumaro.components.project import Project
+
+
+TargetKinds = Enum('TargetKinds',
+    ['project', 'source', 'external_dataset', 'inference', 'image'])
+
+def is_project_name(value, project):
+    return value == project.config.project_name
+
+def is_project_path(value):
+    if value:
+        try:
+            Project.load(value)
+            return True
+        except Exception:
+            pass
+    return False
+
+def is_project(value, project=None):
+    if is_project_path(value):
+        return True
+    elif project is not None:
+        return is_project_name(value, project)
+
+    return False
+
+def is_source(value, project=None):
+    if project is not None:
+        try:
+            project.get_source(value)
+            return True
+        except KeyError:
+            pass
+
+    return False
+
+def is_external_source(value):
+    return False
+
+def is_inference_path(value):
+    return False
+
+def is_image_path(value):
+    return cv2.imread(value) is not None
+
+
+class Target:
+    def __init__(self, kind, test, is_default=False, name=None):
+        self.kind = kind
+        self.test = test
+        self.is_default = is_default
+        self.name = name
+
+    def _get_fields(self):
+        return [self.kind, self.test, self.is_default, self.name]
+
+    def __str__(self):
+        return self.name or str(self.kind)
+
+    def __len__(self):
+        return len(self._get_fields())
+
+    def __iter__(self):
+        return iter(self._get_fields())
+
+def ProjectTarget(kind=TargetKinds.project, test=None,
+        is_default=False, name='project name or path',
+        project=None):
+    if test is None:
+        test = lambda v: is_project(v, project=project)
+    return Target(kind, test, is_default, name)
+
+def SourceTarget(kind=TargetKinds.source, test=None,
+        is_default=False, name='source name',
+        project=None):
+    if test is None:
+        test = lambda v: is_source(v, project=project)
+    return Target(kind, test, is_default, name)
+
+def ExternalDatasetTarget(kind=TargetKinds.external_dataset,
+        test=is_external_source,
+        is_default=False, name='external dataset path'):
+    return Target(kind, test, is_default, name)
+
+def InferenceTarget(kind=TargetKinds.inference, test=is_inference_path,
+        is_default=False, name='inference path'):
+    return Target(kind, test, is_default, name)
+
+def ImageTarget(kind=TargetKinds.image, test=is_image_path,
+            is_default=False, name='image path'):
+    return Target(kind, test, is_default, name)
+
+
+def target_selector(*targets):
+    def selector(value):
+        for (kind, test, is_default, _) in targets:
+            if (is_default and (value == '' or value is None)) or test(value):
+                return (kind, value)
+        raise argparse.ArgumentTypeError('Value should be one of: %s' \
+            % (', '.join([str(t) for t in targets])))
+    return selector

datumaro/datumaro/util/image.py

@@ -0,0 +1,30 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import cv2
+import numpy as np
+
+
+def load_image(path):
+    """
+    Reads an image in the HWC Grayscale/BGR(A) float [0; 255] format.
+    """
+    image = cv2.imread(path)
+    image = image.astype(np.float32)
+
+    assert len(image.shape) == 3
+    assert image.shape[2] in [1, 3, 4]
+    return image
+
+class lazy_image:
+    def __init__(self, path, loader=load_image):
+        self.path = path
+        self.loader = loader
+        self.image = None
+
+    def __call__(self):
+        if self.image is None:
+            self.image = self.loader(self.path)
+        return self.image

datumaro/datumaro/util/mask_tools.py

@@ -0,0 +1,96 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from itertools import groupby
+import numpy as np
+
+from datumaro.util.image import lazy_image, load_image
+
+
+def generate_colormap(length=256):
+    def get_bit(number, index):
+        return (number >> index) & 1
+
+    colormap = np.zeros((length, 3), dtype=int)
+    indices = np.arange(length, dtype=int)
+
+    for j in range(7, -1, -1):
+        for c in range(3):
+            colormap[:, c] |= get_bit(indices, c) << j
+        indices >>= 3
+
+    return {
+        id: tuple(color) for id, color in enumerate(colormap)
+    }
+
+def invert_colormap(colormap):
+    return {
+        tuple(a): index for index, a in colormap.items()
+    }
+
+_default_colormap = generate_colormap()
+_default_unpaint_colormap = invert_colormap(_default_colormap)
+
+def _default_unpaint_colormap_fn(r, g, b):
+    return _default_unpaint_colormap[(r, g, b)]
+
+def unpaint_mask(painted_mask, colormap=None):
+    # expect HWC BGR [0; 255] image
+    # expect RGB->index colormap
+    assert len(painted_mask.shape) == 3
+    if colormap is None:
+        colormap = _default_unpaint_colormap_fn
+    if callable(colormap):
+        map_fn = lambda a: colormap(int(a[2]), int(a[1]), int(a[0]))
+    else:
+        map_fn = lambda a: colormap[(int(a[2]), int(a[1]), int(a[0]))]
+
+    unpainted_mask = np.apply_along_axis(map_fn,
+        1, np.reshape(painted_mask, (-1, 3)))
+    unpainted_mask = np.reshape(unpainted_mask, (painted_mask.shape[:2]))
+    return unpainted_mask.astype(int)
+
+
+def apply_colormap(mask, colormap=None):
+    # expect HW [0; max_index] mask
+    # expect index->RGB colormap
+    assert len(mask.shape) == 2
+
+    if colormap is None:
+        colormap = _default_colormap
+    if callable(colormap):
+        map_fn = lambda p: colormap(int(p[0]))[::-1]
+    else:
+        map_fn = lambda p: colormap[int(p[0])][::-1]
+    painted_mask = np.apply_along_axis(map_fn, 1, np.reshape(mask, (-1, 1)))
+
+    painted_mask = np.reshape(painted_mask, (*mask.shape, 3))
+    return painted_mask.astype(np.float32)
+
+
+def load_mask(path, colormap=None):
+    mask = load_image(path)
+    if colormap is not None:
+        if len(mask.shape) == 3 and mask.shape[2] != 1:
+            mask = unpaint_mask(mask, colormap=colormap)
+    return mask
+
+def lazy_mask(path, colormap=None):
+    return lazy_image(path, lambda path: load_mask(path, colormap))
+
+
+def convert_mask_to_rle(binary_mask):
+    counts = []
+    for i, (value, elements) in enumerate(
+            groupby(binary_mask.ravel(order='F'))):
+        # decoding starts from 0
+        if i == 0 and value == 1:
+            counts.append(0)
+        counts.append(len(list(elements)))
+
+    return {
+        'counts': counts,
+        'size': list(binary_mask.shape)
+    }

datumaro/datumaro/util/test_utils.py

@@ -0,0 +1,39 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import inspect
+import os
+import os.path as osp
+import shutil
+
+
+def current_function_name(depth=1):
+    return inspect.getouterframes(inspect.currentframe())[depth].function
+
+class FileRemover:
+    def __init__(self, path, is_dir=False, ignore_errors=False):
+        self.path = path
+        self.is_dir = is_dir
+        self.ignore_errors = ignore_errors
+
+    def __enter__(self):
+        return self
+
+    # pylint: disable=redefined-builtin
+    def __exit__(self, type=None, value=None, traceback=None):
+        if self.is_dir:
+            shutil.rmtree(self.path, ignore_errors=self.ignore_errors)
+        else:
+            os.remove(self.path)
+    # pylint: enable=redefined-builtin
+
+class TestDir(FileRemover):
+    def __init__(self, path=None, ignore_errors=False):
+        if path is None:
+            path = osp.abspath('temp_%s' % current_function_name(2))
+
+        os.makedirs(path, exist_ok=ignore_errors)
+
+        super().__init__(path, is_dir=True, ignore_errors=ignore_errors)

datumaro/docs/cli_design.mm

@@ -0,0 +1,147 @@
+<map version="1.0.1">
+<!-- To view this file, download free mind mapping software FreeMind from http://freemind.sourceforge.net -->
+<node CREATED="1562588909441" ID="ID_362065379" MODIFIED="1562594436169" TEXT="datum">
+<node COLOR="#669900" CREATED="1562588926230" ID="ID_392208345" MODIFIED="1562594653553" POSITION="right" STYLE="fork" TEXT="project">
+<node CREATED="1562592021703" ID="ID_1131736910" MODIFIED="1567594093533" TEXT="create">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1574330157737" ID="ID_507280937" MODIFIED="1574330158757" TEXT="Creates a Datumaro project"/>
+</node>
+<node CREATED="1562592669910" ID="ID_1273417784" MODIFIED="1567594103605" TEXT="import">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562592677270" ID="ID_1205701076" MODIFIED="1574330175510" TEXT="Generates a project from other project or dataset in a specific format"/>
+</node>
+<node CREATED="1562592764462" ID="ID_724395644" MODIFIED="1569927189023" TEXT="export">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562592918908" ID="ID_44929477" MODIFIED="1574330221398" TEXT="Saves dataset in a specfic format"/>
+</node>
+<node CREATED="1562593914751" ID="ID_378739335" MODIFIED="1574330501157" TEXT="extract">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562593918968" ID="ID_424607257" MODIFIED="1569929409897" TEXT="Extracts subproject by filter"/>
+</node>
+<node CREATED="1569928239212" ID="ID_1246336762" MODIFIED="1574330501159" TEXT="merge">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1569928465766" ID="ID_96716547" MODIFIED="1569928867634" TEXT="Adds new items to project"/>
+</node>
+<node CREATED="1562594882533" ID="ID_487465081" MODIFIED="1567594126105" TEXT="diff">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562594886583" ID="ID_1671375265" MODIFIED="1569928079633" TEXT="Compares two projects"/>
+</node>
+<node COLOR="#ff0000" CREATED="1563435039037" ID="ID_97578583" MODIFIED="1567594117984" TEXT="transform">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1563435074116" ID="ID_695576446" MODIFIED="1574330414686" TEXT="Applies specific transformation to the dataset"/>
+</node>
+<node CREATED="1562592759129" ID="ID_1934152899" MODIFIED="1574330506478" TEXT="build">
+<node CREATED="1562592866813" ID="ID_321145109" MODIFIED="1569929109413" TEXT="Compound operation which executes other required operations. &#xa;Probably, executes some pipeline based on a script provided"/>
+</node>
+<node CREATED="1569928254654" ID="ID_273542545" MODIFIED="1569928258098" TEXT="show">
+<node CREATED="1569928411749" ID="ID_842692369" MODIFIED="1569928852922" TEXT="Visualizes project"/>
+</node>
+<node CREATED="1569928386605" ID="ID_493330514" MODIFIED="1569928388754" TEXT="info">
+<node CREATED="1569928423173" ID="ID_1273620035" MODIFIED="1569928429050" TEXT="Outputs valuable info"/>
+</node>
+<node CREATED="1562593076507" ID="ID_779027516" MODIFIED="1574330511948" TEXT="stats">
+<node CREATED="1562593079585" ID="ID_1498895180" MODIFIED="1562594653556" TEXT="Computes dataset statistics"/>
+</node>
+<node CREATED="1562593105322" ID="ID_117744850" MODIFIED="1574330511947" TEXT="docs">
+<node CREATED="1562593108705" ID="ID_878198723" MODIFIED="1562594653557" TEXT="Generates dataset documentation"/>
+</node>
+</node>
+<node COLOR="#669900" CREATED="1562592073422" ID="ID_1793909666" MODIFIED="1569928300945" POSITION="right" STYLE="fork" TEXT="source">
+<node CREATED="1568023057930" ID="ID_633965389" MODIFIED="1568023077570" TEXT="create">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1569928185077" ID="ID_1231594305" MODIFIED="1574330396647" TEXT="Creates source dataset in project"/>
+</node>
+<node CREATED="1562592085302" ID="ID_199597063" MODIFIED="1568023069817" TEXT="import">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562592138228" ID="ID_1202153971" MODIFIED="1574330391766" TEXT="Adds source dataset by its URL under a name (like git submodule add)"/>
+</node>
+<node CREATED="1562592088238" ID="ID_744367784" MODIFIED="1567594264178" TEXT="remove">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562592316435" ID="ID_810859340" MODIFIED="1574330377694" TEXT="Removes source dataset"/>
+</node>
+<node CREATED="1562592469569" ID="ID_329615614" MODIFIED="1569927769905" TEXT="export">
+<icon BUILTIN="button_ok"/>
+</node>
+<node CREATED="1562593134746" ID="ID_195077187" MODIFIED="1562594652955" TEXT="stats"/>
+<node CREATED="1569928327997" ID="ID_389265529" MODIFIED="1569928330154" TEXT="show"/>
+<node CREATED="1569928398580" ID="ID_348421413" MODIFIED="1569928400642" TEXT="info"/>
+</node>
+<node COLOR="#669900" CREATED="1563434979149" ID="ID_782927311" MODIFIED="1563435233504" POSITION="right" TEXT="model">
+<node CREATED="1563434987574" ID="ID_290716982" MODIFIED="1567594144970" TEXT="add">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1563435018178" ID="ID_1059015375" MODIFIED="1574330372326" TEXT="Registers model for inference"/>
+</node>
+<node CREATED="1564500174410" ID="ID_451702794" MODIFIED="1567594149642" TEXT="remove">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1569928809165" ID="ID_1093915022" MODIFIED="1574330359950" TEXT="Removes model from project"/>
+</node>
+</node>
+<node COLOR="#669900" CREATED="1562593748114" ID="ID_970814064" MODIFIED="1562594652591" POSITION="right" STYLE="fork" TEXT="inference">
+<node CREATED="1562593758235" ID="ID_1984980861" MODIFIED="1563443545700" STYLE="fork" TEXT="run">
+<node CREATED="1562593765978" ID="ID_918840812" MODIFIED="1574330356630" TEXT="Executes network for inference"/>
+</node>
+<node CREATED="1564500277834" ID="ID_1264946351" MODIFIED="1564500279953" TEXT="parse">
+<node CREATED="1569927270764" ID="ID_1995847022" MODIFIED="1569927285793" TEXT="Parses training log file"/>
+</node>
+</node>
+<node COLOR="#669900" CREATED="1562594817022" ID="ID_133277273" MODIFIED="1562674963173" POSITION="right" TEXT="item">
+<font NAME="SansSerif" SIZE="12"/>
+<node CREATED="1562594955691" ID="ID_1344471806" MODIFIED="1569928758939" TEXT="export"/>
+<node CREATED="1562594960747" ID="ID_1898276667" MODIFIED="1562594963201" TEXT="stats"/>
+<node CREATED="1562594983907" ID="ID_218343857" MODIFIED="1562594985561" TEXT="diff"/>
+<node CREATED="1562595454823" ID="ID_1649071450" MODIFIED="1562595456796" TEXT="edit"/>
+</node>
+<node CREATED="1562594240501" ID="ID_1530017548" MODIFIED="1567594340403" POSITION="right" STYLE="fork" TEXT="create">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562594244868" ID="ID_1309935216" MODIFIED="1562594591882" TEXT="Calls project create"/>
+</node>
+<node CREATED="1562594254667" ID="ID_190882752" MODIFIED="1567594344740" POSITION="right" STYLE="fork" TEXT="add">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562594262484" ID="ID_949937557" MODIFIED="1569929674939" TEXT="Calls source add / import"/>
+</node>
+<node CREATED="1562594276540" ID="ID_1430572506" MODIFIED="1567594350421" POSITION="right" STYLE="fork" TEXT="remove">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562594281180" ID="ID_124160415" MODIFIED="1562594591248" TEXT="Calls source remove"/>
+</node>
+<node CREATED="1562594289395" ID="ID_1608995178" MODIFIED="1574330539766" POSITION="right" STYLE="fork" TEXT="export">
+<node CREATED="1562594293699" ID="ID_199067242" MODIFIED="1569930927620" TEXT="[project arg, default]">
+<node CREATED="1562594313250" ID="ID_1243481155" MODIFIED="1569927804137" TEXT="Calls project export"/>
+</node>
+<node CREATED="1562594323035" ID="ID_1281657568" MODIFIED="1569930961981" TEXT="[source/item arg]">
+<node CREATED="1562594338482" ID="ID_1085162426" MODIFIED="1569930968180" TEXT="Calls source/item export"/>
+</node>
+<node CREATED="1562594360266" ID="ID_840060495" MODIFIED="1562594590793" TEXT="[external dataset arg]">
+<node CREATED="1562594370348" ID="ID_778378456" MODIFIED="1569927504041" TEXT="Project import + project export"/>
+</node>
+</node>
+<node CREATED="1562594703543" ID="ID_210248464" MODIFIED="1562594705685" POSITION="right" TEXT="diff">
+<node CREATED="1569927601316" ID="ID_920307385" MODIFIED="1569927934921" TEXT="[2 item/project/source/ext.dataset args]">
+<node CREATED="1569927624724" ID="ID_1503422177" MODIFIED="1569927985130" TEXT="Import + project diff"/>
+</node>
+</node>
+<node CREATED="1569929167198" ID="ID_1583130184" MODIFIED="1569929169274" POSITION="right" TEXT="show"/>
+<node CREATED="1569929169942" ID="ID_912693725" MODIFIED="1569929174043" POSITION="right" TEXT="info"/>
+<node CREATED="1567594310257" ID="ID_995434490" MODIFIED="1567594363999" POSITION="right" TEXT="explain">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1567594365942" ID="ID_1529218756" MODIFIED="1567594404172" TEXT="Runs inference explanation"/>
+</node>
+<node CREATED="1562593914751" ID="ID_925304191" MODIFIED="1569927316928" POSITION="right" TEXT="extract">
+<node CREATED="1562593918968" ID="ID_1746788348" MODIFIED="1569929409897" TEXT="Extracts subproject by filter"/>
+</node>
+<node CREATED="1569928239212" ID="ID_874360504" MODIFIED="1569928241378" POSITION="right" TEXT="merge">
+<node CREATED="1569928465766" ID="ID_332142804" MODIFIED="1569928867634" TEXT="Adds new items to project"/>
+</node>
+<node CREATED="1562593031995" ID="ID_1818638085" MODIFIED="1569930889221" POSITION="right" STYLE="fork" TEXT="stats">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1562593043258" ID="ID_280465436" MODIFIED="1562594682163" STYLE="fork" TEXT="[project arg, default]">
+<node CREATED="1562593064794" ID="ID_1859975421" MODIFIED="1562594682163" STYLE="fork" TEXT="Calls project stats"/>
+</node>
+<node CREATED="1562593187881" ID="ID_815427730" MODIFIED="1569930976940" STYLE="fork" TEXT="[source/item arg]">
+<node CREATED="1562593203687" ID="ID_1958444123" MODIFIED="1569930985172" STYLE="fork" TEXT="Calls source/item stats"/>
+</node>
+<node CREATED="1562593537868" ID="ID_1000873843" MODIFIED="1562594682163" STYLE="fork" TEXT="[external dataset arg]">
+<node CREATED="1562593695074" ID="ID_1931687508" MODIFIED="1569930999660" STYLE="fork" TEXT="Project import + project stats"/>
+</node>
+</node>
+</node>
+</map>

datumaro/docs/design.md

@@ -0,0 +1,228 @@
+# Datumaro
+
+<!--lint disable list-item-indent-->
+
+## Table of contents
+
+- [Concept](#concept)
+- [Design](#design)
+- [RC 1 vision](#rc-1-vision)
+
+## Concept
+
+Datumaro is:
+- a tool to build composite datasets and iterate over them
+- a tool to create and maintain datasets
+  - Version control of annotations and images
+  - Publication (with removal of sensitive information)
+  - Editing
+  - Joining and splitting
+  - Exporting, format changing
+  - Image preprocessing
+- a dataset storage
+- a tool to debug datasets
+  - A network can be used to generate
+    informative data subsets (e.g. with false-positives)
+    to be analyzed further
+
+### Requirements
+
+- User interfaces
+  - a library
+  - a console tool with visualization means
+- Targets: single datasets, composite datasets, single images / videos
+- Built-in support for well-known annotation formats and datasets:
+    CVAT, COCO, PASCAL VOC, Cityscapes, ImageNet
+- Extensibility with user-provided components
+- Lightweightness - it should be easy to start working with Datumaro
+  - Minimal dependency on environment and configuration
+  - It should be easier to use Datumaro than writing own code
+    for computation of statistics or dataset manipulations
+
+### Functionality and ideas
+
+- Blur sensitive areas on dataset images
+- Dataset annotation filters, relabelling etc.
+- Dataset augmentation
+- Calculation of statistics:
+  - Mean & std, custom stats
+- "Edit" command to modify annotations
+- Versioning (for images, annotations, subsets, sources etc., comparison)
+- Documentation generation
+- Provision of iterators for user code
+- Dataset building (export in a specific format, indexation, statistics, documentation)
+- Dataset exporting to other formats
+- Dataset debugging (run inference, generate dataset slices, compute statistics)
+- "Explainable AI" - highlight network attention areas ([paper](https://arxiv.org/abs/1901.04592))
+  - Black-box approach
+    - Classification, Detection, Segmentation, Captioning
+    - White-box approach
+
+### Research topics
+
+- exploration of network prediction uncertainty (aka Bayessian approach)
+  Use case: explanation of network "quality", "stability", "certainty"
+- adversarial attacks on networks
+- dataset minification / reduction
+  Use case: removal of redundant information to reach the same network quality with lesser training time
+- dataset expansion and filtration of additions
+  Use case: add only important data
+- guidance for key frame selection for tracking ([paper](https://arxiv.org/abs/1903.11779))
+  Use case: more effective annotation, better predictions
+
+## Design
+
+### Command-line
+
+Use Docker as an example. Basically, the interface is partitioned
+on contexts and shortcuts. Contexts are semantically grouped commands,
+related to a single topic or target. Shortcuts are handy shorter
+alternatives for the most used commands and also special commands,
+which are hard to be put into specific context.
+
+![cli-design-image](images/cli_design.png)
+
+- [FreeMind tool link](http://freemind.sourceforge.net/wiki/index.php/Main_Page)
+
+### High-level architecture
+
+- Using MVVM UI pattern
+
+![mvvm-image](images/mvvm.png)
+
+### Datumaro project and environment structure
+
+<!--lint disable fenced-code-flag-->
+```
+├── [datumaro module]
+└── [project folder]
+    ├── .datumaro/
+    │   ├── config.yml
+    │   ├── .git/
+    │   ├── importers/
+    │   │   ├── custom_format_importer1.py
+    │   │   └── ...
+    │   ├── statistics/
+    │   │   ├── custom_statistic1.py
+    │   │   └── ...
+    │   ├── visualizers/
+    │   │   ├── custom_visualizer1.py
+    │   │   └── ...
+    │   └── extractors/
+    │       ├── custom_extractor1.py
+    │       └── ...
+    └── sources/
+        ├── source1
+        └── ...
+```
+<!--lint enable fenced-code-flag-->
+
+## RC 1 vision
+
+In the first version Datumaro should be a project manager for CVAT.
+It should only consume data from CVAT. The collected dataset
+can be downloaded by user to be operated on with Datumaro CLI.
+
+<!--lint disable fenced-code-flag-->
+```
+        User
+          |
+          v
+ +------------------+
+ |       CVAT       |
+ +--------v---------+       +------------------+       +--------------+
+ | Datumaro module  | ----> | Datumaro project | <---> | Datumaro CLI | <--- User
+ +------------------+       +------------------+       +--------------+
+```
+<!--lint enable fenced-code-flag-->
+
+### Interfaces
+
+- [x] Python API for user code
+  - [ ] Installation as a package
+- [x] A command-line tool for dataset manipulations
+
+### Features
+
+- Dataset format support (reading, exporting)
+  - [x] Own format
+  - [x] COCO
+  - [x] PASCAL VOC
+  - [ ] Cityscapes
+  - [ ] ImageNet
+  - [ ] CVAT
+
+- Dataset visualization (`show`)
+  - [ ] Ability to visualize a dataset
+    - [ ] with TensorBoard
+
+- Calculation of statistics for datasets
+  - [ ] Pixel mean, std
+  - [ ] Object counts (detection scenario)
+  - [ ] Image-Class distribution (classification scenario)
+  - [ ] Pixel-Class distribution (segmentation scenario)
+  - [ ] Image clusters
+  - [ ] Custom statistics
+
+- Dataset building
+  - [x] Composite dataset building
+  - [ ] Annotation remapping
+  - [ ] Subset splitting
+  - [x] Dataset filtering (`extract`)
+  - [x] Dataset merging (`merge`)
+  - [ ] Dataset item editing (`edit`)
+
+- Dataset comparison (`diff`)
+  - [x] Annotation-annotation comparison
+  - [x] Annotation-inference comparison
+  - [ ] Annotation quality estimation (for CVAT)
+    - Provide a simple method to check
+      annotation quality with a model and generate summary
+
+- Dataset and model debugging
+  - [x] Inference explanation (`explain`)
+  - [x] Black-box approach ([RISE paper](https://arxiv.org/abs/1806.07421))
+  - [x] Ability to run a model on a dataset and read the results
+
+- CVAT-integration features
+  - [x] Task export
+    - [x] Datumaro project export
+    - [x] Dataset export
+    - [ ] Original raw data (images, a video file) can be downloaded (exported)
+      together with annotations or just have links
+      on CVAT server (in the future support S3, etc)
+      - [x] Be able to use local files instead of remote links
+        - [ ] Specify cache directory
+  - [x] Use case "annotate for model training"
+    - create a task
+    - annotate
+    - export the task
+    - convert to a training format
+    - train a DL model
+  - [ ] Use case "annotate and estimate quality"
+    - create a task
+    - annotate
+    - estimate quality of annotations
+
+### Optional features
+
+- Dataset publishing
+  - [ ] Versioning (for annotations, subsets, sources, etc.)
+  - [ ] Blur sensitive areas on images
+  - [ ] Tracking of legal information
+  - [ ] Documentation generation
+
+- Dataset building
+  - [ ] Dataset minification / Extraction of the most representative subset
+    - Use case: generate low-precision calibration dataset
+
+- Dataset and model debugging
+  - [ ] Training visualization
+  - [ ] Inference explanation (`explain`)
+    - [ ] White-box approach
+
+### Properties
+
+- Lightweightness
+- Modularity
+- Extensibility

datumaro/docs/quickstart.md

@@ -0,0 +1,325 @@
+# Quick start guide
+
+## Installation
+
+### Prerequisites
+
+- Python (3.5+)
+- OpenVINO (optional)
+
+### Installation steps
+
+Download the project to any directory.
+
+Set up a virtual environment:
+
+``` bash
+python -m pip install virtualenv
+python -m virtualenv venv
+. venv/bin/activate
+while read -r p; do pip install $p; done < requirements.txt
+```
+
+## Usage
+
+The directory containing the project should be in the
+`PYTHONPATH` environment variable. The other way is to invoke
+commands from that directory.
+
+As a python module:
+
+``` bash
+python -m datumaro --help
+```
+
+As a standalone python script:
+
+``` bash
+python datum.py --help
+```
+
+As a python library:
+
+``` python
+import datumaro
+```
+
+## Workflow
+
+> **Note**: command invocation **syntax is subject to change, refer to --help output**
+
+The key object is the project. It can be created or imported with
+`project create` and `project import` commands. The project is a combination of
+dataset and environment.
+
+If you want to interact with models, you should add them to project first.
+
+Implemented commands ([CLI design doc](images/cli_design.png)):
+- project create
+- project import
+- project diff
+- project transform
+- source add
+- explain
+
+### Create a project
+
+Usage:
+
+``` bash
+python datum.py project create --help
+
+python datum.py project create \
+     -d <project_dir>
+```
+
+Example:
+
+``` bash
+python datum.py project create -d /home/my_dataset
+```
+
+### Import a project
+
+This command creates a project from an existing dataset. Supported formats:
+- MS COCO
+- Custom formats via custom `importers` and `extractors`
+
+Usage:
+
+``` bash
+python -m datumaro project import --help
+
+python -m datumaro project import \
+     <dataset_path> \
+     -d <project_dir> \
+     -t <format>
+```
+
+Example:
+
+``` bash
+python -m datumaro project import \
+     /home/coco_dir \
+     -d /home/project_dir \
+     -t ms_coco
+```
+
+An _MS COCO_-like dataset should have the following directory structure:
+
+<!--lint disable fenced-code-flag-->
+```
+COCO/
+├── annotations/
+│   ├── instances_val2017.json
+│   ├── instances_train2017.json
+├── images/
+│   ├── val2017
+│   ├── train2017
+```
+<!--lint enable fenced-code-flag-->
+
+Everything after the last `_` is considered as a subset name.
+
+### Register a model
+
+Supported models:
+- OpenVINO
+- Custom models via custom `launchers`
+
+Usage:
+
+``` bash
+python -m datumaro model add --help
+```
+
+Example: register OpenVINO model
+
+A model consists of a graph description and weights. There is also a script
+used to convert model outputs to internal data structures.
+
+``` bash
+python -m datumaro model add \
+     <model_name> openvino \
+     -d <path_to_xml> -w <path_to_bin> -i <path_to_interpretation_script>
+```
+
+Interpretation script for an OpenVINO detection model (`convert.py`):
+
+``` python
+from datumaro.components.extractor import *
+
+max_det = 10
+conf_thresh = 0.1
+
+def process_outputs(inputs, outputs):
+     # inputs = model input, array or images, shape = (N, C, H, W)
+     # outputs = model output, shape = (N, 1, K, 7)
+     # results = conversion result, [ [ Annotation, ... ], ... ]
+     results = []
+     for input, output in zip(inputs, outputs):
+          input_height, input_width = input.shape[:2]
+          detections = output[0]
+          image_results = []
+          for i, det in enumerate(detections):
+               label = int(det[1])
+               conf = det[2]
+               if conf <= conf_thresh:
+                    continue
+
+               x = max(int(det[3] * input_width), 0)
+               y = max(int(det[4] * input_height), 0)
+               w = min(int(det[5] * input_width - x), input_width)
+               h = min(int(det[6] * input_height - y), input_height)
+               image_results.append(BboxObject(x, y, w, h,
+                    label=label, attributes={'score': conf} ))
+
+               results.append(image_results[:max_det])
+
+     return results
+
+def get_categories():
+     # Optionally, provide output categories - label map etc.
+     # Example:
+     label_categories = LabelCategories()
+     label_categories.add('person')
+     label_categories.add('car')
+     return { AnnotationType.label: label_categories }
+```
+
+### Run a model inference
+
+This command сreates a new project from the current project. The new
+one annotations are the model outputs.
+
+Usage:
+
+``` bash
+python -m datumaro project transform --help
+
+python -m datumaro project transform \
+     -m <model_name> \
+     -d <save_dir>
+```
+
+Example:
+
+``` bash
+python -m datumaro project import <...>
+python -m datumaro model add mymodel <...>
+python -m datumaro project transform -m mymodel -d ../mymodel_inference
+```
+
+### Compare datasets
+
+The command compares two datasets and saves the results in the
+specified directory. The current project is considered to be
+"ground truth".
+
+``` bash
+python -m datumaro project diff --help
+
+python -m datumaro project diff <other_project_dir> -d <save_dir>
+```
+
+Example: compare a dataset with model inference
+
+``` bash
+python -m datumaro project import <...>
+python -m datumaro model add mymodel <...>
+python -m datumaro project transform <...> -d ../inference
+python -m datumaro project diff ../inference -d ../diff
+```
+
+### Run inference explanation
+
+Usage:
+
+``` bash
+python -m datumaro explain --help
+
+python -m datumaro explain \
+     -m <model_name> \
+     -d <save_dir> \
+     -t <target> \
+     <method> \
+     <method_params>
+```
+
+Example: run inference explanation on a single image with visualization
+
+``` bash
+python -m datumaro project create <...>
+python -m datumaro model add mymodel <...>
+python -m datumaro explain \
+     -m mymodel \
+     -t 'image.png' \
+     rise \
+     -s 1000 --progressive
+```
+
+### Extract data subset based on filter
+
+This command allows to create a subprject form a project, which
+would include only items satisfying some condition. XPath is used as a query
+format.
+
+Usage:
+
+``` bash
+python -m datumaro project extract --help
+
+python -m datumaro project extract \
+     -p <source_project> \
+     -d <destinatin dir> \
+     -f '<filter expression>'
+```
+
+Example:
+
+``` bash
+python -m datumaro project extract \
+     -p ../test_project \
+     -d ../test_project-extract \
+     -f '/item[image/width < image/height]'
+```
+
+Item representation:
+
+``` xml
+<item>
+  <id>290768</id>
+  <subset>minival2014</subset>
+  <image>
+    <width>612</width>
+    <height>612</height>
+    <depth>3</depth>
+  </image>
+  <annotation>
+    <id>80154</id>
+    <type>bbox</type>
+    <label_id>39</label_id>
+    <x>264.59</x>
+    <y>150.25</y>
+    <w>11.199999999999989</w>
+    <h>42.31</h>
+    <area>473.87199999999956</area>
+  </annotation>
+  <annotation>
+    <id>669839</id>
+    <type>bbox</type>
+    <label_id>41</label_id>
+    <x>163.58</x>
+    <y>191.75</y>
+    <w>76.98999999999998</w>
+    <h>73.63</h>
+    <area>5668.773699999998</area>
+  </annotation>
+  ...
+</item>
+```
+
+## Links
+- [TensorFlow detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md)
+- [How to convert model to OpenVINO format](https://docs.openvinotoolkit.org/latest/_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models.html)
+- [Model convert script for this model](https://github.com/opencv/cvat/blob/3e09503ba6c6daa6469a6c4d275a5a8b168dfa2c/components/tf_annotation/install.sh#L23)

datumaro/requirements.txt

@@ -0,0 +1,11 @@
+Cython>=0.27.3 # include before pycocotools
+GitPython>=2.1.11
+lxml>=4.4.1
+matplotlib<3.1 # 3.1+ requires python3.6, but we have 3.5 in cvat
+opencv-python>=4.1.0.25
+Pillow>=6.1.0
+pycocotools>=2.0.0
+PyYAML>=5.1.1
+requests>=2.20.0
+tensorboard>=1.12.0
+tensorboardX>=1.8

datumaro/test.py

@@ -0,0 +1,5 @@
+import unittest
+
+
+if __name__ == '__main__':
+    unittest.main()

datumaro/tests/test_RISE.py

@@ -0,0 +1,230 @@
+from collections import namedtuple
+import cv2
+import numpy as np
+
+from unittest import TestCase
+
+from datumaro.components.extractor import LabelObject, BboxObject
+from datumaro.components.launcher import Launcher
+from datumaro.components.algorithms.rise import RISE
+
+
+class RiseTest(TestCase):
+    def test_rise_can_be_applied_to_classification_model(self):
+        class TestLauncher(Launcher):
+            def __init__(self, class_count, roi, **kwargs):
+                self.class_count = class_count
+                self.roi = roi
+
+            def launch(self, inputs):
+                for inp in inputs:
+                    yield self._process(inp)
+
+            def _process(self, image):
+                roi = self.roi
+                roi_area = (roi[1] - roi[0]) * (roi[3] - roi[2])
+                if 0.5 * roi_area < np.sum(image[roi[0]:roi[1], roi[2]:roi[3], 0]):
+                    cls = 0
+                else:
+                    cls = 1
+
+                cls_conf = 0.5
+                other_conf = (1.0 - cls_conf) / (self.class_count - 1)
+
+                return [
+                    LabelObject(i, attributes={
+                        'score': cls_conf if cls == i else other_conf }) \
+                    for i in range(self.class_count)
+                ]
+
+        roi = [70, 90, 7, 90]
+        model = TestLauncher(class_count=3, roi=roi)
+
+        rise = RISE(model, max_samples=(7 * 7) ** 2, mask_width=7, mask_height=7)
+
+        image = np.ones((100, 100, 3))
+        heatmaps = next(rise.apply(image))
+
+        self.assertEqual(1, len(heatmaps))
+
+        heatmap = heatmaps[0]
+        self.assertEqual(image.shape[:2], heatmap.shape)
+
+        h_sum = np.sum(heatmap)
+        h_area = np.prod(heatmap.shape)
+        roi_sum = np.sum(heatmap[roi[0]:roi[1], roi[2]:roi[3]])
+        roi_area = (roi[1] - roi[0]) * (roi[3] - roi[2])
+        roi_den = roi_sum / roi_area
+        hrest_den = (h_sum - roi_sum) / (h_area - roi_area)
+        self.assertLess(hrest_den, roi_den)
+
+    def test_rise_can_be_applied_to_detection_model(self):
+        ROI = namedtuple('ROI',
+            ['threshold', 'x', 'y', 'w', 'h', 'label'])
+
+        class TestLauncher(Launcher):
+            def __init__(self, rois, class_count, fp_count=4, pixel_jitter=20, **kwargs):
+                self.rois = rois
+                self.roi_base_sums = [None, ] * len(rois)
+                self.class_count = class_count
+                self.fp_count = fp_count
+                self.pixel_jitter = pixel_jitter
+
+            @staticmethod
+            def roi_value(roi, image):
+                return np.sum(
+                    image[roi.y:roi.y + roi.h, roi.x:roi.x + roi.w, :])
+
+            def launch(self, inputs):
+                for inp in inputs:
+                    yield self._process(inp)
+
+            def _process(self, image):
+                detections = []
+                for i, roi in enumerate(self.rois):
+                    roi_sum = self.roi_value(roi, image)
+                    roi_base_sum = self.roi_base_sums[i]
+                    first_run = roi_base_sum is None
+                    if first_run:
+                        roi_base_sum = roi_sum
+                        self.roi_base_sums[i] = roi_base_sum
+
+                    cls_conf = roi_sum / roi_base_sum
+
+                    if roi.threshold < roi_sum / roi_base_sum:
+                        cls = roi.label
+                        detections.append(
+                            BboxObject(roi.x, roi.y, roi.w, roi.h,
+                                label=cls, attributes={'score': cls_conf})
+                        )
+
+                    if first_run:
+                        continue
+                    for j in range(self.fp_count):
+                        if roi.threshold < cls_conf:
+                            cls = roi.label
+                        else:
+                            cls = (i + j) % self.class_count
+                        box = [roi.x, roi.y, roi.w, roi.h]
+                        offset = (np.random.rand(4) - 0.5) * self.pixel_jitter
+                        detections.append(
+                            BboxObject(*(box + offset),
+                                label=cls, attributes={'score': cls_conf})
+                        )
+
+                return detections
+
+        rois = [
+            ROI(0.3, 10, 40, 30, 10, 0),
+            ROI(0.5, 70, 90, 7, 10, 0),
+            ROI(0.7, 5, 20, 40, 60, 2),
+            ROI(0.9, 30, 20, 10, 40, 1),
+        ]
+        model = model = TestLauncher(class_count=3, rois=rois)
+
+        rise = RISE(model, max_samples=(7 * 7) ** 2, mask_width=7, mask_height=7)
+
+        image = np.ones((100, 100, 3))
+        heatmaps = next(rise.apply(image))
+        heatmaps_class_count = len(set([roi.label for roi in rois]))
+        self.assertEqual(heatmaps_class_count + len(rois), len(heatmaps))
+
+        # roi_image = image.copy()
+        # for i, roi in enumerate(rois):
+        #     cv2.rectangle(roi_image, (roi.x, roi.y), (roi.x + roi.w, roi.y + roi.h), (32 * i) * 3)
+        # cv2.imshow('img', roi_image)
+
+        for c in range(heatmaps_class_count):
+            class_roi = np.zeros(image.shape[:2])
+            for i, roi in enumerate(rois):
+                if roi.label != c:
+                    continue
+                class_roi[roi.y:roi.y + roi.h, roi.x:roi.x + roi.w] \
+                    += roi.threshold
+
+            heatmap = heatmaps[c]
+
+            roi_pixels = heatmap[class_roi != 0]
+            h_sum = np.sum(roi_pixels)
+            h_area = np.sum(roi_pixels != 0)
+            h_den = h_sum / h_area
+
+            rest_pixels = heatmap[class_roi == 0]
+            r_sum = np.sum(rest_pixels)
+            r_area = np.sum(rest_pixels != 0)
+            r_den = r_sum / r_area
+
+            # print(r_den, h_den)
+            # cv2.imshow('class %s' % c, heatmap)
+            self.assertLess(r_den, h_den)
+
+        for i, roi in enumerate(rois):
+            heatmap = heatmaps[heatmaps_class_count + i]
+            h_sum = np.sum(heatmap)
+            h_area = np.prod(heatmap.shape)
+            roi_sum = np.sum(heatmap[roi.y:roi.y + roi.h, roi.x:roi.x + roi.w])
+            roi_area = roi.h * roi.w
+            roi_den = roi_sum / roi_area
+            hrest_den = (h_sum - roi_sum) / (h_area - roi_area)
+            # print(hrest_den, h_den)
+            # cv2.imshow('roi %s' % i, heatmap)
+            self.assertLess(hrest_den, roi_den)
+        # cv2.waitKey(0)
+
+    @staticmethod
+    def DISABLED_test_roi_nms():
+        ROI = namedtuple('ROI',
+            ['conf', 'x', 'y', 'w', 'h', 'label'])
+
+        class_count = 3
+        noisy_count = 3
+        rois = [
+            ROI(0.3, 10, 40, 30, 10, 0),
+            ROI(0.5, 70, 90, 7, 10, 0),
+            ROI(0.7, 5, 20, 40, 60, 2),
+            ROI(0.9, 30, 20, 10, 40, 1),
+        ]
+        pixel_jitter = 10
+
+        detections = []
+        for i, roi in enumerate(rois):
+            detections.append(
+                BboxObject(roi.x, roi.y, roi.w, roi.h,
+                    label=roi.label, attributes={'score': roi.conf})
+            )
+
+            for j in range(noisy_count):
+                cls_conf = roi.conf * j / noisy_count
+                cls = (i + j) % class_count
+                box = [roi.x, roi.y, roi.w, roi.h]
+                offset = (np.random.rand(4) - 0.5) * pixel_jitter
+                detections.append(
+                    BboxObject(*(box + offset),
+                        label=cls, attributes={'score': cls_conf})
+                )
+
+        image = np.zeros((100, 100, 3))
+        for i, det in enumerate(detections):
+            roi = ROI(det.attributes['score'], *det.get_bbox(), det.label)
+            p1 = (int(roi.x), int(roi.y))
+            p2 = (int(roi.x + roi.w), int(roi.y + roi.h))
+            c = (0, 1 * (i % (1 + noisy_count) == 0), 1)
+            cv2.rectangle(image, p1, p2, c)
+            cv2.putText(image, 'd%s-%s-%.2f' % (i, roi.label, roi.conf),
+                p1, cv2.FONT_HERSHEY_SIMPLEX, 0.25, c)
+        cv2.imshow('nms_image', image)
+        cv2.waitKey(0)
+
+        nms_boxes = RISE.nms(detections, iou_thresh=0.25)
+        print(len(detections), len(nms_boxes))
+
+        for i, det in enumerate(nms_boxes):
+            roi = ROI(det.attributes['score'], *det.get_bbox(), det.label)
+            p1 = (int(roi.x), int(roi.y))
+            p2 = (int(roi.x + roi.w), int(roi.y + roi.h))
+            c = (0, 1, 0)
+            cv2.rectangle(image, p1, p2, c)
+            cv2.putText(image, 'p%s-%s-%.2f' % (i, roi.label, roi.conf),
+                p1, cv2.FONT_HERSHEY_SIMPLEX, 0.25, c)
+        cv2.imshow('nms_image', image)
+        cv2.waitKey(0)

datumaro/tests/test_coco_format.py

@@ -0,0 +1,389 @@
+import json
+import numpy as np
+import os
+import os.path as osp
+from PIL import Image
+
+from unittest import TestCase
+
+from datumaro.components.project import Project
+from datumaro.components.extractor import (
+    DEFAULT_SUBSET_NAME,
+    Extractor, DatasetItem,
+    AnnotationType, LabelObject, MaskObject, PointsObject, PolygonObject,
+    BboxObject, CaptionObject,
+    LabelCategories, PointsCategories
+)
+from datumaro.components.converters.ms_coco import (
+    CocoConverter,
+    CocoImageInfoConverter,
+    CocoCaptionsConverter,
+    CocoInstancesConverter,
+    CocoPersonKeypointsConverter,
+    CocoLabelsConverter,
+)
+from datumaro.util import find
+from datumaro.util.test_utils import TestDir
+
+
+class CocoImporterTest(TestCase):
+    @staticmethod
+    def generate_annotation():
+        annotation = {
+            'licenses': [],
+            'info': {},
+            'categories': [],
+            'images': [],
+            'annotations': []
+        }
+        annotation['licenses'].append({
+            'name': '',
+            'id': 0,
+            'url': ''
+        })
+        annotation['info'] = {
+            'contributor': '',
+            'date_created': '',
+            'description': '',
+            'url': '',
+            'version': '',
+            'year': ''
+        }
+        annotation['licenses'].append({
+            'name': '',
+            'id': 0,
+            'url': ''
+        })
+        annotation['categories'].append({'id': 0, 'name': 'TEST', 'supercategory': ''})
+        annotation['images'].append({
+            "id": 0,
+            "width": 10,
+            "height": 5,
+            "file_name": '000000000001.jpg',
+            "license": 0,
+            "flickr_url": '',
+            "coco_url": '',
+            "date_captured": 0
+            })
+        annotation['annotations'].append({
+            "id": 0,
+            "image_id": 0,
+            "category_id": 0,
+            "segmentation": [[0, 0, 1, 0, 1, 2, 0, 2]],
+            "area": 2,
+            "bbox": [0, 0, 1, 2],
+            "iscrowd": 0
+        })
+        annotation['annotations'].append({
+            "id": 1,
+            "image_id": 0,
+            "category_id": 0,
+            "segmentation": {
+                "counts": [
+                    0, 10,
+                    5, 5,
+                    5, 5,
+                    0, 10,
+                    10, 0],
+                "size": [10, 5]},
+            "area": 30,
+            "bbox": [0, 0, 10, 4],
+            "iscrowd": 0
+        })
+        return annotation
+
+    def COCO_dataset_generate(self, path):
+        img_dir = osp.join(path, 'images', 'val')
+        ann_dir = osp.join(path, 'annotations')
+        os.makedirs(img_dir)
+        os.makedirs(ann_dir)
+        a = np.random.rand(100, 100, 3) * 255
+        im_out = Image.fromarray(a.astype('uint8')).convert('RGB')
+        im_out.save(osp.join(img_dir, '000000000001.jpg'))
+        annotation = self.generate_annotation()
+        with open(osp.join(ann_dir, 'instances_val.json'), 'w') as outfile:
+            json.dump(annotation, outfile)
+
+    def test_can_import(self):
+        with TestDir() as temp_dir:
+            self.COCO_dataset_generate(temp_dir.path)
+            project = Project.import_from(temp_dir.path, 'ms_coco')
+            dataset = project.make_dataset()
+
+            self.assertListEqual(['val'], sorted(dataset.subsets()))
+            self.assertEqual(1, len(dataset))
+
+            item = next(iter(dataset))
+            self.assertTrue(item.has_image)
+            self.assertEqual(5, len(item.annotations))
+
+            ann_0 = find(item.annotations, lambda x: x.id == 0)
+            ann_0_poly = find(item.annotations, lambda x: \
+                x.group == ann_0.id and x.type == AnnotationType.polygon)
+            ann_0_mask = find(item.annotations, lambda x: \
+                x.group == ann_0.id and x.type == AnnotationType.mask)
+            self.assertFalse(ann_0 is None)
+            self.assertFalse(ann_0_poly is None)
+            self.assertFalse(ann_0_mask is None)
+
+            ann_1 = find(item.annotations, lambda x: x.id == 1)
+            ann_1_mask = find(item.annotations, lambda x: \
+                x.group == ann_1.id and x.type == AnnotationType.mask)
+            self.assertFalse(ann_1 is None)
+            self.assertFalse(ann_1_mask is None)
+
+class CocoConverterTest(TestCase):
+    def _test_save_and_load(self, source_dataset, converter_type, test_dir):
+        converter = converter_type()
+        converter(source_dataset, test_dir.path)
+
+        project = Project.import_from(test_dir.path, 'ms_coco')
+        parsed_dataset = project.make_dataset()
+
+        source_subsets = [s if s else DEFAULT_SUBSET_NAME
+            for s in source_dataset.subsets()]
+        self.assertListEqual(
+            sorted(source_subsets),
+            sorted(parsed_dataset.subsets()),
+        )
+
+        self.assertEqual(len(source_dataset), len(parsed_dataset))
+
+        for item_a in source_dataset:
+            item_b = find(parsed_dataset, lambda x: x.id == item_a.id)
+            self.assertFalse(item_b is None)
+            self.assertEqual(len(item_a.annotations), len(item_b.annotations))
+            for ann_a in item_a.annotations:
+                ann_b = find(item_b.annotations, lambda x: \
+                    x.id == ann_a.id if ann_a.id else \
+                    x.type == ann_a.type and x.group == ann_a.group)
+                self.assertEqual(ann_a, ann_b)
+
+    def test_can_save_and_load_captions(self):
+        class TestExtractor(Extractor):
+            def __iter__(self):
+                items = [
+                    DatasetItem(id=0, subset='train',
+                        annotations=[
+                            CaptionObject('hello', id=1),
+                            CaptionObject('world', id=2),
+                        ]),
+                    DatasetItem(id=1, subset='train',
+                        annotations=[
+                            CaptionObject('test', id=3),
+                        ]),
+
+                    DatasetItem(id=2, subset='val',
+                        annotations=[
+                            CaptionObject('word', id=1),
+                        ]
+                    ),
+                ]
+                return iter(items)
+
+            def subsets(self):
+                return ['train', 'val']
+
+        with TestDir() as test_dir:
+            self._test_save_and_load(TestExtractor(),
+                CocoCaptionsConverter, test_dir)
+
+    def test_can_save_and_load_instances(self):
+        class TestExtractor(Extractor):
+            def __iter__(self):
+                items = [
+                    DatasetItem(id=0, subset='train', image=np.ones((4, 4, 3)),
+                        annotations=[
+                            BboxObject(0, 1, 2, 3, label=2, group=1,
+                                attributes={ 'is_crowd': False }, id=1),
+                            PolygonObject([0, 1, 2, 1, 2, 3, 0, 3],
+                                label=2, group=1),
+                            MaskObject(np.array([[0, 0, 0, 0], [1, 1, 0, 0],
+                                                 [1, 1, 0, 0], [0, 0, 0, 0]],
+                                                 # does not include lower row
+                                                 dtype=np.bool),
+                                label=2, group=1),
+                        ]),
+                    DatasetItem(id=1, subset='train',
+                        annotations=[
+                            BboxObject(0, 1, 3, 3, label=4, group=3,
+                                attributes={ 'is_crowd': True }, id=3),
+                            MaskObject(np.array([[0, 0, 0, 0], [1, 0, 1, 0],
+                                                 [1, 1, 0, 0], [0, 0, 1, 0]],
+                                                 dtype=np.bool),
+                                label=4, group=3),
+                        ]),
+
+                    DatasetItem(id=2, subset='val',
+                        annotations=[
+                            BboxObject(0, 1, 3, 2, label=4, group=3,
+                                attributes={ 'is_crowd': True }, id=3),
+                            MaskObject(np.array([[0, 0, 0, 0], [1, 0, 1, 0],
+                                                 [1, 1, 0, 0], [0, 0, 0, 0]],
+                                                 dtype=np.bool),
+                                label=4, group=3),
+                        ]),
+                ]
+                return iter(items)
+
+            def subsets(self):
+                return ['train', 'val']
+
+            def categories(self):
+                label_categories = LabelCategories()
+                for i in range(10):
+                    label_categories.add(str(i))
+                return {
+                    AnnotationType.label: label_categories,
+                }
+
+        with TestDir() as test_dir:
+            self._test_save_and_load(TestExtractor(),
+                CocoInstancesConverter, test_dir)
+
+    def test_can_save_and_load_images(self):
+        class TestExtractor(Extractor):
+            def __iter__(self):
+                items = [
+                    DatasetItem(id=0, subset='train'),
+                    DatasetItem(id=1, subset='train'),
+
+                    DatasetItem(id=2, subset='val'),
+                    DatasetItem(id=3, subset='val'),
+                    DatasetItem(id=4, subset='val'),
+
+                    DatasetItem(id=5, subset='test'),
+                ]
+                return iter(items)
+
+            def subsets(self):
+                return ['train', 'val', 'test']
+
+        with TestDir() as test_dir:
+            self._test_save_and_load(TestExtractor(),
+                CocoImageInfoConverter, test_dir)
+
+    def test_can_save_and_load_labels(self):
+        class TestExtractor(Extractor):
+            def __iter__(self):
+                items = [
+                    DatasetItem(id=0, subset='train',
+                        annotations=[
+                            LabelObject(4, id=1),
+                            LabelObject(9, id=2),
+                        ]),
+                    DatasetItem(id=1, subset='train',
+                        annotations=[
+                            LabelObject(4, id=4),
+                        ]),
+
+                    DatasetItem(id=2, subset='val',
+                        annotations=[
+                            LabelObject(2, id=1),
+                        ]),
+                ]
+                return iter(items)
+
+            def subsets(self):
+                return ['train', 'val']
+
+            def categories(self):
+                label_categories = LabelCategories()
+                for i in range(10):
+                    label_categories.add(str(i))
+                return {
+                    AnnotationType.label: label_categories,
+                }
+
+        with TestDir() as test_dir:
+            self._test_save_and_load(TestExtractor(),
+                CocoLabelsConverter, test_dir)
+
+    def test_can_save_and_load_keypoints(self):
+        class TestExtractor(Extractor):
+            def __iter__(self):
+                items = [
+                    DatasetItem(id=0, subset='train',
+                        annotations=[
+                            PointsObject([1, 2, 0, 2, 4, 1], [0, 1, 2],
+                                label=3, group=1, id=1),
+                            BboxObject(1, 2, 3, 4, label=3, group=1),
+                            PointsObject([5, 6, 0, 7], group=2, id=2),
+                            BboxObject(1, 2, 3, 4, group=2),
+                        ]),
+                    DatasetItem(id=1, subset='train',
+                        annotations=[
+                            PointsObject([1, 2, 0, 2, 4, 1], label=5,
+                                group=3, id=3),
+                            BboxObject(1, 2, 3, 4, label=5, group=3),
+                        ]),
+
+                    DatasetItem(id=2, subset='val',
+                        annotations=[
+                            PointsObject([0, 2, 0, 2, 4, 1], label=2,
+                                group=3, id=3),
+                            BboxObject(0, 2, 4, 4, label=2, group=3),
+                        ]),
+                ]
+                return iter(items)
+
+            def subsets(self):
+                return ['train', 'val']
+
+            def categories(self):
+                label_categories = LabelCategories()
+                points_categories = PointsCategories()
+                for i in range(10):
+                    label_categories.add(str(i))
+                    points_categories.add(i, [])
+
+                return {
+                    AnnotationType.label: label_categories,
+                    AnnotationType.points: points_categories,
+                }
+
+        with TestDir() as test_dir:
+            self._test_save_and_load(TestExtractor(),
+                CocoPersonKeypointsConverter, test_dir)
+
+    def test_can_save_dataset_with_no_subsets(self):
+        class TestExtractor(Extractor):
+            def __iter__(self):
+                items = [
+                    DatasetItem(id=1, annotations=[
+                        LabelObject(2, id=1),
+                    ]),
+
+                    DatasetItem(id=2, image=np.zeros((5, 5, 3)), annotations=[
+                        LabelObject(3, id=3),
+                        BboxObject(0, 0, 5, 5, label=3,
+                            attributes={ 'is_crowd': False }, id=4, group=4),
+                        PolygonObject([0, 0, 4, 0, 4, 4],
+                                label=3, group=4),
+                        MaskObject(np.array([
+                                [0, 1, 1, 1, 0],
+                                [0, 0, 1, 1, 0],
+                                [0, 0, 0, 1, 0],
+                                [0, 0, 0, 0, 0],
+                                [0, 0, 0, 0, 0]],
+                                # only internal fragment (without the border),
+                                # but not everywhere...
+                                dtype=np.bool),
+                            label=3, group=4),
+                    ]),
+                ]
+
+                for item in items:
+                    yield item
+
+            def categories(self):
+                label_cat = LabelCategories()
+                for label in range(10):
+                    label_cat.add('label_' + str(label))
+                return {
+                    AnnotationType.label: label_cat,
+                }
+
+        with TestDir() as test_dir:
+            self._test_save_and_load(TestExtractor(),
+                CocoConverter, test_dir)

datumaro/tests/test_command_targets.py

@@ -0,0 +1,131 @@
+import cv2
+import numpy as np
+import os.path as osp
+
+from unittest import TestCase
+
+from datumaro.components.project import Project
+from datumaro.util.command_targets import ProjectTarget, \
+    ImageTarget, SourceTarget
+from datumaro.util.test_utils import current_function_name, TestDir
+
+
+class CommandTargetsTest(TestCase):
+    def test_image_false_when_no_file(self):
+        path = '%s.jpg' % current_function_name()
+        target = ImageTarget()
+
+        status = target.test(path)
+
+        self.assertFalse(status)
+
+    def test_image_false_when_false(self):
+        with TestDir() as test_dir:
+            path = osp.join(test_dir.path, 'test.jpg')
+            with open(path, 'w+') as f:
+                f.write('qwerty123')
+
+            target = ImageTarget()
+
+            status = target.test(path)
+
+            self.assertFalse(status)
+
+    def test_image_true_when_true(self):
+        with TestDir() as test_dir:
+            path = osp.join(test_dir.path, 'test.jpg')
+            image = np.random.random_sample([10, 10, 3])
+            cv2.imwrite(path, image)
+
+            target = ImageTarget()
+
+            status = target.test(path)
+
+            self.assertTrue(status)
+
+    def test_project_false_when_no_file(self):
+        path = '%s.jpg' % current_function_name()
+        target = ProjectTarget()
+
+        status = target.test(path)
+
+        self.assertFalse(status)
+
+    def test_project_false_when_no_name(self):
+        target = ProjectTarget(project=Project())
+
+        status = target.test('')
+
+        self.assertFalse(status)
+
+    def test_project_true_when_project_file(self):
+        with TestDir() as test_dir:
+            path = osp.join(test_dir.path, 'test.jpg')
+            Project().save(path)
+
+            target = ProjectTarget()
+
+            status = target.test(path)
+
+            self.assertTrue(status)
+
+    def test_project_true_when_project_name(self):
+        project_name = 'qwerty'
+        project = Project({
+            'project_name': project_name
+        })
+        target = ProjectTarget(project=project)
+
+        status = target.test(project_name)
+
+        self.assertTrue(status)
+
+    def test_project_false_when_not_project_name(self):
+        project_name = 'qwerty'
+        project = Project({
+            'project_name': project_name
+        })
+        target = ProjectTarget(project=project)
+
+        status = target.test(project_name + '123')
+
+        self.assertFalse(status)
+
+    def test_project_true_when_not_project_file(self):
+        with TestDir() as test_dir:
+            path = osp.join(test_dir.path, 'test.jpg')
+            with open(path, 'w+') as f:
+                f.write('wqererw')
+
+            target = ProjectTarget()
+
+            status = target.test(path)
+
+            self.assertFalse(status)
+
+    def test_source_false_when_no_project(self):
+        target = SourceTarget()
+
+        status = target.test('qwerty123')
+
+        self.assertFalse(status)
+
+    def test_source_true_when_source_exists(self):
+        source_name = 'qwerty'
+        project = Project()
+        project.add_source(source_name)
+        target = SourceTarget(project=project)
+
+        status = target.test(source_name)
+
+        self.assertTrue(status)
+
+    def test_source_false_when_source_doesnt_exist(self):
+        source_name = 'qwerty'
+        project = Project()
+        project.add_source(source_name)
+        target = SourceTarget(project=project)
+
+        status = target.test(source_name + '123')
+
+        self.assertFalse(status)

datumaro/tests/test_datumaro_format.py

@@ -0,0 +1,101 @@
+from itertools import zip_longest
+import numpy as np
+
+from unittest import TestCase
+
+from datumaro.components.project import Project
+from datumaro.components.extractor import (Extractor, DatasetItem,
+    AnnotationType, LabelObject, MaskObject, PointsObject, PolygonObject,
+    PolyLineObject, BboxObject, CaptionObject,
+    LabelCategories, MaskCategories, PointsCategories
+)
+from datumaro.components.converters.datumaro import DatumaroConverter
+from datumaro.util.test_utils import TestDir
+from datumaro.util.mask_tools import generate_colormap
+
+
+class DatumaroConverterTest(TestCase):
+    class TestExtractor(Extractor):
+        def __iter__(self):
+            items = [
+                DatasetItem(id=100, subset='train',
+                    annotations=[
+                        CaptionObject('hello', id=1),
+                        CaptionObject('world', id=2, group=5),
+                        LabelObject(2, id=3, attributes={
+                            'x': 1,
+                            'y': '2',
+                        }),
+                        BboxObject(1, 2, 3, 4, label=4, id=4, attributes={
+                            'score': 10.0,
+                        }),
+                        BboxObject(5, 6, 7, 8, id=5, group=5),
+                        PointsObject([1, 2, 2, 0, 1, 1], label=0, id=5),
+                        MaskObject(label=3, id=5, image=np.ones((2, 3))),
+                    ]),
+                DatasetItem(id=21, subset='train',
+                    annotations=[
+                        CaptionObject('test'),
+                        LabelObject(2),
+                        BboxObject(1, 2, 3, 4, 5, id=42, group=42)
+                    ]),
+
+                DatasetItem(id=2, subset='val',
+                    annotations=[
+                        PolyLineObject([1, 2, 3, 4, 5, 6, 7, 8], id=11),
+                        PolygonObject([1, 2, 3, 4, 5, 6, 7, 8], id=12),
+                    ]),
+
+                DatasetItem(id=42, subset='test'),
+            ]
+            return iter(items)
+
+        def subsets(self):
+            return ['train', 'val', 'test']
+
+        def categories(self):
+            label_categories = LabelCategories()
+            for i in range(5):
+                label_categories.add('cat' + str(i))
+
+            mask_categories = MaskCategories(
+                generate_colormap(len(label_categories.items)))
+
+            points_categories = PointsCategories()
+            for index, _ in enumerate(label_categories.items):
+                points_categories.add(index, ['cat1', 'cat2'], adjacent=[0, 1])
+
+            return {
+                AnnotationType.label: label_categories,
+                AnnotationType.mask: mask_categories,
+                AnnotationType.points: points_categories,
+            }
+
+    def test_can_save_and_load(self):
+        with TestDir() as test_dir:
+            source_dataset = self.TestExtractor()
+
+            converter = DatumaroConverter(
+                save_images=True, apply_colormap=True)
+            converter(source_dataset, test_dir.path)
+
+            project = Project.import_from(test_dir.path, 'datumaro')
+            parsed_dataset = project.make_dataset()
+
+            self.assertListEqual(
+                sorted(source_dataset.subsets()),
+                sorted(parsed_dataset.subsets()),
+            )
+
+            self.assertEqual(len(source_dataset), len(parsed_dataset))
+
+            for subset_name in source_dataset.subsets():
+                source_subset = source_dataset.get_subset(subset_name)
+                parsed_subset = parsed_dataset.get_subset(subset_name)
+                for idx, (item_a, item_b) in enumerate(
+                        zip_longest(source_subset, parsed_subset)):
+                    self.assertEqual(item_a, item_b, str(idx))
+
+            self.assertEqual(
+                source_dataset.categories(),
+                parsed_dataset.categories())

datumaro/tests/test_diff.py

@@ -0,0 +1,142 @@
+from unittest import TestCase
+
+from datumaro.components.extractor import DatasetItem, LabelObject, BboxObject
+from datumaro.components.comparator import Comparator
+
+
+class DiffTest(TestCase):
+    def test_no_bbox_diff_with_same_item(self):
+        detections = 3
+        anns = [
+            BboxObject(i * 10, 10, 10, 10, label=i,
+                    attributes={'score': (1.0 + i) / detections}) \
+                for i in range(detections)
+        ]
+        item = DatasetItem(id=0, annotations=anns)
+
+        iou_thresh = 0.5
+        conf_thresh = 0.5
+        comp = Comparator(
+            iou_threshold=iou_thresh, conf_threshold=conf_thresh)
+
+        result = comp.compare_item_bboxes(item, item)
+
+        matches, mispred, a_greater, b_greater = result
+        self.assertEqual(0, len(mispred))
+        self.assertEqual(0, len(a_greater))
+        self.assertEqual(0, len(b_greater))
+        self.assertEqual(len([it for it in item.annotations \
+                if conf_thresh < it.attributes['score']]),
+            len(matches))
+        for a_bbox, b_bbox in matches:
+            self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
+            self.assertEqual(a_bbox.label, b_bbox.label)
+            self.assertLess(conf_thresh, a_bbox.attributes['score'])
+            self.assertLess(conf_thresh, b_bbox.attributes['score'])
+
+    def test_can_find_bbox_with_wrong_label(self):
+        detections = 3
+        class_count = 2
+        item1 = DatasetItem(id=1, annotations=[
+            BboxObject(i * 10, 10, 10, 10, label=i,
+                    attributes={'score': (1.0 + i) / detections}) \
+                for i in range(detections)
+        ])
+        item2 = DatasetItem(id=2, annotations=[
+            BboxObject(i * 10, 10, 10, 10, label=(i + 1) % class_count,
+                    attributes={'score': (1.0 + i) / detections}) \
+                for i in range(detections)
+        ])
+
+        iou_thresh = 0.5
+        conf_thresh = 0.5
+        comp = Comparator(
+            iou_threshold=iou_thresh, conf_threshold=conf_thresh)
+
+        result = comp.compare_item_bboxes(item1, item2)
+
+        matches, mispred, a_greater, b_greater = result
+        self.assertEqual(len([it for it in item1.annotations \
+                if conf_thresh < it.attributes['score']]),
+            len(mispred))
+        self.assertEqual(0, len(a_greater))
+        self.assertEqual(0, len(b_greater))
+        self.assertEqual(0, len(matches))
+        for a_bbox, b_bbox in mispred:
+            self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
+            self.assertEqual((a_bbox.label + 1) % class_count, b_bbox.label)
+            self.assertLess(conf_thresh, a_bbox.attributes['score'])
+            self.assertLess(conf_thresh, b_bbox.attributes['score'])
+
+    def test_can_find_missing_boxes(self):
+        detections = 3
+        class_count = 2
+        item1 = DatasetItem(id=1, annotations=[
+            BboxObject(i * 10, 10, 10, 10, label=i,
+                    attributes={'score': (1.0 + i) / detections}) \
+                for i in range(detections) if i % 2 == 0
+        ])
+        item2 = DatasetItem(id=2, annotations=[
+            BboxObject(i * 10, 10, 10, 10, label=(i + 1) % class_count,
+                    attributes={'score': (1.0 + i) / detections}) \
+                for i in range(detections) if i % 2 == 1
+        ])
+
+        iou_thresh = 0.5
+        conf_thresh = 0.5
+        comp = Comparator(
+            iou_threshold=iou_thresh, conf_threshold=conf_thresh)
+
+        result = comp.compare_item_bboxes(item1, item2)
+
+        matches, mispred, a_greater, b_greater = result
+        self.assertEqual(0, len(mispred))
+        self.assertEqual(len([it for it in item1.annotations \
+                if conf_thresh < it.attributes['score']]),
+            len(a_greater))
+        self.assertEqual(len([it for it in item2.annotations \
+                if conf_thresh < it.attributes['score']]),
+            len(b_greater))
+        self.assertEqual(0, len(matches))
+
+    def test_no_label_diff_with_same_item(self):
+        detections = 3
+        anns = [
+            LabelObject(i, attributes={'score': (1.0 + i) / detections}) \
+                for i in range(detections)
+        ]
+        item = DatasetItem(id=1, annotations=anns)
+
+        conf_thresh = 0.5
+        comp = Comparator(conf_threshold=conf_thresh)
+
+        result = comp.compare_item_labels(item, item)
+
+        matches, a_greater, b_greater = result
+        self.assertEqual(0, len(a_greater))
+        self.assertEqual(0, len(b_greater))
+        self.assertEqual(len([it for it in item.annotations \
+                if conf_thresh < it.attributes['score']]),
+            len(matches))
+
+    def test_can_find_wrong_label(self):
+        item1 = DatasetItem(id=1, annotations=[
+            LabelObject(0),
+            LabelObject(1),
+            LabelObject(2),
+        ])
+        item2 = DatasetItem(id=2, annotations=[
+            LabelObject(2),
+            LabelObject(3),
+            LabelObject(4),
+        ])
+
+        conf_thresh = 0.5
+        comp = Comparator(conf_threshold=conf_thresh)
+
+        result = comp.compare_item_labels(item1, item2)
+
+        matches, a_greater, b_greater = result
+        self.assertEqual(2, len(a_greater))
+        self.assertEqual(2, len(b_greater))
+        self.assertEqual(1, len(matches))

datumaro/tests/test_project.py

@@ -0,0 +1,450 @@
+import os
+import os.path as osp
+
+from unittest import TestCase
+
+from datumaro.components.project import Project, Environment
+from datumaro.components.project import Source, Model
+from datumaro.components.launcher import Launcher, InferenceWrapper
+from datumaro.components.converter import Converter
+from datumaro.components.extractor import Extractor, DatasetItem, LabelObject
+from datumaro.components.config import Config, DefaultConfig, SchemaBuilder
+from datumaro.components.dataset_filter import XPathDatasetFilter
+from datumaro.util.test_utils import TestDir
+
+
+class ProjectTest(TestCase):
+    def test_project_generate(self):
+        src_config = Config({
+            'project_name': 'test_project',
+            'format_version': 1,
+        })
+
+        with TestDir() as test_dir:
+            project_path = test_dir.path
+            Project.generate(project_path, src_config)
+
+            self.assertTrue(osp.isdir(project_path))
+
+            result_config = Project.load(project_path).config
+            self.assertEqual(
+                src_config.project_name, result_config.project_name)
+            self.assertEqual(
+                src_config.format_version, result_config.format_version)
+
+    @staticmethod
+    def test_default_ctor_is_ok():
+        Project()
+
+    @staticmethod
+    def test_empty_config_is_ok():
+        Project(Config())
+
+    def test_add_source(self):
+        source_name = 'source'
+        origin = Source({
+            'url': 'path',
+            'format': 'ext'
+        })
+        project = Project()
+
+        project.add_source(source_name, origin)
+
+        added = project.get_source(source_name)
+        self.assertIsNotNone(added)
+        self.assertEqual(added, origin)
+
+    def test_added_source_can_be_saved(self):
+        source_name = 'source'
+        origin = Source({
+            'url': 'path',
+        })
+        project = Project()
+        project.add_source(source_name, origin)
+
+        saved = project.config
+
+        self.assertEqual(origin, saved.sources[source_name])
+
+    def test_added_source_can_be_dumped(self):
+        source_name = 'source'
+        origin = Source({
+            'url': 'path',
+        })
+        project = Project()
+        project.add_source(source_name, origin)
+
+        with TestDir() as test_dir:
+            project.save(test_dir.path)
+
+            loaded = Project.load(test_dir.path)
+            loaded = loaded.get_source(source_name)
+            self.assertEqual(origin, loaded)
+
+    def test_can_import_with_custom_importer(self):
+        class TestImporter:
+            def __call__(self, path, subset=None):
+                return Project({
+                    'project_filename': path,
+                    'subsets': [ subset ]
+                })
+
+        path = 'path'
+        importer_name = 'test_importer'
+
+        env = Environment()
+        env.importers.register(importer_name, TestImporter)
+
+        project = Project.import_from(path, importer_name, env,
+            subset='train')
+
+        self.assertEqual(path, project.config.project_filename)
+        self.assertListEqual(['train'], project.config.subsets)
+
+    def test_can_dump_added_model(self):
+        model_name = 'model'
+
+        project = Project()
+        saved = Model({ 'launcher': 'name' })
+        project.add_model(model_name, saved)
+
+        with TestDir() as test_dir:
+            project.save(test_dir.path)
+
+            loaded = Project.load(test_dir.path)
+            loaded = loaded.get_model(model_name)
+            self.assertEqual(saved, loaded)
+
+    def test_can_have_project_source(self):
+        with TestDir() as test_dir:
+            Project.generate(test_dir.path)
+
+            project2 = Project()
+            project2.add_source('project1', {
+                'url': test_dir.path,
+            })
+            dataset = project2.make_dataset()
+
+            self.assertTrue('project1' in dataset.sources)
+
+    def test_can_batch_launch_custom_model(self):
+        class TestExtractor(Extractor):
+            def __init__(self, url, n=0):
+                super().__init__(length=n)
+                self.n = n
+
+            def __iter__(self):
+                for i in range(self.n):
+                    yield DatasetItem(id=i, subset='train', image=i)
+
+            def subsets(self):
+                return ['train']
+
+        class TestLauncher(Launcher):
+            def __init__(self, **kwargs):
+                pass
+
+            def launch(self, inputs):
+                for i, inp in enumerate(inputs):
+                    yield [ LabelObject(attributes={'idx': i, 'data': inp}) ]
+
+        model_name = 'model'
+        launcher_name = 'custom_launcher'
+
+        project = Project()
+        project.env.launchers.register(launcher_name, TestLauncher)
+        project.add_model(model_name, { 'launcher': launcher_name })
+        model = project.make_executable_model(model_name)
+        extractor = TestExtractor('', n=5)
+
+        batch_size = 3
+        executor = InferenceWrapper(extractor, model, batch_size=batch_size)
+
+        for item in executor:
+            self.assertEqual(1, len(item.annotations))
+            self.assertEqual(int(item.id) % batch_size,
+                item.annotations[0].attributes['idx'])
+            self.assertEqual(int(item.id),
+                item.annotations[0].attributes['data'])
+
+    def test_can_do_transform_with_custom_model(self):
+        class TestExtractorSrc(Extractor):
+            def __init__(self, url, n=2):
+                super().__init__(length=n)
+                self.n = n
+
+            def __iter__(self):
+                for i in range(self.n):
+                    yield DatasetItem(id=i, subset='train', image=i,
+                        annotations=[ LabelObject(i) ])
+
+            def subsets(self):
+                return ['train']
+
+        class TestLauncher(Launcher):
+            def __init__(self, **kwargs):
+                pass
+
+            def launch(self, inputs):
+                for inp in inputs:
+                    yield [ LabelObject(inp) ]
+
+        class TestConverter(Converter):
+            def __call__(self, extractor, save_dir):
+                for item in extractor:
+                    with open(osp.join(save_dir, '%s.txt' % item.id), 'w+') as f:
+                        f.write(str(item.subset) + '\n')
+                        f.write(str(item.annotations[0].label) + '\n')
+
+        class TestExtractorDst(Extractor):
+            def __init__(self, url):
+                super().__init__()
+                self.items = [osp.join(url, p) for p in sorted(os.listdir(url))]
+
+            def __iter__(self):
+                for path in self.items:
+                    with open(path, 'r') as f:
+                        index = osp.splitext(osp.basename(path))[0]
+                        subset = f.readline()[:-1]
+                        label = int(f.readline()[:-1])
+                        assert(subset == 'train')
+                        yield DatasetItem(id=index, subset=subset,
+                            annotations=[ LabelObject(label) ])
+
+            def __len__(self):
+                return len(self.items)
+
+            def subsets(self):
+                return ['train']
+
+
+        model_name = 'model'
+        launcher_name = 'custom_launcher'
+        extractor_name = 'custom_extractor'
+
+        project = Project()
+        project.env.launchers.register(launcher_name, TestLauncher)
+        project.env.extractors.register(extractor_name, TestExtractorSrc)
+        project.env.converters.register(extractor_name, TestConverter)
+        project.add_model(model_name, { 'launcher': launcher_name })
+        project.add_source('source', { 'format': extractor_name })
+
+        with TestDir() as test_dir:
+            project.make_dataset().transform(model_name, test_dir.path)
+
+            result = Project.load(test_dir.path)
+            result.env.extractors.register(extractor_name, TestExtractorDst)
+            it = iter(result.make_dataset())
+            item1 = next(it)
+            item2 = next(it)
+            self.assertEqual(0, item1.annotations[0].label)
+            self.assertEqual(1, item2.annotations[0].label)
+
+    def test_source_datasets_can_be_merged(self):
+        class TestExtractor(Extractor):
+            def __init__(self, url, n=0, s=0):
+                super().__init__(length=n)
+                self.n = n
+                self.s = s
+
+            def __iter__(self):
+                for i in range(self.n):
+                    yield DatasetItem(id=self.s + i, subset='train')
+
+            def subsets(self):
+                return ['train']
+
+        e_name1 = 'e1'
+        e_name2 = 'e2'
+        n1 = 2
+        n2 = 4
+
+        project = Project()
+        project.env.extractors.register(e_name1, lambda p: TestExtractor(p, n=n1))
+        project.env.extractors.register(e_name2, lambda p: TestExtractor(p, n=n2, s=n1))
+        project.add_source('source1', { 'format': e_name1 })
+        project.add_source('source2', { 'format': e_name2 })
+
+        dataset = project.make_dataset()
+
+        self.assertEqual(n1 + n2, len(dataset))
+
+    def test_project_filter_can_be_applied(self):
+        class TestExtractor(Extractor):
+            def __init__(self, url, n=10):
+                super().__init__(length=n)
+                self.n = n
+
+            def __iter__(self):
+                for i in range(self.n):
+                    yield DatasetItem(id=i, subset='train')
+
+            def subsets(self):
+                return ['train']
+
+        e_type = 'type'
+        project = Project()
+        project.env.extractors.register(e_type, TestExtractor)
+        project.add_source('source', { 'format': e_type })
+        project.set_filter('/item[id < 5]')
+
+        dataset = project.make_dataset()
+
+        self.assertEqual(5, len(dataset))
+
+    def test_project_own_dataset_can_be_modified(self):
+        project = Project()
+        dataset = project.make_dataset()
+
+        item = DatasetItem(id=1)
+        dataset.put(item)
+
+        self.assertEqual(item, next(iter(dataset)))
+
+    def test_project_compound_child_can_be_modified_recursively(self):
+        with TestDir() as test_dir:
+            child1 = Project({
+                'project_dir': osp.join(test_dir.path, 'child1'),
+            })
+            child1.save()
+
+            child2 = Project({
+                'project_dir': osp.join(test_dir.path, 'child2'),
+            })
+            child2.save()
+
+            parent = Project()
+            parent.add_source('child1', {
+                'url': child1.config.project_dir
+            })
+            parent.add_source('child2', {
+                'url': child2.config.project_dir
+            })
+            dataset = parent.make_dataset()
+
+            item1 = DatasetItem(id='ch1', path=['child1'])
+            item2 = DatasetItem(id='ch2', path=['child2'])
+            dataset.put(item1)
+            dataset.put(item2)
+
+            self.assertEqual(2, len(dataset))
+            self.assertEqual(1, len(dataset.sources['child1']))
+            self.assertEqual(1, len(dataset.sources['child2']))
+
+    def test_project_can_merge_item_annotations(self):
+        class TestExtractor(Extractor):
+            def __init__(self, url, v=None):
+                super().__init__()
+                self.v = v
+
+            def __iter__(self):
+                v1_item = DatasetItem(id=1, subset='train', annotations=[
+                    LabelObject(2, id=3),
+                    LabelObject(3, attributes={ 'x': 1 }),
+                ])
+
+                v2_item = DatasetItem(id=1, subset='train', annotations=[
+                    LabelObject(3, attributes={ 'x': 1 }),
+                    LabelObject(4, id=4),
+                ])
+
+                if self.v == 1:
+                    yield v1_item
+                else:
+                    yield v2_item
+
+            def subsets(self):
+                return ['train']
+
+        project = Project()
+        project.env.extractors.register('t1', lambda p: TestExtractor(p, v=1))
+        project.env.extractors.register('t2', lambda p: TestExtractor(p, v=2))
+        project.add_source('source1', { 'format': 't1' })
+        project.add_source('source2', { 'format': 't2' })
+
+        merged = project.make_dataset()
+
+        self.assertEqual(1, len(merged))
+
+        item = next(iter(merged))
+        self.assertEqual(3, len(item.annotations))
+
+class DatasetFilterTest(TestCase):
+    class TestExtractor(Extractor):
+        def __init__(self, url, n=0):
+            super().__init__(length=n)
+            self.n = n
+
+        def __iter__(self):
+            for i in range(self.n):
+                yield DatasetItem(id=i, subset='train')
+
+        def subsets(self):
+            return ['train']
+
+    def test_xpathfilter_can_be_applied(self):
+        extractor = self.TestExtractor('', n=4)
+        dataset_filter = XPathDatasetFilter('/item[id > 1]')
+
+        filtered = extractor.select(dataset_filter)
+
+        self.assertEqual(2, len(filtered))
+
+class ConfigTest(TestCase):
+    def test_can_produce_multilayer_config_from_dict(self):
+        schema_low = SchemaBuilder() \
+            .add('options', dict) \
+            .build()
+        schema_mid = SchemaBuilder() \
+            .add('desc', lambda: Config(schema=schema_low)) \
+            .build()
+        schema_top = SchemaBuilder() \
+            .add('container', lambda: DefaultConfig(
+                lambda v: Config(v, schema=schema_mid))) \
+            .build()
+
+        value = 1
+        source = Config({
+            'container': {
+                'elem': {
+                    'desc': {
+                        'options': {
+                            'k': value
+                        }
+                    }
+                }
+            }
+        }, schema=schema_top)
+
+        self.assertEqual(value, source.container['elem'].desc.options['k'])
+
+class ExtractorTest(TestCase):
+    def test_custom_extractor_can_be_created(self):
+        class CustomExtractor(Extractor):
+            def __init__(self, url):
+                super().__init__()
+
+            def __iter__(self):
+                return iter([
+                    DatasetItem(id=0, subset='train'),
+                    DatasetItem(id=1, subset='train'),
+                    DatasetItem(id=2, subset='train'),
+
+                    DatasetItem(id=3, subset='test'),
+                ])
+
+            def subsets(self):
+                return ['train', 'test']
+
+        extractor_name = 'ext1'
+        project = Project()
+        project.env.extractors.register(extractor_name, CustomExtractor)
+        project.add_source('src1', {
+            'url': 'path',
+            'format': extractor_name,
+        })
+        project.set_subsets(['train'])
+
+        dataset = project.make_dataset()
+
+        self.assertEqual(3, len(dataset))

datumaro/tests/test_voc_format.py

@@ -0,0 +1,487 @@
+import cv2
+from itertools import zip_longest
+import numpy as np
+import os
+import os.path as osp
+from xml.etree import ElementTree as ET
+import shutil
+
+from unittest import TestCase
+
+from datumaro.components.extractor import (Extractor, DatasetItem,
+    AnnotationType, BboxObject, LabelCategories,
+)
+import datumaro.components.formats.voc as VOC
+from datumaro.components.extractors.voc import (
+    VocClassificationExtractor,
+    VocDetectionExtractor,
+    VocSegmentationExtractor,
+    VocLayoutExtractor,
+    VocActionExtractor,
+)
+from datumaro.components.converters.voc import (
+    VocConverter,
+    VocClassificationConverter,
+    VocDetectionConverter,
+    VocLayoutConverter,
+    VocActionConverter,
+    VocSegmentationConverter,
+)
+from datumaro.components.importers.voc import VocImporter
+from datumaro.util import find
+from datumaro.util.test_utils import TestDir
+
+
+class VocTest(TestCase):
+    def test_colormap_generator(self):
+        reference = [
+            [  0,   0,   0],
+            [128,   0,   0],
+            [  0, 128,   0],
+            [128, 128,   0],
+            [  0,   0, 128],
+            [128,   0, 128],
+            [  0, 128, 128],
+            [128, 128, 128],
+            [ 64,   0,   0],
+            [192,   0,   0],
+            [ 64, 128,   0],
+            [192, 128,   0],
+            [ 64,   0, 128],
+            [192,   0, 128],
+            [ 64, 128, 128],
+            [192, 128, 128],
+            [  0,  64,   0],
+            [128,  64,   0],
+            [  0, 192,   0],
+            [128, 192,   0],
+        ]
+
+        self.assertTrue(np.array_equal(reference, list(VOC.VocColormap.values())))
+
+def get_label(extractor, label_id):
+    return extractor.categories()[AnnotationType.label].items[label_id].name
+
+def generate_dummy_voc(path):
+    cls_subsets_dir = osp.join(path, 'ImageSets', 'Main')
+    action_subsets_dir = osp.join(path, 'ImageSets', 'Action')
+    layout_subsets_dir = osp.join(path, 'ImageSets', 'Layout')
+    segm_subsets_dir = osp.join(path, 'ImageSets', 'Segmentation')
+    ann_dir = osp.join(path, 'Annotations')
+    img_dir = osp.join(path, 'JPEGImages')
+    segm_dir = osp.join(path, 'SegmentationClass')
+    inst_dir = osp.join(path, 'SegmentationObject')
+
+    os.makedirs(cls_subsets_dir)
+    os.makedirs(ann_dir)
+    os.makedirs(img_dir)
+    os.makedirs(segm_dir)
+    os.makedirs(inst_dir)
+
+    subsets = {
+        'train': ['2007_000001'],
+        'test': ['2007_000002'],
+    }
+
+    # Subsets
+    for subset_name, subset in subsets.items():
+        for item in subset:
+            with open(osp.join(cls_subsets_dir, subset_name + '.txt'), 'w') as f:
+                for item in subset:
+                    f.write('%s\n' % item)
+    shutil.copytree(cls_subsets_dir, action_subsets_dir)
+    shutil.copytree(cls_subsets_dir, layout_subsets_dir)
+    shutil.copytree(cls_subsets_dir, segm_subsets_dir)
+
+    # Classification
+    subset_name = 'train'
+    subset = subsets[subset_name]
+    for label in VOC.VocLabel:
+        with open(osp.join(cls_subsets_dir, '%s_%s.txt' % \
+                (label.name, subset_name)), 'w') as f:
+            for item in subset:
+                presence = label.value % 2
+                f.write('%s %2d\n' % (item, 1 if presence else -1))
+
+    # Detection + Action + Layout
+    subset_name = 'train'
+    subset = subsets[subset_name]
+    for item in subset:
+        root_elem = ET.Element('annotation')
+        ET.SubElement(root_elem, 'folder').text = 'VOC' + item.split('_')[0]
+        ET.SubElement(root_elem, 'filename').text = item + '.jpg'
+
+        size_elem = ET.SubElement(root_elem, 'size')
+        ET.SubElement(size_elem, 'width').text = '10'
+        ET.SubElement(size_elem, 'height').text = '20'
+        ET.SubElement(size_elem, 'depth').text = '3'
+
+        ET.SubElement(root_elem, 'segmented').text = '1'
+
+        obj1_elem = ET.SubElement(root_elem, 'object')
+        ET.SubElement(obj1_elem, 'name').text = VOC.VocLabel(1).name
+        ET.SubElement(obj1_elem, 'pose').text = VOC.VocPose(1).name
+        ET.SubElement(obj1_elem, 'truncated').text = '1'
+        ET.SubElement(obj1_elem, 'difficult').text = '0'
+        obj1bb_elem = ET.SubElement(obj1_elem, 'bndbox')
+        ET.SubElement(obj1bb_elem, 'xmin').text = '1'
+        ET.SubElement(obj1bb_elem, 'ymin').text = '2'
+        ET.SubElement(obj1bb_elem, 'xmax').text = '3'
+        ET.SubElement(obj1bb_elem, 'ymax').text = '4'
+
+        obj2_elem = ET.SubElement(root_elem, 'object')
+        ET.SubElement(obj2_elem, 'name').text = VOC.VocLabel.person.name
+        obj2bb_elem = ET.SubElement(obj2_elem, 'bndbox')
+        ET.SubElement(obj2bb_elem, 'xmin').text = '4'
+        ET.SubElement(obj2bb_elem, 'ymin').text = '5'
+        ET.SubElement(obj2bb_elem, 'xmax').text = '6'
+        ET.SubElement(obj2bb_elem, 'ymax').text = '7'
+        obj2head_elem = ET.SubElement(obj2_elem, 'part')
+        ET.SubElement(obj2head_elem, 'name').text = VOC.VocBodyPart(1).name
+        obj2headbb_elem = ET.SubElement(obj2head_elem, 'bndbox')
+        ET.SubElement(obj2headbb_elem, 'xmin').text = '5'
+        ET.SubElement(obj2headbb_elem, 'ymin').text = '6'
+        ET.SubElement(obj2headbb_elem, 'xmax').text = '7'
+        ET.SubElement(obj2headbb_elem, 'ymax').text = '8'
+        obj2act_elem = ET.SubElement(obj2_elem, 'actions')
+        for act in VOC.VocAction:
+            ET.SubElement(obj2act_elem, act.name).text = '%s' % (act.value % 2)
+
+        with open(osp.join(ann_dir, item + '.xml'), 'w') as f:
+            f.write(ET.tostring(root_elem, encoding='unicode'))
+
+    # Segmentation + Instances
+    subset_name = 'train'
+    subset = subsets[subset_name]
+    for item in subset:
+        cv2.imwrite(osp.join(segm_dir, item + '.png'),
+            np.ones([10, 20, 3]) * VOC.VocColormap[2])
+        cv2.imwrite(osp.join(inst_dir, item + '.png'),
+            np.ones([10, 20, 3]) * VOC.VocColormap[2])
+
+    # Test images
+    subset_name = 'test'
+    subset = subsets[subset_name]
+    for item in subset:
+        cv2.imwrite(osp.join(img_dir, item + '.jpg'),
+            np.ones([10, 20, 3]))
+
+    return subsets
+
+class VocExtractorTest(TestCase):
+    def test_can_load_voc_cls(self):
+        with TestDir() as test_dir:
+            generated_subsets = generate_dummy_voc(test_dir.path)
+
+            extractor = VocClassificationExtractor(test_dir.path)
+
+            self.assertEqual(len(generated_subsets), len(extractor.subsets()))
+
+            subset_name = 'train'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                count = 0
+                for label in VOC.VocLabel:
+                    if label.value % 2 == 1:
+                        count += 1
+                        ann = find(item.annotations,
+                            lambda x: x.type == AnnotationType.label and \
+                                x.label == label.value)
+                        self.assertFalse(ann is None)
+                self.assertEqual(count, len(item.annotations))
+
+            subset_name = 'test'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                self.assertEqual(0, len(item.annotations))
+
+    def test_can_load_voc_det(self):
+        with TestDir() as test_dir:
+            generated_subsets = generate_dummy_voc(test_dir.path)
+
+            extractor = VocDetectionExtractor(test_dir.path)
+
+            self.assertEqual(len(generated_subsets), len(extractor.subsets()))
+
+            subset_name = 'train'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                obj1 = find(item.annotations,
+                    lambda x: x.type == AnnotationType.bbox and \
+                        get_label(extractor, x.label) == VOC.VocLabel(1).name)
+                self.assertFalse(obj1 is None)
+                self.assertListEqual([1, 2, 2, 2], obj1.get_bbox())
+                self.assertDictEqual(
+                    {
+                        'pose': VOC.VocPose(1).name,
+                        'truncated': True,
+                        'difficult': False,
+                    },
+                    obj1.attributes)
+
+                obj2 = find(item.annotations,
+                    lambda x: x.type == AnnotationType.bbox and \
+                        get_label(extractor, x.label) == VOC.VocLabel.person.name)
+                self.assertFalse(obj2 is None)
+                self.assertListEqual([4, 5, 2, 2], obj2.get_bbox())
+
+                self.assertEqual(2, len(item.annotations))
+
+            subset_name = 'test'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                self.assertEqual(0, len(item.annotations))
+
+    def test_can_load_voc_segm(self):
+        with TestDir() as test_dir:
+            generated_subsets = generate_dummy_voc(test_dir.path)
+
+            extractor = VocSegmentationExtractor(test_dir.path)
+
+            self.assertEqual(len(generated_subsets), len(extractor.subsets()))
+
+            subset_name = 'train'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                cls_mask = find(item.annotations,
+                    lambda x: x.type == AnnotationType.mask and \
+                        x.attributes.get('class') == True)
+                self.assertFalse(cls_mask is None)
+                self.assertFalse(cls_mask.image is None)
+
+                inst_mask = find(item.annotations,
+                    lambda x: x.type == AnnotationType.mask and \
+                        x.attributes.get('instances') == True)
+                self.assertFalse(inst_mask is None)
+                self.assertFalse(inst_mask.image is None)
+
+                self.assertEqual(2, len(item.annotations))
+
+            subset_name = 'test'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                self.assertEqual(0, len(item.annotations))
+
+    def test_can_load_voc_layout(self):
+        with TestDir() as test_dir:
+            generated_subsets = generate_dummy_voc(test_dir.path)
+
+            extractor = VocLayoutExtractor(test_dir.path)
+
+            self.assertEqual(len(generated_subsets), len(extractor.subsets()))
+
+            subset_name = 'train'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                obj2 = find(item.annotations,
+                    lambda x: x.type == AnnotationType.bbox and \
+                        get_label(extractor, x.label) == VOC.VocLabel.person.name)
+                self.assertFalse(obj2 is None)
+                self.assertListEqual([4, 5, 2, 2], obj2.get_bbox())
+
+                obj2head = find(item.annotations,
+                    lambda x: x.type == AnnotationType.bbox and \
+                        get_label(extractor, x.label) == VOC.VocBodyPart(1).name)
+                self.assertTrue(obj2.id == obj2head.group)
+                self.assertListEqual([5, 6, 2, 2], obj2head.get_bbox())
+
+                self.assertEqual(2, len(item.annotations))
+
+            subset_name = 'test'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                self.assertEqual(0, len(item.annotations))
+
+    def test_can_load_voc_action(self):
+        with TestDir() as test_dir:
+            generated_subsets = generate_dummy_voc(test_dir.path)
+
+            extractor = VocActionExtractor(test_dir.path)
+
+            self.assertEqual(len(generated_subsets), len(extractor.subsets()))
+
+            subset_name = 'train'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                obj2 = find(item.annotations,
+                    lambda x: x.type == AnnotationType.bbox and \
+                        get_label(extractor, x.label) == VOC.VocLabel.person.name)
+                self.assertFalse(obj2 is None)
+                self.assertListEqual([4, 5, 2, 2], obj2.get_bbox())
+
+                count = 1
+                for action in VOC.VocAction:
+                    if action.value % 2 == 1:
+                        count += 1
+                        ann = find(item.annotations,
+                            lambda x: x.type == AnnotationType.label and \
+                                get_label(extractor, x.label) == action.name)
+                        self.assertFalse(ann is None)
+                        self.assertTrue(obj2.id == ann.group)
+                self.assertEqual(count, len(item.annotations))
+
+            subset_name = 'test'
+            generated_subset = generated_subsets[subset_name]
+            for id_ in generated_subset:
+                parsed_subset = extractor.get_subset(subset_name)
+                self.assertEqual(len(generated_subset), len(parsed_subset))
+
+                item = find(parsed_subset, lambda x: x.id == id_)
+                self.assertFalse(item is None)
+
+                self.assertEqual(0, len(item.annotations))
+
+class VocConverterTest(TestCase):
+    def _test_can_save_voc(self, extractor_type, converter_type, test_dir):
+        dummy_dir = osp.join(test_dir, 'dummy')
+        generate_dummy_voc(dummy_dir)
+        gen_extractor = extractor_type(dummy_dir)
+
+        conv_dir = osp.join(test_dir, 'converted')
+        converter = converter_type()
+        converter(gen_extractor, conv_dir)
+
+        conv_extractor = extractor_type(conv_dir)
+        for item_a, item_b in zip_longest(gen_extractor, conv_extractor):
+            self.assertEqual(item_a.id, item_b.id)
+            self.assertEqual(len(item_a.annotations), len(item_b.annotations))
+            for ann_a, ann_b in zip(item_a.annotations, item_b.annotations):
+                self.assertEqual(ann_a.type, ann_b.type)
+
+    def test_can_save_voc_cls(self):
+        with TestDir() as test_dir:
+            self._test_can_save_voc(
+                VocClassificationExtractor, VocClassificationConverter,
+                test_dir.path)
+
+    def test_can_save_voc_det(self):
+        with TestDir() as test_dir:
+            self._test_can_save_voc(
+                VocDetectionExtractor, VocDetectionConverter,
+                test_dir.path)
+
+    def test_can_save_voc_segm(self):
+        with TestDir() as test_dir:
+            self._test_can_save_voc(
+                VocSegmentationExtractor, VocSegmentationConverter,
+                test_dir.path)
+
+    def test_can_save_voc_layout(self):
+        with TestDir() as test_dir:
+            self._test_can_save_voc(
+                VocLayoutExtractor, VocLayoutConverter,
+                test_dir.path)
+
+    def test_can_save_voc_action(self):
+        with TestDir() as test_dir:
+            self._test_can_save_voc(
+                VocActionExtractor, VocActionConverter,
+                test_dir.path)
+
+    def test_can_save_dataset_with_no_subsets(self):
+        class TestExtractor(Extractor):
+            def __iter__(self):
+                items = [
+                    DatasetItem(id=1, annotations=[
+                        BboxObject(2, 3, 4, 5, label=2, id=1),
+                        BboxObject(2, 3, 4, 5, label=3, id=2),
+                    ]),
+
+                    DatasetItem(id=2, annotations=[
+                        BboxObject(5, 4, 6, 5, label=3, id=1),
+                    ]),
+                ]
+
+                for item in items:
+                    yield item
+
+            def categories(self):
+                label_cat = LabelCategories()
+                for label in VOC.VocLabel:
+                    label_cat.add(label.name)
+                return {
+                    AnnotationType.label: label_cat,
+                }
+
+        with TestDir() as test_dir:
+            src_extractor = TestExtractor()
+            converter = VocConverter()
+
+            converter(src_extractor, test_dir.path)
+
+            dst_extractor = VocImporter()(test_dir.path).make_dataset()
+
+            self.assertEqual(len(src_extractor), len(dst_extractor))
+            for item_a, item_b in zip_longest(src_extractor, dst_extractor):
+                self.assertEqual(item_a.id, item_b.id)
+                self.assertEqual(len(item_a.annotations), len(item_b.annotations))
+                for ann_a, ann_b in zip(item_a.annotations, item_b.annotations):
+                    self.assertEqual(ann_a.type, ann_b.type)
+
+class VocImporterTest(TestCase):
+    def test_can_import(self):
+        with TestDir() as test_dir:
+            dummy_dir = osp.join(test_dir.path, 'dummy')
+            subsets = generate_dummy_voc(dummy_dir)
+
+            dataset = VocImporter()(dummy_dir).make_dataset()
+
+            self.assertEqual(len(VOC.VocTask), len(dataset.sources))
+            self.assertEqual(set(subsets), set(dataset.subsets()))
+            self.assertEqual(
+                sum([len(s) for _, s in subsets.items()]),
+                len(dataset))

supervisord.conf

@@ -41,6 +41,12 @@ command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \
 environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
 numprocs=1
 
+[program:rqscheduler]
+command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \
+    "/usr/bin/python3 /usr/local/bin/rqscheduler --host redis -i 30"
+environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock"
+numprocs=1
+
 [program:runserver]
 ; Here need to run a couple of commands to initialize DB and copy static files.
 ; We cannot initialize DB on build because the DB should be online. Also some