Update Datumaro dependency to 0.2.0 (#3813)
Kirill Sizov
4 years ago
committed by
GitHub
parent
78158cbcf5
commit
b57467937a
@ -0,0 +1,67 @@
|
|||||||
|
# Copyright (C) 2019 Intel Corporation
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
|
|
||||||
|
from datumaro.components.dataset import Dataset
|
||||||
|
from datumaro.components.extractor import ItemTransform
|
||||||
|
from datumaro.util.image import Image
|
||||||
|
from pyunpack import Archive
|
||||||
|
|
||||||
|
from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor,
|
||||||
|
import_dm_annotations)
|
||||||
|
from cvat.apps.dataset_manager.util import make_zip_archive
|
||||||
|
from cvat.apps.engine.models import DimensionType
|
||||||
|
|
||||||
|
from .registry import dm_env, exporter, importer
|
||||||
|
|
||||||
|
class DeleteImagePath(ItemTransform):
|
||||||
|
def transform_item(self, item):
|
||||||
|
image = None
|
||||||
|
if item.has_image and item.image.has_data:
|
||||||
|
image = Image(data=item.image.data, size=item.image.size)
|
||||||
|
return item.wrap(image=image, point_cloud='', related_images=[])
|
||||||
|
|
||||||
|
|
||||||
|
@exporter(name="Datumaro", ext="ZIP", version="1.0")
|
||||||
|
def _export(dst_file, instance_data, save_images=False):
|
||||||
|
dataset = Dataset.from_extractors(GetCVATDataExtractor(
|
||||||
|
instance_data=instance_data, include_images=save_images), env=dm_env)
|
||||||
|
if not save_images:
|
||||||
|
dataset.transform(DeleteImagePath)
|
||||||
|
with TemporaryDirectory() as tmp_dir:
|
||||||
|
dataset.export(tmp_dir, 'datumaro', save_images=save_images)
|
||||||
|
|
||||||
|
make_zip_archive(tmp_dir, dst_file)
|
||||||
|
|
||||||
|
@importer(name="Datumaro", ext="ZIP", version="1.0")
|
||||||
|
def _import(src_file, instance_data):
|
||||||
|
with TemporaryDirectory() as tmp_dir:
|
||||||
|
Archive(src_file.name).extractall(tmp_dir)
|
||||||
|
|
||||||
|
dataset = Dataset.import_from(tmp_dir, 'datumaro', env=dm_env)
|
||||||
|
|
||||||
|
import_dm_annotations(dataset, instance_data)
|
||||||
|
|
||||||
|
@exporter(name="Datumaro 3D", ext="ZIP", version="1.0", dimension=DimensionType.DIM_3D)
|
||||||
|
def _export(dst_file, instance_data, save_images=False):
|
||||||
|
dataset = Dataset.from_extractors(GetCVATDataExtractor(
|
||||||
|
instance_data=instance_data, include_images=save_images,
|
||||||
|
dimension=DimensionType.DIM_3D), env=dm_env)
|
||||||
|
|
||||||
|
if not save_images:
|
||||||
|
dataset.transform(DeleteImagePath)
|
||||||
|
with TemporaryDirectory() as tmp_dir:
|
||||||
|
dataset.export(tmp_dir, 'datumaro', save_images=save_images)
|
||||||
|
|
||||||
|
make_zip_archive(tmp_dir, dst_file)
|
||||||
|
|
||||||
|
@importer(name="Datumaro 3D", ext="ZIP", version="1.0", dimension=DimensionType.DIM_3D)
|
||||||
|
def _import(src_file, instance_data):
|
||||||
|
with TemporaryDirectory() as tmp_dir:
|
||||||
|
Archive(src_file.name).extractall(tmp_dir)
|
||||||
|
|
||||||
|
dataset = Dataset.import_from(tmp_dir, 'datumaro', env=dm_env)
|
||||||
|
|
||||||
|
import_dm_annotations(dataset, instance_data)
|
||||||
@ -1,102 +0,0 @@
|
|||||||
# Copyright (C) 2019 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import os.path as osp
|
|
||||||
import shutil
|
|
||||||
from tempfile import TemporaryDirectory
|
|
||||||
|
|
||||||
from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor,
|
|
||||||
import_dm_annotations, ProjectData)
|
|
||||||
from cvat.apps.dataset_manager.util import make_zip_archive
|
|
||||||
from cvat.settings.base import BASE_DIR
|
|
||||||
from datumaro.components.project import Project
|
|
||||||
|
|
||||||
from ..registry import dm_env, exporter
|
|
||||||
|
|
||||||
|
|
||||||
@exporter(name="Datumaro", ext="ZIP", version="1.0")
|
|
||||||
class DatumaroProjectExporter:
|
|
||||||
_REMOTE_IMAGES_EXTRACTOR = 'cvat_rest_api_task_images'
|
|
||||||
_TEMPLATES_DIR = osp.join(osp.dirname(__file__), 'export_templates')
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _save_image_info(save_dir, instance_data):
|
|
||||||
os.makedirs(save_dir, exist_ok=True)
|
|
||||||
|
|
||||||
config = {
|
|
||||||
'server_url': instance_data._host or 'localhost'
|
|
||||||
}
|
|
||||||
if isinstance(instance_data, ProjectData):
|
|
||||||
config['project_id'] = instance_data.db_project.id
|
|
||||||
else:
|
|
||||||
config['task_id'] = instance_data.db_task.id
|
|
||||||
|
|
||||||
images = []
|
|
||||||
images_meta = { 'images': images, }
|
|
||||||
for frame_id, frame in enumerate(instance_data.frame_info.values()):
|
|
||||||
image_info = {
|
|
||||||
'id': frame_id,
|
|
||||||
'name': osp.basename(frame['path']),
|
|
||||||
'width': frame['width'],
|
|
||||||
'height': frame['height'],
|
|
||||||
}
|
|
||||||
if isinstance(instance_data, ProjectData):
|
|
||||||
image_info['subset'] = frame['subset']
|
|
||||||
|
|
||||||
with open(osp.join(save_dir, 'config.json'),
|
|
||||||
'w', encoding='utf-8') as config_file:
|
|
||||||
json.dump(config, config_file)
|
|
||||||
with open(osp.join(save_dir, 'images_meta.json'),
|
|
||||||
'w', encoding='utf-8') as images_file:
|
|
||||||
json.dump(images_meta, images_file)
|
|
||||||
|
|
||||||
def _export(self, instance_data, save_dir, save_images=False):
|
|
||||||
dataset = GetCVATDataExtractor(instance_data, include_images=save_images)
|
|
||||||
db_instance = instance_data.db_project if isinstance(instance_data, ProjectData) else instance_data.db_task
|
|
||||||
dm_env.converters.get('datumaro_project').convert(dataset,
|
|
||||||
save_dir=save_dir, save_images=save_images,
|
|
||||||
project_config={ 'project_name': db_instance.name, }
|
|
||||||
)
|
|
||||||
|
|
||||||
project = Project.load(save_dir)
|
|
||||||
target_dir = project.config.project_dir
|
|
||||||
os.makedirs(target_dir, exist_ok=True)
|
|
||||||
shutil.copyfile(
|
|
||||||
osp.join(self._TEMPLATES_DIR, 'README.md'),
|
|
||||||
osp.join(target_dir, 'README.md'))
|
|
||||||
|
|
||||||
if not save_images:
|
|
||||||
# add remote links to images
|
|
||||||
source_name = '{}_{}_images'.format(
|
|
||||||
'project' if isinstance(instance_data, ProjectData) else 'task',
|
|
||||||
db_instance.id,
|
|
||||||
)
|
|
||||||
project.add_source(source_name, {
|
|
||||||
'format': self._REMOTE_IMAGES_EXTRACTOR,
|
|
||||||
})
|
|
||||||
self._save_image_info(
|
|
||||||
osp.join(save_dir, project.local_source_dir(source_name)),
|
|
||||||
instance_data)
|
|
||||||
project.save()
|
|
||||||
|
|
||||||
templates_dir = osp.join(self._TEMPLATES_DIR, 'plugins')
|
|
||||||
target_dir = osp.join(project.config.project_dir,
|
|
||||||
project.config.env_dir, project.config.plugins_dir)
|
|
||||||
os.makedirs(target_dir, exist_ok=True)
|
|
||||||
shutil.copyfile(
|
|
||||||
osp.join(templates_dir, self._REMOTE_IMAGES_EXTRACTOR + '.py'),
|
|
||||||
osp.join(target_dir, self._REMOTE_IMAGES_EXTRACTOR + '.py'))
|
|
||||||
|
|
||||||
# Make CVAT CLI module available to the user
|
|
||||||
cvat_utils_dst_dir = osp.join(save_dir, 'cvat', 'utils')
|
|
||||||
os.makedirs(cvat_utils_dst_dir)
|
|
||||||
shutil.copytree(osp.join(BASE_DIR, 'utils', 'cli'),
|
|
||||||
osp.join(cvat_utils_dst_dir, 'cli'))
|
|
||||||
|
|
||||||
def __call__(self, dst_file, instance_data, save_images=False):
|
|
||||||
with TemporaryDirectory() as temp_dir:
|
|
||||||
self._export(instance_data, save_dir=temp_dir, save_images=save_images)
|
|
||||||
make_zip_archive(temp_dir, dst_file)
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
# Quick start
|
|
||||||
|
|
||||||
``` bash
|
|
||||||
# optionally make a virtualenv
|
|
||||||
python -m virtualenv .venv
|
|
||||||
. .venv/bin/activate
|
|
||||||
|
|
||||||
# install dependencies
|
|
||||||
pip install 'git+https://github.com/openvinotoolkit/datumaro'
|
|
||||||
pip install -r cvat/utils/cli/requirements.txt
|
|
||||||
|
|
||||||
# set up environment
|
|
||||||
PYTHONPATH=':'
|
|
||||||
export PYTHONPATH
|
|
||||||
|
|
||||||
# use Datumaro
|
|
||||||
datum --help
|
|
||||||
```
|
|
||||||
|
|
||||||
Check [Datumaro docs](https://github.com/openvinotoolkit/datumaro/README.md) for more info.
|
|
||||||
@ -1,105 +0,0 @@
|
|||||||
# Copyright (C) 2020-2021 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
import getpass
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import os.path as osp
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
from cvat.utils.cli.core import CLI as CVAT_CLI
|
|
||||||
from cvat.utils.cli.core import CVAT_API_V1
|
|
||||||
from datumaro.components.config import Config, SchemaBuilder
|
|
||||||
from datumaro.components.extractor import SourceExtractor, DatasetItem
|
|
||||||
from datumaro.util.image import Image, lazy_image, load_image
|
|
||||||
|
|
||||||
CONFIG_SCHEMA = SchemaBuilder() \
|
|
||||||
.add('task_id', int) \
|
|
||||||
.add('server_url', str) \
|
|
||||||
.build()
|
|
||||||
|
|
||||||
class cvat_rest_api_task_images(SourceExtractor):
|
|
||||||
def _image_local_path(self, item_id):
|
|
||||||
task_id = self._config.task_id
|
|
||||||
return osp.join(self._cache_dir,
|
|
||||||
'task_{}_frame_{:06d}.jpg'.format(task_id, int(item_id)))
|
|
||||||
|
|
||||||
def _make_image_loader(self, item_id):
|
|
||||||
return lazy_image(item_id,
|
|
||||||
lambda item_id: self._image_loader(item_id, self))
|
|
||||||
|
|
||||||
def _is_image_cached(self, item_id):
|
|
||||||
return osp.isfile(self._image_local_path(item_id))
|
|
||||||
|
|
||||||
def _download_image(self, item_id):
|
|
||||||
self._connect()
|
|
||||||
os.makedirs(self._cache_dir, exist_ok=True)
|
|
||||||
self._cvat_cli.tasks_frame(task_id=self._config.task_id,
|
|
||||||
frame_ids=[item_id], outdir=self._cache_dir, quality='original')
|
|
||||||
|
|
||||||
def _connect(self):
|
|
||||||
if self._cvat_cli is not None:
|
|
||||||
return
|
|
||||||
|
|
||||||
print("Enter credentials for '%s' to read task data:" % \
|
|
||||||
(self._config.server_url))
|
|
||||||
username = input('User: ')
|
|
||||||
password = getpass.getpass()
|
|
||||||
|
|
||||||
session = requests.Session()
|
|
||||||
|
|
||||||
api = CVAT_API_V1(self._config.server_url)
|
|
||||||
cli = CVAT_CLI(session, api, credentials=(username, password))
|
|
||||||
self._cvat_cli = cli
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _image_loader(item_id, extractor):
|
|
||||||
if not extractor._is_image_cached(item_id):
|
|
||||||
extractor._download_image(item_id)
|
|
||||||
local_path = extractor._image_local_path(item_id)
|
|
||||||
return load_image(local_path)
|
|
||||||
|
|
||||||
def __init__(self, url):
|
|
||||||
super().__init__()
|
|
||||||
|
|
||||||
local_dir = url
|
|
||||||
self._local_dir = local_dir
|
|
||||||
self._cache_dir = osp.join(local_dir, 'images')
|
|
||||||
|
|
||||||
with open(osp.join(url, 'config.json'),
|
|
||||||
'r', encoding='utf-8') as config_file:
|
|
||||||
config = json.load(config_file)
|
|
||||||
config = Config(config, schema=CONFIG_SCHEMA)
|
|
||||||
self._config = config
|
|
||||||
|
|
||||||
with open(osp.join(url, 'images_meta.json'),
|
|
||||||
'r', encoding='utf-8') as images_file:
|
|
||||||
images_meta = json.load(images_file)
|
|
||||||
image_list = images_meta['images']
|
|
||||||
|
|
||||||
items = []
|
|
||||||
for entry in image_list:
|
|
||||||
item_id = entry['id']
|
|
||||||
item_filename = entry.get('name', str(item_id))
|
|
||||||
size = None
|
|
||||||
if entry.get('height') and entry.get('width'):
|
|
||||||
size = (entry['height'], entry['width'])
|
|
||||||
image = Image(data=self._make_image_loader(item_id),
|
|
||||||
path=self._image_local_path(item_id), size=size)
|
|
||||||
item = DatasetItem(id=osp.splitext(item_filename)[0], image=image)
|
|
||||||
items.append((item.id, item))
|
|
||||||
|
|
||||||
items = OrderedDict(items)
|
|
||||||
self._items = items
|
|
||||||
|
|
||||||
self._cvat_cli = None
|
|
||||||
|
|
||||||
def __iter__(self):
|
|
||||||
for item in self._items.values():
|
|
||||||
yield item
|
|
||||||
|
|
||||||
def __len__(self):
|
|
||||||
return len(self._items)
|
|
||||||
Loading…
Reference in New Issue