Update Datumaro dependency to 0.2.0 (#3813)
Kirill Sizov
4 years ago
committed by
GitHub
parent
78158cbcf5
commit
b57467937a
@ -0,0 +1,67 @@
|
||||
# Copyright (C) 2019 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from datumaro.components.dataset import Dataset
|
||||
from datumaro.components.extractor import ItemTransform
|
||||
from datumaro.util.image import Image
|
||||
from pyunpack import Archive
|
||||
|
||||
from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor,
|
||||
import_dm_annotations)
|
||||
from cvat.apps.dataset_manager.util import make_zip_archive
|
||||
from cvat.apps.engine.models import DimensionType
|
||||
|
||||
from .registry import dm_env, exporter, importer
|
||||
|
||||
class DeleteImagePath(ItemTransform):
|
||||
def transform_item(self, item):
|
||||
image = None
|
||||
if item.has_image and item.image.has_data:
|
||||
image = Image(data=item.image.data, size=item.image.size)
|
||||
return item.wrap(image=image, point_cloud='', related_images=[])
|
||||
|
||||
|
||||
@exporter(name="Datumaro", ext="ZIP", version="1.0")
|
||||
def _export(dst_file, instance_data, save_images=False):
|
||||
dataset = Dataset.from_extractors(GetCVATDataExtractor(
|
||||
instance_data=instance_data, include_images=save_images), env=dm_env)
|
||||
if not save_images:
|
||||
dataset.transform(DeleteImagePath)
|
||||
with TemporaryDirectory() as tmp_dir:
|
||||
dataset.export(tmp_dir, 'datumaro', save_images=save_images)
|
||||
|
||||
make_zip_archive(tmp_dir, dst_file)
|
||||
|
||||
@importer(name="Datumaro", ext="ZIP", version="1.0")
|
||||
def _import(src_file, instance_data):
|
||||
with TemporaryDirectory() as tmp_dir:
|
||||
Archive(src_file.name).extractall(tmp_dir)
|
||||
|
||||
dataset = Dataset.import_from(tmp_dir, 'datumaro', env=dm_env)
|
||||
|
||||
import_dm_annotations(dataset, instance_data)
|
||||
|
||||
@exporter(name="Datumaro 3D", ext="ZIP", version="1.0", dimension=DimensionType.DIM_3D)
|
||||
def _export(dst_file, instance_data, save_images=False):
|
||||
dataset = Dataset.from_extractors(GetCVATDataExtractor(
|
||||
instance_data=instance_data, include_images=save_images,
|
||||
dimension=DimensionType.DIM_3D), env=dm_env)
|
||||
|
||||
if not save_images:
|
||||
dataset.transform(DeleteImagePath)
|
||||
with TemporaryDirectory() as tmp_dir:
|
||||
dataset.export(tmp_dir, 'datumaro', save_images=save_images)
|
||||
|
||||
make_zip_archive(tmp_dir, dst_file)
|
||||
|
||||
@importer(name="Datumaro 3D", ext="ZIP", version="1.0", dimension=DimensionType.DIM_3D)
|
||||
def _import(src_file, instance_data):
|
||||
with TemporaryDirectory() as tmp_dir:
|
||||
Archive(src_file.name).extractall(tmp_dir)
|
||||
|
||||
dataset = Dataset.import_from(tmp_dir, 'datumaro', env=dm_env)
|
||||
|
||||
import_dm_annotations(dataset, instance_data)
|
||||
@ -1,102 +0,0 @@
|
||||
# Copyright (C) 2019 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import json
|
||||
import os
|
||||
import os.path as osp
|
||||
import shutil
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor,
|
||||
import_dm_annotations, ProjectData)
|
||||
from cvat.apps.dataset_manager.util import make_zip_archive
|
||||
from cvat.settings.base import BASE_DIR
|
||||
from datumaro.components.project import Project
|
||||
|
||||
from ..registry import dm_env, exporter
|
||||
|
||||
|
||||
@exporter(name="Datumaro", ext="ZIP", version="1.0")
|
||||
class DatumaroProjectExporter:
|
||||
_REMOTE_IMAGES_EXTRACTOR = 'cvat_rest_api_task_images'
|
||||
_TEMPLATES_DIR = osp.join(osp.dirname(__file__), 'export_templates')
|
||||
|
||||
@staticmethod
|
||||
def _save_image_info(save_dir, instance_data):
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
|
||||
config = {
|
||||
'server_url': instance_data._host or 'localhost'
|
||||
}
|
||||
if isinstance(instance_data, ProjectData):
|
||||
config['project_id'] = instance_data.db_project.id
|
||||
else:
|
||||
config['task_id'] = instance_data.db_task.id
|
||||
|
||||
images = []
|
||||
images_meta = { 'images': images, }
|
||||
for frame_id, frame in enumerate(instance_data.frame_info.values()):
|
||||
image_info = {
|
||||
'id': frame_id,
|
||||
'name': osp.basename(frame['path']),
|
||||
'width': frame['width'],
|
||||
'height': frame['height'],
|
||||
}
|
||||
if isinstance(instance_data, ProjectData):
|
||||
image_info['subset'] = frame['subset']
|
||||
|
||||
with open(osp.join(save_dir, 'config.json'),
|
||||
'w', encoding='utf-8') as config_file:
|
||||
json.dump(config, config_file)
|
||||
with open(osp.join(save_dir, 'images_meta.json'),
|
||||
'w', encoding='utf-8') as images_file:
|
||||
json.dump(images_meta, images_file)
|
||||
|
||||
def _export(self, instance_data, save_dir, save_images=False):
|
||||
dataset = GetCVATDataExtractor(instance_data, include_images=save_images)
|
||||
db_instance = instance_data.db_project if isinstance(instance_data, ProjectData) else instance_data.db_task
|
||||
dm_env.converters.get('datumaro_project').convert(dataset,
|
||||
save_dir=save_dir, save_images=save_images,
|
||||
project_config={ 'project_name': db_instance.name, }
|
||||
)
|
||||
|
||||
project = Project.load(save_dir)
|
||||
target_dir = project.config.project_dir
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
shutil.copyfile(
|
||||
osp.join(self._TEMPLATES_DIR, 'README.md'),
|
||||
osp.join(target_dir, 'README.md'))
|
||||
|
||||
if not save_images:
|
||||
# add remote links to images
|
||||
source_name = '{}_{}_images'.format(
|
||||
'project' if isinstance(instance_data, ProjectData) else 'task',
|
||||
db_instance.id,
|
||||
)
|
||||
project.add_source(source_name, {
|
||||
'format': self._REMOTE_IMAGES_EXTRACTOR,
|
||||
})
|
||||
self._save_image_info(
|
||||
osp.join(save_dir, project.local_source_dir(source_name)),
|
||||
instance_data)
|
||||
project.save()
|
||||
|
||||
templates_dir = osp.join(self._TEMPLATES_DIR, 'plugins')
|
||||
target_dir = osp.join(project.config.project_dir,
|
||||
project.config.env_dir, project.config.plugins_dir)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
shutil.copyfile(
|
||||
osp.join(templates_dir, self._REMOTE_IMAGES_EXTRACTOR + '.py'),
|
||||
osp.join(target_dir, self._REMOTE_IMAGES_EXTRACTOR + '.py'))
|
||||
|
||||
# Make CVAT CLI module available to the user
|
||||
cvat_utils_dst_dir = osp.join(save_dir, 'cvat', 'utils')
|
||||
os.makedirs(cvat_utils_dst_dir)
|
||||
shutil.copytree(osp.join(BASE_DIR, 'utils', 'cli'),
|
||||
osp.join(cvat_utils_dst_dir, 'cli'))
|
||||
|
||||
def __call__(self, dst_file, instance_data, save_images=False):
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
self._export(instance_data, save_dir=temp_dir, save_images=save_images)
|
||||
make_zip_archive(temp_dir, dst_file)
|
||||
@ -1,20 +0,0 @@
|
||||
# Quick start
|
||||
|
||||
``` bash
|
||||
# optionally make a virtualenv
|
||||
python -m virtualenv .venv
|
||||
. .venv/bin/activate
|
||||
|
||||
# install dependencies
|
||||
pip install 'git+https://github.com/openvinotoolkit/datumaro'
|
||||
pip install -r cvat/utils/cli/requirements.txt
|
||||
|
||||
# set up environment
|
||||
PYTHONPATH=':'
|
||||
export PYTHONPATH
|
||||
|
||||
# use Datumaro
|
||||
datum --help
|
||||
```
|
||||
|
||||
Check [Datumaro docs](https://github.com/openvinotoolkit/datumaro/README.md) for more info.
|
||||
@ -1,105 +0,0 @@
|
||||
# Copyright (C) 2020-2021 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import getpass
|
||||
import json
|
||||
import os
|
||||
import os.path as osp
|
||||
from collections import OrderedDict
|
||||
|
||||
import requests
|
||||
|
||||
from cvat.utils.cli.core import CLI as CVAT_CLI
|
||||
from cvat.utils.cli.core import CVAT_API_V1
|
||||
from datumaro.components.config import Config, SchemaBuilder
|
||||
from datumaro.components.extractor import SourceExtractor, DatasetItem
|
||||
from datumaro.util.image import Image, lazy_image, load_image
|
||||
|
||||
CONFIG_SCHEMA = SchemaBuilder() \
|
||||
.add('task_id', int) \
|
||||
.add('server_url', str) \
|
||||
.build()
|
||||
|
||||
class cvat_rest_api_task_images(SourceExtractor):
|
||||
def _image_local_path(self, item_id):
|
||||
task_id = self._config.task_id
|
||||
return osp.join(self._cache_dir,
|
||||
'task_{}_frame_{:06d}.jpg'.format(task_id, int(item_id)))
|
||||
|
||||
def _make_image_loader(self, item_id):
|
||||
return lazy_image(item_id,
|
||||
lambda item_id: self._image_loader(item_id, self))
|
||||
|
||||
def _is_image_cached(self, item_id):
|
||||
return osp.isfile(self._image_local_path(item_id))
|
||||
|
||||
def _download_image(self, item_id):
|
||||
self._connect()
|
||||
os.makedirs(self._cache_dir, exist_ok=True)
|
||||
self._cvat_cli.tasks_frame(task_id=self._config.task_id,
|
||||
frame_ids=[item_id], outdir=self._cache_dir, quality='original')
|
||||
|
||||
def _connect(self):
|
||||
if self._cvat_cli is not None:
|
||||
return
|
||||
|
||||
print("Enter credentials for '%s' to read task data:" % \
|
||||
(self._config.server_url))
|
||||
username = input('User: ')
|
||||
password = getpass.getpass()
|
||||
|
||||
session = requests.Session()
|
||||
|
||||
api = CVAT_API_V1(self._config.server_url)
|
||||
cli = CVAT_CLI(session, api, credentials=(username, password))
|
||||
self._cvat_cli = cli
|
||||
|
||||
@staticmethod
|
||||
def _image_loader(item_id, extractor):
|
||||
if not extractor._is_image_cached(item_id):
|
||||
extractor._download_image(item_id)
|
||||
local_path = extractor._image_local_path(item_id)
|
||||
return load_image(local_path)
|
||||
|
||||
def __init__(self, url):
|
||||
super().__init__()
|
||||
|
||||
local_dir = url
|
||||
self._local_dir = local_dir
|
||||
self._cache_dir = osp.join(local_dir, 'images')
|
||||
|
||||
with open(osp.join(url, 'config.json'),
|
||||
'r', encoding='utf-8') as config_file:
|
||||
config = json.load(config_file)
|
||||
config = Config(config, schema=CONFIG_SCHEMA)
|
||||
self._config = config
|
||||
|
||||
with open(osp.join(url, 'images_meta.json'),
|
||||
'r', encoding='utf-8') as images_file:
|
||||
images_meta = json.load(images_file)
|
||||
image_list = images_meta['images']
|
||||
|
||||
items = []
|
||||
for entry in image_list:
|
||||
item_id = entry['id']
|
||||
item_filename = entry.get('name', str(item_id))
|
||||
size = None
|
||||
if entry.get('height') and entry.get('width'):
|
||||
size = (entry['height'], entry['width'])
|
||||
image = Image(data=self._make_image_loader(item_id),
|
||||
path=self._image_local_path(item_id), size=size)
|
||||
item = DatasetItem(id=osp.splitext(item_filename)[0], image=image)
|
||||
items.append((item.id, item))
|
||||
|
||||
items = OrderedDict(items)
|
||||
self._items = items
|
||||
|
||||
self._cvat_cli = None
|
||||
|
||||
def __iter__(self):
|
||||
for item in self._items.values():
|
||||
yield item
|
||||
|
||||
def __len__(self):
|
||||
return len(self._items)
|
||||
Loading…
Reference in New Issue