wangchunlin
/
cvat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move project dir to .datumaro (#1207)

Browse Source
main
zhiltsov-max 6 years ago committed by GitHub
parent c2d58e5b7d
commit 79c30696dd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 74 additions and 51 deletions
Show Stats Download Patch File Download Diff File

46
datumaro/datumaro/cli/contexts/project/__init__.py
Unescape Escape View File

@ -10,7 +10,8 @@ import os
import os.path as osp import os.path as osp
import shutil import shutil
from datumaro.components.project import Project, Environment from datumaro.components.project import Project, Environment, \
PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG
from datumaro.components.comparator import Comparator from datumaro.components.comparator import Comparator
from datumaro.components.dataset_filter import DatasetItemEncoder from datumaro.components.dataset_filter import DatasetItemEncoder
from datumaro.components.extractor import AnnotationType from datumaro.components.extractor import AnnotationType
@ -18,8 +19,7 @@ from datumaro.components.cli_plugin import CliPlugin
from .diff import DiffVisualizer from .diff import DiffVisualizer
from ...util import add_subparser, CliException, MultilineFormatter, \ from ...util import add_subparser, CliException, MultilineFormatter, \
make_file_name make_file_name
from ...util.project import make_project_path, load_project, \ from ...util.project import load_project, generate_next_dir_name
generate_next_dir_name
def build_create_parser(parser_ctor=argparse.ArgumentParser): def build_create_parser(parser_ctor=argparse.ArgumentParser):
@ -48,19 +48,23 @@ def build_create_parser(parser_ctor=argparse.ArgumentParser):
def create_command(args): def create_command(args):
project_dir = osp.abspath(args.dst_dir) project_dir = osp.abspath(args.dst_dir)
project_path = make_project_path(project_dir)
if osp.isdir(project_dir) and os.listdir(project_dir): project_env_dir = osp.join(project_dir, DEFAULT_CONFIG.env_dir)
if osp.isdir(project_env_dir) and os.listdir(project_env_dir):
if not args.overwrite: if not args.overwrite:
raise CliException("Directory '%s' already exists " raise CliException("Directory '%s' already exists "
"(pass --overwrite to force creation)" % project_dir) "(pass --overwrite to force creation)" % project_env_dir)
else: else:
shutil.rmtree(project_dir) shutil.rmtree(project_env_dir, ignore_errors=True)
os.makedirs(project_dir, exist_ok=True)
if not args.overwrite and osp.isfile(project_path): own_dataset_dir = osp.join(project_dir, DEFAULT_CONFIG.dataset_dir)
raise CliException("Project file '%s' already exists " if osp.isdir(own_dataset_dir) and os.listdir(own_dataset_dir):
"(pass --overwrite to force creation)" % project_path) if not args.overwrite:
raise CliException("Directory '%s' already exists "
"(pass --overwrite to force creation)" % own_dataset_dir)
else:
# NOTE: remove the dir to avoid using data from previous project
shutil.rmtree(own_dataset_dir)
project_name = args.name project_name = args.name
if project_name is None: if project_name is None:
@ -138,19 +142,23 @@ def build_import_parser(parser_ctor=argparse.ArgumentParser):
def import_command(args): def import_command(args):
project_dir = osp.abspath(args.dst_dir) project_dir = osp.abspath(args.dst_dir)
project_path = make_project_path(project_dir)
if osp.isdir(project_dir) and os.listdir(project_dir): project_env_dir = osp.join(project_dir, DEFAULT_CONFIG.env_dir)
if osp.isdir(project_env_dir) and os.listdir(project_env_dir):
if not args.overwrite: if not args.overwrite:
raise CliException("Directory '%s' already exists " raise CliException("Directory '%s' already exists "
"(pass --overwrite to force creation)" % project_dir) "(pass --overwrite to force creation)" % project_env_dir)
else: else:
shutil.rmtree(project_dir) shutil.rmtree(project_env_dir, ignore_errors=True)
os.makedirs(project_dir, exist_ok=True)
if not args.overwrite and osp.isfile(project_path): own_dataset_dir = osp.join(project_dir, DEFAULT_CONFIG.dataset_dir)
raise CliException("Project file '%s' already exists " if osp.isdir(own_dataset_dir) and os.listdir(own_dataset_dir):
"(pass --overwrite to force creation)" % project_path) if not args.overwrite:
raise CliException("Directory '%s' already exists "
"(pass --overwrite to force creation)" % own_dataset_dir)
else:
# NOTE: remove the dir to avoid using data from previous project
shutil.rmtree(own_dataset_dir)
project_name = args.name project_name = args.name
if project_name is None: if project_name is None:

13
datumaro/datumaro/cli/util/project.py
Unescape Escape View File

@ -4,20 +4,11 @@
# SPDX-License-Identifier: MIT # SPDX-License-Identifier: MIT
import os import os
import os.path as osp
from datumaro.components.project import Project, \ from datumaro.components.project import Project
PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG
def make_project_path(project_dir, project_filename=None): def load_project(project_dir):
if project_filename is None:
project_filename = DEFAULT_CONFIG.project_filename
return osp.join(project_dir, project_filename)
def load_project(project_dir, project_filename=None):
if project_filename:
project_dir = osp.join(project_dir, project_filename)
return Project.load(project_dir) return Project.load(project_dir)
def generate_next_dir_name(dirname, basedir='.', sep='.'): def generate_next_dir_name(dirname, basedir='.', sep='.'):

66
datumaro/datumaro/components/project.py
Unescape Escape View File

@ -12,11 +12,13 @@ import inspect
import logging as log import logging as log
import os import os
import os.path as osp import os.path as osp
import shutil
import sys import sys
from datumaro.components.config import Config, DEFAULT_FORMAT from datumaro.components.config import Config, DEFAULT_FORMAT
from datumaro.components.config_model import * from datumaro.components.config_model import (Model, Source,
from datumaro.components.extractor import DatasetItem, Extractor PROJECT_DEFAULT_CONFIG, PROJECT_SCHEMA)
from datumaro.components.extractor import Extractor
from datumaro.components.launcher import InferenceWrapper from datumaro.components.launcher import InferenceWrapper
from datumaro.components.dataset_filter import \ from datumaro.components.dataset_filter import \
XPathDatasetFilter, XPathAnnotationsFilter XPathDatasetFilter, XPathAnnotationsFilter
@ -672,16 +674,21 @@ class ProjectDataset(Dataset):
def export_project(self, save_dir, converter, def export_project(self, save_dir, converter,
filter_expr=None, filter_annotations=False, remove_empty=False): filter_expr=None, filter_annotations=False, remove_empty=False):
# NOTE: probably this function should be in the ViewModel layer # NOTE: probably this function should be in the ViewModel layer
save_dir = osp.abspath(save_dir)
os.makedirs(save_dir, exist_ok=True)
dataset = self dataset = self
if filter_expr: if filter_expr:
dataset = dataset.extract(filter_expr, dataset = dataset.extract(filter_expr,
filter_annotations=filter_annotations, filter_annotations=filter_annotations,
remove_empty=remove_empty) remove_empty=remove_empty)
converter(dataset, save_dir) save_dir = osp.abspath(save_dir)
save_dir_existed = osp.exists(save_dir)
try:
os.makedirs(save_dir, exist_ok=True)
converter(dataset, save_dir)
except Exception:
if not save_dir_existed:
shutil.rmtree(save_dir)
raise
def extract_project(self, filter_expr, filter_annotations=False, def extract_project(self, filter_expr, filter_annotations=False,
save_dir=None, remove_empty=False): save_dir=None, remove_empty=False):
@ -694,24 +701,41 @@ class ProjectDataset(Dataset):
self._save_branch_project(filtered, save_dir=save_dir) self._save_branch_project(filtered, save_dir=save_dir)
class Project: class Project:
@staticmethod @classmethod
def load(path): def load(cls, path):
path = osp.abspath(path) path = osp.abspath(path)
if osp.isdir(path): config_path = osp.join(path, PROJECT_DEFAULT_CONFIG.env_dir,
path = osp.join(path, PROJECT_DEFAULT_CONFIG.project_filename) PROJECT_DEFAULT_CONFIG.project_filename)
config = Config.parse(path) config = Config.parse(config_path)
config.project_dir = osp.dirname(path) config.project_dir = path
config.project_filename = osp.basename(path) config.project_filename = osp.basename(config_path)
return Project(config) return Project(config)
def save(self, save_dir=None): def save(self, save_dir=None):
config = self.config config = self.config
if save_dir is None: if save_dir is None:
assert config.project_dir assert config.project_dir
save_dir = osp.abspath(config.project_dir) project_dir = config.project_dir
os.makedirs(save_dir, exist_ok=True) else:
config_path = osp.join(save_dir, config.project_filename) project_dir = save_dir
config.dump(config_path)
env_dir = osp.join(project_dir, config.env_dir)
save_dir = osp.abspath(env_dir)
project_dir_existed = osp.exists(project_dir)
env_dir_existed = osp.exists(env_dir)
try:
os.makedirs(save_dir, exist_ok=True)
config_path = osp.join(save_dir, config.project_filename)
config.dump(config_path)
except Exception:
if not env_dir_existed:
shutil.rmtree(save_dir, ignore_errors=True)
if not project_dir_existed:
shutil.rmtree(project_dir, ignore_errors=True)
raise
@staticmethod @staticmethod
def generate(save_dir, config=None): def generate(save_dir, config=None):
@ -735,8 +759,8 @@ class Project:
def make_dataset(self): def make_dataset(self):
return ProjectDataset(self) return ProjectDataset(self)
def add_source(self, name, value=Source()): def add_source(self, name, value=None):
if isinstance(value, (dict, Config)): if value is None or isinstance(value, (dict, Config)):
value = Source(value) value = Source(value)
self.config.sources[name] = value self.config.sources[name] = value
self.env.sources.register(name, value) self.env.sources.register(name, value)
@ -760,8 +784,8 @@ class Project:
else: else:
self.config.subsets = value self.config.subsets = value
def add_model(self, name, value=Model()): def add_model(self, name, value=None):
if isinstance(value, (dict, Config)): if value is None or isinstance(value, (dict, Config)):
value = Model(value) value = Model(value)
self.env.register_model(name, value) self.env.register_model(name, value)
self.config.models[name] = value self.config.models[name] = value

Write Preview
Loading…
Cancel
Save