From fd4a1307b575fdb25357f415d713117e1b06b798 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 18 Jun 2022 06:19:31 +0300 Subject: [PATCH] CLI package (#59) --- .coveragerc | 2 +- .github/workflows/main.yml | 2 +- .vscode/launch.json | 2 +- CHANGELOG.md | 1 + Dockerfile | 1 - Dockerfile.ci | 6 +- cvat-cli/MANIFEST.in | 2 + cvat-cli/README.md | 11 +++ cvat-cli/developer_guide.md | 19 ++++++ cvat-cli/pyproject.toml | 4 ++ {utils/cli => cvat-cli}/requirements/base.txt | 0 cvat-cli/requirements/testing.txt | 4 ++ cvat-cli/setup.py | 67 +++++++++++++++++++ cvat-cli/src/cvat_cli/__init__.py | 3 + .../src/cvat_cli/__main__.py | 11 +-- .../src/cvat_cli}/core/__init__.py | 3 + .../src/cvat_cli}/core/core.py | 0 .../src/cvat_cli}/core/definition.py | 5 +- .../src/cvat_cli}/core/utils.py | 0 cvat-cli/src/cvat_cli/version.py | 1 + cvat-cli/tests/__init__.py | 3 + {utils/cli => cvat-cli}/tests/test_cli.py | 6 +- site/content/en/docs/manual/advanced/cli.md | 46 ++++++------- utils/cli/requirements/testing.txt | 4 -- utils/cli/tests/__init__.py | 0 25 files changed, 160 insertions(+), 43 deletions(-) create mode 100644 cvat-cli/MANIFEST.in create mode 100644 cvat-cli/README.md create mode 100644 cvat-cli/developer_guide.md create mode 100644 cvat-cli/pyproject.toml rename {utils/cli => cvat-cli}/requirements/base.txt (100%) create mode 100644 cvat-cli/requirements/testing.txt create mode 100644 cvat-cli/setup.py create mode 100644 cvat-cli/src/cvat_cli/__init__.py rename utils/cli/cli.py => cvat-cli/src/cvat_cli/__main__.py (88%) rename {utils/cli => cvat-cli/src/cvat_cli}/core/__init__.py (72%) rename {utils/cli => cvat-cli/src/cvat_cli}/core/core.py (100%) rename {utils/cli => cvat-cli/src/cvat_cli}/core/definition.py (98%) rename {utils/cli => cvat-cli/src/cvat_cli}/core/utils.py (100%) create mode 100644 cvat-cli/src/cvat_cli/version.py create mode 100644 cvat-cli/tests/__init__.py rename {utils/cli => cvat-cli}/tests/test_cli.py (98%) delete mode 100644 utils/cli/requirements/testing.txt delete mode 100644 utils/cli/tests/__init__.py diff --git a/.coveragerc b/.coveragerc index c669baf7..c2e5ab67 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ branch = true source = cvat/apps/ - utils/cli/ + cvat-cli/ utils/dataset_manifest omit = diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 08ec96d5..9624807d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -133,7 +133,7 @@ jobs: CONTAINER_COVERAGE_DATA_DIR: "/coverage_data" run: | docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ - -c 'coverage run -a manage.py test cvat/apps utils/cli -k 'tasks_id' -k 'lambda' -k 'share' && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}' + -c 'coverage run -a manage.py test cvat/apps cvat-cli -k 'tasks_id' -k 'lambda' -k 'share' && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}' docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \ -c 'cd cvat-data && npm ci --ignore-scripts && cd ../cvat-core && npm ci --ignore-scripts && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info' diff --git a/.vscode/launch.json b/.vscode/launch.json index 903ed413..79cb5743 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -169,7 +169,7 @@ "--settings", "cvat.settings.testing", "cvat/apps", - "utils/cli" + "cvat-cli/" ], "django": true, "cwd": "${workspaceFolder}", diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c8aea8f..b6cebc54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Project/task backups uploading via chunk uploads () - Fixed UX bug when jobs pagination is reset after changing a job () - Progressbars in CLI for file uploading and downloading () +- `utils/cli` changed to `cvat-cli` package () ### Changed - Bumped nuclio version to 1.8.14 () diff --git a/Dockerfile b/Dockerfile index 1f6410f5..ba312c2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -147,7 +147,6 @@ COPY --chown=${USER} ssh ${HOME}/.ssh COPY --chown=${USER} supervisord.conf mod_wsgi.conf wait-for-it.sh manage.py ${HOME}/ COPY --chown=${USER} cvat/ ${HOME}/cvat COPY --chown=${USER} utils/ ${HOME}/utils -COPY --chown=${USER} tests/ ${HOME}/tests # RUN all commands below as 'django' user USER ${USER} diff --git a/Dockerfile.ci b/Dockerfile.ci index ca594138..7b0ff2a9 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -22,14 +22,14 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/*; COPY cvat/requirements/ /tmp/cvat/requirements/ -COPY utils/cli/requirements/ /tmp/utils/cli/requirements/ +COPY cvat-cli ${HOME}/cvat-cli RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/cvat/requirements/${DJANGO_CONFIGURATION}.txt && \ - python3 -m pip install --no-cache-dir -r /tmp/utils/cli/requirements/testing.txt && \ + python3 -m pip install --no-cache-dir -r ${HOME}/cvat-cli/requirements/testing.txt && \ + python3 -m pip install --no-cache-dir ${HOME}/cvat-cli && \ python3 -m pip install --no-cache-dir coveralls RUN gem install coveralls-lcov -COPY utils ${HOME}/utils COPY cvat-core ${HOME}/cvat-core COPY cvat-data ${HOME}/cvat-data COPY package*.json ${HOME}/ diff --git a/cvat-cli/MANIFEST.in b/cvat-cli/MANIFEST.in new file mode 100644 index 00000000..e56c92e0 --- /dev/null +++ b/cvat-cli/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include requirements/base.txt \ No newline at end of file diff --git a/cvat-cli/README.md b/cvat-cli/README.md new file mode 100644 index 00000000..7caa57da --- /dev/null +++ b/cvat-cli/README.md @@ -0,0 +1,11 @@ +# Command-line client for CVAT + +## Installation + +`pip install cvat-cli/` + +## Usage + +```bash +$ cvat-cli --help +``` diff --git a/cvat-cli/developer_guide.md b/cvat-cli/developer_guide.md new file mode 100644 index 00000000..0bd47bd1 --- /dev/null +++ b/cvat-cli/developer_guide.md @@ -0,0 +1,19 @@ +# Developer guide + +Install testing requirements: + +```bash +pip install -r requirements/testing.txt +``` + +Run unit tests: +``` +cd cvat/ +python manage.py test --settings cvat.settings.testing cvat-cli/ +``` + +Install package in the editable mode: + +```bash +pip install -e . +``` diff --git a/cvat-cli/pyproject.toml b/cvat-cli/pyproject.toml new file mode 100644 index 00000000..1870a2ed --- /dev/null +++ b/cvat-cli/pyproject.toml @@ -0,0 +1,4 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + diff --git a/utils/cli/requirements/base.txt b/cvat-cli/requirements/base.txt similarity index 100% rename from utils/cli/requirements/base.txt rename to cvat-cli/requirements/base.txt diff --git a/cvat-cli/requirements/testing.txt b/cvat-cli/requirements/testing.txt new file mode 100644 index 00000000..ed69e1b5 --- /dev/null +++ b/cvat-cli/requirements/testing.txt @@ -0,0 +1,4 @@ +-r base.txt + +# We depend on the server in the tests +-r ../../cvat/requirements/testing.txt diff --git a/cvat-cli/setup.py b/cvat-cli/setup.py new file mode 100644 index 00000000..ba23a65a --- /dev/null +++ b/cvat-cli/setup.py @@ -0,0 +1,67 @@ +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os.path as osp +import re + +import setuptools + + +def find_version(project_dir=None): + if not project_dir: + project_dir = osp.dirname(osp.abspath(__file__)) + + file_path = osp.join(project_dir, "version.py") + + with open(file_path, "r") as version_file: + version_text = version_file.read() + + # PEP440: + # https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions + pep_regex = r"([1-9]\d*!)?(0|[1-9]\d*)(\.(0|[1-9]\d*))*((a|b|rc)(0|[1-9]\d*))?(\.post(0|[1-9]\d*))?(\.dev(0|[1-9]\d*))?" + version_regex = r"VERSION\s*=\s*.(" + pep_regex + ")." + match = re.match(version_regex, version_text) + if not match: + raise RuntimeError("Failed to find version string in '%s'" % file_path) + + version = version_text[match.start(1) : match.end(1)] + return version + + +BASE_REQUIREMENTS_FILE = "requirements/base.txt" + + +def parse_requirements(filename=BASE_REQUIREMENTS_FILE): + with open(filename) as fh: + return fh.readlines() + + +BASE_REQUIREMENTS = parse_requirements(BASE_REQUIREMENTS_FILE) + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="cvat-cli", + version=find_version(project_dir="src/cvat_cli"), + description="Command-line client for CVAT", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/openvinotoolkit/cvat/", + package_dir={"": "src"}, + packages=setuptools.find_packages(where='src'), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires=">=3.7", + install_requires=BASE_REQUIREMENTS, + entry_points={ + "console_scripts": [ + "cvat-cli=cvat_cli.__main__:main", + ], + }, + include_package_data=True, +) diff --git a/cvat-cli/src/cvat_cli/__init__.py b/cvat-cli/src/cvat_cli/__init__.py new file mode 100644 index 00000000..eed40849 --- /dev/null +++ b/cvat-cli/src/cvat_cli/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2020-2022 Intel Corporation +# +# SPDX-License-Identifier: MIT diff --git a/utils/cli/cli.py b/cvat-cli/src/cvat_cli/__main__.py similarity index 88% rename from utils/cli/cli.py rename to cvat-cli/src/cvat_cli/__main__.py index dd02d076..db7d8557 100755 --- a/utils/cli/cli.py +++ b/cvat-cli/src/cvat_cli/__main__.py @@ -1,12 +1,13 @@ -#!/usr/bin/env python3 +# Copyright (C) 2020-2022 Intel Corporation # # SPDX-License-Identifier: MIT + import logging import requests import sys from http.client import HTTPConnection -from core.core import CLI, CVAT_API_V2 -from core.definition import parser +from cvat_cli.core.core import CLI, CVAT_API_V2 +from cvat_cli.core.definition import parser log = logging.getLogger(__name__) @@ -45,6 +46,8 @@ def main(): requests.exceptions.RequestException) as e: log.critical(e) + return 0 + if __name__ == '__main__': - main() + sys.exit(main()) diff --git a/utils/cli/core/__init__.py b/cvat-cli/src/cvat_cli/core/__init__.py similarity index 72% rename from utils/cli/core/__init__.py rename to cvat-cli/src/cvat_cli/core/__init__.py index c97e610a..f5a79293 100644 --- a/utils/cli/core/__init__.py +++ b/cvat-cli/src/cvat_cli/core/__init__.py @@ -1,3 +1,6 @@ +# Copyright (C) 2020-2022 Intel Corporation +# # SPDX-License-Identifier: MIT + from .definition import parser, ResourceType # noqa from .core import CLI, CVAT_API_V2 # noqa diff --git a/utils/cli/core/core.py b/cvat-cli/src/cvat_cli/core/core.py similarity index 100% rename from utils/cli/core/core.py rename to cvat-cli/src/cvat_cli/core/core.py diff --git a/utils/cli/core/definition.py b/cvat-cli/src/cvat_cli/core/definition.py similarity index 98% rename from utils/cli/core/definition.py rename to cvat-cli/src/cvat_cli/core/definition.py index ca2a3c55..9749ffc3 100644 --- a/utils/cli/core/definition.py +++ b/cvat-cli/src/cvat_cli/core/definition.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2021-2022 Intel Corporation # # SPDX-License-Identifier: MIT import argparse @@ -7,6 +7,7 @@ import json import logging import os from enum import Enum +from ..version import VERSION def get_auth(s): @@ -53,6 +54,8 @@ class ResourceType(Enum): parser = argparse.ArgumentParser( description='Perform common operations related to CVAT tasks.\n\n' ) +parser.add_argument("--version", action="version", version=VERSION) + task_subparser = parser.add_subparsers(dest='action') ####################################################################### diff --git a/utils/cli/core/utils.py b/cvat-cli/src/cvat_cli/core/utils.py similarity index 100% rename from utils/cli/core/utils.py rename to cvat-cli/src/cvat_cli/core/utils.py diff --git a/cvat-cli/src/cvat_cli/version.py b/cvat-cli/src/cvat_cli/version.py new file mode 100644 index 00000000..72bdd01c --- /dev/null +++ b/cvat-cli/src/cvat_cli/version.py @@ -0,0 +1 @@ +VERSION = "0.1" diff --git a/cvat-cli/tests/__init__.py b/cvat-cli/tests/__init__.py new file mode 100644 index 00000000..eed40849 --- /dev/null +++ b/cvat-cli/tests/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2020-2022 Intel Corporation +# +# SPDX-License-Identifier: MIT diff --git a/utils/cli/tests/test_cli.py b/cvat-cli/tests/test_cli.py similarity index 98% rename from utils/cli/tests/test_cli.py rename to cvat-cli/tests/test_cli.py index 867ccb46..e329b3ac 100644 --- a/utils/cli/tests/test_cli.py +++ b/cvat-cli/tests/test_cli.py @@ -16,7 +16,7 @@ from datumaro.util.scope import scoped, on_exit_do import cvat.apps.engine.log as log from cvat.apps.engine.tests.test_rest_api import (create_db_users, generate_image_file) -from utils.cli.core import CLI, CVAT_API_V2, ResourceType +from cvat_cli.core import CLI, CVAT_API_V2, ResourceType from tqdm import tqdm @@ -24,7 +24,7 @@ class TestCLI(APITestCase): def setUp(self): self._stdout_handler = redirect_stdout(io.StringIO()) mock_stdout = self._stdout_handler.__enter__() - log = logging.getLogger("utils.cli") + log = logging.getLogger("cvat_cli") log.propagate = False log.setLevel(logging.INFO) log.handlers.clear() @@ -75,7 +75,7 @@ class TestTaskOperations(APITestCase): def setUp(self): self._stdout_handler = redirect_stdout(io.StringIO()) mock_stdout = self._stdout_handler.__enter__() - log = logging.getLogger("utils.cli") + log = logging.getLogger("cvat_cli") log.propagate = False log.setLevel(logging.INFO) log.handlers.clear() diff --git a/site/content/en/docs/manual/advanced/cli.md b/site/content/en/docs/manual/advanced/cli.md index 9b37dde0..90cdd5e7 100644 --- a/site/content/en/docs/manual/advanced/cli.md +++ b/site/content/en/docs/manual/advanced/cli.md @@ -2,7 +2,7 @@ title: 'Command line interface (CLI)' linkTitle: 'CLI' weight: 29 -description: 'Guide to working with CVAT tasks in the command line interface. This section on [GitHub](https://github.com/openvinotoolkit/cvat/tree/develop/utils/cli).' +description: 'Guide to working with CVAT tasks in the command line interface. This section on [GitHub](https://github.com/openvinotoolkit/cvat/tree/develop/cvat-cli).' --- ## Description @@ -28,15 +28,13 @@ To access the CLI, you need to have python in environment, as well as a clone of the CVAT repository and the necessary modules: ```bash -git clone https://github.com/openvinotoolkit/cvat.git -cd cvat/utils/cli -pip install -r requirements.txt +pip install 'git+https://github.com/openvinotoolkit/cvat#subdirectory=cvat-cli' ``` -You will get help with `cli.py`. +You can get help with `cvat-cli --help`. ``` -usage: cli.py [-h] [--auth USER:[PASS]] [--server-host SERVER_HOST] +usage: cvat-cli [-h] [--auth USER:[PASS]] [--server-host SERVER_HOST] [--server-port SERVER_PORT] [--debug] {create,delete,ls,frames,dump,upload,export,import} ... @@ -61,10 +59,10 @@ optional arguments: You can get help for each positional argument, e.g. `ls`: ```bash -cli.py ls -h +cvat-cli ls -h ``` ``` -usage: cli.py ls [-h] [--json] +usage: cvat-cli ls [-h] [--json] List all CVAT tasks in simple or JSON format. @@ -103,25 +101,25 @@ by using the [label constructor](/docs/manual/basics/creating_an_annotation_task - Create a task named "new task" on the default server "localhost:8080", labels from the file "labels.json" and local images "file1.jpg" and "file2.jpg", the task will be created as current user: ```bash - cli.py create "new task" --labels labels.json local file1.jpg file2.jpg + cvat-cli create "new task" --labels labels.json local file1.jpg file2.jpg ``` - Create a task named "task 1" on the server "example.com" labels from the file "labels.json" and local image "image1.jpg", the task will be created as user "user-1": ```bash - cli.py --server-host example.com --auth user-1 create "task 1" \ + cvat-cli --server-host example.com --auth user-1 create "task 1" \ --labels labels.json local image1.jpg ``` - Create a task named "task 1", labels from the project with id 1 and with a remote video file, the task will be created as user "user-1": ```bash - cli.py --auth user-1:password create "task 1" --project_id 1 \ + cvat-cli --auth user-1:password create "task 1" --project_id 1 \ remote https://github.com/opencv/opencv/blob/master/samples/data/vtest.avi?raw=true ``` - Create a task named "task 1 sort random", with labels "cat" and "dog", with chunk size 8, with sorting-method random, frame step 10, copy the data on the CVAT server, with use zip chunks and the video file will be taken from the shared resource: ```bash - cli.py create "task 1 sort random" --labels '[{"name": "cat"},{"name": "dog"}]' --chunk_size 8 \ + cvat-cli create "task 1 sort random" --labels '[{"name": "cat"},{"name": "dog"}]' --chunk_size 8 \ --sorting-method random --frame_step 10 --copy_data --use_zip_chunks share //share/dataset_1/video.avi ``` - Create a task named "task from dataset_1", labels from the file "labels.json", with link to bug tracker, @@ -129,21 +127,21 @@ by using the [label constructor](/docs/manual/basics/creating_an_annotation_task from the file "annotation.xml", the data will be loaded from "dataset_1/images/", the task will be created as user "user-2", and the password will need to be entered additionally: ```bash - cli.py --auth user-2 create "task from dataset_1" --labels labels.json \ + cvat-cli --auth user-2 create "task from dataset_1" --labels labels.json \ --bug_tracker https://bug-tracker.com/0001 --image_quality 75 --annotation_path annotation.xml \ --annotation_format "CVAT 1.1" local dataset_1/images/ ``` - Create a task named "segmented task 1", labels from the file "labels.json", with overlay size 5, segment size 100, with frames 5 through 705, using cache and with a remote video file: ```bash - cli.py create "segmented task 1" --labels labels.json --overlap 5 --segment_size 100 \ + cvat-cli create "segmented task 1" --labels labels.json --overlap 5 --segment_size 100 \ --start_frame 5 --stop_frame 705 --use_cache \ remote https://github.com/opencv/opencv/blob/master/samples/data/vtest.avi?raw=true ``` - Create a task named "task 1 with sync annotation", with label "person", with annotation storage in `git` repository, enable `lfs` and the image files from the shared resource: ```bash - cli.py create "task 1 with sync annotation" --labels '[{"name": "person"}]' \ + cvat-cli create "task 1 with sync annotation" --labels '[{"name": "person"}]' \ --dataset_repository_url https://github.com/user/dataset/blob/main/annotation/anno_file_name.zip \ --lfs share //share/large_dataset/images/ ``` @@ -152,55 +150,55 @@ by using the [label constructor](/docs/manual/basics/creating_an_annotation_task - Delete tasks with id "100", "101", "102" , the command will be executed from "user-1" having delete permissions: ```bash - cli.py --auth user-1:password delete 100 101 102 + cvat-cli --auth user-1:password delete 100 101 102 ``` ### List - List all tasks: ```bash - cli.py ls + cvat-cli ls ``` - Save list of all tasks into file "list_of_tasks.json": ```bash - cli.py ls --json > list_of_tasks.json + cvat-cli ls --json > list_of_tasks.json ``` ### Frames - Save frame 12, 15, 22 from task with id 119, into "images" folder with compressed quality: ```bash - cli.py frames --outdir images --quality compressed 119 12 15 22 + cvat-cli frames --outdir images --quality compressed 119 12 15 22 ``` ### Dump annotation - Dump annotation task with id 103, in the format `CVAT for images 1.1` and save to the file "output.zip": ```bash - cli.py dump --format "CVAT for images 1.1" 103 output.zip + cvat-cli dump --format "CVAT for images 1.1" 103 output.zip ``` - Dump annotation task with id 104, in the format `COCO 1.0` and save to the file "output.zip": ```bash - cli.py dump --format "COCO 1.0" 104 output.zip + cvat-cli dump --format "COCO 1.0" 104 output.zip ``` ### Upload annotation - Upload annotation into task with id 105, in the format `CVAT 1.1` from the file "annotation.xml": ```bash - cli.py upload --format "CVAT 1.1" 105 annotation.xml + cvat-cli upload --format "CVAT 1.1" 105 annotation.xml ``` ### Export task - Export task with id 136 to file "task_136.zip": ```bash - cli.py export 136 task_136.zip + cvat-cli export 136 task_136.zip ``` ### Import - Import task from file "task_backup.zip": ```bash - cli.py import task_backup.zip + cvat-cli import task_backup.zip ``` diff --git a/utils/cli/requirements/testing.txt b/utils/cli/requirements/testing.txt deleted file mode 100644 index 01b4c5f5..00000000 --- a/utils/cli/requirements/testing.txt +++ /dev/null @@ -1,4 +0,0 @@ --r base.txt - -# We depend on the server in tests --r ../../../cvat/requirements/testing.txt diff --git a/utils/cli/tests/__init__.py b/utils/cli/tests/__init__.py deleted file mode 100644 index e69de29b..00000000