From 87f07fd3918611a440f58fdb7d0fcaa02539c4c6 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Tue, 14 Feb 2023 12:39:55 +0300 Subject: [PATCH] Various improvements to the SDK generation process (#5678) --- .github/workflows/full.yml | 4 +- .github/workflows/helm.yml | 4 +- .github/workflows/main.yml | 4 +- .github/workflows/schedule.yml | 4 +- cvat-sdk/.gitignore | 4 +- cvat-sdk/.openapi-generator-ignore | 2 + cvat-sdk/.remarkignore | 1 - cvat-sdk/{gen/templates => }/MANIFEST.in | 0 .../README.md.template => README.md} | 0 cvat-sdk/gen/generate.sh | 33 ++-- cvat-sdk/gen/generator-config.yml | 1 - cvat-sdk/gen/postprocess.py | 2 + .../.openapi-generator-ignore | 40 ---- .../openapi-generator/README.mustache | 36 ---- .../openapi-generator/README_common.mustache | 176 ------------------ cvat-sdk/gen/templates/version.py.template | 1 - .../{gen/templates => }/requirements/base.txt | 0 17 files changed, 24 insertions(+), 288 deletions(-) rename cvat-sdk/{gen/templates => }/MANIFEST.in (100%) rename cvat-sdk/{gen/templates/README.md.template => README.md} (100%) delete mode 100644 cvat-sdk/gen/templates/openapi-generator/.openapi-generator-ignore delete mode 100644 cvat-sdk/gen/templates/openapi-generator/README.mustache delete mode 100644 cvat-sdk/gen/templates/openapi-generator/README_common.mustache delete mode 100644 cvat-sdk/gen/templates/version.py.template rename cvat-sdk/{gen/templates => }/requirements/base.txt (100%) diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index 6143c4db..b6fd0362 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -78,9 +78,7 @@ jobs: --entrypoint /bin/bash -u root cvat/server \ -c 'python manage.py spectacular --file /transfer/schema.yml' pip3 install --user -r cvat-sdk/gen/requirements.txt - cd cvat-sdk/ - gen/generate.sh - cd .. + ./cvat-sdk/gen/generate.sh cp -r cvat-sdk/* /tmp/cvat_sdk/ diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index f936e8eb..6ac0fcab 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -72,9 +72,7 @@ jobs: kubectl exec $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}') -- /bin/bash -c "python manage.py spectacular --file /tmp/schema.yml" kubectl cp $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}'):/tmp/schema.yml cvat-sdk/schema/schema.yml pip3 install --user -r cvat-sdk/gen/requirements.txt - cd cvat-sdk/ - gen/generate.sh - cd .. + ./cvat-sdk/gen/generate.sh - name: Install test requirements run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5282b979..f62ba91f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,9 +79,7 @@ jobs: --entrypoint /bin/bash -u root cvat/server \ -c 'python manage.py spectacular --file /transfer/schema.yml' pip3 install --user -r cvat-sdk/gen/requirements.txt - cd cvat-sdk/ - gen/generate.sh - cd .. + ./cvat-sdk/gen/generate.sh cp -r cvat-sdk/* /tmp/cvat_sdk/ diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index da28cc7e..0bb46cce 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -203,9 +203,7 @@ jobs: --entrypoint /bin/bash -u root cvat/server \ -c 'python manage.py spectacular --file /transfer/schema.yml' pip3 install --user -r cvat-sdk/gen/requirements.txt - cd cvat-sdk/ - gen/generate.sh - cd .. + ./cvat-sdk/gen/generate.sh pip3 install --user 'cvat-sdk/[pytorch]' pip3 install --user cvat-cli/ diff --git a/cvat-sdk/.gitignore b/cvat-sdk/.gitignore index d01a61d1..867ef353 100644 --- a/cvat-sdk/.gitignore +++ b/cvat-sdk/.gitignore @@ -72,8 +72,6 @@ schema/ # Generated code /cvat_sdk/api_client/ /cvat_sdk/version.py -/requirements/ /docs/ +/requirements/api_client.txt /setup.py -/README.md -/MANIFEST.in \ No newline at end of file diff --git a/cvat-sdk/.openapi-generator-ignore b/cvat-sdk/.openapi-generator-ignore index 44b0733a..7133b340 100644 --- a/cvat-sdk/.openapi-generator-ignore +++ b/cvat-sdk/.openapi-generator-ignore @@ -23,6 +23,7 @@ #!docs/README.md # For safety +/MANIFEST.in /cvat_sdk/__init__.py /config /gen @@ -31,6 +32,7 @@ /types.py # Don't generate these files +/README.md /git_push.sh /setup.cfg /test-requirements.txt diff --git a/cvat-sdk/.remarkignore b/cvat-sdk/.remarkignore index 4f1b9bbf..0b28d478 100644 --- a/cvat-sdk/.remarkignore +++ b/cvat-sdk/.remarkignore @@ -1,2 +1 @@ cvat-sdk/docs/ -cvat-sdk/README.md diff --git a/cvat-sdk/gen/templates/MANIFEST.in b/cvat-sdk/MANIFEST.in similarity index 100% rename from cvat-sdk/gen/templates/MANIFEST.in rename to cvat-sdk/MANIFEST.in diff --git a/cvat-sdk/gen/templates/README.md.template b/cvat-sdk/README.md similarity index 100% rename from cvat-sdk/gen/templates/README.md.template rename to cvat-sdk/README.md diff --git a/cvat-sdk/gen/generate.sh b/cvat-sdk/gen/generate.sh index bdb76867..9b2068e7 100755 --- a/cvat-sdk/gen/generate.sh +++ b/cvat-sdk/gen/generate.sh @@ -11,35 +11,35 @@ GENERATOR_VERSION="v6.0.1" VERSION="2.3.0" LIB_NAME="cvat_sdk" LAYER1_LIB_NAME="${LIB_NAME}/api_client" -DST_DIR="." -TEMPLATE_DIR="gen" +DST_DIR="$(cd "$(dirname -- "$0")/.." && pwd)" +TEMPLATE_DIR_NAME="gen" +TEMPLATE_DIR="$DST_DIR/$TEMPLATE_DIR_NAME" POST_PROCESS_SCRIPT="${TEMPLATE_DIR}/postprocess.py" -mkdir -p "${DST_DIR}/" -rm -f -r "${DST_DIR}/docs" "${DST_DIR}/${LAYER1_LIB_NAME}" "requirements/" -cp "${TEMPLATE_DIR}/templates/openapi-generator/.openapi-generator-ignore" "${DST_DIR}/" +rm -f -r "${DST_DIR}/docs" "${DST_DIR}/${LAYER1_LIB_NAME}" \ + "${DST_DIR}/requirements/api_client.txt" # Pass template dir here # https://github.com/OpenAPITools/openapi-generator/issues/8420 -docker run --rm -v "$PWD":"/local" -u "$(id -u)":"$(id -g)" \ +docker run --rm -v "$DST_DIR:/local" -u "$(id -u)":"$(id -g)" \ openapitools/openapi-generator-cli:${GENERATOR_VERSION} generate \ - -t "/local/${TEMPLATE_DIR}/templates/openapi-generator/" \ + -t "/local/${TEMPLATE_DIR_NAME}/templates/openapi-generator/" \ -i "/local/schema/schema.yml" \ - --config "/local/${TEMPLATE_DIR}/generator-config.yml" \ + --config "/local/${TEMPLATE_DIR_NAME}/generator-config.yml" \ + -p "packageVersion=$VERSION" \ -g python \ - -o "/local/${DST_DIR}/" + -o "/local/" -sed -e "s|{{packageVersion}}|${VERSION}|g" "${TEMPLATE_DIR}/templates/version.py.template" > "${DST_DIR}/${LIB_NAME}/version.py" -cp -r "${TEMPLATE_DIR}/templates/requirements" "${DST_DIR}/" -cp -r "${TEMPLATE_DIR}/templates/MANIFEST.in" "${DST_DIR}/" +echo "VERSION = \"$VERSION\"" > "${DST_DIR}/${LIB_NAME}/version.py" mv "${DST_DIR}/requirements.txt" "${DST_DIR}/requirements/api_client.txt" # Do custom postprocessing for code files -"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/${LIB_NAME}" +"${POST_PROCESS_SCRIPT}" --schema "${DST_DIR}/schema/schema.yml" \ + --input-path "${DST_DIR}/${LIB_NAME}" # Do custom postprocessing for docs files -"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/docs" --file-ext '.md' -"${POST_PROCESS_SCRIPT}" --schema "schema/schema.yml" --input-path "${DST_DIR}/README.md" +"${POST_PROCESS_SCRIPT}" --schema "${DST_DIR}/schema/schema.yml" \ + --input-path "${DST_DIR}/docs" --file-ext '.md' API_DOCS_DIR="${DST_DIR}/docs/apis/" MODEL_DOCS_DIR="${DST_DIR}/docs/models/" @@ -47,6 +47,3 @@ mkdir "${API_DOCS_DIR}" mkdir "${MODEL_DOCS_DIR}" mv "${DST_DIR}/docs/"*Api.md "${API_DOCS_DIR}" mv "${DST_DIR}/docs/"*.md "${MODEL_DOCS_DIR}" -mv "${DST_DIR}/README.md" "${DST_DIR}/docs/" - -cp "${TEMPLATE_DIR}/templates/README.md.template" "${DST_DIR}/README.md" diff --git a/cvat-sdk/gen/generator-config.yml b/cvat-sdk/gen/generator-config.yml index 614ecb3c..b171facb 100644 --- a/cvat-sdk/gen/generator-config.yml +++ b/cvat-sdk/gen/generator-config.yml @@ -1,6 +1,5 @@ additionalProperties: projectName: "cvat_sdk" - packageVersion: "2.0-alpha" packageUrl: "https://github.com/cvat-ai/cvat" packageName: "cvat_sdk.api_client" initRequiredVars: true diff --git a/cvat-sdk/gen/postprocess.py b/cvat-sdk/gen/postprocess.py index 376ad2f3..cb0bfbbe 100755 --- a/cvat-sdk/gen/postprocess.py +++ b/cvat-sdk/gen/postprocess.py @@ -161,6 +161,8 @@ def main(args=None): processor.process_dir(args.input_path, file_ext=args.file_ext) elif osp.isfile(args.input_path): processor.process_file(args.input_path) + else: + return f"error: input {args.input_path} is neither a file nor a directory" return 0 diff --git a/cvat-sdk/gen/templates/openapi-generator/.openapi-generator-ignore b/cvat-sdk/gen/templates/openapi-generator/.openapi-generator-ignore deleted file mode 100644 index 44b0733a..00000000 --- a/cvat-sdk/gen/templates/openapi-generator/.openapi-generator-ignore +++ /dev/null @@ -1,40 +0,0 @@ -# OpenAPI Generator Ignore -# Generated by openapi-generator https://github.com/openapitools/openapi-generator - -# Use this file to prevent files from being overwritten by the generator. -# The patterns follow closely to .gitignore or .dockerignore. - -# As an example, the C# client generator defines ApiClient.cs. -# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: -#ApiClient.cs - -# You can match any string of characters against a directory, file or extension with a single asterisk (*): -#foo/*/qux -# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux - -# You can recursively match patterns against a directory, file or extension with a double asterisk (**): -#foo/**/qux -# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux - -# You can also negate patterns with an exclamation (!). -# For example, you can ignore all files in a docs folder with the file extension .md: -#docs/*.md -# Then explicitly reverse the ignore rule for a single file: -#!docs/README.md - -# For safety -/cvat_sdk/__init__.py -/config -/gen -/helpers.py -/utils.py -/types.py - -# Don't generate these files -/git_push.sh -/setup.cfg -/test-requirements.txt -/tox.ini -/.gitlab-ci.yml -/.travis.yml -/.gitignore diff --git a/cvat-sdk/gen/templates/openapi-generator/README.mustache b/cvat-sdk/gen/templates/openapi-generator/README.mustache deleted file mode 100644 index 95480268..00000000 --- a/cvat-sdk/gen/templates/openapi-generator/README.mustache +++ /dev/null @@ -1,36 +0,0 @@ -# {{{projectName}}} -{{#appDescriptionWithNewLines}} -{{{.}}} -{{/appDescriptionWithNewLines}} - -This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project: - -- API version: {{appVersion}} -- Package version: {{packageVersion}} -{{^hideGenerationTimestamp}} -- Build date: {{generatedDate}} -{{/hideGenerationTimestamp}} -- Build package: {{generatorClass}} -{{#infoUrl}} -For more information, please visit [{{{infoUrl}}}]({{{infoUrl}}}) -{{/infoUrl}} - -## Installation & Usage - -To install a prebuilt package, run the following command in the terminal: - -```sh -pip install cvat-sdk -``` - -To install from the local directory, follow [the developer guide](https://opencv.github.io/cvat/docs/api_sdk/sdk/developer_guide). - -After installation you can import the package: - -```python -import cvat_sdk -``` - -## Getting Started - -{{> README_common }} diff --git a/cvat-sdk/gen/templates/openapi-generator/README_common.mustache b/cvat-sdk/gen/templates/openapi-generator/README_common.mustache deleted file mode 100644 index a14bbfe3..00000000 --- a/cvat-sdk/gen/templates/openapi-generator/README_common.mustache +++ /dev/null @@ -1,176 +0,0 @@ -API includes 2 layers: -- REST API wrappers (`ApiClient`). Located in at `cvat_sdk.api_client` -- high-level tools (`core`). Located at `cvat_sdk.core` - -### `ApiClient` (low level) - -This layer is useful if you need to work directly with REST API, but want -to have data validation and syntax assistance from your code editor. The code -on this layer is autogenerated. - -#### Example - -Let's see how a task with local files can be created. We will use the basic auth -to make things simpler. - -```python -from time import sleep -from cvat_sdk.api_client import Configuration, ApiClient, models, apis, exceptions - -configuration = Configuration( - host="{{{basePath}}}", - username='YOUR_USERNAME', - password='YOUR_PASSWORD', -) - -# Enter a context with an instance of the API client -with ApiClient(configuration) as api_client: - # Parameters can be passed as a plain dict with JSON-serialized data - # or as model objects (from cvat_sdk.api_client.models), including - # mixed variants. - # - # In case of dicts, keys must be the same as members of models.I - # interfaces and values must be convertible to the corresponding member - # value types (e.g. a date or string enum value can be parsed from a string). - # - # In case of model objects, data must be of the corresponding - # models. types. - # - # Let's use a dict here. It should look like models.ITaskWriteRequest - task_spec = { - 'name': 'example task', - "labels": [{ - "name": "car", - "color": "#ff00ff", - "attributes": [ - { - "name": "a", - "mutable": True, - "input_type": "number", - "default_value": "5", - "values": ["4", "5", "6"] - } - ] - }], - } - - try: - # Apis can be accessed as ApiClient class members - # We use different models for input and output data. For input data, - # models are typically called like "*Request". Output data models have - # no suffix. - (task, response) = api_client.tasks_api.create(task_spec) - except exceptions.ApiException as e: - # We can catch the basic exception type, or a derived type - print("Exception when trying to create a task: %s\n" % e) - - # Here we will use models instead of a dict - task_data = models.DataRequest( - image_quality=75, - start_frame=2, - stop_frame=5, - client_files=[ - open('image1.jpg', 'rb'), - open('image2.jpg', 'rb'), - ], - ) - - # If we pass binary file objects, we need to specify content type. - # For this endpoint, we don't have response data - (_, response) = api_client.tasks_api.create_data(task.id, - data_request=task_data, - _content_type="multipart/form-data", - - # we can choose to check the response status manually - # and disable the response data parsing - _check_status=False, _parse_response=False - ) - assert response.status == 202, response.msg - - # Wait till task data is processed - for _ in range(100): - (status, _) = api_client.tasks_api.retrieve_status(task.id) - if status.state.value in ['Finished', 'Failed']: - break - sleep(0.1) - assert status.state.value == 'Finished', status.message - - # Update the task object and check the task size - (task, _) = api_client.tasks_api.retrieve(task.id) - assert task.size == 4 -``` - -### `Core` (high-level) - -This layer provides high-level APIs, allowing easier access to server operations. -API includes *Repositories* and *Entities*. Repositories provide management -operations for Entitites. Entitites represent separate objects on the server -(e.g. tasks, jobs etc). - -#### Example - -```python -from cvat_sdk import make_client, models -from cvat_sdk.core.proxies.tasks import ResourceType, Task - -with make_client(host="{{{basePath}}}") as client: - # Authorize using the basic auth - client.login(('YOUR_USERNAME', 'YOUR_PASSWORD')) - - # Models are used the same way as in the layer 1 - task_spec = { - "name": "example task 2", - "labels": [ - { - "name": "car", - "color": "#ff00ff", - "attributes": [ - { - "name": "a", - "mutable": True, - "input_type": "number", - "default_value": "5", - "values": ["4", "5", "6"], - } - ], - } - ], - } - - # Different repositories can be accessed as the Client class members. - # They may provide both simple and complex operations, - # such as entity creation, retrieval and removal. - task = client.tasks.create_from_data( - spec=task_spec, - resource_type=ResourceType.LOCAL, - resources=['image1.jpg', 'image2.png'], - ) - - # Task object is already up-to-date with its server counterpart - assert task.size == 2 - - # An entity needs to be fetch()-ed to reflect the latest changes. - # It can be update()-d and remove()-d depending on the entity type. - task.update({'name': 'mytask'}) - task.remove() -``` - -## Available API Endpoints - -All URIs are relative to _{{basePath}}_ - -Class | Method | HTTP request | Description ------------- | ------------- | ------------- | ------------- -{{#apiInfo}}{{#apis}}{{#operations}}{{#operation}}_{{classname}}_ | [**{{>operation_name}}**](apis/{{classname}}#{{>operation_name}}) | **{{httpMethod}}** {{path}} | {{summary}} -{{/operation}}{{/operations}}{{/apis}}{{/apiInfo}} - -## Available Models - -{{#models}}{{#model}} - {{{classname}}} -{{/model}}{{/models}} - - -## Author - -{{#apiInfo}}{{#apis}}{{#-last}}{{infoEmail}} -{{/-last}}{{/apis}}{{/apiInfo}} diff --git a/cvat-sdk/gen/templates/version.py.template b/cvat-sdk/gen/templates/version.py.template deleted file mode 100644 index 9d40f77c..00000000 --- a/cvat-sdk/gen/templates/version.py.template +++ /dev/null @@ -1 +0,0 @@ -VERSION = "{{packageVersion}}" diff --git a/cvat-sdk/gen/templates/requirements/base.txt b/cvat-sdk/requirements/base.txt similarity index 100% rename from cvat-sdk/gen/templates/requirements/base.txt rename to cvat-sdk/requirements/base.txt