Migrate to using pathlib in most of the SDK (#5435)

For user-facing functions, keep accepting `str` paths to maintain
compatibility and flexibility, but add support for arbitrary path-like
objects. For internal functions (in `downloading.py` and
`uploading.py`), don't bother and require `pathlib.Path`.

The only code that isn't converted is build-time code (e.g. `setup.py`)
and code that came from openapi-generator.

CHANGELOG.md

@@ -29,6 +29,8 @@ from online detectors & interactors) (<https://github.com/opencv/cvat/pull/4543>
 - Allowed trailing slashes in the SDK host address (<https://github.com/opencv/cvat/pull/5057>)
 - Adjusted initial camera position, enabled 'Reset zoom' option for 3D canvas (<https://github.com/opencv/cvat/pull/5395>)
 - Enabled authentication via email (<https://github.com/opencv/cvat/pull/5037>)
+- In the SDK, functions taking paths as strings now also accept path-like objects
+  (<https://github.com/opencv/cvat/pull/5435>)
 
 ### Deprecated
 - TDB

cvat-sdk/cvat_sdk/core/downloading.py

@@ -5,8 +5,8 @@
 
 from __future__ import annotations
 
-import os.path as osp
 from contextlib import closing
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, Optional
 
 from cvat_sdk.api_client.api_client import Endpoint
@@ -28,7 +28,7 @@ class Downloader:
     def download_file(
         self,
         url: str,
-        output_path: str,
+        output_path: Path,
         *,
         timeout: int = 60,
         pbar: Optional[ProgressReporter] = None,
@@ -39,7 +39,7 @@ class Downloader:
 
         CHUNK_SIZE = 10 * 2**20
 
-        assert not osp.exists(output_path)
+        assert not output_path.exists()
 
         response = self._client.api_client.rest_client.GET(
             url,
@@ -70,7 +70,7 @@ class Downloader:
     def prepare_and_download_file_from_endpoint(
         self,
         endpoint: Endpoint,
-        filename: str,
+        filename: Path,
         *,
         url_params: Optional[Dict[str, Any]] = None,
         query_params: Optional[Dict[str, Any]] = None,

cvat-sdk/cvat_sdk/core/proxies/jobs.py

@@ -6,9 +6,8 @@ from __future__ import annotations
 
 import io
 import mimetypes
-import os
+from pathlib import Path
-import os.path as osp
+from typing import TYPE_CHECKING, List, Optional, Sequence
-from typing import List, Optional, Sequence
 
 from PIL import Image
 
@@ -26,6 +25,9 @@ from cvat_sdk.core.proxies.model_proxy import (
 )
 from cvat_sdk.core.uploading import AnnotationUploader
 
+if TYPE_CHECKING:
+    from _typeshed import StrPath
+
 _JobEntityBase, _JobRepoBase = build_model_bases(
     models.JobRead, apis.JobsApi, api_member_name="jobs_api"
 )
@@ -43,7 +45,7 @@ class Job(
     def import_annotations(
         self,
         format_name: str,
-        filename: str,
+        filename: StrPath,
         *,
         status_check_period: Optional[int] = None,
         pbar: Optional[ProgressReporter] = None,
@@ -52,6 +54,8 @@ class Job(
         Upload annotations for a job in the specified format (e.g. 'YOLO ZIP 1.0').
         """
 
+        filename = Path(filename)
+
         AnnotationUploader(self._client).upload_file_and_wait(
             self.api.create_annotations_endpoint,
             filename,
@@ -66,7 +70,7 @@ class Job(
     def export_dataset(
         self,
         format_name: str,
-        filename: str,
+        filename: StrPath,
         *,
         pbar: Optional[ProgressReporter] = None,
         status_check_period: Optional[int] = None,
@@ -75,6 +79,9 @@ class Job(
         """
         Download annotations for a job in the specified format (e.g. 'YOLO ZIP 1.0').
         """
+
+        filename = Path(filename)
+
         if include_images:
             endpoint = self.api.retrieve_dataset_endpoint
         else:
@@ -112,7 +119,7 @@ class Job(
         self,
         frame_ids: Sequence[int],
         *,
-        outdir: str = "",
+        outdir: StrPath = ".",
         quality: str = "original",
         filename_pattern: str = "frame_{frame_id:06d}{frame_ext}",
     ) -> Optional[List[Image.Image]]:
@@ -120,7 +127,9 @@ class Job(
         Download the requested frame numbers for a job and save images as outdir/filename_pattern
         """
         # TODO: add arg descriptions in schema
-        os.makedirs(outdir, exist_ok=True)
+
+        outdir = Path(outdir)
+        outdir.mkdir(parents=True, exist_ok=True)
 
         for frame_id in frame_ids:
             frame_bytes = self.get_frame(frame_id, quality=quality)
@@ -136,7 +145,7 @@ class Job(
                 im_ext = ".jpg"
 
             outfile = filename_pattern.format(frame_id=frame_id, frame_ext=im_ext)
-            im.save(osp.join(outdir, outfile))
+            im.save(outdir / outfile)
 
     def get_meta(self) -> models.IDataMetaRead:
         (meta, _) = self.api.retrieve_data_meta(self.id)

cvat-sdk/cvat_sdk/core/proxies/projects.py

@@ -5,8 +5,8 @@
 from __future__ import annotations
 
 import json
-import os.path as osp
+from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from cvat_sdk.api_client import apis, models
 from cvat_sdk.core.downloading import Downloader
@@ -21,6 +21,9 @@ from cvat_sdk.core.proxies.model_proxy import (
 )
 from cvat_sdk.core.uploading import DatasetUploader, Uploader
 
+if TYPE_CHECKING:
+    from _typeshed import StrPath
+
 _ProjectEntityBase, _ProjectRepoBase = build_model_bases(
     models.ProjectRead, apis.ProjectsApi, api_member_name="projects_api"
 )
@@ -34,7 +37,7 @@ class Project(
     def import_dataset(
         self,
         format_name: str,
-        filename: str,
+        filename: StrPath,
         *,
         status_check_period: Optional[int] = None,
         pbar: Optional[ProgressReporter] = None,
@@ -43,6 +46,8 @@ class Project(
         Import dataset for a project in the specified format (e.g. 'YOLO ZIP 1.0').
         """
 
+        filename = Path(filename)
+
         DatasetUploader(self._client).upload_file_and_wait(
             self.api.create_dataset_endpoint,
             filename,
@@ -57,7 +62,7 @@ class Project(
     def export_dataset(
         self,
         format_name: str,
-        filename: str,
+        filename: StrPath,
         *,
         pbar: Optional[ProgressReporter] = None,
         status_check_period: Optional[int] = None,
@@ -66,6 +71,9 @@ class Project(
         """
         Download annotations for a project in the specified format (e.g. 'YOLO ZIP 1.0').
         """
+
+        filename = Path(filename)
+
         if include_images:
             endpoint = self.api.retrieve_dataset_endpoint
         else:
@@ -84,7 +92,7 @@ class Project(
 
     def download_backup(
         self,
-        filename: str,
+        filename: StrPath,
         *,
         status_check_period: int = None,
         pbar: Optional[ProgressReporter] = None,
@@ -93,6 +101,8 @@ class Project(
         Download a project backup
         """
 
+        filename = Path(filename)
+
         Downloader(self._client).prepare_and_download_file_from_endpoint(
             self.api.retrieve_backup_endpoint,
             filename=filename,
@@ -148,7 +158,7 @@ class ProjectsRepo(
 
     def create_from_backup(
         self,
-        filename: str,
+        filename: StrPath,
         *,
         status_check_period: int = None,
         pbar: Optional[ProgressReporter] = None,
@@ -156,10 +166,13 @@ class ProjectsRepo(
         """
         Import a project from a backup file
         """
+
+        filename = Path(filename)
+
         if status_check_period is None:
             status_check_period = self.config.status_check_period
 
-        params = {"filename": osp.basename(filename)}
+        params = {"filename": filename.name}
         url = self.api_map.make_endpoint_url(self.api.create_backup_endpoint.path)
 
         uploader = Uploader(self)

cvat-sdk/cvat_sdk/core/proxies/tasks.py

@@ -7,10 +7,9 @@ from __future__ import annotations
 import io
 import json
 import mimetypes
-import os
-import os.path as osp
 import shutil
 from enum import Enum
+from pathlib import Path
 from time import sleep
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence
 
@@ -34,7 +33,7 @@ from cvat_sdk.core.uploading import AnnotationUploader, DataUploader, Uploader
 from cvat_sdk.core.utils import filter_dict
 
 if TYPE_CHECKING:
-    from _typeshed import SupportsWrite
+    from _typeshed import StrPath, SupportsWrite
 
 
 class ResourceType(Enum):
@@ -67,7 +66,7 @@ class Task(
     def upload_data(
         self,
         resource_type: ResourceType,
-        resources: Sequence[str],
+        resources: Sequence[StrPath],
         *,
         pbar: Optional[ProgressReporter] = None,
         params: Optional[Dict[str, Any]] = None,
@@ -77,15 +76,8 @@ class Task(
         """
         params = params or {}
 
-        data = {}
+        data = {"image_quality": 70}
-        if resource_type is ResourceType.LOCAL:
-            pass  # handled later
-        elif resource_type is ResourceType.REMOTE:
-            data["remote_files"] = resources
-        elif resource_type is ResourceType.SHARE:
-            data["server_files"] = resources
 
-        data["image_quality"] = 70
         data.update(
             filter_dict(
                 params,
@@ -105,6 +97,15 @@ class Task(
             data["frame_filter"] = f"step={params.get('frame_step')}"
 
         if resource_type in [ResourceType.REMOTE, ResourceType.SHARE]:
+            for resource in resources:
+                if not isinstance(resource, str):
+                    raise TypeError(f"resources: expected instances of str, got {type(resource)}")
+
+            if resource_type is ResourceType.REMOTE:
+                data["remote_files"] = resources
+            elif resource_type is ResourceType.SHARE:
+                data["server_files"] = resources
+
             self.api.create_data(
                 self.id,
                 data_request=models.DataRequest(**data),
@@ -114,12 +115,14 @@ class Task(
                 self.api.create_data_endpoint.path, kwsub={"id": self.id}
             )
 
-            DataUploader(self._client).upload_files(url, resources, pbar=pbar, **data)
+            DataUploader(self._client).upload_files(
+                url, list(map(Path, resources)), pbar=pbar, **data
+            )
 
     def import_annotations(
         self,
         format_name: str,
-        filename: str,
+        filename: StrPath,
         *,
         status_check_period: Optional[int] = None,
         pbar: Optional[ProgressReporter] = None,
@@ -128,6 +131,8 @@ class Task(
         Upload annotations for a task in the specified format (e.g. 'YOLO ZIP 1.0').
         """
 
+        filename = Path(filename)
+
         AnnotationUploader(self._client).upload_file_and_wait(
             self.api.create_annotations_endpoint,
             filename,
@@ -178,7 +183,7 @@ class Task(
         self,
         frame_ids: Sequence[int],
         *,
-        outdir: str = "",
+        outdir: StrPath = ".",
         quality: str = "original",
         filename_pattern: str = "frame_{frame_id:06d}{frame_ext}",
     ) -> Optional[List[Image.Image]]:
@@ -186,7 +191,9 @@ class Task(
         Download the requested frame numbers for a task and save images as outdir/filename_pattern
         """
         # TODO: add arg descriptions in schema
-        os.makedirs(outdir, exist_ok=True)
+
+        outdir = Path(outdir)
+        outdir.mkdir(exist_ok=True)
 
         for frame_id in frame_ids:
             frame_bytes = self.get_frame(frame_id, quality=quality)
@@ -202,12 +209,12 @@ class Task(
                 im_ext = ".jpg"
 
             outfile = filename_pattern.format(frame_id=frame_id, frame_ext=im_ext)
-            im.save(osp.join(outdir, outfile))
+            im.save(outdir / outfile)
 
     def export_dataset(
         self,
         format_name: str,
-        filename: str,
+        filename: StrPath,
         *,
         pbar: Optional[ProgressReporter] = None,
         status_check_period: Optional[int] = None,
@@ -216,6 +223,9 @@ class Task(
         """
         Download annotations for a task in the specified format (e.g. 'YOLO ZIP 1.0').
         """
+
+        filename = Path(filename)
+
         if include_images:
             endpoint = self.api.retrieve_dataset_endpoint
         else:
@@ -234,7 +244,7 @@ class Task(
 
     def download_backup(
         self,
-        filename: str,
+        filename: StrPath,
         *,
         status_check_period: int = None,
         pbar: Optional[ProgressReporter] = None,
@@ -243,6 +253,8 @@ class Task(
         Download a task backup
         """
 
+        filename = Path(filename)
+
         Downloader(self._client).prepare_and_download_file_from_endpoint(
             self.api.retrieve_backup_endpoint,
             filename=filename,
@@ -370,7 +382,7 @@ class TasksRepo(
 
     def create_from_backup(
         self,
-        filename: str,
+        filename: StrPath,
         *,
         status_check_period: int = None,
         pbar: Optional[ProgressReporter] = None,
@@ -378,10 +390,13 @@ class TasksRepo(
         """
         Import a task from a backup file
         """
+
+        filename = Path(filename)
+
         if status_check_period is None:
             status_check_period = self._client.config.status_check_period
 
-        params = {"filename": osp.basename(filename)}
+        params = {"filename": filename.name}
         url = self._client.api_map.make_endpoint_url(self.api.create_backup_endpoint.path)
         uploader = Uploader(self._client)
         response = uploader.upload_file(

cvat-sdk/cvat_sdk/core/uploading.py

@@ -5,8 +5,8 @@
 from __future__ import annotations
 
 import os
-import os.path as osp
 from contextlib import ExitStack, closing
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple
 
 import requests
@@ -144,7 +144,7 @@ class Uploader:
     def upload_file(
         self,
         url: str,
-        filename: str,
+        filename: Path,
         *,
         meta: Dict[str, Any],
         query_params: Dict[str, Any] = None,
@@ -207,15 +207,15 @@ class Uploader:
         )
 
     def _split_files_by_requests(
-        self, filenames: List[str]
+        self, filenames: List[Path]
-    ) -> Tuple[List[Tuple[List[str], int]], List[str], int]:
+    ) -> Tuple[List[Tuple[List[Path], int]], List[Path], int]:
         bulk_files: Dict[str, int] = {}
         separate_files: Dict[str, int] = {}
 
         # sort by size
         for filename in filenames:
-            filename = os.path.abspath(filename)
+            filename = filename.resolve()
-            file_size = os.stat(filename).st_size
+            file_size = filename.stat().st_size
             if MAX_REQUEST_SIZE < file_size:
                 separate_files[filename] = file_size
             else:
@@ -252,7 +252,7 @@ class Uploader:
         return _MyTusUploader(client=client, api_client=api_client, **kwargs)
 
     def _upload_file_data_with_tus(self, url, filename, *, meta=None, pbar=None, logger=None):
-        file_size = os.stat(filename).st_size
+        file_size = filename.stat().st_size
         if pbar is None:
             pbar = NullProgressReporter()
 
@@ -299,7 +299,7 @@ class AnnotationUploader(Uploader):
     def upload_file_and_wait(
         self,
         endpoint: Endpoint,
-        filename: str,
+        filename: Path,
         format_name: str,
         *,
         url_params: Optional[Dict[str, Any]] = None,
@@ -307,7 +307,7 @@ class AnnotationUploader(Uploader):
         status_check_period: Optional[int] = None,
     ):
         url = self._client.api_map.make_endpoint_url(endpoint.path, kwsub=url_params)
-        params = {"format": format_name, "filename": osp.basename(filename)}
+        params = {"format": format_name, "filename": filename.name}
         self.upload_file(
             url, filename, pbar=pbar, query_params=params, meta={"filename": params["filename"]}
         )
@@ -326,7 +326,7 @@ class DatasetUploader(Uploader):
     def upload_file_and_wait(
         self,
         endpoint: Endpoint,
-        filename: str,
+        filename: Path,
         format_name: str,
         *,
         url_params: Optional[Dict[str, Any]] = None,
@@ -334,7 +334,7 @@ class DatasetUploader(Uploader):
         status_check_period: Optional[int] = None,
     ):
         url = self._client.api_map.make_endpoint_url(endpoint.path, kwsub=url_params)
-        params = {"format": format_name, "filename": osp.basename(filename)}
+        params = {"format": format_name, "filename": filename.name}
         self.upload_file(
             url, filename, pbar=pbar, query_params=params, meta={"filename": params["filename"]}
         )
@@ -353,7 +353,7 @@ class DataUploader(Uploader):
     def upload_files(
         self,
         url: str,
-        resources: List[str],
+        resources: List[Path],
         *,
         pbar: Optional[ProgressReporter] = None,
         **kwargs,
@@ -370,7 +370,7 @@ class DataUploader(Uploader):
                 files = {}
                 for i, filename in enumerate(group):
                     files[f"client_files[{i}]"] = (
-                        filename,
+                        os.fspath(filename),
                         es.enter_context(closing(open(filename, "rb"))).read(),
                     )
                 response = self._client.api_client.rest_client.POST(
@@ -392,7 +392,7 @@ class DataUploader(Uploader):
             self._upload_file_data_with_tus(
                 url,
                 filename,
-                meta={"filename": osp.basename(filename)},
+                meta={"filename": filename.name},
                 pbar=pbar,
                 logger=self._client.logger.debug,
             )

tests/python/cli/test_cli.py

@@ -65,7 +65,7 @@ class TestCLI:
         backup_path = self.tmp_path / "backup.zip"
 
         fxt_new_task.import_annotations("COCO 1.0", filename=fxt_coco_file)
-        fxt_new_task.download_backup(str(backup_path))
+        fxt_new_task.download_backup(backup_path)
 
         yield backup_path
 
@@ -79,7 +79,7 @@ class TestCLI:
                 "labels": [{"name": "car"}, {"name": "person"}],
             },
             resource_type=ResourceType.LOCAL,
-            resources=list(map(os.fspath, files)),
+            resources=files,
         )
 
         return task

tests/python/sdk/test_issues_comments.py

@@ -42,7 +42,7 @@ class TestIssuesUsecases:
                 "labels": [{"name": "car"}, {"name": "person"}],
             },
             resource_type=ResourceType.LOCAL,
-            resources=[str(fxt_image_file)],
+            resources=[fxt_image_file],
             data_params={"image_quality": 80},
         )
 
@@ -162,7 +162,7 @@ class TestCommentsUsecases:
                 "labels": [{"name": "car"}, {"name": "person"}],
             },
             resource_type=ResourceType.LOCAL,
-            resources=[str(fxt_image_file)],
+            resources=[fxt_image_file],
             data_params={"image_quality": 80},
         )
 

tests/python/sdk/test_jobs.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: MIT
 
 import io
-import os
 from logging import Logger
 from pathlib import Path
 from typing import Tuple
@@ -46,7 +45,7 @@ class TestJobUsecases:
                 "labels": [{"name": "car"}, {"name": "person"}],
             },
             resource_type=ResourceType.LOCAL,
-            resources=[str(fxt_image_file)],
+            resources=[fxt_image_file],
             data_params={"image_quality": 80},
         )
 
@@ -108,7 +107,7 @@ class TestJobUsecases:
         job = self.client.jobs.retrieve(job_id)
         job.export_dataset(
             format_name="CVAT for images 1.1",
-            filename=os.fspath(path),
+            filename=path,
             pbar=pbar,
             include_images=include_images,
         )
@@ -135,7 +134,7 @@ class TestJobUsecases:
         fxt_new_task.get_jobs()[0].download_frames(
             [0],
             quality=quality,
-            outdir=str(self.tmp_path),
+            outdir=self.tmp_path,
             filename_pattern="frame-{frame_id}{frame_ext}",
         )
 
@@ -147,7 +146,7 @@ class TestJobUsecases:
         pbar = make_pbar(file=pbar_out)
 
         fxt_new_task.get_jobs()[0].import_annotations(
-            format_name="COCO 1.0", filename=str(fxt_coco_file), pbar=pbar
+            format_name="COCO 1.0", filename=fxt_coco_file, pbar=pbar
         )
 
         assert "uploaded" in self.logger_stream.getvalue()

tests/python/sdk/test_tasks.py

@@ -4,7 +4,6 @@
 
 import io
 import json
-import os
 import zipfile
 from logging import Logger
 from pathlib import Path
@@ -48,7 +47,7 @@ class TestTaskUsecases:
         backup_path = self.tmp_path / "backup.zip"
 
         fxt_new_task.import_annotations("COCO 1.0", filename=fxt_coco_file)
-        fxt_new_task.download_backup(str(backup_path))
+        fxt_new_task.download_backup(backup_path)
 
         yield backup_path
 
@@ -60,7 +59,7 @@ class TestTaskUsecases:
                 "labels": [{"name": "car"}, {"name": "person"}],
             },
             resource_type=ResourceType.LOCAL,
-            resources=[str(fxt_image_file)],
+            resources=[fxt_image_file],
             data_params={"image_quality": 80},
         )
 
@@ -113,9 +112,8 @@ class TestTaskUsecases:
         task_files = generate_image_files(7)
         for i, f in enumerate(task_files):
             fname = self.tmp_path / f.name
-            with fname.open("wb") as fd:
+            fname.write_bytes(f.getvalue())
-                fd.write(f.getvalue())
+            task_files[i] = fname
-                task_files[i] = str(fname)
 
         task = self.client.tasks.create_from_data(
             spec=task_spec,
@@ -184,7 +182,7 @@ class TestTaskUsecases:
         task = self.client.tasks.create_from_data(
             spec=task_spec,
             resource_type=ResourceType.LOCAL,
-            resources=[str(fxt_image_file)],
+            resources=[fxt_image_file],
             pbar=pbar,
             dataset_repository_url=repository_url,
         )
@@ -256,7 +254,7 @@ class TestTaskUsecases:
         task = self.client.tasks.retrieve(task_id)
         task.export_dataset(
             format_name="CVAT for images 1.1",
-            filename=os.fspath(path),
+            filename=path,
             pbar=pbar,
             include_images=include_images,
         )
@@ -272,7 +270,7 @@ class TestTaskUsecases:
         task_id = fxt_new_task.id
         path = self.tmp_path / f"task_{task_id}-backup.zip"
         task = self.client.tasks.retrieve(task_id)
-        task.download_backup(filename=os.fspath(path), pbar=pbar)
+        task.download_backup(filename=path, pbar=pbar)
 
         assert "100%" in pbar_out.getvalue().strip("\r").split("\r")[-1]
         assert path.is_file()
@@ -296,7 +294,7 @@ class TestTaskUsecases:
         fxt_new_task.download_frames(
             [0],
             quality=quality,
-            outdir=str(self.tmp_path),
+            outdir=self.tmp_path,
             filename_pattern="frame-{frame_id}{frame_ext}",
         )
 
@@ -319,9 +317,7 @@ class TestTaskUsecases:
         pbar_out = io.StringIO()
         pbar = make_pbar(file=pbar_out)
 
-        fxt_new_task.import_annotations(
+        fxt_new_task.import_annotations(format_name="COCO 1.0", filename=fxt_coco_file, pbar=pbar)
-            format_name="COCO 1.0", filename=str(fxt_coco_file), pbar=pbar
-        )
 
         assert "uploaded" in self.logger_stream.getvalue()
         assert "100%" in pbar_out.getvalue().strip("\r").split("\r")[-1]
@@ -331,7 +327,7 @@ class TestTaskUsecases:
         pbar_out = io.StringIO()
         pbar = make_pbar(file=pbar_out)
 
-        task = self.client.tasks.create_from_backup(str(fxt_backup_file), pbar=pbar)
+        task = self.client.tasks.create_from_backup(fxt_backup_file, pbar=pbar)
 
         assert task.id
         assert task.id != fxt_new_task.id