[Datumaro] Pip installation (#881)

* Add version file
* Remove unnecessary dependencies
* Add lxml use motivation
* Add pip setup script
* Reduce opencv dependency
* Fix cli command
* Codacy

datumaro/datumaro/__init__.py

@@ -23,7 +23,7 @@ from .cli import (
     stats_command as stats_command_module,
     explain_command as explain_command_module,
 )
-from .components.config import VERSION
+from .version import VERSION
 
 
 KNOWN_COMMANDS = {

datumaro/datumaro/cli/explain_command.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: MIT
 
 import argparse
-import cv2
 import logging as log
 import os
 import os.path as osp
@@ -13,7 +12,7 @@ from datumaro.components.project import Project
 from datumaro.components.algorithms.rise import RISE
 from datumaro.util.command_targets import (TargetKinds, target_selector,
     ProjectTarget, SourceTarget, ImageTarget, is_project_path)
-from datumaro.util.image import load_image
+from datumaro.util.image import load_image, save_image
 from .util.project import load_project
 
 
@@ -60,6 +59,7 @@ def build_parser(parser=argparse.ArgumentParser()):
     return parser
 
 def explain_command(args):
+    import cv2
     from matplotlib import cm
 
     project = load_project(args.project_dir)
@@ -110,7 +110,7 @@ def explain_command(args):
             for j, heatmap in enumerate(heatmaps):
                 save_path = osp.join(args.save_dir,
                     file_name + '-heatmap-%s.png' % j)
-                cv2.imwrite(save_path, heatmap * 255.0)
+                save_image(save_path, heatmap * 255.0)
         else:
             for j, heatmap in enumerate(heatmaps):
                 disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2
@@ -151,7 +151,7 @@ def explain_command(args):
                 for j, heatmap in enumerate(heatmaps):
                     save_path = osp.join(args.save_dir,
                         file_name + '-heatmap-%s.png' % j)
-                    cv2.imwrite(save_path, heatmap * 255.0)
+                    save_image(save_path, heatmap * 255.0)
 
             if args.progressive:
                 for j, heatmap in enumerate(heatmaps):

datumaro/datumaro/cli/project/diff.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: MIT
 
 from collections import Counter
-import cv2
 from enum import Enum
 import numpy as np
 import os
@@ -19,6 +18,7 @@ with warnings.catch_warnings():
     _formats.append('tensorboard')
 
 from datumaro.components.extractor import AnnotationType
+from datumaro.util.image import save_image
 
 
 Format = Enum('Formats', _formats)
@@ -135,8 +135,13 @@ class DiffVisualizer:
 
     @classmethod
     def draw_text_with_background(cls, frame, text, origin,
-            font=cv2.FONT_HERSHEY_SIMPLEX, scale=1.0,
+            font=None, scale=1.0,
             color=(0, 0, 0), thickness=1, bgcolor=(1, 1, 1)):
+        import cv2
+
+        if not font:
+            font = cv2.FONT_HERSHEY_SIMPLEX
+
         text_size, baseline = cv2.getTextSize(text, font, scale, thickness)
         cv2.rectangle(frame,
             tuple((origin + (0, baseline)).astype(int)),
@@ -148,6 +153,8 @@ class DiffVisualizer:
         return text_size, baseline
 
     def draw_detection_roi(self, frame, x, y, w, h, label, conf, color):
+        import cv2
+
         cv2.rectangle(frame, (x, y), (x + w, y + h), color, 2)
 
         text = '%s %.2f%%' % (label, 100.0 * conf)
@@ -216,7 +223,7 @@ class DiffVisualizer:
             path = osp.join(self.save_dir, 'diff_%s' % item_a.id)
 
             if self.output_format is Format.simple:
-                cv2.imwrite(path + '.png', img)
+                save_image(path + '.png', img)
             elif self.output_format is Format.tensorboard:
                 self.save_as_tensorboard(img, path)
 

datumaro/datumaro/components/algorithms/rise.py

@@ -5,7 +5,6 @@
 
 # pylint: disable=unused-variable
 
-import cv2
 import numpy as np
 from math import ceil
 
@@ -79,6 +78,8 @@ class RISE:
         return np.reshape(mhmaps, heatmaps.shape)
 
     def apply(self, image, progressive=False):
+        import cv2
+
         assert len(image.shape) == 3, \
             "Expected an input image in (H, W, C) format"
         assert image.shape[2] in [3, 4], \

datumaro/datumaro/components/config.py

@@ -234,5 +234,4 @@ class DefaultConfig(Config):
             return super().set(key, value)
 
 
-VERSION = '0.1.0'
 DEFAULT_FORMAT = 'datumaro'

datumaro/datumaro/components/converters/datumaro.py

@@ -5,7 +5,6 @@
 
 # pylint: disable=no-self-use
 
-import cv2
 import json
 import os
 import os.path as osp
@@ -19,6 +18,7 @@ from datumaro.components.extractor import (
     LabelCategories, MaskCategories, PointsCategories
 )
 from datumaro.components.formats.datumaro import DatumaroPath
+from datumaro.util.image import save_image
 from datumaro.util.mask_tools import apply_colormap
 
 
@@ -133,7 +133,7 @@ class _SubsetWriter:
             DatumaroPath.MASKS_DIR)
         os.makedirs(masks_dir, exist_ok=True)
         path = osp.join(masks_dir, filename)
-        cv2.imwrite(path, mask)
+        save_image(path, mask)
         return mask_id
 
     def _convert_mask_object(self, obj):
@@ -279,7 +279,7 @@ class _Converter:
 
         image_path = osp.join(self._images_dir,
             str(item.id) + DatumaroPath.IMAGE_EXT)
-        cv2.imwrite(image_path, image)
+        save_image(image_path, image)
 
 class DatumaroConverter(Converter):
     def __init__(self, save_images=False, apply_colormap=False):

datumaro/datumaro/components/converters/ms_coco.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: MIT
 
-import cv2
 import json
 import numpy as np
 import os
@@ -17,6 +16,7 @@ from datumaro.components.extractor import (
 )
 from datumaro.components.formats.ms_coco import CocoAnnotationType, CocoPath
 from datumaro.util import find
+from datumaro.util.image import save_image
 import datumaro.util.mask_tools as mask_tools
 
 
@@ -374,7 +374,7 @@ class _Converter:
 
     def save_image(self, item, filename):
         path = osp.join(self._images_dir, filename)
-        cv2.imwrite(path, item.image)
+        save_image(path, item.image)
 
         return path
 

datumaro/datumaro/components/converters/voc.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: MIT
 
-import cv2
 from collections import OrderedDict, defaultdict
 import os
 import os.path as osp
@@ -14,6 +13,7 @@ from datumaro.components.extractor import DEFAULT_SUBSET_NAME, AnnotationType
 from datumaro.components.formats.voc import VocLabel, VocAction, \
     VocBodyPart, VocPose, VocTask, VocPath, VocColormap, VocInstColormap
 from datumaro.util import find
+from datumaro.util.image import save_image
 from datumaro.util.mask_tools import apply_colormap
 
 
@@ -111,7 +111,7 @@ class _Converter:
                 if self._save_images:
                     data = item.image
                     if data is not None:
-                        cv2.imwrite(osp.join(self._images_dir,
+                        save_image(osp.join(self._images_dir,
                                 str(item_id) + VocPath.IMAGE_EXT),
                             data)
 
@@ -334,7 +334,7 @@ class _Converter:
             if colormap is None:
                 colormap = VocColormap
             data = apply_colormap(data, colormap)
-        cv2.imwrite(path, data)
+        save_image(path, data)
 
 class VocConverter(Converter):
     def __init__(self, task=None, save_images=False, apply_colormap=False):

datumaro/datumaro/components/dataset_filter.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: MIT
 
-from lxml import etree as ET
+from lxml import etree as ET # NOTE: lxml has proper XPath implementation
 from datumaro.components.extractor import (DatasetItem, Annotation,
     LabelObject, MaskObject, PointsObject, PolygonObject,
     PolyLineObject, BboxObject, CaptionObject,

datumaro/datumaro/components/extractor.py

@@ -182,7 +182,8 @@ class MaskObject(Annotation):
             return False
         return \
             (self.label == other.label) and \
-            (np.all(self.image == other.image))
+            (self.image is not None and other.image is not None and \
+                np.all(self.image == other.image))
 
 def compute_iou(bbox_a, bbox_b):
     aX, aY, aW, aH = bbox_a
@@ -461,7 +462,9 @@ class DatasetItem:
             (self.id == other.id) and \
             (self.subset == other.subset) and \
             (self.annotations == other.annotations) and \
-            (self.image == other.image)
+            (self.has_image == other.has_image) and \
+            (self.has_image and np.all(self.image == other.image) or \
+                not self.has_image)
 
 class IExtractor:
     def __iter__(self):

datumaro/datumaro/components/launchers/openvino.py

@@ -5,7 +5,6 @@
 
 # pylint: disable=exec-used
 
-import cv2
 import os
 import os.path as osp
 import numpy as np
@@ -142,6 +141,8 @@ class OpenVinoLauncher(Launcher):
         self._net = plugin.load(network=network, num_requests=1)
 
     def infer(self, inputs):
+        import cv2
+
         assert len(inputs.shape) == 4, \
             "Expected an input image in (N, H, W, C) format, got %s" % \
                 (inputs.shape)

datumaro/datumaro/util/command_targets.py

@@ -4,10 +4,10 @@
 # SPDX-License-Identifier: MIT
 
 import argparse
-import cv2
 from enum import Enum
 
 from datumaro.components.project import Project
+from datumaro.util.image import load_image
 
 
 TargetKinds = Enum('TargetKinds',
@@ -50,7 +50,10 @@ def is_inference_path(value):
     return False
 
 def is_image_path(value):
-    return cv2.imread(value) is not None
+    try:
+        return load_image(value) is not None
+    except Exception:
+        return False
 
 
 class Target:

datumaro/datumaro/util/image.py

@@ -3,9 +3,20 @@
 #
 # SPDX-License-Identifier: MIT
 
-import cv2
+# pylint: disable=unused-import
+
 import numpy as np
 
+from enum import Enum
+_IMAGE_BACKENDS = Enum('_IMAGE_BACKENDS', ['cv2', 'PIL'])
+_IMAGE_BACKEND = None
+try:
+    import cv2
+    _IMAGE_BACKEND = _IMAGE_BACKENDS.cv2
+except ModuleNotFoundError:
+    import PIL
+    _IMAGE_BACKEND = _IMAGE_BACKENDS.PIL
+
 from datumaro.util.image_cache import ImageCache as _ImageCache
 
 
@@ -13,13 +24,39 @@ def load_image(path):
     """
     Reads an image in the HWC Grayscale/BGR(A) float [0; 255] format.
     """
-    image = cv2.imread(path)
+
-    image = image.astype(np.float32)
+    if _IMAGE_BACKEND == _IMAGE_BACKENDS.cv2:
+        import cv2
+        image = cv2.imread(path)
+        image = image.astype(np.float32)
+    elif _IMAGE_BACKEND == _IMAGE_BACKENDS.PIL:
+        from PIL import Image
+        image = Image.open(path)
+        image = np.asarray(image, dtype=np.float32)
+        if len(image.shape) == 3 and image.shape[2] in [3, 4]:
+            image[:, :, :3] = image[:, :, 2::-1] # RGB to BGR
+    else:
+        raise NotImplementedError()
 
     assert len(image.shape) == 3
     assert image.shape[2] in [1, 3, 4]
     return image
 
+def save_image(path, image):
+    if _IMAGE_BACKEND == _IMAGE_BACKENDS.cv2:
+        import cv2
+        cv2.imwrite(path, image)
+    elif _IMAGE_BACKEND == _IMAGE_BACKENDS.PIL:
+        from PIL import Image
+        image = image.astype(np.uint8)
+        if len(image.shape) == 3 and image.shape[2] in [3, 4]:
+            image[:, :, :3] = image[:, :, 2::-1] # BGR to RGB
+        image = Image.fromarray(image)
+        image.save(path)
+    else:
+        raise NotImplementedError()
+
+
 class lazy_image:
     def __init__(self, path, loader=load_image, cache=None):
         self.path = path

datumaro/datumaro/version.py

@@ -0,0 +1 @@
+VERSION = '0.1.0'

datumaro/requirements.txt

@@ -6,6 +6,4 @@ opencv-python>=4.1.0.25
 Pillow>=6.1.0
 pycocotools>=2.0.0
 PyYAML>=5.1.1
-requests>=2.20.0
-tensorboard>=1.12.0
 tensorboardX>=1.8

datumaro/setup.py

@@ -0,0 +1,66 @@
+
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import os.path as osp
+import re
+import setuptools
+
+
+def find_version(file_path=None):
+    if not file_path:
+        file_path = osp.join(osp.dirname(osp.abspath(__file__)),
+            'datumaro', 'version.py')
+
+    with open(file_path, 'r') as version_file:
+        version_text = version_file.read()
+
+    # PEP440:
+    # https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions
+    pep_regex = r'([1-9]\d*!)?(0|[1-9]\d*)(\.(0|[1-9]\d*))*((a|b|rc)(0|[1-9]\d*))?(\.post(0|[1-9]\d*))?(\.dev(0|[1-9]\d*))?'
+    version_regex = r'VERSION\s*=\s*.(' + pep_regex + ').'
+    match = re.match(version_regex, version_text)
+    if not match:
+        raise RuntimeError("Failed to find version string in '%s'" % file_path)
+
+    version = version_text[match.start(1) : match.end(1)]
+    return version
+
+
+with open('README.md', 'r') as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name="datumaro",
+    version=find_version(),
+    author="Intel",
+    author_email="maxim.zhiltsov@intel.com",
+    description="Dataset Framework",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/opencv/cvat/datumaro",
+    packages=setuptools.find_packages(exclude=['tests*']),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.5',
+    install_requires=[
+        'GitPython',
+        'lxml',
+        'matplotlib',
+        'numpy',
+        'opencv-python',
+        'Pillow',
+        'PyYAML',
+        'pycocotools',
+        'tensorboardX',
+    ],
+    entry_points={
+        'console_scripts': [
+            'datum=datumaro:main',
+        ],
+    },
+)

datumaro/tests/test_image.py

@@ -0,0 +1,39 @@
+from itertools import product
+import numpy as np
+import os.path as osp
+
+from unittest import TestCase
+
+import datumaro.util.image as image_module
+from datumaro.util.test_utils import TestDir
+
+
+class ImageTest(TestCase):
+    def setUp(self):
+        self.default_backend = image_module._IMAGE_BACKEND
+
+    def tearDown(self):
+        image_module._IMAGE_BACKEND = self.default_backend
+
+    def _test_can_save_and_load(self, src_image, path,
+            save_backend=None, load_backend=None):
+        if save_backend:
+            image_module._IMAGE_BACKEND = save_backend
+        image_module.save_image(path, src_image)
+
+        if load_backend:
+            image_module._IMAGE_BACKEND = load_backend
+        dst_image = image_module.load_image(path)
+
+        self.assertTrue(np.all(src_image == dst_image), 'save: %s, load: %s' % \
+            (save_backend, load_backend))
+
+    def test_save_and_load_backends(self):
+        backends = image_module._IMAGE_BACKENDS
+        for save_backend, load_backend in product(backends, backends):
+            with TestDir() as test_dir:
+                src_image = np.random.random_integers(0, 255, (2, 4, 3))
+                image_path = osp.join(test_dir.path, 'img.png')
+
+                self._test_can_save_and_load(src_image, image_path,
+                    save_backend, load_backend)