Fix tensorflow installation (#1129)

* Make tf dependency optional

* Reduce opencv dependency

* Import tf eagerly as it is a plugin

* Do not install TF with Datumaro

Dockerfile

@@ -124,7 +124,7 @@ COPY cvat-core/ ${HOME}/cvat-core
 COPY tests ${HOME}/tests
 COPY datumaro/ ${HOME}/datumaro
 
-RUN sed -r "s/^(.*)#.*$/\1/g" ${HOME}/datumaro/requirements.txt | xargs -n 1 -L 1 python3 -m pip install --no-cache-dir
+RUN python3 -m pip install --no-cache-dir -r ${HOME}/datumaro/requirements.txt
 
 # Binary option is necessary to correctly apply the patch on Windows platform.
 # https://unix.stackexchange.com/questions/239364/how-to-fix-hunk-1-failed-at-1-different-line-endings-message

datumaro/datumaro/plugins/openvino_launcher.py

@@ -5,11 +5,12 @@
 
 # pylint: disable=exec-used
 
+import cv2
+import numpy as np
 import os
 import os.path as osp
-import numpy as np
-import subprocess
 import platform
+import subprocess
 
 from openvino.inference_engine import IENetwork, IEPlugin
 
@@ -141,8 +142,6 @@ class OpenVinoLauncher(Launcher):
         self._net = plugin.load(network=network, num_requests=1)
 
     def infer(self, inputs):
-        import cv2
-
         assert len(inputs.shape) == 4, \
             "Expected an input image in (N, H, W, C) format, got %s" % \
                 (inputs.shape)

datumaro/datumaro/plugins/tf_detection_api_format/converter.py

@@ -17,6 +17,7 @@ from datumaro.util.image import encode_image
 from datumaro.util.tf_util import import_tf as _import_tf
 
 from .format import DetectionApiPath
+tf = _import_tf()
 
 
 # we need it to filter out non-ASCII characters, otherwise training will crash
@@ -25,8 +26,6 @@ def _make_printable(s):
     return ''.join(filter(lambda x: x in _printable, s))
 
 def _make_tf_example(item, get_label_id, get_label, save_images=False):
-    tf = _import_tf()
-
     def int64_feature(value):
         return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
 
@@ -118,8 +117,6 @@ class TfDetectionApiConverter(Converter, CliPlugin):
         self._save_images = save_images
 
     def __call__(self, extractor, save_dir):
-        tf = _import_tf()
-
         os.makedirs(save_dir, exist_ok=True)
 
         subsets = extractor.subsets()

datumaro/datumaro/plugins/tf_detection_api_format/extractor.py

@@ -16,6 +16,7 @@ from datumaro.util.image import lazy_image, decode_image
 from datumaro.util.tf_util import import_tf as _import_tf
 
 from .format import DetectionApiPath
+tf = _import_tf()
 
 
 def clamp(value, _min, _max):
@@ -92,8 +93,6 @@ class TfDetectionApiExtractor(SourceExtractor):
 
     @classmethod
     def _parse_tfrecord_file(cls, filepath, subset_name, images_dir):
-        tf = _import_tf()
-
         dataset = tf.data.TFRecordDataset(filepath)
         features = {
             'image/filename': tf.io.FixedLenFeature([], tf.string),

datumaro/requirements.txt

@@ -8,4 +8,3 @@ pycocotools>=2.0.0
 PyYAML>=5.1.1
 scikit-image>=0.15.0
 tensorboardX>=1.8
-tensorflow==1.13.1

datumaro/setup.py

@@ -58,8 +58,11 @@ setuptools.setup(
         'pycocotools',
         'scikit-image',
         'tensorboardX',
-        'tensorflow',
     ],
+    extras_require={
+        'tf': ['tensorflow'],
+        'tf-gpu': ['tensorflow-gpu'],
+    },
     entry_points={
         'console_scripts': [
             'datum=datumaro.cli.__main__:main',

datumaro/tests/test_RISE.py

@@ -1,5 +1,4 @@
 from collections import namedtuple
-import cv2
 import numpy as np
 
 from unittest import TestCase
@@ -129,6 +128,7 @@ class RiseTest(TestCase):
         heatmaps_class_count = len(set([roi.label for roi in rois]))
         self.assertEqual(heatmaps_class_count + len(rois), len(heatmaps))
 
+        # import cv2
         # roi_image = image.copy()
         # for i, roi in enumerate(rois):
         #     cv2.rectangle(roi_image, (roi.x, roi.y), (roi.x + roi.w, roi.y + roi.h), (32 * i) * 3)
@@ -203,6 +203,7 @@ class RiseTest(TestCase):
                         label=cls, attributes={'score': cls_conf})
                 )
 
+        import cv2
         image = np.zeros((100, 100, 3))
         for i, det in enumerate(detections):
             roi = ROI(det.attributes['score'], *det.get_bbox(), det.label)

datumaro/tests/test_command_targets.py

@@ -1,4 +1,3 @@
-import cv2
 import numpy as np
 import os.path as osp
 
@@ -7,15 +6,15 @@ from unittest import TestCase
 from datumaro.components.project import Project
 from datumaro.util.command_targets import ProjectTarget, \
     ImageTarget, SourceTarget
-from datumaro.util.test_utils import current_function_name, TestDir
+from datumaro.util.image import save_image
+from datumaro.util.test_utils import TestDir
 
 
 class CommandTargetsTest(TestCase):
     def test_image_false_when_no_file(self):
-        path = '%s.jpg' % current_function_name()
         target = ImageTarget()
 
-        status = target.test(path)
+        status = target.test('somepath.jpg')
 
         self.assertFalse(status)
 
@@ -34,8 +33,7 @@ class CommandTargetsTest(TestCase):
     def test_image_true_when_true(self):
         with TestDir() as test_dir:
             path = osp.join(test_dir, 'test.jpg')
-            image = np.random.random_sample([10, 10, 3])
+            save_image(path, np.ones([10, 7, 3]))
-            cv2.imwrite(path, image)
 
             target = ImageTarget()
 
@@ -44,10 +42,9 @@ class CommandTargetsTest(TestCase):
             self.assertTrue(status)
 
     def test_project_false_when_no_file(self):
-        path = '%s.jpg' % current_function_name()
         target = ProjectTarget()
 
-        status = target.test(path)
+        status = target.test('somepath.jpg')
 
         self.assertFalse(status)
 

datumaro/tests/test_voc_format.py

@@ -1,4 +1,3 @@
-import cv2
 import numpy as np
 import os
 import os.path as osp
@@ -28,6 +27,7 @@ from datumaro.plugins.voc_format.converter import (
 )
 from datumaro.plugins.voc_format.importer import VocImporter
 from datumaro.components.project import Project
+from datumaro.util.image import save_image
 from datumaro.util.test_utils import TestDir, compare_datasets
 
 
@@ -155,17 +155,17 @@ def generate_dummy_voc(path):
     subset_name = 'train'
     subset = subsets[subset_name]
     for item in subset:
-        cv2.imwrite(osp.join(segm_dir, item + '.png'),
+        save_image(osp.join(segm_dir, item + '.png'),
             np.tile(VOC.VocColormap[2][::-1], (5, 10, 1))
         )
-        cv2.imwrite(osp.join(inst_dir, item + '.png'),
+        save_image(osp.join(inst_dir, item + '.png'),
             np.tile(1, (5, 10, 1)))
 
     # Test images
     subset_name = 'test'
     subset = subsets[subset_name]
     for item in subset:
-        cv2.imwrite(osp.join(img_dir, item + '.jpg'),
+        save_image(osp.join(img_dir, item + '.jpg'),
             np.ones([10, 20, 3]))
 
     return subsets