From 29d65b0cda247d351d77a09a159f4e84819a7be8 Mon Sep 17 00:00:00 2001 From: zhiltsov-max Date: Fri, 7 Feb 2020 18:51:16 +0300 Subject: [PATCH] Fix tensorflow installation (#1129) * Make tf dependency optional * Reduce opencv dependency * Import tf eagerly as it is a plugin * Do not install TF with Datumaro --- Dockerfile | 2 +- datumaro/datumaro/plugins/openvino_launcher.py | 7 +++---- .../plugins/tf_detection_api_format/converter.py | 5 +---- .../plugins/tf_detection_api_format/extractor.py | 3 +-- datumaro/requirements.txt | 1 - datumaro/setup.py | 5 ++++- datumaro/tests/test_RISE.py | 3 ++- datumaro/tests/test_command_targets.py | 13 +++++-------- datumaro/tests/test_voc_format.py | 8 ++++---- 9 files changed, 21 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index 29a4cb4f..57a9bc74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -124,7 +124,7 @@ COPY cvat-core/ ${HOME}/cvat-core COPY tests ${HOME}/tests COPY datumaro/ ${HOME}/datumaro -RUN sed -r "s/^(.*)#.*$/\1/g" ${HOME}/datumaro/requirements.txt | xargs -n 1 -L 1 python3 -m pip install --no-cache-dir +RUN python3 -m pip install --no-cache-dir -r ${HOME}/datumaro/requirements.txt # Binary option is necessary to correctly apply the patch on Windows platform. # https://unix.stackexchange.com/questions/239364/how-to-fix-hunk-1-failed-at-1-different-line-endings-message diff --git a/datumaro/datumaro/plugins/openvino_launcher.py b/datumaro/datumaro/plugins/openvino_launcher.py index 613203b9..b0e8360d 100644 --- a/datumaro/datumaro/plugins/openvino_launcher.py +++ b/datumaro/datumaro/plugins/openvino_launcher.py @@ -5,11 +5,12 @@ # pylint: disable=exec-used +import cv2 +import numpy as np import os import os.path as osp -import numpy as np -import subprocess import platform +import subprocess from openvino.inference_engine import IENetwork, IEPlugin @@ -141,8 +142,6 @@ class OpenVinoLauncher(Launcher): self._net = plugin.load(network=network, num_requests=1) def infer(self, inputs): - import cv2 - assert len(inputs.shape) == 4, \ "Expected an input image in (N, H, W, C) format, got %s" % \ (inputs.shape) diff --git a/datumaro/datumaro/plugins/tf_detection_api_format/converter.py b/datumaro/datumaro/plugins/tf_detection_api_format/converter.py index 7c240626..87618079 100644 --- a/datumaro/datumaro/plugins/tf_detection_api_format/converter.py +++ b/datumaro/datumaro/plugins/tf_detection_api_format/converter.py @@ -17,6 +17,7 @@ from datumaro.util.image import encode_image from datumaro.util.tf_util import import_tf as _import_tf from .format import DetectionApiPath +tf = _import_tf() # we need it to filter out non-ASCII characters, otherwise training will crash @@ -25,8 +26,6 @@ def _make_printable(s): return ''.join(filter(lambda x: x in _printable, s)) def _make_tf_example(item, get_label_id, get_label, save_images=False): - tf = _import_tf() - def int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) @@ -118,8 +117,6 @@ class TfDetectionApiConverter(Converter, CliPlugin): self._save_images = save_images def __call__(self, extractor, save_dir): - tf = _import_tf() - os.makedirs(save_dir, exist_ok=True) subsets = extractor.subsets() diff --git a/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py b/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py index 3592072c..0d86d0f3 100644 --- a/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py +++ b/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py @@ -16,6 +16,7 @@ from datumaro.util.image import lazy_image, decode_image from datumaro.util.tf_util import import_tf as _import_tf from .format import DetectionApiPath +tf = _import_tf() def clamp(value, _min, _max): @@ -92,8 +93,6 @@ class TfDetectionApiExtractor(SourceExtractor): @classmethod def _parse_tfrecord_file(cls, filepath, subset_name, images_dir): - tf = _import_tf() - dataset = tf.data.TFRecordDataset(filepath) features = { 'image/filename': tf.io.FixedLenFeature([], tf.string), diff --git a/datumaro/requirements.txt b/datumaro/requirements.txt index 1fc9c3d5..b81db6a5 100644 --- a/datumaro/requirements.txt +++ b/datumaro/requirements.txt @@ -8,4 +8,3 @@ pycocotools>=2.0.0 PyYAML>=5.1.1 scikit-image>=0.15.0 tensorboardX>=1.8 -tensorflow==1.13.1 diff --git a/datumaro/setup.py b/datumaro/setup.py index 7880e644..c39f38d2 100644 --- a/datumaro/setup.py +++ b/datumaro/setup.py @@ -58,8 +58,11 @@ setuptools.setup( 'pycocotools', 'scikit-image', 'tensorboardX', - 'tensorflow', ], + extras_require={ + 'tf': ['tensorflow'], + 'tf-gpu': ['tensorflow-gpu'], + }, entry_points={ 'console_scripts': [ 'datum=datumaro.cli.__main__:main', diff --git a/datumaro/tests/test_RISE.py b/datumaro/tests/test_RISE.py index 7dab28dd..04772287 100644 --- a/datumaro/tests/test_RISE.py +++ b/datumaro/tests/test_RISE.py @@ -1,5 +1,4 @@ from collections import namedtuple -import cv2 import numpy as np from unittest import TestCase @@ -129,6 +128,7 @@ class RiseTest(TestCase): heatmaps_class_count = len(set([roi.label for roi in rois])) self.assertEqual(heatmaps_class_count + len(rois), len(heatmaps)) + # import cv2 # roi_image = image.copy() # for i, roi in enumerate(rois): # cv2.rectangle(roi_image, (roi.x, roi.y), (roi.x + roi.w, roi.y + roi.h), (32 * i) * 3) @@ -203,6 +203,7 @@ class RiseTest(TestCase): label=cls, attributes={'score': cls_conf}) ) + import cv2 image = np.zeros((100, 100, 3)) for i, det in enumerate(detections): roi = ROI(det.attributes['score'], *det.get_bbox(), det.label) diff --git a/datumaro/tests/test_command_targets.py b/datumaro/tests/test_command_targets.py index e9f4167f..5b8a69f3 100644 --- a/datumaro/tests/test_command_targets.py +++ b/datumaro/tests/test_command_targets.py @@ -1,4 +1,3 @@ -import cv2 import numpy as np import os.path as osp @@ -7,15 +6,15 @@ from unittest import TestCase from datumaro.components.project import Project from datumaro.util.command_targets import ProjectTarget, \ ImageTarget, SourceTarget -from datumaro.util.test_utils import current_function_name, TestDir +from datumaro.util.image import save_image +from datumaro.util.test_utils import TestDir class CommandTargetsTest(TestCase): def test_image_false_when_no_file(self): - path = '%s.jpg' % current_function_name() target = ImageTarget() - status = target.test(path) + status = target.test('somepath.jpg') self.assertFalse(status) @@ -34,8 +33,7 @@ class CommandTargetsTest(TestCase): def test_image_true_when_true(self): with TestDir() as test_dir: path = osp.join(test_dir, 'test.jpg') - image = np.random.random_sample([10, 10, 3]) - cv2.imwrite(path, image) + save_image(path, np.ones([10, 7, 3])) target = ImageTarget() @@ -44,10 +42,9 @@ class CommandTargetsTest(TestCase): self.assertTrue(status) def test_project_false_when_no_file(self): - path = '%s.jpg' % current_function_name() target = ProjectTarget() - status = target.test(path) + status = target.test('somepath.jpg') self.assertFalse(status) diff --git a/datumaro/tests/test_voc_format.py b/datumaro/tests/test_voc_format.py index d64e467b..ca2733fc 100644 --- a/datumaro/tests/test_voc_format.py +++ b/datumaro/tests/test_voc_format.py @@ -1,4 +1,3 @@ -import cv2 import numpy as np import os import os.path as osp @@ -28,6 +27,7 @@ from datumaro.plugins.voc_format.converter import ( ) from datumaro.plugins.voc_format.importer import VocImporter from datumaro.components.project import Project +from datumaro.util.image import save_image from datumaro.util.test_utils import TestDir, compare_datasets @@ -155,17 +155,17 @@ def generate_dummy_voc(path): subset_name = 'train' subset = subsets[subset_name] for item in subset: - cv2.imwrite(osp.join(segm_dir, item + '.png'), + save_image(osp.join(segm_dir, item + '.png'), np.tile(VOC.VocColormap[2][::-1], (5, 10, 1)) ) - cv2.imwrite(osp.join(inst_dir, item + '.png'), + save_image(osp.join(inst_dir, item + '.png'), np.tile(1, (5, 10, 1))) # Test images subset_name = 'test' subset = subsets[subset_name] for item in subset: - cv2.imwrite(osp.join(img_dir, item + '.jpg'), + save_image(osp.join(img_dir, item + '.jpg'), np.ones([10, 20, 3])) return subsets