Add recursive importers (#1290)

datumaro/datumaro/plugins/coco_format/importer.py

@@ -11,7 +11,7 @@ import os.path as osp
 from datumaro.components.extractor import Importer
 from datumaro.util.log_utils import logging_disabled
 
-from .format import CocoTask, CocoPath
+from .format import CocoTask
 
 
 class CocoImporter(Importer):
@@ -55,11 +55,8 @@ class CocoImporter(Importer):
         if path.endswith('.json') and osp.isfile(path):
             subset_paths = [path]
         else:
-            subset_paths = glob(osp.join(path, '*_*.json'))
-
-            if osp.basename(osp.normpath(path)) != CocoPath.ANNOTATIONS_DIR:
-                path = osp.join(path, CocoPath.ANNOTATIONS_DIR)
-                subset_paths += glob(osp.join(path, '*_*.json'))
+            subset_paths = glob(osp.join(path, '**', '*_*.json'),
+                recursive=True)
 
         subsets = defaultdict(dict)
         for subset_path in subset_paths:

datumaro/datumaro/plugins/cvat_format/importer.py

@@ -9,8 +9,6 @@ import os.path as osp
 
 from datumaro.components.extractor import Importer
 
-from .format import CvatPath
-
 
 class CvatImporter(Importer):
     EXTRACTOR_NAME = 'cvat'
@@ -49,9 +47,5 @@ class CvatImporter(Importer):
         if path.endswith('.xml') and osp.isfile(path):
             subset_paths = [path]
         else:
-            subset_paths = glob(osp.join(path, '*.xml'))
-
-            if osp.basename(osp.normpath(path)) != CvatPath.ANNOTATIONS_DIR:
-                path = osp.join(path, CvatPath.ANNOTATIONS_DIR)
-                subset_paths += glob(osp.join(path, '*.xml'))
+            subset_paths = glob(osp.join(path, '**', '*.xml'), recursive=True)
         return subset_paths

datumaro/datumaro/plugins/tf_detection_api_format/importer.py

@@ -47,5 +47,6 @@ class TfDetectionApiImporter(Importer):
         if path.endswith('.tfrecord') and osp.isfile(path):
             subset_paths = [path]
         else:
-            subset_paths = glob(osp.join(path, '*.tfrecord'))
+            subset_paths = glob(osp.join(path, '**', '*.tfrecord'),
+                recursive=True)
         return subset_paths

datumaro/datumaro/plugins/yolo_format/importer.py

@@ -42,5 +42,5 @@ class YoloImporter(Importer):
         if path.endswith('.data') and osp.isfile(path):
             config_paths = [path]
         else:
-            config_paths = glob(osp.join(path, '*.data'))
+            config_paths = glob(osp.join(path, '**', '*.data'), recursive=True)
         return config_paths