Fix ValidationError when using multiple * in path pattern

HippocampusGirl · HippocampusGirl · commit 90f9edb8fc9f · 2022-02-13T18:48:03.000+01:00
- Change to `fnmatch` introduces named groups, that need to be filtered out from the `groupdict` (python/cpython#19908)
diff --git a/halfpipe/ingest/glob.py b/halfpipe/ingest/glob.py
@@ -27,7 +27,6 @@ def tag_glob(pathname, entities=None, dironly=False) -> Generator[tuple[str, dic
     """
     dirname, basename = op.split(pathname)
     if not dirname:
-        # print(repr(dirname), repr(basename))
         if _isrecursive(basename):
             dir_generator = _rlistdir(dirname, dironly)
         else:
@@ -40,7 +39,6 @@ def tag_glob(pathname, entities=None, dironly=False) -> Generator[tuple[str, dic
     else:
         dirs = [(dirname, dict())]
     for dirname, dirtagdict in dirs:
-        # print("40", repr(dirname), repr(dirtagdict))
         for name, tagdict in _tag_glob_in_dir(dirname, basename, entities, dironly, dirtagdict):
             yield (op.join(dirname, name), _combine_tagdict(dirtagdict, tagdict))
 
@@ -60,13 +58,16 @@ def _tag_glob_in_dir(dirname, basename, entities, dironly, parenttagdict):
     adapted from cpython glob
     only basename can contain magic
     """
-    # print("60", repr(dirname), repr(basename), repr(entities), repr(parenttagdict))
     assert not has_magic(dirname)
-    match = _translate(basename, entities, parenttagdict)
+    fullmatch = _translate(basename, entities, parenttagdict)
     for x in _iterdir(dirname, dironly):
-        matchobj = match(x)
+        matchobj = fullmatch(x)
         if matchobj is not None:
-            yield x, matchobj.groupdict()
+            yield x, {
+                entity: value
+                for entity, value in matchobj.groupdict().items()
+                if entity in entities  # filter out groups added by fnmatch such as "g0"
+            }
 
 
 def get_entities_in_path(pat):
@@ -178,7 +179,6 @@ def _rlistdir(dirname, dironly):
     for x in names:
         path = op.join(dirname, x) if dirname else x
         yield path
-        # print("176", repr(dirname), repr(path))
         yield from _rlistdir(path, dironly)
 
 
diff --git a/halfpipe/ingest/tests/test_glob.py b/halfpipe/ingest/tests/test_glob.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# vi: set ft=python sts=4 ts=4 sw=4 et:
+
+from string import ascii_lowercase, digits
+from random import seed, choices
+
+from ..glob import tag_glob
+
+
+def _random_string(length: int = 5) -> str:
+    return "".join(choices(ascii_lowercase + digits, k=length))
+
+
+def test_tag_glob(tmp_path):
+    seed(a=0x5e6128c4)
+
+    ground_truth: dict[str, dict[str, int]] = dict()
+
+    for i in range(10):
+        for j in range(1, 3):
+            file_path = tmp_path / f"sub-{i:02d}" / f"{_random_string()}_T1w_{_random_string()}_run-{j:02d}.txt"
+            file_path.parent.mkdir(exist_ok=True)
+            file_path.touch()
+
+            ground_truth[str(file_path)] = dict(subject=i, run=j)
+
+            file_path = tmp_path / f"sub-{i:02d}" / f"{_random_string()}_T2w_{_random_string()}_run-{j:02d}.txt"
+            file_path.parent.mkdir(exist_ok=True)
+            file_path.touch()
+
+    path_pattern = str(tmp_path / "sub-{subject}" / "*_T1w_*_run-{run}.txt")
+
+    matched_file_paths = set()
+
+    for file_path, tag_dict in tag_glob(path_pattern, entities=["subject", "run"]):
+        assert file_path in ground_truth
+
+        assert set(ground_truth[file_path].keys()) == set(tag_dict.keys())
+
+        a = set(ground_truth[file_path].items())
+        b = set((entity, int(value)) for entity, value in tag_dict.items())
+        assert a == b
+
+        matched_file_paths.add(file_path)
+
+    assert set(ground_truth.keys()) == matched_file_paths
