Fixes dpath-maintainers#30: Allow usage of lists as paths

akesterson · akesterson · commit 1cd46d540ec2 · 2015-02-04T06:30:59.000-08:00
diff --git a/README.rst b/README.rst
@@ -408,6 +408,19 @@ setting a library-wide dpath option:
 Again, by default, this behavior is OFF, and empty string keys will
 result in ``dpath.exceptions.InvalidKeyName`` being thrown.
 
+Separator got you down? Use lists as paths
+==========================================
+
+The default behavior in dpath is to assume that the path given is a string, which must be tokenized by splitting at the separator to yield a distinct set of path components against which dictionary keys can be individually glob tested. However, this presents a problem when you want to use paths that have a separator in their name; the tokenizer cannot properly understand what you mean by '/a/b/c' if it is possible for '/' to exist as a valid character in a key name.
+
+To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.util.* support the use of a list instead of a string as a path. So for example:
+
+.. code-block: python
+
+   >>> x = { 'a': {'b/c': 0}}
+   >>> dpath.util.get(['a', 'b/c'])
+   0
+
 dpath.path : The Undocumented Backend
 =====================================
 
diff --git a/dpath/path.py b/dpath/path.py
@@ -42,27 +42,22 @@ def paths_only(path):
         l.append(p[0])
     return l
 
-def validate(path, separator="/", regex=None):
+def validate(path, regex=None):
     """
     Validate that all the keys in the given list of path components are valid, given that they do not contain the separator, and match any optional regex given.
     """
     validated = []
     for elem in path:
         key = elem[0]
         strkey = str(key)
-        if (separator and (separator in strkey)):
-            raise dpath.exceptions.InvalidKeyName("{0} at {1} contains the separator {2}"
-                                                  "".format(strkey,
-                                                            separator.join(validated),
-                                                            separator))
-        elif (regex and (not regex.findall(strkey))):
+        if (regex and (not regex.findall(strkey))):
             raise dpath.exceptions.InvalidKeyName("{} at {} does not match the expression {}"
                                                   "".format(strkey,
-                                                            separator.join(validated),
+                                                            validated,
                                                             regex.pattern))
         validated.append(strkey)
 
-def paths(obj, dirs=True, leaves=True, path=[], skip=False, separator="/"):
+def paths(obj, dirs=True, leaves=True, path=[], skip=False):
     """Yield all paths of the object.
 
     Arguments:
@@ -94,17 +89,17 @@ def paths(obj, dirs=True, leaves=True, path=[], skip=False, separator="/"):
                 elif (skip and k[0] == '+'):
                     continue
             newpath = path + [[k, v.__class__]]
-            validate(newpath, separator=separator)
+            validate(newpath)
             if dirs:
                 yield newpath
-            for child in paths(v, dirs, leaves, newpath, skip, separator=separator):
+            for child in paths(v, dirs, leaves, newpath, skip):
                 yield child
     elif isinstance(obj, (list, tuple)):
         for (i, v) in enumerate(obj):
             newpath = path + [[i, v.__class__]]
             if dirs:
                 yield newpath
-            for child in paths(obj[i], dirs, leaves, newpath, skip, separator=separator):
+            for child in paths(obj[i], dirs, leaves, newpath, skip):
                 yield child
     elif leaves:
         yield path + [[obj, obj.__class__]]
@@ -150,7 +145,7 @@ def match(path, glob):
 def is_glob(string):
     return any([c in string for c in '*?[]!'])
 
-def set(obj, path, value, create_missing=True, separator="/", afilter=None):
+def set(obj, path, value, create_missing=True, afilter=None):
     """Set the value of the given path in the object. Path
     must be a list of specific path elements, not a glob.
     You can use dpath.util.set for globs, but the paths must
@@ -208,7 +203,7 @@ def _assigner_list(obj, elem, value):
             if not str(elem_value).isdigit():
                 raise TypeError("Can only create integer indexes in lists, "
                                 "not {}, in {}".format(type(obj),
-                                                       separator.join(traversed)
+                                                       traversed
                                                        )
                                 )
             tester = _presence_test_list
@@ -217,15 +212,15 @@ def _assigner_list(obj, elem, value):
             assigner = _assigner_list
         else:
             raise TypeError("Unable to path into elements of type {} "
-                            "at {}".format(obj, separator.join(traversed)))
+                            "at {}".format(obj, traversed))
 
         if (not tester(obj, elem)) and (create_missing):
             creator(obj, elem)
         elif (not tester(obj, elem)):
             raise dpath.exceptions.PathNotFound(
                 "{} does not exist in {}".format(
                     elem,
-                    separator.join(traversed)
+                    traversed
                     )
                 )
         traversed.append(elem_value)
diff --git a/dpath/util.py b/dpath/util.py
@@ -1,10 +1,36 @@
 import dpath.path
 import dpath.exceptions
+import traceback
 
 MERGE_REPLACE=(1 << 1)
 MERGE_ADDITIVE=(1 << 2)
 MERGE_TYPESAFE=(1 << 3)
 
+def __safe_path__(path, separator):
+    """
+    Given a path and separator, return a list of path components. If path
+    is already a list, return it.
+
+    Note that a string path with the separator at index[0] will have the
+    separator stripped off. If you pass a list path, the separator is
+    ignored, and is assumed to be part of each key glob. It will not be
+    stripped.
+    """
+    if issubclass(path.__class__, (list)):
+        return path
+    path = path.lstrip(separator).split(separator)
+    validated = []
+    for elem in path:
+        key = elem[0]
+        strkey = str(key)
+        if (separator and (separator in strkey)):
+            raise dpath.exceptions.InvalidKeyName("{0} at {1} contains the separator {2}"
+                                                  "".format(strkey,
+                                                            separator.join(validated),
+                                                            separator))
+        validated.append(strkey)
+    return path
+
 def new(obj, path, value, separator="/"):
     """
     Set the element at the terminus of path to value, and create
@@ -15,7 +41,8 @@ def new(obj, path, value, separator="/"):
     characters in it, they will become part of the resulting
     keys
     """
-    pathobj = dpath.path.path_types(obj, path.lstrip(separator).split(separator))
+    pathlist = __safe_path__(path, separator)
+    pathobj = dpath.path.path_types(obj, pathlist)
     return dpath.path.set(obj, pathobj, value, create_missing=True)
 
 def delete(obj, glob, separator="/", afilter=None):
@@ -27,7 +54,8 @@ def delete(obj, glob, separator="/", afilter=None):
     """
     deleted = 0
     paths = []
-    for path in _inner_search(obj, glob.lstrip(separator).split(separator), separator):
+    globlist = __safe_path__(glob, separator)
+    for path in _inner_search(obj, globlist, separator):
         # These are yielded back, don't mess up the dict.
         paths.append(path)
 
@@ -55,7 +83,8 @@ def set(obj, glob, value, separator="/", afilter=None):
     to the given value. Returns the number of elements changed.
     """
     changed = 0
-    for path in _inner_search(obj, glob.lstrip(separator).split(separator), separator):
+    globlist = __safe_path__(glob, separator)
+    for path in _inner_search(obj, globlist, separator):
         changed += 1
         dpath.path.set(obj, path, value, create_missing=False, afilter=afilter)
     return changed
@@ -98,7 +127,8 @@ def search(obj, glob, yielded=False, separator="/", afilter=None, dirs = True):
 
     def _search_view(obj, glob, separator, afilter, dirs):
         view = {}
-        for path in _inner_search(obj, glob.lstrip(separator).split(separator), separator, dirs=dirs):
+        globlist = __safe_path__(glob, separator)
+        for path in _inner_search(obj, globlist, separator, dirs=dirs):
             try:
                 val = dpath.path.get(obj, path, afilter=afilter, view=True)
                 merge(view, val)
@@ -107,7 +137,8 @@ def _search_view(obj, glob, separator, afilter, dirs):
         return view
 
     def _search_yielded(obj, glob, separator, afilter, dirs):
-        for path in _inner_search(obj, glob.lstrip(separator).split(separator), separator, dirs=dirs):
+        globlist = __safe_path__(glob, separator)
+        for path in _inner_search(obj, globlist, separator, dirs=dirs):
             try:
                 val = dpath.path.get(obj, path, view=False, afilter=afilter)
                 yield (separator.join(map(str, dpath.path.paths_only(path))), val)
@@ -122,7 +153,7 @@ def _search_yielded(obj, glob, separator, afilter, dirs):
 
 def _inner_search(obj, glob, separator, dirs=True, leaves=False):
     """Search the object paths that match the glob."""
-    for path in dpath.path.paths(obj, dirs, leaves, skip=True, separator = separator):
+    for path in dpath.path.paths(obj, dirs, leaves, skip=True):
         if dpath.path.match(path, glob):
             yield path
 
diff --git a/tests/test_path_get.py b/tests/test_path_get.py
@@ -16,4 +16,3 @@ def test_path_get_list_of_dicts():
     assert(isinstance(res['a']['b'], list))
     assert(len(res['a']['b']) == 1)
     assert(res['a']['b'][0][0] == 0)
-
diff --git a/tests/test_path_paths.py b/tests/test_path_paths.py
@@ -4,14 +4,6 @@
 import dpath.exceptions
 import dpath.options
 
-@raises(dpath.exceptions.InvalidKeyName)
-def test_path_paths_invalid_keyname():
-    tdict = {
-        "I/contain/the/separator": 0
-        }
-    for x in dpath.path.paths(tdict):
-        pass
-
 @raises(dpath.exceptions.InvalidKeyName)
 def test_path_paths_empty_key_disallowed():
     tdict = {
diff --git a/tests/test_util_get_values.py b/tests/test_util_get_values.py
@@ -16,6 +16,7 @@ def test_get_explicit_single():
             }
         }
     assert(dpath.util.get(ehash, '/a/b/c/f') == 2)
+    assert(dpath.util.get(ehash, ['a', 'b', 'c', 'f']) == 2)
 
 def test_get_glob_single():
     ehash = {
@@ -30,6 +31,7 @@ def test_get_glob_single():
             }
         }
     assert(dpath.util.get(ehash, '/a/b/*/f') == 2)
+    assert(dpath.util.get(ehash, ['a', 'b', '*', 'f']) == 2)
 
 def test_get_glob_multiple():
     ehash = {
@@ -45,10 +47,12 @@ def test_get_glob_multiple():
         }
     }
     assert_raises(ValueError, dpath.util.get, ehash, '/a/b/*/d')
+    assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd'])
 
 def test_get_absent():
     ehash = {}
     assert_raises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f')
+    assert_raises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f'])
 
 def test_values():
     ehash = {
@@ -68,10 +72,18 @@ def test_values():
     assert(1 in ret)
     assert(2 in ret)
 
+    ret = dpath.util.values(ehash, ['a', 'b', 'c', '*'])
+    assert(isinstance(ret, list))
+    assert(0 in ret)
+    assert(1 in ret)
+    assert(2 in ret)
+
 @mock.patch('dpath.util.search')
 def test_values_passes_through(searchfunc):
     searchfunc.return_value = []
     def y():
         pass
     dpath.util.values({}, '/a/b', ':', y, False)
     searchfunc.assert_called_with({}, '/a/b', dirs=False, yielded=True, separator=':', afilter=y)
+    dpath.util.values({}, ['a', 'b'], ':', y, False)
+    searchfunc.assert_called_with({}, ['a', 'b'], dirs=False, yielded=True, separator=':', afilter=y)
diff --git a/tests/test_util_new.py b/tests/test_util_new.py
@@ -8,6 +8,8 @@ def test_set_new_separator():
         }
     dpath.util.new(dict, ';a;b', 1, separator=";")
     assert(dict['a']['b'] == 1)
+    dpath.util.new(dict, ['a', 'b'], 1, separator=";")
+    assert(dict['a']['b'] == 1)
 
 def test_set_new_dict():
     dict = {
@@ -16,6 +18,8 @@ def test_set_new_dict():
         }
     dpath.util.new(dict, '/a/b', 1)
     assert(dict['a']['b'] == 1)
+    dpath.util.new(dict, ['a', 'b'], 1)
+    assert(dict['a']['b'] == 1)
 
 def test_set_new_list():
     dict = {
@@ -25,3 +29,17 @@ def test_set_new_list():
     dpath.util.new(dict, '/a/1', 1)
     assert(dict['a'][1] == 1)
     assert(dict['a'][0] == None)
+    dpath.util.new(dict, ['a', '1'], 1)
+    assert(dict['a'][1] == 1)
+    assert(dict['a'][0] == None)
+
+def test_set_new_list_path_with_separator():
+    # This test kills many birds with one stone, forgive me
+    dict = {
+        "a": {}
+        }
+    dpath.util.new(dict, ['a', 'b/c/d', 0], 1)
+    assert(len(dict['a']) == 1)
+    assert(len(dict['a']['b/c/d']) == 1)
+    assert(dict['a']['b/c/d'][0] == 1)
+    
diff --git a/tests/test_util_search.py b/tests/test_util_search.py
diff --git a/tests/test_util_set.py b/tests/test_util_set.py
