diff --git a/.github/tag-changelog-config.js b/.github/tag-changelog-config.js new file mode 100644 index 0000000..bd30aa0 --- /dev/null +++ b/.github/tag-changelog-config.js @@ -0,0 +1,20 @@ +module.exports = { + types: [ + { types: ["other"], label: "Commits" }, + ], + + renderTypeSection: function (label, commits) { + let text = `\n## ${label}\n`; + + commits.forEach((commit) => { + text += `- ${commit.subject}\n`; + }); + + return text; + }, + + renderChangelog: function (release, changes) { + const now = new Date(); + return `# ${release} - ${now.toISOString().substr(0, 10)}\n` + changes + "\n\n"; + }, +}; diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..5c1386e --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,68 @@ +name: Deploy and Release + +# Controls when the workflow will run +on: + # Triggers the workflow on version change + push: + branches: + - master + paths: + - dpath/version.py + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "deploy" + deploy: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Get Version + id: get-version + run: | + python -c "from dpath.version import VERSION; print(f'::set-output name=version::v{VERSION}');" + + - name: Check Tag + uses: mukunku/tag-exists-action@v1.0.0 + id: check-tag + with: + tag: ${{ steps.get-version.outputs.version }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create Tag + if: steps.check-tag.outputs.exists == 'false' + uses: negz/create-tag@v1 + with: + version: ${{ steps.get-version.outputs.version }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate Changelog + id: generate-changelog + uses: loopwerk/tag-changelog@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + config_file: .github/tag-changelog-config.js + + - name: PyPI Deployment + uses: casperdcl/deploy-pypi@v2 + with: + # PyPI username + user: ${{ secrets.PYPI_USER }} + # PyPI password or API token + password: ${{ secrets.PYPI_PASS }} + # `setup.py` command to run ("true" is a shortcut for "clean sdist -d bdist_wheel -d ") + build: clean sdist -d dist/ + # `pip` command to run ("true" is a shortcut for "wheel -w --no-deps .") + pip: true + + - name: Github Release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ steps.get-version.outputs.version }} + body: ${{ steps.generate-changelog.outputs.changes }} + files: dist/* diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..d350624 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,86 @@ +name: Run tests + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for important files + push: + branches: + - master + paths: + - "dpath/" + - "**.py" + - "tox.ini" + pull_request: + paths: + - "dpath/" + - "**.py" + - "tox.ini" + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + + # Run flake8 linter + flake8: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@main + + - name: Set up Python 3.12 + uses: actions/setup-python@main + with: + python-version: "3.12" + + - name: Setup flake8 annotations + uses: TrueBrain/actions-flake8@v2.3 + with: + path: setup.py dpath/ tests/ + + # Generate a common hashseed for all tests + generate-hashseed: + runs-on: ubuntu-latest + + outputs: + hashseed: ${{ steps.generate.outputs.hashseed }} + + steps: + - name: Generate Hashseed + id: generate + run: | + python -c "import os + from random import randint + hashseed = randint(0, 4294967295) + print(f'{hashseed=}') + open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" + + # Tests job + tests: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + needs: [generate-hashseed, flake8] + + strategy: + matrix: + # Match versions specified in tox.ini + python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.7', '3.12'] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Check out code + uses: actions/checkout@main + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@main + with: + python-version: ${{ matrix.python-version }} + + - name: Run tox with tox-gh-actions + uses: ymyzk/run-tox-gh-actions@main + with: + tox-args: -vv --hashseed=${{ needs.generate-hashseed.outputs.hashseed }} diff --git a/.gitignore b/.gitignore index fb9ae3a..52fd738 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ /MANIFEST /.tox -/.hypothesis /build /env +.hypothesis *.pyc +.vscode +venv_39 +.idea/ +dpath.egg-info/ +dist/ +tests/.hypothesis \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c6a8bd5..0000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -deploy: - provider: pypi - user: akesterson - password: - secure: RlZ3kw0x7/5QNBUIqj5wjkpS11TuSdH7o1S41m4Vea8HiryQjB2+CFf+uBXj8nb1OtYDRaeKTSnk3qVlUvaUglFd7sV+Hh2DDo4pYomd0xDFHnK2LB+hzbYftkOptmwdS2bc0X7vr+xdY+7b2DPTlyowrs3Xp2WvzRWM4hWnA3Q= - true: - tags: true - branch: master -sudo: false -language: python -python: -- '3.6' -- '3.8' -- pypy3 -install: travis_retry pip install tox-travis -script: tox diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ba67806..ee327f6 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,25 +1,22 @@ Who Maintains DPATH =================== -dpath is primarily maintained by Andrew Kesterson and Caleb Case . These two individuals collectively govern the project. +dpath was created by and originally maintained by Andrew Kesterson and Caleb Case . In July +of 2020 they put out a call for new maintainers. [@bigsablept](https://github.com/bigsablept) and +[@moomoohk](https://github.com/moomoohk) stepped up to become the new maintainers. There are several individuals in the community who have taken an active role in helping to maintain the project and submit fixes. Those individuals are shown in the git changelog. -Becoming a Maintainer -===================== - -Nobody has to become a maintainer to submit a patch against dpath. Simply send the pull request on github. - -If you would like to help triage issues, attend monthly meetings, and become a regular part of the team working on the roadmap, send an email to andrew@aklabs.net and/or calebcase@gmail.com. - Where and How do we communicate =============================== -The dpath maintainers communcate in 3 primary ways: +The dpath maintainers communicate in 3 primary ways: 1. Email, directly to each other. 2. Github via issue and pull request comments -3. A monthly maintainers meeting via telephone +3. A monthly maintainers meeting via Zoom + +The remainder of this document is subject to change after further discussion among the new maintainers. What is the roadmap =================== @@ -76,11 +73,23 @@ The more complete process goes: 6. Send your pull request 7. If accepted, the maintainers will merge your pull request and close the issue. +Branching Strategy +================== + +We run a clean bleeding edge master. Long term support for major version numbers are broken out into version branches. + +* master : Current 3.x (bleeding edge) development +* version/1.x : 1.x series bugfixes +* version/2.x : 2.x series features and bugfixes + +We name bugfixes as "bugfix/ISSUENUMBER_shortname"; features are named "feature/ISSUENUMBER_shortname". All branches representing work against an issue must have the issue number in the branch name. Cutting a New Release ===================== -Releases for dpath occur automatically from travis-ci based on tags on the master branch. +Releases for dpath occur automatically from Github Actions based on version changes on the master branch. + +Due to legacy reasons older tag names do not follow a uniform format: akesterson@akesterson:~/dpath-python$ git tag 1.0-0 @@ -101,18 +110,33 @@ Releases for dpath occur automatically from travis-ci based on tags on the maste build,1.5,0 build,2.0,0 -Once upon a time, the version string was automatially computed based on the content of these tags. Now, however, the version string is stored statically in dpath/version.py +Moving forward version numbers and tag names will be identical and follow the standard semver format. + +The version string is stored in `dpath/version.py` and tag names/release versions are generated using this string. akesterson@akesterson:~/dpath-python$ cat dpath/version.py VERSION = "2.0.0" To cut a new release, follow this procedure: -1. Commit a new dpath/version.py on the master branch with the format "MAJOR.MINOR.RELEASE" -2. Add a new tag of the form "build,MAJOR.MINOR,RELEASE" to the master branch. This tag must have the same version number as the one commmited in dpath/version.py or we will fill your desk drawers with cockroaches. -3. Push the new master version and the associated tag to github. -4. travis-ci SHOULD push the new release to pypi. +1. Commit a new `dpath/version.py` on the appropriate branch with the format "MAJOR.MINOR.RELEASE". +2. Github Actions SHOULD push the new release to PyPI on merge to `master`. + +See `.github/workflows/deploy.yml` for more information. + +If the Github workflow fails to update pypi, follow the instructions on manually creating a release, here: + +https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives + +Deployment CI was previously implemented using [Travis CI](https://travis-ci.org/github/akesterson/dpath-python). + +Running Tests +============= + +Tests are managed using [tox](https://tox.readthedocs.io/en/latest/). + +Environment creation and dependency installation is managed by this tool, all one has to do is install it with `pip` and run `tox` in this repo's root directory. -If travis-ci fails to update pypi, follow the instructions on manually creating a release, here: +Tests can also be run with Github Actions via the [tests.yml](https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml) workflow. -https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives \ No newline at end of file +This workflow will run automatically on pretty much any commit to any branch of this repo but manual runs are also available. diff --git a/README.rst b/README.rst index c8c3032..0ad3ad2 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,9 @@ dpath-python ============ |PyPI| +|Python Version| |Build Status| +|Gitter| A python library for accessing and searching dictionaries via /slashed/paths ala xpath @@ -29,7 +31,7 @@ Using Dpath .. code-block:: python - import dpath.util + import dpath Separators ========== @@ -48,10 +50,10 @@ Suppose we have a dictionary like this: x = { "a": { "b": { - "3": 2, - "43": 30, - "c": [], - "d": ['red', 'buggy', 'bumpers'], + "3": 2, + "43": 30, + "c": [], + "d": ['red', 'buggy', 'bumpers'], } } } @@ -61,8 +63,8 @@ key '43' in the 'b' hash which is in the 'a' hash". That's easy. .. code-block:: pycon - >>> help(dpath.util.get) - Help on function get in module dpath.util: + >>> help(dpath.get) + Help on function get in module dpath: get(obj, glob, separator='/') Given an object which contains only one possible match for the given glob, @@ -71,7 +73,7 @@ key '43' in the 'b' hash which is in the 'a' hash". That's easy. If more than one leaf matches the glob, ValueError is raised. If the glob is not found, KeyError is raised. - >>> dpath.util.get(x, '/a/b/43') + >>> dpath.get(x, '/a/b/43') 30 Or you could say "Give me a new dictionary with the values of all @@ -79,8 +81,8 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon - >>> help(dpath.util.search) - Help on function search in module dpath.util: + >>> help(dpath.search) + Help on function search in module dpath: search(obj, glob, yielded=False) Given a path glob, return a dictionary containing all keys @@ -94,27 +96,27 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon - >>> result = dpath.util.search(x, "a/b/[cd]") - >>> print json.dumps(result, indent=4, sort_keys=True) + >>> result = dpath.search(x, "a/b/[cd]") + >>> print(json.dumps(result, indent=4, sort_keys=True)) { - "a": { - "b": { - "c": [], - "d": [ - "red", - "buggy", - "bumpers" - ] + "a": { + "b": { + "c": [], + "d": [ + "red", + "buggy", + "bumpers" + ] + } } } - } ... Wow that was easy. What if I want to iterate over the results, and not get a merged view? .. code-block:: pycon - >>> for x in dpath.util.search(x, "a/b/[cd]", yielded=True): print x + >>> for x in dpath.search(x, "a/b/[cd]", yielded=True): print(x) ... ('a/b/c', []) ('a/b/d', ['red', 'buggy', 'bumpers']) @@ -124,8 +126,8 @@ don't care about the paths they were found at: .. code-block:: pycon - >>> help(dpath.util.values) - Help on function values in module dpath.util: + >>> help(dpath.values) + Help on function values in module dpath: values(obj, glob, separator='/', afilter=None, dirs=True) Given an object and a path glob, return an array of all values which match @@ -133,7 +135,7 @@ don't care about the paths they were found at: and it is primarily a shorthand for a list comprehension over a yielded search call. - >>> dpath.util.values(x, '/a/b/d/*') + >>> dpath.values(x, '/a/b/d/*') ['red', 'buggy', 'bumpers'] Example: Setting existing keys @@ -144,23 +146,23 @@ value 'Waffles'. .. code-block:: pycon - >>> help(dpath.util.set) - Help on function set in module dpath.util: + >>> help(dpath.set) + Help on function set in module dpath: set(obj, glob, value) Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. - >>> dpath.util.set(x, 'a/b/[cd]', 'Waffles') + >>> dpath.set(x, 'a/b/[cd]', 'Waffles') 2 - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { - "3": 2, - "43": 30, - "c": "Waffles", - "d": "Waffles" + "3": 2, + "43": 30, + "c": "Waffles", + "d": "Waffles" } } } @@ -174,8 +176,8 @@ necessary to get to the terminus. .. code-block:: pycon - >>> help(dpath.util.new) - Help on function new in module dpath.util: + >>> help(dpath.new) + Help on function new in module dpath: new(obj, path, value) Set the element at the terminus of path to value, and create @@ -186,8 +188,8 @@ necessary to get to the terminus. characters in it, they will become part of the resulting keys - >>> dpath.util.new(x, 'a/b/e/f/g', "Roffle") - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> dpath.new(x, 'a/b/e/f/g', "Roffle") + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { @@ -210,9 +212,9 @@ object with None entries in order to make it big enough: .. code-block:: pycon - >>> dpath.util.new(x, 'a/b/e/f/h', []) - >>> dpath.util.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> dpath.new(x, 'a/b/e/f/h', []) + >>> dpath.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { @@ -250,11 +252,11 @@ Handy! Example: Deleting Existing Keys =============================== -To delete keys in an object, use dpath.util.delete, which accepts the same globbing syntax as the other methods. +To delete keys in an object, use dpath.delete, which accepts the same globbing syntax as the other methods. .. code-block:: pycon - >>> help(dpath.util.delete) + >>> help(dpath.delete) delete(obj, glob, separator='/', afilter=None): Given a path glob, delete all elements that match the glob. @@ -265,84 +267,83 @@ To delete keys in an object, use dpath.util.delete, which accepts the same globb Example: Merging ================ -Also, check out dpath.util.merge. The python dict update() method is +Also, check out dpath.merge. The python dict update() method is great and all but doesn't handle merging dictionaries deeply. This one does. .. code-block:: pycon - >>> help(dpath.util.merge) - Help on function merge in module dpath.util: + >>> help(dpath.merge) + Help on function merge in module dpath: merge(dst, src, afilter=None, flags=4, _path='') Merge source into destination. Like dict.update() but performs deep merging. - flags is an OR'ed combination of MERGE_ADDITIVE, MERGE_REPLACE - MERGE_TYPESAFE. - * MERGE_ADDITIVE : List objects are combined onto one long + flags is an OR'ed combination of MergeType enum members. + * ADDITIVE : List objects are combined onto one long list (NOT a set). This is the default flag. - * MERGE_REPLACE : Instead of combining list objects, when + * REPLACE : Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination with the source. - * MERGE_TYPESAFE : When 2 keys at equal levels are of different + * TYPESAFE : When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source replaces the destination in this situation. >>> y = {'a': {'b': { 'e': {'f': {'h': [None, 0, 1, None, 13, 14]}}}, 'c': 'RoffleWaffles'}} - >>> print json.dumps(y, indent=4, sort_keys=True) + >>> print(json.dumps(y, indent=4, sort_keys=True)) { - "a": { - "b": { - "e": { - "f": { - "h": [ - null, - 0, - 1, - null, - 13, - 14 - ] - } + "a": { + "b": { + "e": { + "f": { + "h": [ + null, + 0, + 1, + null, + 13, + 14 + ] + } + } + }, + "c": "RoffleWaffles" } - }, - "c": "RoffleWaffles" - } } - >>> dpath.util.merge(x, y) - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> dpath.merge(x, y) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { - "a": { - "b": { - "3": 2, - "43": 30, - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle", - "h": [ - null, - 0, - 1, - null, - 13, - 14, - null, - null, - null, - null, - null, - null, - null, - "Wow this is a big array, it sure is lonely in here by myself" - ] - } + "a": { + "b": { + "3": 2, + "43": 30, + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle", + "h": [ + null, + 0, + 1, + null, + 13, + 14, + null, + null, + null, + null, + null, + null, + null, + "Wow this is a big array, it sure is lonely in here by myself" + ] + } + } + }, + "c": "RoffleWaffles" } - }, - "c": "RoffleWaffles" - } } Now that's handy. You shouldn't try to use this as a replacement for the @@ -369,41 +370,41 @@ them: .. code-block:: pycon - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { - "a": { - "b": { - "3": 2, - "43": 30, - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle" + "a": { + "b": { + "3": 2, + "43": 30, + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle" + } + } } } - } - } } >>> def afilter(x): ... if "ffle" in str(x): ... return True ... return False ... - >>> result = dpath.util.search(x, '**', afilter=afilter) - >>> print json.dumps(result, indent=4, sort_keys=True) + >>> result = dpath.search(x, '**', afilter=afilter) + >>> print(json.dumps(result, indent=4, sort_keys=True)) { - "a": { - "b": { - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle" + "a": { + "b": { + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle" + } + } } } - } - } } Obviously filtering functions can perform more advanced tests (regular @@ -429,18 +430,18 @@ Separator got you down? Use lists as paths The default behavior in dpath is to assume that the path given is a string, which must be tokenized by splitting at the separator to yield a distinct set of path components against which dictionary keys can be individually glob tested. However, this presents a problem when you want to use paths that have a separator in their name; the tokenizer cannot properly understand what you mean by '/a/b/c' if it is possible for '/' to exist as a valid character in a key name. -To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.util.* support the use of a list instead of a string as a path. So for example: +To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.* support the use of a list instead of a string as a path. So for example: .. code-block:: python >>> x = { 'a': {'b/c': 0}} - >>> dpath.util.get(['a', 'b/c']) + >>> dpath.get(['a', 'b/c']) 0 dpath.segments : The Low-Level Backend ====================================== -dpath.util is where you want to spend your time: this library has the friendly +dpath is where you want to spend your time: this library has the friendly functions that will understand simple string globs, afilter functions, etc. dpath.segments is the backend pathing library. It passes around tuples of path @@ -450,8 +451,16 @@ components instead of string globs. :target: https://pypi.python.org/pypi/dpath/ :alt: PyPI: Latest Version -.. |Build Status| image:: https://travis-ci.org/akesterson/dpath-python.svg?branch=travisci - :target: https://travis-ci.org/akesterson/dpath-python +.. |Python Version| image:: https://img.shields.io/pypi/pyversions/dpath?style=flat + :target: https://pypi.python.org/pypi/dpath/ + :alt: Supported Python Version + +.. |Build Status| image:: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml/badge.svg + :target: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml + +.. |Gitter| image:: https://badges.gitter.im/dpath-python/chat.svg + :target: https://gitter.im/dpath-python/chat?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge + :alt: Gitter Contributors ============ diff --git a/dpath/__init__.py b/dpath/__init__.py index e69de29..2bc9942 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -0,0 +1,364 @@ +# Needed for pre-3.10 versions +from __future__ import annotations + +__all__ = [ + "new", + "delete", + "set", + "get", + "values", + "search", + "merge", + "exceptions", + "options", + "segments", + "types", + "version", + "MergeType", + "PathSegment", + "Filter", + "Glob", + "Path", + "Hints", + "Creator", +] + +from collections.abc import MutableMapping, MutableSequence +from typing import Union, List, Any, Callable, Optional + +from dpath import segments, options +from dpath.exceptions import InvalidKeyName, PathNotFound +from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, Path, Hints + +_DEFAULT_SENTINEL = object() + + +def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: + """ + Given a path and separator, return a tuple of segments. If path is + already a non-leaf thing, return it. + + Note that a string path with the separator at index[0] will have the + separator stripped off. If you pass a list path, the separator is + ignored, and is assumed to be part of each key glob. It will not be + stripped. + """ + if not segments.leaf(path): + split_segments = path + else: + split_segments = path.lstrip(separator).split(separator) + + return split_segments + + +def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator | None = None) -> MutableMapping: + """ + Set the element at the terminus of path to value, and create + it if it does not exist (as opposed to 'set' that can only + change existing keys). + + path will NOT be treated like a glob. If it has globbing + characters in it, they will become part of the resulting + keys + + creator allows you to pass in a creator method that is + responsible for creating missing keys at arbitrary levels of + the path (see the help for dpath.path.set) + """ + split_segments = _split_path(path, separator) + if creator: + return segments.set(obj, split_segments, value, creator=creator) + return segments.set(obj, split_segments, value) + + +def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter | None = None) -> int: + """ + Given a obj, delete all elements that match the glob. + + Returns the number of deleted objects. Raises PathNotFound if no paths are + found to delete. + """ + globlist = _split_path(glob, separator) + + def f(obj, pair, counter): + (path_segments, value) = pair + + # Skip segments if they no longer exist in obj. + if not segments.has(obj, path_segments): + return + + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(value) and afilter(value) + + if (matched and not afilter) or selected: + key = path_segments[-1] + parent = segments.get(obj, path_segments[:-1]) + + # Deletion behavior depends on parent type + if isinstance(parent, MutableMapping): + del parent[key] + + else: + # Handle sequence types + # TODO: Consider cases where type isn't a simple list (e.g. set) + + if len(parent) - 1 == key: + # Removing the last element of a sequence. It can be + # truly removed without affecting the ordering of + # remaining items. + # + # Note: In order to achieve proper behavior we are + # relying on the reverse iteration of + # non-dictionaries from segments.kvs(). + # Otherwise we'd be unable to delete all the tails + # of a list and end up with None values when we + # don't need them. + del parent[key] + + else: + # This key can't be removed completely because it + # would affect the order of items that remain in our + # result. + parent[key] = None + + counter[0] += 1 + + [deleted] = segments.foldm(obj, f, [0]) + if not deleted: + raise PathNotFound(f"Could not find {glob} to delete it") + + return deleted + + +def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter | None = None) -> int: + """ + Given a path glob, set all existing elements in the document + to the given value. Returns the number of elements changed. + """ + globlist = _split_path(glob, separator) + + def f(obj, pair, counter): + (path_segments, found) = pair + + # Skip segments if they no longer exist in obj. + if not segments.has(obj, path_segments): + return + + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(found) and afilter(found) + + if (matched and not afilter) or (matched and selected): + segments.set(obj, path_segments, value, creator=None) + counter[0] += 1 + + [changed] = segments.foldm(obj, f, [0]) + return changed + + +def get( + obj: MutableMapping, + glob: Glob, + separator="/", + default: Any = _DEFAULT_SENTINEL +) -> Union[MutableMapping, object, Callable]: + """ + Given an object which contains only one possible match for the given glob, + return the value for the leaf matching the given glob. + If the glob is not found and a default is provided, + the default is returned. + + If more than one leaf matches the glob, ValueError is raised. If the glob is + not found and a default is not provided, KeyError is raised. + """ + if isinstance(glob, str) and glob == "/" or len(glob) == 0: + return obj + + globlist = _split_path(glob, separator) + + def f(_, pair, results): + (path_segments, found) = pair + + if segments.match(path_segments, globlist): + results.append(found) + if len(results) > 1: + return False + + results = segments.fold(obj, f, []) + + if len(results) == 0: + if default is not _DEFAULT_SENTINEL: + return default + + raise KeyError(glob) + elif len(results) > 1: + raise ValueError(f"dpath.get() globs must match only one leaf: {glob}") + + return results[0] + + +def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter | None = None, dirs=True): + """ + Given an object and a path glob, return an array of all values which match + the glob. The arguments to this function are identical to those of search(). + """ + yielded = True + + return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] + + +def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter | None = None, dirs=True): + """ + Given a path glob, return a dictionary containing all keys + that matched the given glob. + + If 'yielded' is true, then a dictionary will not be returned. + Instead, tuples will be yielded in the form of (path, value) for + every element in the document that matched the glob. + """ + + split_glob = _split_path(glob, separator) + + def keeper(path, found): + """ + Generalized test for use in both yielded and folded cases. + Returns True if we want this result. Otherwise, returns False. + """ + if not dirs and not segments.leaf(found): + return False + + matched = segments.match(path, split_glob) + selected = afilter and afilter(found) + + return (matched and not afilter) or (matched and selected) + + if yielded: + def yielder(): + for path, found in segments.walk(obj): + if keeper(path, found): + yield separator.join(map(segments.int_str, path)), found + + return yielder() + else: + def f(obj, pair, result): + (path, found) = pair + + if keeper(path, found): + segments.set(result, path, found, hints=segments.types(obj, path)) + + return segments.fold(obj, f, {}) + + +def merge( + dst: MutableMapping, + src: MutableMapping, + separator="/", + afilter: Filter | None = None, + flags=MergeType.ADDITIVE +): + """ + Merge source into destination. Like dict.update() but performs deep + merging. + + NOTE: This does not do a deep copy of the source object. Applying merge + will result in references to src being present in the dst tree. If you do + not want src to potentially be modified by other changes in dst (e.g. more + merge calls), then use a deep copy of src. + + NOTE that merge() does NOT copy objects - it REFERENCES. If you merge + take these two dictionaries: + + >>> a = {'a': [0] } + >>> b = {'a': [1] } + + ... and you merge them into an empty dictionary, like so: + + >>> d = {} + >>> dpath.merge(d, a) + >>> dpath.merge(d, b) + + ... you might be surprised to find that a['a'] now contains [0, 1]. + This is because merge() says (d['a'] = a['a']), and thus creates a reference. + This reference is then modified when b is merged, causing both d and + a to have ['a'][0, 1]. To avoid this, make your own deep copies of source + objects that you intend to merge. For further notes see + https://github.com/akesterson/dpath-python/issues/58 + + flags is an OR'ed combination of MergeType enum members. + """ + filtered_src = search(src, '**', afilter=afilter, separator='/') + + def are_both_mutable(o1, o2): + mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) + seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) + + if mapP or seqP: + return True + + return False + + def merger(dst, src, _segments=()): + for key, found in segments.make_walkable(src): + # Our current path in the source. + current_path = _segments + (key,) + + if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: + raise InvalidKeyName("Empty string keys not allowed without " + "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " + f"{current_path}") + + # Validate src and dst types match. + if flags & MergeType.TYPESAFE: + if segments.has(dst, current_path): + target = segments.get(dst, current_path) + tt = type(target) + ft = type(found) + if tt != ft: + path = separator.join(current_path) + raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") + + # Path not present in destination, create it. + if not segments.has(dst, current_path): + segments.set(dst, current_path, found) + continue + + # Retrieve the value in the destination. + target = segments.get(dst, current_path) + + # If the types don't match, replace it. + if type(found) is not type(target) and not are_both_mutable(found, target): + segments.set(dst, current_path, found) + continue + + # If target is a leaf, the replace it. + if segments.leaf(target): + segments.set(dst, current_path, found) + continue + + # At this point we know: + # + # * The target exists. + # * The types match. + # * The target isn't a leaf. + # + # Pretend we have a sequence and account for the flags. + try: + if flags & MergeType.ADDITIVE: + target += found + continue + + if flags & MergeType.REPLACE: + try: + target[""] + except TypeError: + segments.set(dst, current_path, found) + continue + except Exception: + raise + except Exception: + # We have a dictionary like thing and we need to attempt to + # recursively merge it. + merger(dst, found, current_path) + + merger(dst, filtered_src) + + return dst diff --git a/dpath/py.typed b/dpath/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/dpath/segments.py b/dpath/segments.py index 65f8920..f1443f0 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,89 +1,110 @@ from copy import deepcopy -from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath import options from fnmatch import fnmatchcase +from typing import Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping, MutableSequence + +from dpath import options +from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound +from dpath.types import PathSegment, Creator, Hints, Glob, Path, ListIndex + +def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: + """ + Returns an iterator which yields tuple pairs of (node index, node value), regardless of node type. -def kvs(node): - ''' - Return a (key, value) iterator for the node. + * For dict nodes `node.items()` will be returned. + * For sequence nodes (lists/tuples/etc.) a zip between index number and index value will be returned. + * Edge cases will result in an empty iterator being returned. - kvs(node) -> (generator -> (key, value)) - ''' + make_walkable(node) -> (generator -> (key, value)) + """ try: return iter(node.items()) except AttributeError: - return zip(range(len(node)), node) + try: + indices = range(len(node)) + # Convert all list indices to objects so negative indices are supported. + indices = map(lambda i: ListIndex(i, len(node)), indices) + return zip(indices, node) + except TypeError: + # This can happen in cases where the node isn't leaf(node) == True, + # but also isn't actually iterable. Instead of this being an error + # we will treat this node as if it has no children. + return enumerate([]) def leaf(thing): - ''' + """ Return True if thing is a leaf, otherwise False. - - leaf(thing) -> bool - ''' + """ leaves = (bytes, str, int, float, bool, type(None)) return isinstance(thing, leaves) def leafy(thing): - ''' + """ Same as leaf(thing), but also treats empty sequences and dictionaries as True. + """ - leafy(thing) -> bool - ''' - return leaf(thing) or len(thing) == 0 + try: + return leaf(thing) or len(thing) == 0 + except TypeError: + # In case thing has no len() + return False def walk(obj, location=()): - ''' + """ Yield all valid (segments, value) pairs (from a breadth-first search, right-to-left on sequences). walk(obj) -> (generator -> (segments, value)) - ''' + """ if not leaf(obj): - for k, v in kvs(obj): + for k, v in make_walkable(obj): length = None try: length = len(k) - except: + except TypeError: pass if length is not None and length == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(location + (k,))) - yield ((location + (k,)), v) - for k, v in kvs(obj): + f"{location + (k,)}") + yield (location + (k,)), v + + for k, v in make_walkable(obj): for found in walk(v, location + (k,)): yield found -def get(obj, segments): - ''' +def get(obj, segments: Path): + """ Return the value at the path indicated by segments. get(obj, segments) -> value - ''' + """ current = obj - for (i, segment) in enumerate(segments): + for i, segment in enumerate(segments): if leaf(current): - raise PathNotFound('Path: {}[{}]'.format(segments, i)) + raise PathNotFound(f"Path: {segments}[{i}]") + + if isinstance(current, Sequence) and isinstance(segment, str) and segment.isdecimal(): + segment = int(segment) current = current[segment] return current def has(obj, segments): - ''' + """ Return True if the path exists in the obj. Otherwise return False. has(obj, segments) -> bool - ''' + """ try: get(obj, segments) return True @@ -92,24 +113,24 @@ def has(obj, segments): def expand(segments): - ''' + """ Yield a tuple of segments for each possible length of segments. Starting from the shortest length of segments and increasing by 1. expand(keys) -> (..., keys[:-2], keys[:-1]) - ''' + """ index = 0 - for segment in segments: + for _ in segments: index += 1 yield segments[:index] def types(obj, segments): - ''' + """ For each segment produce a tuple of (segment, type(value)). types(obj, segments) -> ((segment[0], type0), (segment[1], type1), ...) - ''' + """ result = [] for depth in expand(segments): result.append((depth[-1], type(get(obj, depth)))) @@ -117,43 +138,39 @@ def types(obj, segments): def leaves(obj): - ''' + """ Yield all leaves as (segment, value) pairs. leaves(obj) -> (generator -> (segment, value)) - ''' + """ return filter(lambda p: leafy(p[1]), walk(obj)) -def int_str(segment): - ''' +def int_str(segment: PathSegment) -> PathSegment: + """ If the segment is an integer, return the string conversion. - Otherwise return the segment unchanged. The conversion uses 'str' - which means that: - - * in Python 2.x: 0 -> b'0' - * in Python 3.x: 0 -> u'0' + Otherwise return the segment unchanged. The conversion uses 'str'. int_str(segment) -> str - ''' + """ if isinstance(segment, int): return str(segment) return segment class Star(object): - ''' + """ Used to create a global STAR symbol for tracking stars added when expanding star-star globs. - ''' + """ pass STAR = Star() -def match(segments, glob): - ''' +def match(segments: Path, glob: Glob): + """ Return True if the segments match the given glob, otherwise False. For the purposes of matching, integers are converted to their string @@ -169,33 +186,23 @@ def match(segments, glob): fnmatch.fnmatchcase returns True. If fnmatchcase returns False or throws an exception the result will be False. - A Word of Caution: - - *** Globs can have different results between Python 2.x and 3.x. *** - - fnmatchcase differs in behavior between Python 2.x and 3.x with - regard to handling comparison between different types (e.g. bytes - and unicode). In 2.x it will attempt the comparison implicitly - converting, but in 3.x it will throw an exception. - match(segments, glob) -> bool - ''' + """ segments = tuple(segments) glob = tuple(glob) path_len = len(segments) glob_len = len(glob) - # Index of the star-star in the glob. - ss = -1 # The star-star normalized glob ('**' has been removed). ss_glob = glob if '**' in glob: + # Index of the star-star in the glob. ss = glob.index('**') + if '**' in glob[ss + 1:]: - raise InvalidGlob("Invalid glob. Only one '**' is permitted per glob: {}" - "".format(glob)) + raise InvalidGlob(f"Invalid glob. Only one '**' is permitted per glob: {glob}") # Convert '**' segment into multiple '*' segments such that the # lengths of the path and glob match. '**' also can collapse and @@ -213,19 +220,30 @@ def match(segments, glob): # If we were successful in matching up the lengths, then we can # compare them using fnmatch. if path_len == len(ss_glob): - for (s, g) in zip(map(int_str, segments), map(int_str, ss_glob)): + i = zip(segments, ss_glob) + for s, g in i: # Match the stars we added to the glob to the type of the # segment itself. if g is STAR: if isinstance(s, bytes): g = b'*' else: - g = u'*' + g = '*' + + try: + # If search path segment (s) is an int then assume currently evaluated index (g) might be a sequence + # index as well. Try converting it to an int. + if isinstance(s, int) and s == int(g): + continue + except: + # Will reach this point if g can't be converted to an int (e.g. when g is a RegEx pattern). + # In this case convert s to a str so fnmatch can work on it. + s = str(s) - # Let's see if the glob matches. We will turn any kind of - # exception while attempting to match into a False for the - # match. try: + # Let's see if the glob matches. We will turn any kind of + # exception while attempting to match into a False for the + # match. if not fnmatchcase(s, g): return False except: @@ -239,15 +257,15 @@ def match(segments, glob): return False -def extend(thing, index, value=None): - ''' +def extend(thing: MutableSequence, index: int, value=None): + """ Extend a sequence like thing such that it contains at least index + 1 many elements. The extension values will be None (default). extend(thing, int) -> [thing..., None, ...] - ''' + """ try: - expansion = (type(thing)()) + expansion = type(thing)() # Using this rather than the multiply notation in order to support a # wider variety of sequence like things. @@ -264,17 +282,25 @@ def extend(thing, index, value=None): return thing -def __default_creator__(current, segments, i, hints=()): - ''' +def _default_creator( + current: Union[MutableMapping, Sequence], + segments: Sequence[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () +): + """ Create missing path components. If the segment is an int, then it will create a list. Otherwise a dictionary is created. set(obj, segments, value) -> obj - ''' + """ segment = segments[i] length = len(segments) - if isinstance(segment, int): + if isinstance(current, Sequence): + segment = int(segment) + + if isinstance(current, MutableSequence): extend(current, segment) # Infer the type from the hints provided. @@ -288,26 +314,37 @@ def __default_creator__(current, segments, i, hints=()): else: segment_next = None - if isinstance(segment_next, int): + if isinstance(segment_next, int) or (isinstance(segment_next, str) and segment_next.isdecimal()): current[segment] = [] else: current[segment] = {} -def set(obj, segments, value, creator=__default_creator__, hints=()): - ''' +def set( + obj: MutableMapping, + segments: Sequence[PathSegment], + value, + creator: Optional[Creator] = _default_creator, + hints: Hints = () +) -> MutableMapping: + """ Set the value in obj at the place indicated by segments. If creator is not - None (default __default_creator__), then call the creator function to + None (default _default_creator), then call the creator function to create any missing path components. set(obj, segments, value) -> obj - ''' + """ current = obj length = len(segments) # For everything except the last value, walk down the path and # create if creator is set. for (i, segment) in enumerate(segments[:-1]): + + # If segment is non-int but supposed to be a sequence index + if isinstance(segment, str) and isinstance(current, Sequence) and segment.isdecimal(): + segment = int(segment) + try: # Optimistically try to get the next value. This makes the # code agnostic to whether current is a list or a dict. @@ -316,24 +353,30 @@ def set(obj, segments, value, creator=__default_creator__, hints=()): current[segment] except: if creator is not None: - creator(current, segments, i, hints=hints) + creator(current, segments, i, hints) else: raise current = current[segment] if i != length - 1 and leaf(current): - raise PathNotFound('Path: {}[{}]'.format(segments, i)) + raise PathNotFound(f"Path: {segments}[{i}]") + + last_segment = segments[-1] + + # Resolve ambiguity of last segment + if isinstance(last_segment, str) and isinstance(current, Sequence) and last_segment.isdecimal(): + last_segment = int(last_segment) - if isinstance(segments[-1], int): - extend(current, segments[-1]) + if isinstance(last_segment, int): + extend(current, last_segment) - current[segments[-1]] = value + current[last_segment] = value return obj def fold(obj, f, acc): - ''' + """ Walk obj applying f to each path and returning accumulator acc. The function f will be called, for each result in walk(obj): @@ -345,7 +388,7 @@ def fold(obj, f, acc): retrieved from the walk. fold(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc - ''' + """ for pair in walk(obj): if f(obj, pair, acc) is False: break @@ -353,33 +396,34 @@ def fold(obj, f, acc): def foldm(obj, f, acc): - ''' + """ Same as fold(), but permits mutating obj. This requires all paths in walk(obj) to be loaded into memory (whereas fold does not). foldm(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc - ''' + """ pairs = tuple(walk(obj)) for pair in pairs: - (segments, value) = pair if f(obj, pair, acc) is False: break return acc -def view(obj, glob): - ''' +def view(obj: MutableMapping, glob: Glob): + """ Return a view of the object where the glob matches. A view retains the same form as the obj, but is limited to only the paths that matched. Views are new objects (a deepcopy of the matching values). view(obj, glob) -> obj' - ''' + """ + def f(obj, pair, result): (segments, value) = pair if match(segments, glob): if not has(result, segments): set(result, segments, deepcopy(value), hints=types(obj, segments)) + return fold(obj, f, type(obj)()) diff --git a/dpath/types.py b/dpath/types.py new file mode 100644 index 0000000..4f10162 --- /dev/null +++ b/dpath/types.py @@ -0,0 +1,74 @@ +from enum import IntFlag, auto +from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping + + +class ListIndex(int): + """Same as a normal int but mimics the behavior of list indices (can be compared to a negative number).""" + + def __new__(cls, value: int, list_length: int, *args, **kwargs): + if value >= list_length: + raise TypeError( + f"Tried to initiate a {cls.__name__} with a value ({value}) " + f"greater than the provided max value ({list_length})" + ) + + obj = super().__new__(cls, value) + obj.list_length = list_length + + return obj + + def __eq__(self, other): + if not isinstance(other, int): + return False + + # Based on how Python sequences handle negative indices as described in footnote (3) of https://docs.python.org/3/library/stdtypes.html#common-sequence-operations + return other == int(self) or self.list_length + other == int(self) + + def __repr__(self): + return f"<{self.__class__.__name__} {int(self)}/{self.list_length}>" + + def __str__(self): + return str(int(self)) + + +class MergeType(IntFlag): + ADDITIVE = auto() + """List objects are combined onto one long list (NOT a set). This is the default flag.""" + + REPLACE = auto() + """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination \ + with the source.""" + + TYPESAFE = auto() + """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source \ + replaces the destination in this situation.""" + + +PathSegment = Union[int, str, bytes] +"""Type alias for dict path segments where integers are explicitly casted.""" + +Filter = Callable[[Any], bool] +"""Type alias for filter functions. + +(Any) -> bool""" + +Glob = Union[str, Sequence[str]] +"""Type alias for glob parameters.""" + +Path = Union[str, Sequence[PathSegment]] +"""Type alias for path parameters.""" + +Hints = Sequence[Tuple[PathSegment, type]] +"""Type alias for creator function hint sequences.""" + +Creator = Callable[[Union[MutableMapping, List], Path, int, Optional[Hints]], None] +"""Type alias for creator functions. + +Example creator function signature: + + def creator( + current: Union[MutableMapping, List], + segments: Sequence[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () + )""" diff --git a/dpath/util.py b/dpath/util.py index f90fd6e..138a5e5 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,353 +1,51 @@ -from collections.abc import MutableMapping -from collections.abc import MutableSequence -from dpath import options -from dpath.exceptions import InvalidKeyName -import dpath.segments +import warnings -_DEFAULT_SENTINAL = object() -MERGE_REPLACE = (1 << 1) -MERGE_ADDITIVE = (1 << 2) -MERGE_TYPESAFE = (1 << 3) +import dpath +from dpath import _DEFAULT_SENTINEL +from dpath.types import MergeType -def __safe_path__(path, separator): - ''' - Given a path and separator, return a tuple of segments. If path is - already a non-leaf thing, return it. - Note that a string path with the separator at index[0] will have the - separator stripped off. If you pass a list path, the separator is - ignored, and is assumed to be part of each key glob. It will not be - stripped. - ''' - if not dpath.segments.leaf(path): - segments = path - else: - segments = path.lstrip(separator).split(separator) +def deprecated(func): + message = \ + "The dpath.util package is being deprecated. All util functions have been moved to dpath package top level." - # FIXME: This check was in the old internal library, but I can't - # see a way it could fail... - for i, segment in enumerate(segments): - if (separator and (separator in segment)): - raise InvalidKeyName("{} at {}[{}] contains the separator '{}'" - "".format(segment, segments, i, separator)) + def wrapper(*args, **kwargs): + warnings.warn(message, DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) - # Attempt to convert integer segments into actual integers. - final = [] - for segment in segments: - try: - final.append(int(segment)) - except: - final.append(segment) - segments = final + return wrapper - return segments +@deprecated +def new(obj, path, value, separator="/", creator=None): + return dpath.new(obj, path, value, separator, creator) -def new(obj, path, value, separator='/', creator=None): - ''' - Set the element at the terminus of path to value, and create - it if it does not exist (as opposed to 'set' that can only - change existing keys). - path will NOT be treated like a glob. If it has globbing - characters in it, they will become part of the resulting - keys +@deprecated +def delete(obj, glob, separator="/", afilter=None): + return dpath.delete(obj, glob, separator, afilter) - creator allows you to pass in a creator method that is - responsible for creating missing keys at arbitrary levels of - the path (see the help for dpath.path.set) - ''' - segments = __safe_path__(path, separator) - if creator: - return dpath.segments.set(obj, segments, value, creator=creator) - return dpath.segments.set(obj, segments, value) +@deprecated +def set(obj, glob, value, separator="/", afilter=None): + return dpath.set(obj, glob, value, separator, afilter) -def delete(obj, glob, separator='/', afilter=None): - ''' - Given a obj, delete all elements that match the glob. - Returns the number of deleted objects. Raises PathNotFound if no paths are - found to delete. - ''' - globlist = __safe_path__(glob, separator) +@deprecated +def get(obj, glob, separator="/", default=_DEFAULT_SENTINEL): + return dpath.get(obj, glob, separator, default) - def f(obj, pair, counter): - (segments, value) = pair - # Skip segments if they no longer exist in obj. - if not dpath.segments.has(obj, segments): - return +@deprecated +def values(obj, glob, separator="/", afilter=None, dirs=True): + return dpath.values(obj, glob, separator, afilter, dirs) - matched = dpath.segments.match(segments, globlist) - selected = afilter and dpath.segments.leaf(value) and afilter(value) - if (matched and not afilter) or selected: - key = segments[-1] - parent = dpath.segments.get(obj, segments[:-1]) +@deprecated +def search(obj, glob, yielded=False, separator="/", afilter=None, dirs=True): + return dpath.search(obj, glob, yielded, separator, afilter, dirs) - try: - # Attempt to treat parent like a sequence. - parent[0] - if len(parent) - 1 == key: - # Removing the last element of a sequence. It can be - # truly removed without affecting the ordering of - # remaining items. - # - # Note: In order to achieve proper behavior we are - # relying on the reverse iteration of - # non-dictionaries from dpath.segments.kvs(). - # Otherwise we'd be unable to delete all the tails - # of a list and end up with None values when we - # don't need them. - del parent[key] - else: - # This key can't be removed completely because it - # would affect the order of items that remain in our - # result. - parent[key] = None - except: - # Attempt to treat parent like a dictionary instead. - del parent[key] - - counter[0] += 1 - - [deleted] = dpath.segments.foldm(obj, f, [0]) - if not deleted: - raise dpath.exceptions.PathNotFound("Could not find {0} to delete it".format(glob)) - - return deleted - - -def set(obj, glob, value, separator='/', afilter=None): - ''' - Given a path glob, set all existing elements in the document - to the given value. Returns the number of elements changed. - ''' - globlist = __safe_path__(glob, separator) - - def f(obj, pair, counter): - (segments, found) = pair - - # Skip segments if they no longer exist in obj. - if not dpath.segments.has(obj, segments): - return - - matched = dpath.segments.match(segments, globlist) - selected = afilter and dpath.segments.leaf(found) and afilter(found) - - if (matched and not afilter) or (matched and selected): - dpath.segments.set(obj, segments, value, creator=None) - counter[0] += 1 - - [changed] = dpath.segments.foldm(obj, f, [0]) - return changed - - -def get(obj, glob, separator='/', default=_DEFAULT_SENTINAL): - ''' - Given an object which contains only one possible match for the given glob, - return the value for the leaf matching the given glob. - If the glob is not found and a default is provided, - the default is returned. - - If more than one leaf matches the glob, ValueError is raised. If the glob is - not found and a default is not provided, KeyError is raised. - ''' - if glob == '/': - return obj - - globlist = __safe_path__(glob, separator) - - def f(obj, pair, results): - (segments, found) = pair - - if dpath.segments.match(segments, globlist): - results.append(found) - if len(results) > 1: - return False - - results = dpath.segments.fold(obj, f, []) - - if len(results) == 0: - if default is not _DEFAULT_SENTINAL: - return default - - raise KeyError(glob) - elif len(results) > 1: - raise ValueError("dpath.util.get() globs must match only one leaf : %s" % glob) - - return results[0] - - -def values(obj, glob, separator='/', afilter=None, dirs=True): - ''' - Given an object and a path glob, return an array of all values which match - the glob. The arguments to this function are identical to those of search(). - ''' - yielded = True - - return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] - - -def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): - ''' - Given a path glob, return a dictionary containing all keys - that matched the given glob. - - If 'yielded' is true, then a dictionary will not be returned. - Instead tuples will be yielded in the form of (path, value) for - every element in the document that matched the glob. - ''' - - globlist = __safe_path__(glob, separator) - - def keeper(segments, found): - ''' - Generalized test for use in both yielded and folded cases. - Returns True if we want this result. Otherwise returns False. - ''' - if not dirs and not dpath.segments.leaf(found): - return False - - matched = dpath.segments.match(segments, globlist) - selected = afilter and afilter(found) - - return (matched and not afilter) or (matched and selected) - - if yielded: - def yielder(): - for segments, found in dpath.segments.walk(obj): - if keeper(segments, found): - yield (separator.join(map(dpath.segments.int_str, segments)), found) - return yielder() - else: - def f(obj, pair, result): - (segments, found) = pair - - if keeper(segments, found): - dpath.segments.set(result, segments, found, hints=dpath.segments.types(obj, segments)) - - return dpath.segments.fold(obj, f, {}) - - -def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): - ''' - Merge source into destination. Like dict.update() but performs deep - merging. - - NOTE: This does not do a deep copy of the source object. Applying merge - will result in references to src being present in the dst tree. If you do - not want src to potentially be modified by other changes in dst (e.g. more - merge calls), then use a deep copy of src. - - NOTE that merge() does NOT copy objects - it REFERENCES. If you merge - take these two dictionaries: - - >>> a = {'a': [0] } - >>> b = {'a': [1] } - - ... and you merge them into an empty dictionary, like so: - - >>> d = {} - >>> dpath.util.merge(d, a) - >>> dpath.util.merge(d, b) - - ... you might be surprised to find that a['a'] now contains [0, 1]. - This is because merge() says (d['a'] = a['a']), and thus creates a reference. - This reference is then modified when b is merged, causing both d and - a to have ['a'][0, 1]. To avoid this, make your own deep copies of source - objects that you intend to merge. For further notes see - https://github.com/akesterson/dpath-python/issues/58 - - flags is an OR'ed combination of MERGE_ADDITIVE, MERGE_REPLACE, - MERGE_TYPESAFE. - * MERGE_ADDITIVE : List objects are combined onto one long - list (NOT a set). This is the default flag. - * MERGE_REPLACE : Instead of combining list objects, when - 2 list objects are at an equal depth of merge, replace - the destination with the source. - * MERGE_TYPESAFE : When 2 keys at equal levels are of different - types, raise a TypeError exception. By default, the source - replaces the destination in this situation. - ''' - filtered_src = search(src, '**', afilter=afilter, separator='/') - - def are_both_mutable(o1, o2): - mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) - seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) - - if mapP or seqP: - return True - - return False - - def merger(dst, src, _segments=()): - for key, found in dpath.segments.kvs(src): - # Our current path in the source. - segments = _segments + (key,) - - if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: - raise InvalidKeyName("Empty string keys not allowed without " - "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(segments)) - - # Validate src and dst types match. - if flags & MERGE_TYPESAFE: - if dpath.segments.has(dst, segments): - target = dpath.segments.get(dst, segments) - tt = type(target) - ft = type(found) - if tt != ft: - path = separator.join(segments) - raise TypeError("Cannot merge objects of type" - "{0} and {1} at {2}" - "".format(tt, ft, path)) - - # Path not present in destination, create it. - if not dpath.segments.has(dst, segments): - dpath.segments.set(dst, segments, found) - continue - - # Retrieve the value in the destination. - target = dpath.segments.get(dst, segments) - - # If the types don't match, replace it. - if ((type(found) != type(target)) and (not are_both_mutable(found, target))): - dpath.segments.set(dst, segments, found) - continue - - # If target is a leaf, the replace it. - if dpath.segments.leaf(target): - dpath.segments.set(dst, segments, found) - continue - - # At this point we know: - # - # * The target exists. - # * The types match. - # * The target isn't a leaf. - # - # Pretend we have a sequence and account for the flags. - try: - if flags & MERGE_ADDITIVE: - target += found - continue - - if flags & MERGE_REPLACE: - try: - target[''] - except TypeError: - dpath.segments.set(dst, segments, found) - continue - except: - raise - except: - # We have a dictionary like thing and we need to attempt to - # recursively merge it. - merger(dst, found, segments) - - merger(dst, filtered_src) - - return dst +@deprecated +def merge(dst, src, separator="/", afilter=None, flags=MergeType.ADDITIVE): + return dpath.merge(dst, src, separator, afilter, flags) diff --git a/dpath/version.py b/dpath/version.py index b46c2e7..3c00bb4 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.1" +VERSION = "2.2.0" diff --git a/flake8.ini b/flake8.ini new file mode 100644 index 0000000..92830d3 --- /dev/null +++ b/flake8.ini @@ -0,0 +1,5 @@ +[flake8] +filename= + setup.py, + dpath/, + tests/ diff --git a/maintainers_log.md b/maintainers_log.md new file mode 100644 index 0000000..6f932a3 --- /dev/null +++ b/maintainers_log.md @@ -0,0 +1,17 @@ +# 03/29/2020 + +Attendees : Caleb, Andrew + +## Old business : + +* Need to onboard new member Vladimir Ulogov + * No movement +* Need to make project board for 1.5 open bugs + * Done + +## New business : + +* Andrew to define maintainers meeting process and establish log of decisions, process for filing open action items +* Andrew to forward maintainers invite to Vladimir and include in next monthly maintainers meeting +* Andrew to set followup for 1wk from now to check for comments on PRs and cut release version for 1.x / 2.x +* Andrew to rename LTS branches from version/1.0 version/2.0 to version/1.x and version/2.x diff --git a/setup.py b/setup.py index f85ae47..28eb69c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ -from distutils.core import setup -import dpath.version import os +from setuptools import setup +import dpath.version long_description = open( os.path.join( @@ -13,7 +13,7 @@ if __name__ == "__main__": setup( name="dpath", - url="https://www.github.com/akesterson/dpath-python", + url="https://github.com/dpath-maintainers/dpath-python", version=dpath.version.VERSION, description="Filesystem-like pathing and searching for dictionaries", long_description=long_description, @@ -25,14 +25,32 @@ scripts=[], packages=["dpath"], data_files=[], + package_data={"dpath": ["py.typed"]}, + + # Type hints are great. + # Function annotations were added in Python 3.0. + # Typing module was added in Python 3.5. + # Variable annotations were added in Python 3.6. + # Python versions that are >=3.6 are more popular. + # (Source: https://github.com/hugovk/pypi-tools/blob/master/README.md) + # + # Conclusion: In order to accommodate type hinting support must be limited to Python versions >=3.6. + # 3.6 was dropped because of EOL and this issue: https://github.com/actions/setup-python/issues/544 + python_requires=">=3.7", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Natural Language :: English', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Software Development :: Libraries :: Python Modules', + 'Typing :: Typed', ], ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e188ff3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +import warnings + +warnings.simplefilter("always", DeprecationWarning) diff --git a/tests/test_broken_afilter.py b/tests/test_broken_afilter.py index a59454e..683c727 100644 --- a/tests/test_broken_afilter.py +++ b/tests/test_broken_afilter.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath import sys @@ -25,15 +25,15 @@ def afilter(x): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c']) + for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): + assert path in paths + assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c']) + for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): + assert path in paths + assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] def filter(x): sys.stderr.write(str(x)) @@ -52,7 +52,7 @@ def filter(x): ], } - results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', yielded=True)] - results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', afilter=filter, yielded=True)] - assert(len(results) == 1) - assert(results[0][1]['type'] == 'correct') + results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', yielded=True)] + results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', afilter=filter, yielded=True)] + assert len(results) == 1 + assert results[0][1]['type'] == 'correct' diff --git a/tests/test_util_delete.py b/tests/test_delete.py similarity index 54% rename from tests/test_util_delete.py rename to tests/test_delete.py index c14acf7..c7879b0 100644 --- a/tests/test_util_delete.py +++ b/tests/test_delete.py @@ -1,5 +1,6 @@ -from nose.tools import raises -import dpath.util +from nose2.tools.such import helper + +import dpath import dpath.exceptions @@ -10,8 +11,8 @@ def test_delete_separator(): }, } - dpath.util.delete(dict, ';a;b', separator=";") - assert('b' not in dict['a']) + dpath.delete(dict, ';a;b', separator=";") + assert 'b' not in dict['a'] def test_delete_existing(): @@ -21,18 +22,18 @@ def test_delete_existing(): }, } - dpath.util.delete(dict, '/a/b') - assert('b' not in dict['a']) + dpath.delete(dict, '/a/b') + assert 'b' not in dict['a'] -@raises(dpath.exceptions.PathNotFound) def test_delete_missing(): dict = { "a": { }, } - dpath.util.delete(dict, '/a/b') + with helper.assertRaises(dpath.exceptions.PathNotFound): + dpath.delete(dict, '/a/b') def test_delete_filter(): @@ -49,7 +50,7 @@ def afilter(x): }, } - dpath.util.delete(dict, '/a/*', afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert ('d' not in dict['a']) + dpath.delete(dict, '/a/*', afilter=afilter) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert 'd' not in dict['a'] diff --git a/tests/test_get_values.py b/tests/test_get_values.py new file mode 100644 index 0000000..68d5d3b --- /dev/null +++ b/tests/test_get_values.py @@ -0,0 +1,214 @@ +import datetime +import decimal +import time + +from unittest import mock + +from nose2.tools.such import helper + +import dpath + + +def test_util_get_root(): + x = {'p': {'a': {'t': {'h': 'value'}}}} + + ret = dpath.get(x, '/p/a/t/h') + assert ret == 'value' + + ret = dpath.get(x, '/') + assert ret == x + + ret = dpath.get(x, []) + assert ret == x + + +def test_get_explicit_single(): + ehash = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + + assert dpath.get(ehash, '/a/b/c/f') == 2 + assert dpath.get(ehash, ['a', 'b', 'c', 'f']) == 2 + assert dpath.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2 + assert dpath.get(ehash, ['does', 'not', 'exist'], default=None) is None + assert dpath.get(ehash, ['doesnt', 'exist'], default=5) == 5 + + +def test_get_glob_single(): + ehash = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + + assert dpath.get(ehash, '/a/b/*/f') == 2 + assert dpath.get(ehash, ['a', 'b', '*', 'f']) == 2 + assert dpath.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2 + assert dpath.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6 + + +def test_get_glob_multiple(): + ehash = { + "a": { + "b": { + "c": { + "d": 0, + }, + "e": { + "d": 0, + }, + }, + }, + } + + helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd'], default=3) + + +def test_get_absent(): + ehash = {} + + helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) + + +def test_values(): + ehash = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + + ret = dpath.values(ehash, '/a/b/c/*') + assert isinstance(ret, list) + assert 0 in ret + assert 1 in ret + assert 2 in ret + + ret = dpath.values(ehash, ['a', 'b', 'c', '*']) + assert isinstance(ret, list) + assert 0 in ret + assert 1 in ret + assert 2 in ret + + +@mock.patch('dpath.search') +def test_values_passes_through(searchfunc): + searchfunc.return_value = [] + + def y(): + return False + + dpath.values({}, '/a/b', ':', y, False) + searchfunc.assert_called_with({}, '/a/b', True, ':', y, False) + + dpath.values({}, ['a', 'b'], ':', y, False) + searchfunc.assert_called_with({}, ['a', 'b'], True, ':', y, False) + + +def test_none_values(): + d = {'p': {'a': {'t': {'h': None}}}} + + v = dpath.get(d, 'p/a/t/h') + assert v is None + + +def test_values_list(): + a = { + 'actions': [ + { + 'type': 'correct', + }, + { + 'type': 'incorrect', + }, + ], + } + + ret = dpath.values(a, 'actions/*') + assert isinstance(ret, list) + assert len(ret) == 2 + + +def test_non_leaf_leaf(): + # The leaves in this test aren't leaf(thing) == True, but we should still + # be able to get them. They should also not prevent fetching other values. + + def func(x): + return x + + testdict = { + 'a': func, + 'b': lambda x: x, + 'c': [ + { + 'a', + 'b', + }, + ], + 'd': [ + decimal.Decimal(1.5), + decimal.Decimal(2.25), + ], + 'e': datetime.datetime(2020, 1, 1), + 'f': { + 'config': 'something', + }, + } + + # It should be possible to get the callables: + assert dpath.get(testdict, 'a') == func + assert dpath.get(testdict, 'b')(42) == 42 + + # It should be possible to get other values: + assert dpath.get(testdict, 'c/0') == testdict['c'][0] + assert dpath.get(testdict, 'd')[0] == testdict['d'][0] + assert dpath.get(testdict, 'd/0') == testdict['d'][0] + assert dpath.get(testdict, 'd/1') == testdict['d'][1] + assert dpath.get(testdict, 'e') == testdict['e'] + + # Values should also still work: + assert dpath.values(testdict, 'f/config') == ['something'] + + # Data classes should also be retrievable: + try: + import dataclasses + except: + return + + @dataclasses.dataclass + class Connection: + group_name: str + channel_name: str + last_seen: float + + testdict['g'] = { + 'my-key': Connection( + group_name='foo', + channel_name='bar', + last_seen=time.time(), + ), + } + + assert dpath.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] diff --git a/tests/test_util_merge.py b/tests/test_merge.py similarity index 59% rename from tests/test_util_merge.py rename to tests/test_merge.py index 968a4fa..a8b638c 100644 --- a/tests/test_util_merge.py +++ b/tests/test_merge.py @@ -1,9 +1,10 @@ -import nose import copy -from nose.tools import raises +from nose2.tools.such import helper -import dpath.util + +import dpath +from dpath import MergeType def test_merge_typesafe_and_separator(): @@ -19,9 +20,9 @@ def test_merge_typesafe_and_separator(): } try: - dpath.util.merge(dst, src, flags=(dpath.util.MERGE_ADDITIVE | dpath.util.MERGE_TYPESAFE), separator=";") + dpath.merge(dst, src, flags=(dpath.MergeType.ADDITIVE | dpath.MergeType.TYPESAFE), separator=";") except TypeError as e: - assert(str(e).endswith("dict;integer")) + assert str(e).endswith("dict;integer") return raise Exception("MERGE_TYPESAFE failed to raise an exception when merging between str and int!") @@ -35,8 +36,8 @@ def test_merge_simple_int(): "integer": 3, } - dpath.util.merge(dst, src) - nose.tools.eq_(dst["integer"], src["integer"]) + dpath.merge(dst, src) + assert dst["integer"] == src["integer"], "%r != %r" % (dst["integer"], src["integer"]) def test_merge_simple_string(): @@ -47,8 +48,8 @@ def test_merge_simple_string(): "string": "lol I am a string", } - dpath.util.merge(dst, src) - nose.tools.eq_(dst["string"], src["string"]) + dpath.merge(dst, src) + assert dst["string"] == src["string"], "%r != %r" % (dst["string"], src["string"]) def test_merge_simple_list_additive(): @@ -59,8 +60,8 @@ def test_merge_simple_list_additive(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_ADDITIVE) - nose.tools.eq_(dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) + dpath.merge(dst, src, flags=MergeType.ADDITIVE) + assert dst["list"] == [0, 1, 2, 3, 7, 8, 9, 10], "%r != %r" % (dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) def test_merge_simple_list_replace(): @@ -71,8 +72,8 @@ def test_merge_simple_list_replace(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) - nose.tools.eq_(dst["list"], [7, 8, 9, 10]) + dpath.merge(dst, src, flags=dpath.MergeType.REPLACE) + assert dst["list"] == [7, 8, 9, 10], "%r != %r" % (dst["list"], [7, 8, 9, 10]) def test_merge_simple_dict(): @@ -87,8 +88,8 @@ def test_merge_simple_dict(): }, } - dpath.util.merge(dst, src) - nose.tools.eq_(dst["dict"]["key"], src["dict"]["key"]) + dpath.merge(dst, src) + assert dst["dict"]["key"] == src["dict"]["key"], "%r != %r" % (dst["dict"]["key"], src["dict"]["key"]) def test_merge_filter(): @@ -106,13 +107,12 @@ def afilter(x): } dst = {} - dpath.util.merge(dst, src, afilter=afilter) - assert ("key2" in dst) - assert ("key" not in dst) - assert ("otherdict" not in dst) + dpath.merge(dst, src, afilter=afilter) + assert "key2" in dst + assert "key" not in dst + assert "otherdict" not in dst -@raises(TypeError) def test_merge_typesafe(): src = { "dict": { @@ -123,10 +123,9 @@ def test_merge_typesafe(): ], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) -@raises(TypeError) def test_merge_mutables(): class tcid(dict): pass @@ -150,28 +149,28 @@ class tcis(list): "ms": tcis(['a', 'b', 'c']), } - dpath.util.merge(dst, src) + dpath.merge(dst, src) print(dst) - assert(dst["mm"]["a"] == src["mm"]["a"]) - assert(dst['ms'][2] == 'c') - assert("casserole" in dst["mm"]) + assert dst["mm"]["a"] == src["mm"]["a"] + assert dst['ms'][2] == 'c' + assert "casserole" in dst["mm"] - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} - dpath.util.merge(dct_a, dct_b, flags=dpath.util.MERGE_REPLACE) - assert(len(dct_a['a']['b']) == 1) + dpath.merge(dct_a, dct_b, flags=dpath.MergeType.REPLACE) + assert len(dct_a['a']['b']) == 1 def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} - dpath.util.merge(d1, d2, flags=dpath.util.MERGE_REPLACE) - assert(len(d1['a']) == 1) - assert(d1['a'][0] == 'a') + dpath.merge(d1, d2, flags=dpath.MergeType.REPLACE) + assert len(d1['a']) == 1 + assert d1['a'][0] == 'a' def test_merge_list(): @@ -181,18 +180,18 @@ def test_merge_list(): dst1 = {} for d in [copy.deepcopy(src), copy.deepcopy(p1)]: - dpath.util.merge(dst1, d) + dpath.merge(dst1, d) dst2 = {} for d in [copy.deepcopy(src), copy.deepcopy(p2)]: - dpath.util.merge(dst2, d) + dpath.merge(dst2, d) assert dst1["l"] == [1, 2] assert dst2["l"] == [1] dst1 = {} for d in [src, p1]: - dpath.util.merge(dst1, d) + dpath.merge(dst1, d) dst2 = {} for d in [src, p2]: - dpath.util.merge(dst2, d) + dpath.merge(dst2, d) assert dst1["l"] == [1, 2] assert dst2["l"] == [1, 2] diff --git a/tests/test_new.py b/tests/test_new.py new file mode 100644 index 0000000..ac47e7d --- /dev/null +++ b/tests/test_new.py @@ -0,0 +1,114 @@ +import dpath + + +def test_set_new_separator(): + dict = { + "a": { + }, + } + + dpath.new(dict, ';a;b', 1, separator=";") + assert dict['a']['b'] == 1 + + dpath.new(dict, ['a', 'b'], 1, separator=";") + assert dict['a']['b'] == 1 + + +def test_set_new_dict(): + dict = { + "a": { + }, + } + + dpath.new(dict, '/a/b', 1) + assert dict['a']['b'] == 1 + + dpath.new(dict, ['a', 'b'], 1) + assert dict['a']['b'] == 1 + + +def test_set_new_list(): + dict = { + "a": [ + ], + } + + dpath.new(dict, '/a/1', 1) + assert dict['a'][1] == 1 + assert dict['a'][0] is None + + dpath.new(dict, ['a', 1], 1) + assert dict['a'][1] == 1 + assert dict['a'][0] is None + + +def test_set_list_with_dict_int_ambiguity(): + d = {"list": [{"root": {"1": {"k": None}}}]} + + dpath.new(d, "list/0/root/1/k", "new") + + expected = {"list": [{"root": {"1": {"k": "new"}}}]} + + assert d == expected + + +def test_int_segment_list_type_check(): + d = {} + dpath.new(d, "a/b/0/c/0", "hello") + assert 'b' in d.get("a", {}) + assert isinstance(d["a"]["b"], list) + assert len(d["a"]["b"]) == 1 + assert 'c' in d["a"]["b"][0] + assert isinstance(d["a"]["b"][0]["c"], list) + assert len(d["a"]["b"][0]["c"]) == 1 + + +def test_int_segment_dict_type_check(): + d = {"a": {"b": {"0": {}}}} + dpath.new(d, "a/b/0/c/0", "hello") + assert "b" in d.get("a", {}) + assert isinstance(d["a"]["b"], dict) + assert '0' in d["a"]["b"] + assert 'c' in d["a"]["b"]["0"] + assert isinstance(d["a"]["b"]["0"]["c"], list) + + +def test_set_new_list_path_with_separator(): + # This test kills many birds with one stone, forgive me + dict = { + "a": { + }, + } + + dpath.new(dict, ['a', 'b/c/d', 0], 1) + assert len(dict['a']) == 1 + assert len(dict['a']['b/c/d']) == 1 + assert dict['a']['b/c/d'][0] == 1 + + +def test_set_new_list_integer_path_with_creator(): + d = {} + + def mycreator(obj, pathcomp, nextpathcomp, hints): + print(hints) + print(pathcomp) + print(nextpathcomp) + print("...") + + target = pathcomp[0] + if isinstance(obj, list) and (target.isdigit()): + target = int(target) + + if ((nextpathcomp is not None) and (isinstance(nextpathcomp, int) or str(nextpathcomp).isdigit())): + obj[target] = [None] * (int(nextpathcomp) + 1) + print("Created new list in target") + else: + print("Created new dict in target") + obj[target] = {} + print(obj) + + dpath.new(d, '/a/2', 3, creator=mycreator) + print(d) + assert isinstance(d['a'], list) + assert len(d['a']) == 3 + assert d['a'][2] == 3 diff --git a/tests/test_path_get.py b/tests/test_path_get.py index 96347f2..b2a4657 100644 --- a/tests/test_path_get.py +++ b/tests/test_path_get.py @@ -15,6 +15,6 @@ def test_path_get_list_of_dicts(): segments = ['a', 'b', 0, 0] res = dpath.segments.view(tdict, segments) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 1) - assert(res['a']['b'][0][0] == 0) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 1 + assert res['a']['b'][0][0] == 0 diff --git a/tests/test_path_paths.py b/tests/test_path_paths.py index 3364e6d..56595d2 100644 --- a/tests/test_path_paths.py +++ b/tests/test_path_paths.py @@ -1,10 +1,10 @@ -from nose.tools import raises +from nose2.tools.such import helper + import dpath.segments import dpath.exceptions import dpath.options -@raises(dpath.exceptions.InvalidKeyName) def test_path_paths_empty_key_disallowed(): tdict = { "Empty": { @@ -14,8 +14,9 @@ def test_path_paths_empty_key_disallowed(): } } - for x in dpath.segments.walk(tdict): - pass + with helper.assertRaises(dpath.exceptions.InvalidKeyName): + for x in dpath.segments.walk(tdict): + pass def test_path_paths_empty_key_allowed(): @@ -34,4 +35,4 @@ def test_path_paths_empty_key_allowed(): pass dpath.options.ALLOW_EMPTY_STRING_KEYS = False - assert("/".join(segments) == "Empty//Key") + assert "/".join(segments) == "Empty//Key" diff --git a/tests/test_paths.py b/tests/test_paths.py new file mode 100644 index 0000000..c63d728 --- /dev/null +++ b/tests/test_paths.py @@ -0,0 +1,9 @@ +import dpath + + +def test_util_safe_path_list(): + res = dpath._split_path(["Ignore", "the/separator"], None) + + assert len(res) == 2 + assert res[0] == "Ignore" + assert res[1] == "the/separator" diff --git a/tests/test_search.py b/tests/test_search.py new file mode 100644 index 0000000..7219d2b --- /dev/null +++ b/tests/test_search.py @@ -0,0 +1,245 @@ +import dpath + + +def test_search_paths_with_separator(): + dict = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + paths = [ + 'a', + 'a;b', + 'a;b;c', + 'a;b;c;d', + 'a;b;c;e', + 'a;b;c;f', + ] + + for (path, value) in dpath.search(dict, '/**', yielded=True, separator=";"): + assert path in paths + + for (path, value) in dpath.search(dict, ['**'], yielded=True, separator=";"): + assert path in paths + + +def test_search_paths(): + dict = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + paths = [ + 'a', + 'a/b', + 'a/b/c', + 'a/b/c/d', + 'a/b/c/e', + 'a/b/c/f', + ] + + for (path, value) in dpath.search(dict, '/**', yielded=True): + assert path in paths + + for (path, value) in dpath.search(dict, ['**'], yielded=True): + assert path in paths + + +def test_search_afilter(): + def afilter(x): + if x in [1, 2]: + return True + return False + + dict = { + "a": { + "view_failure": "a", + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + paths = [ + 'a/b/c/e', + 'a/b/c/f', + ] + + for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): + assert path in paths + assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] + + for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): + assert path in paths + assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] + + +def test_search_globbing(): + dict = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + paths = [ + 'a/b/c/d', + 'a/b/c/f', + ] + + for (path, value) in dpath.search(dict, '/a/**/[df]', yielded=True): + assert path in paths + + for (path, value) in dpath.search(dict, ['a', '**', '[df]'], yielded=True): + assert path in paths + + +def test_search_return_dict_head(): + tdict = { + "a": { + "b": { + 0: 0, + 1: 1, + 2: 2, + }, + }, + } + res = dpath.search(tdict, '/a/b') + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == {0: 0, 1: 1, 2: 2} + + res = dpath.search(tdict, ['a', 'b']) + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == {0: 0, 1: 1, 2: 2} + + +def test_search_return_dict_globbed(): + tdict = { + "a": { + "b": { + 0: 0, + 1: 1, + 2: 2, + }, + }, + } + + res = dpath.search(tdict, '/a/b/[02]') + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 2 + assert res['a']['b'] == {0: 0, 2: 2} + + res = dpath.search(tdict, ['a', 'b', '[02]']) + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 2 + assert res['a']['b'] == {0: 0, 2: 2} + + +def test_search_return_list_head(): + tdict = { + "a": { + "b": [ + 0, + 1, + 2, + ], + }, + } + + res = dpath.search(tdict, '/a/b') + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, 1, 2] + + res = dpath.search(tdict, ['a', 'b']) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, 1, 2] + + +def test_search_return_list_globbed(): + tdict = { + "a": { + "b": [ + 0, + 1, + 2, + ] + } + } + + res = dpath.search(tdict, '/a/b/[02]') + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, None, 2] + + res = dpath.search(tdict, ['a', 'b', '[02]']) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, None, 2] + + +def test_search_list_key_with_separator(): + tdict = { + "a": { + "b": { + "d": 'failure', + }, + "/b/d": 'success', + }, + } + + res = dpath.search(tdict, ['a', '/b/d']) + assert 'b' not in res['a'] + assert res['a']['/b/d'] == 'success' + + +def test_search_multiple_stars(): + testdata = { + 'a': [ + { + 'b': [ + {'c': 1}, + {'c': 2}, + {'c': 3}, + ], + }, + ], + } + testpath = 'a/*/b/*/c' + + res = dpath.search(testdata, testpath) + assert len(res['a'][0]['b']) == 3 + assert res['a'][0]['b'][0]['c'] == 1 + assert res['a'][0]['b'][1]['c'] == 2 + assert res['a'][0]['b'][2]['c'] == 3 + + +def test_search_negative_index(): + d = {'a': {'b': [1, 2, 3]}} + res = dpath.search(d, 'a/b/-1') + + assert res == dpath.search(d, "a/b/2") diff --git a/tests/test_segments.py b/tests/test_segments.py index af9df85..fb6a8bc 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -1,8 +1,11 @@ -from dpath import options +import os +from unittest import TestCase + +import hypothesis.strategies as st from hypothesis import given, assume, settings, HealthCheck + import dpath.segments as api -import hypothesis.strategies as st -import os +from dpath import options settings.register_profile("default", suppress_health_check=(HealthCheck.too_slow,)) settings.load_profile(os.getenv(u'HYPOTHESIS_PROFILE', 'default')) @@ -27,115 +30,6 @@ random_mutable_node = random_mutable_thing.filter(lambda thing: isinstance(thing, (list, dict))) -def setup(): - # Allow empty strings in segments. - options.ALLOW_EMPTY_STRING_KEYS = True - - -def teardown(): - # Revert back to default. - options.ALLOW_EMPTY_STRING_KEYS = False - - -@given(random_node) -def test_kvs(node): - ''' - Given a node, kvs should produce a key that when used to extract - from the node renders the exact same value given. - ''' - for k, v in api.kvs(node): - assert node[k] is v - - -@given(random_leaf) -def test_leaf_with_leaf(leaf): - ''' - Given a leaf, leaf should return True. - ''' - assert api.leaf(leaf) is True - - -@given(random_node) -def test_leaf_with_node(node): - ''' - Given a node, leaf should return False. - ''' - assert api.leaf(node) is False - - -@given(random_thing) -def test_walk(thing): - ''' - Given a thing to walk, walk should yield key, value pairs where key - is a tuple of non-zero length. - ''' - for k, v in api.walk(thing): - assert isinstance(k, tuple) - assert len(k) > 0 - - -@given(random_node) -def test_get(node): - ''' - Given a node, get should return the exact value given a key for all - key, value pairs in the node. - ''' - for k, v in api.walk(node): - assert api.get(node, k) is v - - -@given(random_node) -def test_has(node): - ''' - Given a node, has should return True for all paths, False otherwise. - ''' - for k, v in api.walk(node): - assert api.has(node, k) is True - - # If we are at a leaf, then we can create a value that isn't - # present easily. - if api.leaf(v): - assert api.has(node, k + (0,)) is False - - -@given(random_segments) -def test_expand(segments): - ''' - Given segments expand should produce as many results are there were - segments and the last result should equal the given segments. - ''' - count = len(segments) - result = list(api.expand(segments)) - - assert count == len(result) - - if count > 0: - assert segments == result[-1] - - -@given(random_node) -def test_types(node): - ''' - Given a node, types should yield a tuple of key, type pairs and the - type indicated should equal the type of the value. - ''' - for k, v in api.walk(node): - ts = api.types(node, k) - ta = () - for tk, tt in ts: - ta += (tk,) - assert type(api.get(node, ta)) is tt - - -@given(random_node) -def test_leaves(node): - ''' - Given a node, leaves should yield only leaf key, value pairs. - ''' - for k, v in api.leaves(node): - assert api.leafy(v) - - @st.composite def mutate(draw, segment): # Convert number segments. @@ -216,7 +110,7 @@ def random_segments_with_glob(draw): stop = draw(st.integers(start, len(glob))) glob[start:stop] = ['**'] - return (segments, glob) + return segments, glob @st.composite @@ -243,25 +137,6 @@ def random_segments_with_nonmatching_glob(draw): return (segments, glob) -@given(random_segments_with_glob()) -def test_match(pair): - ''' - Given segments and a known good glob, match should be True. - ''' - (segments, glob) = pair - assert api.match(segments, glob) is True - - -@given(random_segments_with_nonmatching_glob()) -def test_match_nonmatching(pair): - ''' - Given segments and a known bad glob, match should be False. - ''' - print(pair) - (segments, glob) = pair - assert api.match(segments, glob) is False - - @st.composite def random_walk(draw): node = draw(random_mutable_node) @@ -278,64 +153,179 @@ def random_leaves(draw): return (node, draw(st.sampled_from(found))) -@given(walkable=random_walk(), value=random_thing) -def test_set_walkable(walkable, value): - ''' - Given a walkable location, set should be able to update any value. - ''' - (node, (segments, found)) = walkable - api.set(node, segments, value) - assert api.get(node, segments) is value - - -@given(walkable=random_leaves(), - kstr=random_key_str, - kint=random_key_int, - value=random_thing, - extension=random_segments) -def test_set_create_missing(walkable, kstr, kint, value, extension): - ''' - Given a walkable non-leaf, set should be able to create missing - nodes and set a new value. - ''' - (node, (segments, found)) = walkable - assume(api.leaf(found)) - - parent_segments = segments[:-1] - parent = api.get(node, parent_segments) - - if isinstance(parent, list): - assume(len(parent) < kint) - destination = parent_segments + (kint,) + tuple(extension) - elif isinstance(parent, dict): - assume(kstr not in parent) - destination = parent_segments + (kstr,) + tuple(extension) - else: - raise Exception('mad mad world') - - api.set(node, destination, value) - assert api.get(node, destination) is value +class TestSegments(TestCase): + @classmethod + def setUpClass(cls): + # Allow empty strings in segments. + options.ALLOW_EMPTY_STRING_KEYS = True + + @classmethod + def tearDownClass(cls): + # Revert back to default. + options.ALLOW_EMPTY_STRING_KEYS = False + + @given(random_node) + def test_kvs(self, node): + ''' + Given a node, kvs should produce a key that when used to extract + from the node renders the exact same value given. + ''' + for k, v in api.make_walkable(node): + assert node[k] is v + + @given(random_leaf) + def test_leaf_with_leaf(self, leaf): + ''' + Given a leaf, leaf should return True. + ''' + assert api.leaf(leaf) is True + + @given(random_node) + def test_leaf_with_node(self, node): + ''' + Given a node, leaf should return False. + ''' + assert api.leaf(node) is False + + @given(random_thing) + def test_walk(self, thing): + ''' + Given a thing to walk, walk should yield key, value pairs where key + is a tuple of non-zero length. + ''' + for k, v in api.walk(thing): + assert isinstance(k, tuple) + assert len(k) > 0 + + @given(random_node) + def test_get(self, node): + ''' + Given a node, get should return the exact value given a key for all + key, value pairs in the node. + ''' + for k, v in api.walk(node): + assert api.get(node, k) is v + + @given(random_node) + def test_has(self, node): + ''' + Given a node, has should return True for all paths, False otherwise. + ''' + for k, v in api.walk(node): + assert api.has(node, k) is True + + # If we are at a leaf, then we can create a value that isn't + # present easily. + if api.leaf(v): + assert api.has(node, k + (0,)) is False + + @given(random_segments) + def test_expand(self, segments): + ''' + Given segments expand should produce as many results are there were + segments and the last result should equal the given segments. + ''' + count = len(segments) + result = list(api.expand(segments)) + + assert count == len(result) + + if count > 0: + assert segments == result[-1] + + @given(random_node) + def test_types(self, node): + ''' + Given a node, types should yield a tuple of key, type pairs and the + type indicated should equal the type of the value. + ''' + for k, v in api.walk(node): + ts = api.types(node, k) + ta = () + for tk, tt in ts: + ta += (tk,) + assert type(api.get(node, ta)) is tt + + @given(random_node) + def test_leaves(self, node): + ''' + Given a node, leaves should yield only leaf key, value pairs. + ''' + for k, v in api.leaves(node): + assert api.leafy(v) + + @given(random_segments_with_glob()) + def test_match(self, pair): + ''' + Given segments and a known good glob, match should be True. + ''' + (segments, glob) = pair + assert api.match(segments, glob) is True + + @given(random_segments_with_nonmatching_glob()) + def test_match_nonmatching(self, pair): + ''' + Given segments and a known bad glob, match should be False. + ''' + (segments, glob) = pair + assert api.match(segments, glob) is False + + @given(walkable=random_walk(), value=random_thing) + def test_set_walkable(self, walkable, value): + ''' + Given a walkable location, set should be able to update any value. + ''' + (node, (segments, found)) = walkable + api.set(node, segments, value) + assert api.get(node, segments) is value + + @given(walkable=random_leaves(), + kstr=random_key_str, + kint=random_key_int, + value=random_thing, + extension=random_segments) + def test_set_create_missing(self, walkable, kstr, kint, value, extension): + ''' + Given a walkable non-leaf, set should be able to create missing + nodes and set a new value. + ''' + (node, (segments, found)) = walkable + assume(api.leaf(found)) + + parent_segments = segments[:-1] + parent = api.get(node, parent_segments) + + if isinstance(parent, list): + assume(len(parent) < kint) + destination = parent_segments + (kint,) + tuple(extension) + elif isinstance(parent, dict): + assume(kstr not in parent) + destination = parent_segments + (kstr,) + tuple(extension) + else: + raise Exception('mad mad world') + api.set(node, destination, value) + assert api.get(node, destination) is value -@given(thing=random_thing) -def test_fold(thing): - ''' - Given a thing, count paths with fold. - ''' - def f(o, p, a): - a[0] += 1 + @given(thing=random_thing) + def test_fold(self, thing): + ''' + Given a thing, count paths with fold. + ''' - [count] = api.fold(thing, f, [0]) - assert count == len(tuple(api.walk(thing))) + def f(o, p, a): + a[0] += 1 + [count] = api.fold(thing, f, [0]) + assert count == len(tuple(api.walk(thing))) -@given(walkable=random_walk()) -def test_view(walkable): - ''' - Given a walkable location, view that location. - ''' - (node, (segments, found)) = walkable - assume(found == found) # Hello, nan! We don't want you here. + @given(walkable=random_walk()) + def test_view(self, walkable): + ''' + Given a walkable location, view that location. + ''' + (node, (segments, found)) = walkable + assume(found == found) # Hello, nan! We don't want you here. - view = api.view(node, segments) - assert api.get(view, segments) == api.get(node, segments) + view = api.view(node, segments) + assert api.get(view, segments) == api.get(node, segments) diff --git a/tests/test_set.py b/tests/test_set.py new file mode 100644 index 0000000..ef2dd96 --- /dev/null +++ b/tests/test_set.py @@ -0,0 +1,91 @@ +import dpath + + +def test_set_existing_separator(): + dict = { + "a": { + "b": 0, + }, + } + + dpath.set(dict, ';a;b', 1, separator=";") + assert dict['a']['b'] == 1 + + dict['a']['b'] = 0 + dpath.set(dict, ['a', 'b'], 1, separator=";") + assert dict['a']['b'] == 1 + + +def test_set_existing_dict(): + dict = { + "a": { + "b": 0, + }, + } + + dpath.set(dict, '/a/b', 1) + assert dict['a']['b'] == 1 + + dict['a']['b'] = 0 + dpath.set(dict, ['a', 'b'], 1) + assert dict['a']['b'] == 1 + + +def test_set_existing_list(): + dict = { + "a": [ + 0, + ], + } + + dpath.set(dict, '/a/0', 1) + assert dict['a'][0] == 1 + + dict['a'][0] = 0 + dpath.set(dict, ['a', '0'], 1) + assert dict['a'][0] == 1 + + +def test_set_filter(): + def afilter(x): + if int(x) == 31: + return True + return False + + dict = { + "a": { + "b": 0, + "c": 1, + "d": 31, + } + } + + dpath.set(dict, '/a/*', 31337, afilter=afilter) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert dict['a']['d'] == 31337 + + dict = { + "a": { + "b": 0, + "c": 1, + "d": 31, + } + } + + dpath.set(dict, ['a', '*'], 31337, afilter=afilter) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert dict['a']['d'] == 31337 + + +def test_set_existing_path_with_separator(): + dict = { + "a": { + 'b/c/d': 0, + }, + } + + dpath.set(dict, ['a', 'b/c/d'], 1) + assert len(dict['a']) == 1 + assert dict['a']['b/c/d'] == 1 diff --git a/tests/test_types.py b/tests/test_types.py index 82f8c05..39993f3 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -1,19 +1,16 @@ -import nose -import dpath.util -from nose.tools import assert_raises +from collections.abc import MutableSequence, MutableMapping -try: - # python3, especially 3.8 - from collections.abc import MutableSequence - from collections.abc import MutableMapping -except ImportError: - # python2 - from collections import MutableSequence - from collections import MutableMapping +from nose2.tools.such import helper + +import dpath +from dpath import MergeType class TestMapping(MutableMapping): - def __init__(self, data={}): + def __init__(self, data=None): + if data is None: + data = {} + self._mapping = {} self._mapping.update(data) @@ -37,7 +34,10 @@ def __delitem__(self, key): class TestSequence(MutableSequence): - def __init__(self, data=list()): + def __init__(self, data=None): + if data is None: + data = list() + self._list = [] + data def __len__(self): @@ -71,13 +71,13 @@ def append(self, value): def test_types_set(): data = TestMapping({"a": TestSequence([0])}) - dpath.util.set(data, '/a/0', 1) - assert(data['a'][0] == 1) + dpath.set(data, '/a/0', 1) + assert data['a'][0] == 1 data['a'][0] = 0 - dpath.util.set(data, ['a', '0'], 1) - assert(data['a'][0] == 1) + dpath.set(data, ['a', '0'], 1) + assert data['a'][0] == 1 def test_types_get_list_of_dicts(): @@ -93,9 +93,9 @@ def test_types_get_list_of_dicts(): res = dpath.segments.view(tdict, ['a', 'b', 0, 0]) - assert(isinstance(res['a']['b'], TestSequence)) - assert(len(res['a']['b']) == 1) - assert(res['a']['b'][0][0] == 0) + assert isinstance(res['a']['b'], TestSequence) + assert len(res['a']['b']) == 1 + assert res['a']['b'][0][0] == 0 def test_types_merge_simple_list_replace(): @@ -106,14 +106,14 @@ def test_types_merge_simple_list_replace(): "list": TestSequence([0, 1, 2, 3]) }) - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) - nose.tools.eq_(dst["list"], TestSequence([7, 8, 9, 10])) + dpath.merge(dst, src, flags=MergeType.REPLACE) + assert dst["list"] == TestSequence([7, 8, 9, 10]), "%r != %r" % (dst["list"], TestSequence([7, 8, 9, 10])) def test_types_get_absent(): ehash = TestMapping() - assert_raises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - assert_raises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_types_get_glob_multiple(): @@ -130,8 +130,8 @@ def test_types_get_glob_multiple(): }), }) - assert_raises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) def test_delete_filter(): @@ -148,7 +148,7 @@ def afilter(x): }), }) - dpath.util.delete(data, '/a/*', afilter=afilter) - assert (data['a']['b'] == 0) - assert (data['a']['c'] == 1) - assert ('d' not in data['a']) + dpath.delete(data, '/a/*', afilter=afilter) + assert data['a']['b'] == 0 + assert data['a']['c'] == 1 + assert 'd' not in data['a'] diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 104e108..d4e8033 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,32 +1,32 @@ -import dpath.util +import dpath def test_unicode_merge(): a = {'中': 'zhong'} b = {'文': 'wen'} - dpath.util.merge(a, b) - assert(len(a.keys()) == 2) - assert(a['中'] == 'zhong') - assert(a['文'] == 'wen') + dpath.merge(a, b) + assert len(a.keys()) == 2 + assert a['中'] == 'zhong' + assert a['文'] == 'wen' def test_unicode_search(): a = {'中': 'zhong'} - results = [[x[0], x[1]] for x in dpath.util.search(a, '*', yielded=True)] - assert(len(results) == 1) - assert(results[0][0] == '中') - assert(results[0][1] == 'zhong') + results = [[x[0], x[1]] for x in dpath.search(a, '*', yielded=True)] + assert len(results) == 1 + assert results[0][0] == '中' + assert results[0][1] == 'zhong' def test_unicode_str_hybrid(): a = {'first': u'1'} b = {u'second': '2'} - dpath.util.merge(a, b) - assert(len(a.keys()) == 2) - assert(a[u'second'] == '2') - assert(a['second'] == u'2') - assert(a[u'first'] == '1') - assert(a['first'] == u'1') + dpath.merge(a, b) + assert len(a.keys()) == 2 + assert a[u'second'] == '2' + assert a['second'] == u'2' + assert a[u'first'] == '1' + assert a['first'] == u'1' diff --git a/tests/test_util_get_values.py b/tests/test_util_get_values.py deleted file mode 100644 index b4938d1..0000000 --- a/tests/test_util_get_values.py +++ /dev/null @@ -1,142 +0,0 @@ -from nose.tools import assert_raises -import dpath.util -import mock - - -def test_util_get_root(): - x = {'p': {'a': {'t': {'h': 'value'}}}} - - ret = dpath.util.get(x, '/p/a/t/h') - assert(ret == 'value') - - ret = dpath.util.get(x, '/') - assert(ret == x) - - -def test_get_explicit_single(): - ehash = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - - assert(dpath.util.get(ehash, '/a/b/c/f') == 2) - assert(dpath.util.get(ehash, ['a', 'b', 'c', 'f']) == 2) - assert(dpath.util.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2) - assert(dpath.util.get(ehash, ['does', 'not', 'exist'], default=None) is None) - assert(dpath.util.get(ehash, ['doesnt', 'exist'], default=5) == 5) - - -def test_get_glob_single(): - ehash = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - - assert(dpath.util.get(ehash, '/a/b/*/f') == 2) - assert(dpath.util.get(ehash, ['a', 'b', '*', 'f']) == 2) - assert(dpath.util.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2) - assert(dpath.util.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6) - - -def test_get_glob_multiple(): - ehash = { - "a": { - "b": { - "c": { - "d": 0, - }, - "e": { - "d": 0, - }, - }, - }, - } - - assert_raises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd'], default=3) - - -def test_get_absent(): - ehash = {} - - assert_raises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - assert_raises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) - - -def test_values(): - ehash = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - - ret = dpath.util.values(ehash, '/a/b/c/*') - assert(isinstance(ret, list)) - assert(0 in ret) - assert(1 in ret) - assert(2 in ret) - - ret = dpath.util.values(ehash, ['a', 'b', 'c', '*']) - assert(isinstance(ret, list)) - assert(0 in ret) - assert(1 in ret) - assert(2 in ret) - - -@mock.patch('dpath.util.search') -def test_values_passes_through(searchfunc): - searchfunc.return_value = [] - - def y(): - pass - - dpath.util.values({}, '/a/b', ':', y, False) - searchfunc.assert_called_with({}, '/a/b', True, ':', y, False) - - dpath.util.values({}, ['a', 'b'], ':', y, False) - searchfunc.assert_called_with({}, ['a', 'b'], True, ':', y, False) - - -def test_none_values(): - d = {'p': {'a': {'t': {'h': None}}}} - - v = dpath.util.get(d, 'p/a/t/h') - assert(v is None) - - -def test_values_list(): - a = { - 'actions': [ - { - 'type': 'correct', - }, - { - 'type': 'incorrect', - }, - ], - } - - ret = dpath.util.values(a, 'actions/*') - assert(isinstance(ret, list)) - assert(len(ret) == 2) diff --git a/tests/test_util_new.py b/tests/test_util_new.py deleted file mode 100644 index d04b056..0000000 --- a/tests/test_util_new.py +++ /dev/null @@ -1,83 +0,0 @@ -import dpath.util - - -def test_set_new_separator(): - dict = { - "a": { - }, - } - - dpath.util.new(dict, ';a;b', 1, separator=";") - assert(dict['a']['b'] == 1) - - dpath.util.new(dict, ['a', 'b'], 1, separator=";") - assert(dict['a']['b'] == 1) - - -def test_set_new_dict(): - dict = { - "a": { - }, - } - - dpath.util.new(dict, '/a/b', 1) - assert(dict['a']['b'] == 1) - - dpath.util.new(dict, ['a', 'b'], 1) - assert(dict['a']['b'] == 1) - - -def test_set_new_list(): - dict = { - "a": [ - ], - } - - dpath.util.new(dict, '/a/1', 1) - assert(dict['a'][1] == 1) - assert(dict['a'][0] is None) - - dpath.util.new(dict, ['a', 1], 1) - assert(dict['a'][1] == 1) - assert(dict['a'][0] is None) - - -def test_set_new_list_path_with_separator(): - # This test kills many birds with one stone, forgive me - dict = { - "a": { - }, - } - - dpath.util.new(dict, ['a', 'b/c/d', 0], 1) - assert(len(dict['a']) == 1) - assert(len(dict['a']['b/c/d']) == 1) - assert(dict['a']['b/c/d'][0] == 1) - - -def test_set_new_list_integer_path_with_creator(): - d = {} - - def mycreator(obj, pathcomp, nextpathcomp, hints): - print(hints) - print(pathcomp) - print(nextpathcomp) - print("...") - - target = pathcomp[0] - if isinstance(obj, list) and (target.isdigit()): - target = int(target) - - if ((nextpathcomp is not None) and (isinstance(nextpathcomp, int) or str(nextpathcomp).isdigit())): - obj[target] = [None] * (int(nextpathcomp) + 1) - print("Created new list in target") - else: - print("Created new dict in target") - obj[target] = {} - print(obj) - - dpath.util.new(d, '/a/2', 3, creator=mycreator) - print(d) - assert(isinstance(d['a'], list)) - assert(len(d['a']) == 3) - assert(d['a'][2] == 3) diff --git a/tests/test_util_paths.py b/tests/test_util_paths.py deleted file mode 100644 index 27260fe..0000000 --- a/tests/test_util_paths.py +++ /dev/null @@ -1,9 +0,0 @@ -import dpath.util - - -def test_util_safe_path_list(): - res = dpath.util.__safe_path__(["Ignore", "the/separator"], None) - - assert(len(res) == 2) - assert(res[0] == "Ignore") - assert(res[1] == "the/separator") diff --git a/tests/test_util_search.py b/tests/test_util_search.py deleted file mode 100644 index e7a4d43..0000000 --- a/tests/test_util_search.py +++ /dev/null @@ -1,238 +0,0 @@ -import dpath.util - - -def test_search_paths_with_separator(): - dict = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - paths = [ - 'a', - 'a;b', - 'a;b;c', - 'a;b;c;d', - 'a;b;c;e', - 'a;b;c;f', - ] - - for (path, value) in dpath.util.search(dict, '/**', yielded=True, separator=";"): - assert(path in paths) - - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, separator=";"): - assert(path in paths) - - -def test_search_paths(): - dict = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - paths = [ - 'a', - 'a/b', - 'a/b/c', - 'a/b/c/d', - 'a/b/c/e', - 'a/b/c/f', - ] - - for (path, value) in dpath.util.search(dict, '/**', yielded=True): - assert(path in paths) - - for (path, value) in dpath.util.search(dict, ['**'], yielded=True): - assert(path in paths) - - -def test_search_afilter(): - def afilter(x): - if x in [1, 2]: - return True - return False - - dict = { - "a": { - "view_failure": "a", - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - paths = [ - 'a/b/c/e', - 'a/b/c/f', - ] - - for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c']) - - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c']) - - -def test_search_globbing(): - dict = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - paths = [ - 'a/b/c/d', - 'a/b/c/f', - ] - - for (path, value) in dpath.util.search(dict, '/a/**/[df]', yielded=True): - assert(path in paths) - - for (path, value) in dpath.util.search(dict, ['a', '**', '[df]'], yielded=True): - assert(path in paths) - - -def test_search_return_dict_head(): - tdict = { - "a": { - "b": { - 0: 0, - 1: 1, - 2: 2, - }, - }, - } - res = dpath.util.search(tdict, '/a/b') - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == {0: 0, 1: 1, 2: 2}) - - res = dpath.util.search(tdict, ['a', 'b']) - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == {0: 0, 1: 1, 2: 2}) - - -def test_search_return_dict_globbed(): - tdict = { - "a": { - "b": { - 0: 0, - 1: 1, - 2: 2, - }, - }, - } - - res = dpath.util.search(tdict, '/a/b/[02]') - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 2) - assert(res['a']['b'] == {0: 0, 2: 2}) - - res = dpath.util.search(tdict, ['a', 'b', '[02]']) - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 2) - assert(res['a']['b'] == {0: 0, 2: 2}) - - -def test_search_return_list_head(): - tdict = { - "a": { - "b": [ - 0, - 1, - 2, - ], - }, - } - - res = dpath.util.search(tdict, '/a/b') - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, 1, 2]) - - res = dpath.util.search(tdict, ['a', 'b']) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, 1, 2]) - - -def test_search_return_list_globbed(): - tdict = { - "a": { - "b": [ - 0, - 1, - 2, - ] - } - } - - res = dpath.util.search(tdict, '/a/b/[02]') - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, None, 2]) - - res = dpath.util.search(tdict, ['a', 'b', '[02]']) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, None, 2]) - - -def test_search_list_key_with_separator(): - tdict = { - "a": { - "b": { - "d": 'failure', - }, - "/b/d": 'success', - }, - } - - res = dpath.util.search(tdict, ['a', '/b/d']) - assert('b' not in res['a']) - assert(res['a']['/b/d'] == 'success') - - -def test_search_multiple_stars(): - testdata = { - 'a': [ - { - 'b': [ - {'c': 1}, - {'c': 2}, - {'c': 3}, - ], - }, - ], - } - testpath = 'a/*/b/*/c' - - res = dpath.util.search(testdata, testpath) - assert(len(res['a'][0]['b']) == 3) - assert(res['a'][0]['b'][0]['c'] == 1) - assert(res['a'][0]['b'][1]['c'] == 2) - assert(res['a'][0]['b'][2]['c'] == 3) diff --git a/tests/test_util_set.py b/tests/test_util_set.py deleted file mode 100644 index 3684a56..0000000 --- a/tests/test_util_set.py +++ /dev/null @@ -1,91 +0,0 @@ -import dpath.util - - -def test_set_existing_separator(): - dict = { - "a": { - "b": 0, - }, - } - - dpath.util.set(dict, ';a;b', 1, separator=";") - assert(dict['a']['b'] == 1) - - dict['a']['b'] = 0 - dpath.util.set(dict, ['a', 'b'], 1, separator=";") - assert(dict['a']['b'] == 1) - - -def test_set_existing_dict(): - dict = { - "a": { - "b": 0, - }, - } - - dpath.util.set(dict, '/a/b', 1) - assert(dict['a']['b'] == 1) - - dict['a']['b'] = 0 - dpath.util.set(dict, ['a', 'b'], 1) - assert(dict['a']['b'] == 1) - - -def test_set_existing_list(): - dict = { - "a": [ - 0, - ], - } - - dpath.util.set(dict, '/a/0', 1) - assert(dict['a'][0] == 1) - - dict['a'][0] = 0 - dpath.util.set(dict, ['a', '0'], 1) - assert(dict['a'][0] == 1) - - -def test_set_filter(): - def afilter(x): - if int(x) == 31: - return True - return False - - dict = { - "a": { - "b": 0, - "c": 1, - "d": 31, - } - } - - dpath.util.set(dict, '/a/*', 31337, afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert (dict['a']['d'] == 31337) - - dict = { - "a": { - "b": 0, - "c": 1, - "d": 31, - } - } - - dpath.util.set(dict, ['a', '*'], 31337, afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert (dict['a']['d'] == 31337) - - -def test_set_existing_path_with_separator(): - dict = { - "a": { - 'b/c/d': 0, - }, - } - - dpath.util.set(dict, ['a', 'b/c/d'], 1) - assert(len(dict['a']) == 1) - assert(dict['a']['b/c/d'] == 1) diff --git a/tox.ini b/tox.ini index 8969270..ff9a86d 100644 --- a/tox.ini +++ b/tox.ini @@ -7,16 +7,19 @@ ignore = E501,E722 [tox] -envlist = flake8, py36, py38, pypy3 +envlist = pypy37, py38, py39, py310, py311, py312 + +[gh-actions] +python = + pypy-3.7: pypy37 + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 [testenv] deps = hypothesis - mock - nose -commands = nosetests {posargs} - -[testenv:flake8] -deps = - flake8 -commands = flake8 setup.py dpath/ tests/ + nose2 +commands = nose2 {posargs}