From fa6f0a819af86b5ac05560ac13eee1f2ef820e20 Mon Sep 17 00:00:00 2001 From: Andrew Kesterson Date: Sun, 29 Mar 2020 12:28:55 -0700 Subject: [PATCH 001/133] Fix travis branch recognition to support version/x.x branches for LTS --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c6a8bd5..435e304 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,11 @@ deploy: user: akesterson password: secure: RlZ3kw0x7/5QNBUIqj5wjkpS11TuSdH7o1S41m4Vea8HiryQjB2+CFf+uBXj8nb1OtYDRaeKTSnk3qVlUvaUglFd7sV+Hh2DDo4pYomd0xDFHnK2LB+hzbYftkOptmwdS2bc0X7vr+xdY+7b2DPTlyowrs3Xp2WvzRWM4hWnA3Q= - true: + on: tags: true - branch: master + all_branches: true + condition: $TRAVIS_BRANCH =~ ^master$|^version/[0-9].[0-9]$ + sudo: false language: python python: From a8e7a95a46ab53073b77eab2865e71bc1aefdd4c Mon Sep 17 00:00:00 2001 From: ccase Date: Sun, 29 Mar 2020 16:24:27 -0400 Subject: [PATCH 002/133] Handle nodes that aren't leafy. This relates to: https://github.com/akesterson/dpath-python/issues/119 And incorporates some of the changes from: https://github.com/akesterson/dpath-python/pull/120 https://github.com/akesterson/dpath-python/pull/122 https://github.com/akesterson/dpath-python/pull/123 --- dpath/segments.py | 14 ++++++-- tests/test_util_get_values.py | 67 +++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 65f8920..2ebd485 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -13,7 +13,13 @@ def kvs(node): try: return iter(node.items()) except AttributeError: - return zip(range(len(node)), node) + try: + return zip(range(len(node)), node) + except TypeError: + # This can happen in cases where the node isn't leaf(node) == True, + # but also isn't actually iterable. Instead of this being an error + # we will treat this node as if it has no children. + return enumerate([]) def leaf(thing): @@ -34,7 +40,11 @@ def leafy(thing): leafy(thing) -> bool ''' - return leaf(thing) or len(thing) == 0 + + try: + return leaf(thing) or len(thing) == 0 + except: + return false def walk(obj, location=()): diff --git a/tests/test_util_get_values.py b/tests/test_util_get_values.py index b4938d1..5a14f1e 100644 --- a/tests/test_util_get_values.py +++ b/tests/test_util_get_values.py @@ -1,6 +1,10 @@ from nose.tools import assert_raises + +import datetime +import decimal import dpath.util import mock +import time def test_util_get_root(): @@ -140,3 +144,66 @@ def test_values_list(): ret = dpath.util.values(a, 'actions/*') assert(isinstance(ret, list)) assert(len(ret) == 2) + + +def test_non_leaf_leaf(): + # The leaves in this test aren't leaf(thing) == True, but we should still + # be able to get them. They should also not prevent fetching other values. + + def func(x): + return x + + testdict = { + 'a': func, + 'b': lambda x: x, + 'c': [ + { + 'a', + 'b', + }, + ], + 'd': [ + decimal.Decimal(1.5), + decimal.Decimal(2.25), + ], + 'e': datetime.datetime(2020, 1, 1), + 'f': { + 'config': 'something', + }, + } + + # It should be possible to get the callables: + assert dpath.util.get(testdict, 'a') == func + assert dpath.util.get(testdict, 'b')(42) == 42 + + # It should be possible to get other values: + assert dpath.util.get(testdict, 'c/0') == testdict['c'][0] + assert dpath.util.get(testdict, 'd')[0] == testdict['d'][0] + assert dpath.util.get(testdict, 'd/0') == testdict['d'][0] + assert dpath.util.get(testdict, 'd/1') == testdict['d'][1] + assert dpath.util.get(testdict, 'e') == testdict['e'] + + # Values should also still work: + assert dpath.util.values(testdict, 'f/config') == ['something'] + + # Data classes should also be retrievable: + try: + import dataclasses + except: + return + + @dataclasses.dataclass + class Connection: + group_name: str + channel_name: str + last_seen: float + + testdict['g'] = { + 'my-key': Connection( + group_name='foo', + channel_name='bar', + last_seen=time.time(), + ), + } + + assert dpath.util.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] From d3d8573cd544872be779ff847b04cc9182585040 Mon Sep 17 00:00:00 2001 From: Andrew Kesterson Date: Sun, 29 Mar 2020 15:03:18 -0700 Subject: [PATCH 003/133] Add maintainers log and update MAINTAINERS.md --- .travis.yml | 2 +- MAINTAINERS.md | 18 ++++++++++++++---- maintainers_log.md | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 maintainers_log.md diff --git a/.travis.yml b/.travis.yml index 435e304..20119d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ deploy: on: tags: true all_branches: true - condition: $TRAVIS_BRANCH =~ ^master$|^version/[0-9].[0-9]$ + condition: $TRAVIS_BRANCH =~ ^master$|^version/[0-9].x$ sudo: false language: python diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ba67806..7fa4def 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -76,11 +76,21 @@ The more complete process goes: 6. Send your pull request 7. If accepted, the maintainers will merge your pull request and close the issue. +Branching Strategy +================== + +We run a clean bleeding edge master. Long term support for major version numbers are broken out into version branches. + +* master : Current 3.x (bleeding edge) development +* version/1.x : 1.x series bugfixes +* version/2.x : 2.x series features and bugfixes + +We name bugfixes as "bugfix/ISSUENUMBER_shortname"; features are named "feature/ISSUENUMBER_shortname". All branches representing work against an issue must have the issue number in the branch name. Cutting a New Release ===================== -Releases for dpath occur automatically from travis-ci based on tags on the master branch. +Releases for dpath occur automatically from travis-ci based on tags on the master branch, or on the version/[0-9].x branches for major version LTS. akesterson@akesterson:~/dpath-python$ git tag 1.0-0 @@ -108,9 +118,9 @@ Once upon a time, the version string was automatially computed based on the cont To cut a new release, follow this procedure: -1. Commit a new dpath/version.py on the master branch with the format "MAJOR.MINOR.RELEASE" -2. Add a new tag of the form "build,MAJOR.MINOR,RELEASE" to the master branch. This tag must have the same version number as the one commmited in dpath/version.py or we will fill your desk drawers with cockroaches. -3. Push the new master version and the associated tag to github. +1. Commit a new dpath/version.py on the appropriate branch with the format "MAJOR.MINOR.RELEASE" +2. Add a new tag of the form "build,MAJOR.MINOR,RELEASE" to the appropriate branch. This tag must have the same version number as the one commmited in dpath/version.py or we will fill your desk drawers with cockroaches. +3. Push the new branch version and the associated tag to github. 4. travis-ci SHOULD push the new release to pypi. If travis-ci fails to update pypi, follow the instructions on manually creating a release, here: diff --git a/maintainers_log.md b/maintainers_log.md new file mode 100644 index 0000000..6f932a3 --- /dev/null +++ b/maintainers_log.md @@ -0,0 +1,17 @@ +# 03/29/2020 + +Attendees : Caleb, Andrew + +## Old business : + +* Need to onboard new member Vladimir Ulogov + * No movement +* Need to make project board for 1.5 open bugs + * Done + +## New business : + +* Andrew to define maintainers meeting process and establish log of decisions, process for filing open action items +* Andrew to forward maintainers invite to Vladimir and include in next monthly maintainers meeting +* Andrew to set followup for 1wk from now to check for comments on PRs and cut release version for 1.x / 2.x +* Andrew to rename LTS branches from version/1.0 version/2.0 to version/1.x and version/2.x From 3def7e0dd914e16e74ee3cb86ae691863e391bc8 Mon Sep 17 00:00:00 2001 From: Andrew Kesterson Date: Mon, 27 Jul 2020 08:17:03 -0700 Subject: [PATCH 004/133] Update MAINTAINERS.md --- MAINTAINERS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 7fa4def..65e97dd 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,6 +5,8 @@ dpath is primarily maintained by Andrew Kesterson and Caleb There are several individuals in the community who have taken an active role in helping to maintain the project and submit fixes. Those individuals are shown in the git changelog. +DPATH NEEDS NEW MAINTAINERS. SEE https://github.com/akesterson/dpath-python/issues/136 + Becoming a Maintainer ===================== @@ -125,4 +127,4 @@ To cut a new release, follow this procedure: If travis-ci fails to update pypi, follow the instructions on manually creating a release, here: -https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives \ No newline at end of file +https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives From df947f9db109817df23393b89451167773ac14e7 Mon Sep 17 00:00:00 2001 From: Andrew Kesterson Date: Mon, 27 Jul 2020 08:18:21 -0700 Subject: [PATCH 005/133] Update README.rst --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index c8c3032..3aa0038 100644 --- a/README.rst +++ b/README.rst @@ -14,6 +14,8 @@ provides some facility for filtering those results. sdists are available on pypi: http://pypi.python.org/pypi/dpath +DPATH NEEDS NEW MAINTAINERS. SEE https://github.com/akesterson/dpath-python/issues/136 + Installing ========== @@ -456,6 +458,8 @@ components instead of string globs. Contributors ============ +DPATH NEEDS NEW MAINTAINERS. SEE https://github.com/akesterson/dpath-python/issues/136 + We would like to thank the community for their interest and involvement. You have all made this project significantly better than the sum of its parts, and your continued feedback makes it better every day. Thank you so much! From 0de9dcb0df3df5cac70b87c98cb5a64757bccaf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20K=C3=A1lm=C3=A1n?= Date: Thu, 1 Oct 2020 10:19:07 +0200 Subject: [PATCH 006/133] fix json indentation in README --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 3aa0038..2c1ca9e 100644 --- a/README.rst +++ b/README.rst @@ -159,10 +159,10 @@ value 'Waffles'. { "a": { "b": { - "3": 2, - "43": 30, - "c": "Waffles", - "d": "Waffles" + "3": 2, + "43": 30, + "c": "Waffles", + "d": "Waffles" } } } From e20271984f7009ed041d32b7a4b9f7b9d405e373 Mon Sep 17 00:00:00 2001 From: Brian Westphal Date: Sun, 6 Jun 2021 13:10:42 -0400 Subject: [PATCH 007/133] Removing the need for new maintainers --- README.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.rst b/README.rst index 3aa0038..f6e730a 100644 --- a/README.rst +++ b/README.rst @@ -14,8 +14,6 @@ provides some facility for filtering those results. sdists are available on pypi: http://pypi.python.org/pypi/dpath -DPATH NEEDS NEW MAINTAINERS. SEE https://github.com/akesterson/dpath-python/issues/136 - Installing ========== From f0bbb63952881b7562e3fba2c5aea6c83744bca1 Mon Sep 17 00:00:00 2001 From: Brian Westphal Date: Sun, 6 Jun 2021 13:37:07 -0400 Subject: [PATCH 008/133] Tweak README.rst and MAINTAINERS.md to mention new maintainers and remove need for new maintainers. --- .gitignore | 2 ++ MAINTAINERS.md | 17 ++++++----------- README.rst | 2 -- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index fb9ae3a..0533f46 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ /build /env *.pyc +.vscode +venv_39 \ No newline at end of file diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 65e97dd..cc5b817 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,19 +1,12 @@ Who Maintains DPATH =================== -dpath is primarily maintained by Andrew Kesterson and Caleb Case . These two individuals collectively govern the project. +dpath was created by and originally maintained by Andrew Kesterson and Caleb Case . In July +of 2020 they put out a call for new maintainers. [@bigsablept](https://github.com/bigsablept) and +[@moomoohk](https://github.com/moomoohk) stepped up to become the new maintainers. There are several individuals in the community who have taken an active role in helping to maintain the project and submit fixes. Those individuals are shown in the git changelog. -DPATH NEEDS NEW MAINTAINERS. SEE https://github.com/akesterson/dpath-python/issues/136 - -Becoming a Maintainer -===================== - -Nobody has to become a maintainer to submit a patch against dpath. Simply send the pull request on github. - -If you would like to help triage issues, attend monthly meetings, and become a regular part of the team working on the roadmap, send an email to andrew@aklabs.net and/or calebcase@gmail.com. - Where and How do we communicate =============================== @@ -21,7 +14,9 @@ The dpath maintainers communcate in 3 primary ways: 1. Email, directly to each other. 2. Github via issue and pull request comments -3. A monthly maintainers meeting via telephone +3. A monthly maintainers meeting via Zoom + +The remainder of this document is subject to change after further discussion among the new maintainers. What is the roadmap =================== diff --git a/README.rst b/README.rst index f6e730a..c8c3032 100644 --- a/README.rst +++ b/README.rst @@ -456,8 +456,6 @@ components instead of string globs. Contributors ============ -DPATH NEEDS NEW MAINTAINERS. SEE https://github.com/akesterson/dpath-python/issues/136 - We would like to thank the community for their interest and involvement. You have all made this project significantly better than the sum of its parts, and your continued feedback makes it better every day. Thank you so much! From 96188ac37ad99913fc9cdd86d1ebf046563bd071 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 15 Jun 2021 22:27:40 +0300 Subject: [PATCH 009/133] Fix typo --- dpath/segments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/segments.py b/dpath/segments.py index 2ebd485..b59dfed 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -44,7 +44,7 @@ def leafy(thing): try: return leaf(thing) or len(thing) == 0 except: - return false + return False def walk(obj, location=()): From 52e7413c160905f49d71d31687c32be0c8dd17ce Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 15 Jun 2021 22:29:12 +0300 Subject: [PATCH 010/133] Catch TypeError --- dpath/segments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dpath/segments.py b/dpath/segments.py index b59dfed..9fd017b 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -43,7 +43,8 @@ def leafy(thing): try: return leaf(thing) or len(thing) == 0 - except: + except TypeError: + # In case thing has no len() return False From f15a6fe267e10ffd35e30bbb1cea1673f3163263 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 20 Jun 2021 20:14:23 +0300 Subject: [PATCH 011/133] Update gitignore --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fb9ae3a..fbecdf8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ /MANIFEST /.tox -/.hypothesis /build /env +.hypothesis *.pyc +.vscode +venv_39 +.idea/ +dpath.egg-info/ \ No newline at end of file From 9f7c354b0824083854717a84f5df59e234f66067 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 23 Jun 2021 19:27:31 +0300 Subject: [PATCH 012/133] Bump version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index b46c2e7..311aff2 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.1" +VERSION = "2.0.2" From 3ddb1367484142a76b9ccea6605fdb10ffd44357 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 2 Sep 2021 21:54:19 +0300 Subject: [PATCH 013/133] Update gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0533f46..51ef4ba 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ /env *.pyc .vscode -venv_39 \ No newline at end of file +venv_39 +.idea/ +dpath.egg-info/ +dist/ \ No newline at end of file From 88e9d488a79cbe08d4d0ca54549b94afb9fbcad4 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 2 Sep 2021 23:45:45 +0300 Subject: [PATCH 014/133] Add deploy action --- .github/workflows/deploy.yml | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..e7b9706 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,37 @@ +# This is a basic workflow to help you get started with Actions + +name: Deploy to PyPI + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + release: + types: [published] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + deploy: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: PyPI Deployment + # You may pin to the exact commit or the version. + # uses: casperdcl/deploy-pypi@bb869aafd89f657ceaafe9561d3b5584766c0f95 + uses: casperdcl/deploy-pypi@v2 + with: + # PyPI username + user: ${{ secrets.PYPI_USER }} + # PyPI password or API token + password: ${{ secrets.TEST_PYPI_PASS }} + # Build requirements + # requirements: # optional, default is twine wheel + # `setup.py` command to run ("true" is a shortcut for "clean sdist -d bdist_wheel -d ") + build: clean sdist -d dist/ + # `pip` command to run ("true" is a shortcut for "wheel -w --no-deps .") + pip: true From 7b4efc94310b4b515fe02f203cb4fb8f813d977a Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 2 Sep 2021 23:45:58 +0300 Subject: [PATCH 015/133] Add tests action --- .github/workflows/tests.yml | 42 +++++++++++++++++++++++++++++++++++++ tox.ini | 10 ++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..471d2a2 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,42 @@ +# This is a basic workflow to help you get started with Actions + +name: Run tests + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + pull_request: + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "tests" + tests: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + strategy: + matrix: + # Match versions specified in tox.ini + python-version: [3.6, 3.8, 3.9, pypy-3.7] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Check out code + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@main + with: + python-version: ${{ matrix.python-version }} + + # Instead of installing and running tox and tox-gh-actions, + # you can simply use this composite run steps action. + - name: Run tox with tox-gh-actions + uses: ymyzk/run-tox-gh-actions@main + with: + tox-args: -vv \ No newline at end of file diff --git a/tox.ini b/tox.ini index 8969270..583b884 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,15 @@ ignore = E501,E722 [tox] -envlist = flake8, py36, py38, pypy3 +envlist = py36, pypy37, py38, py39, flake8 + +[gh-actions] +python = + 3.6: py36 + # TODO(moomoohk): Get pypy to work properly. https://github.com/ymyzk/run-tox-gh-actions/issues/3 + pypi-3.7: pypy37 + 3.8: py38 + 3.9: py39, flake8 [testenv] deps = From 9f1d57ea89c4eedea6a8a5c6bd01a4ba8d29627a Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 2 Sep 2021 23:47:56 +0300 Subject: [PATCH 016/133] Remove travis config --- .travis.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 20119d3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -deploy: - provider: pypi - user: akesterson - password: - secure: RlZ3kw0x7/5QNBUIqj5wjkpS11TuSdH7o1S41m4Vea8HiryQjB2+CFf+uBXj8nb1OtYDRaeKTSnk3qVlUvaUglFd7sV+Hh2DDo4pYomd0xDFHnK2LB+hzbYftkOptmwdS2bc0X7vr+xdY+7b2DPTlyowrs3Xp2WvzRWM4hWnA3Q= - on: - tags: true - all_branches: true - condition: $TRAVIS_BRANCH =~ ^master$|^version/[0-9].x$ - -sudo: false -language: python -python: -- '3.6' -- '3.8' -- pypy3 -install: travis_retry pip install tox-travis -script: tox From 95866dd26158a21af6fa9e133eb2025001f36d22 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 2 Sep 2021 23:55:06 +0300 Subject: [PATCH 017/133] PEP8 compliance --- .gitignore | 3 ++- dpath/util.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 51ef4ba..57fe7c7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ venv_39 .idea/ dpath.egg-info/ -dist/ \ No newline at end of file +dist/ +tests/.hypothesis \ No newline at end of file diff --git a/dpath/util.py b/dpath/util.py index f90fd6e..48c03be 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -9,6 +9,7 @@ MERGE_ADDITIVE = (1 << 2) MERGE_TYPESAFE = (1 << 3) + def __safe_path__(path, separator): ''' Given a path and separator, return a tuple of segments. If path is @@ -171,7 +172,7 @@ def f(obj, pair, results): results = dpath.segments.fold(obj, f, []) if len(results) == 0: - if default is not _DEFAULT_SENTINAL: + if default is not _DEFAULT_SENTINAL: return default raise KeyError(glob) From 11329869100e6858ec8108dd8b5c6e888ec3210d Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 00:05:29 +0300 Subject: [PATCH 018/133] Github Action status badge --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2964071..51bedd8 100644 --- a/README.rst +++ b/README.rst @@ -450,8 +450,8 @@ components instead of string globs. :target: https://pypi.python.org/pypi/dpath/ :alt: PyPI: Latest Version -.. |Build Status| image:: https://travis-ci.org/akesterson/dpath-python.svg?branch=travisci - :target: https://travis-ci.org/akesterson/dpath-python +.. |Build Status| image:: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml/badge.svg + :target: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml Contributors ============ From ec052aadcbc51cf44d51e81206c031f08a87d4cd Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 00:08:06 +0300 Subject: [PATCH 019/133] Run tests action only on relevant updates --- .github/workflows/deploy.yml | 10 ++-------- .github/workflows/tests.yml | 9 +++++---- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e7b9706..5a95c36 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,16 +1,14 @@ -# This is a basic workflow to help you get started with Actions - name: Deploy to PyPI # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the master branch + # Triggers the workflow on new releases release: types: [published] # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" + # This workflow contains a single job called "deploy" deploy: # The type of runner that the job will run on runs-on: ubuntu-latest @@ -21,16 +19,12 @@ jobs: - uses: actions/checkout@v2 - name: PyPI Deployment - # You may pin to the exact commit or the version. - # uses: casperdcl/deploy-pypi@bb869aafd89f657ceaafe9561d3b5584766c0f95 uses: casperdcl/deploy-pypi@v2 with: # PyPI username user: ${{ secrets.PYPI_USER }} # PyPI password or API token password: ${{ secrets.TEST_PYPI_PASS }} - # Build requirements - # requirements: # optional, default is twine wheel # `setup.py` command to run ("true" is a shortcut for "clean sdist -d bdist_wheel -d ") build: clean sdist -d dist/ # `pip` command to run ("true" is a shortcut for "wheel -w --no-deps .") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 471d2a2..11105ad 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,11 +1,14 @@ -# This is a basic workflow to help you get started with Actions - name: Run tests # Controls when the workflow will run on: # Triggers the workflow on push or pull request events but only for the master branch push: + # Only run if source code or test configurations are changed + paths: + - "dpath/" + - "**.py" + - "tox.ini" pull_request: # Allows you to run this workflow manually from the Actions tab @@ -34,8 +37,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - # Instead of installing and running tox and tox-gh-actions, - # you can simply use this composite run steps action. - name: Run tox with tox-gh-actions uses: ymyzk/run-tox-gh-actions@main with: From 9418ce2cfc71172af768ac6e39b66944096c701c Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 00:40:45 +0300 Subject: [PATCH 020/133] Add Gitter badge to README --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 51bedd8..ea88f5a 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,7 @@ dpath-python |PyPI| |Build Status| +|Gitter| A python library for accessing and searching dictionaries via /slashed/paths ala xpath @@ -452,6 +453,10 @@ components instead of string globs. .. |Build Status| image:: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml/badge.svg :target: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml + +.. |Gitter| image:: https://badges.gitter.im/dpath-python/chat.svg + :target: https://gitter.im/dpath-python/chat?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge + :alt: Gitter Contributors ============ From 114731eef987005a21870bd0d5414a98216372c5 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 01:32:14 +0300 Subject: [PATCH 021/133] Add release creation to workflow --- .github/workflows/deploy.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5a95c36..6ddaadf 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,9 +2,10 @@ name: Deploy to PyPI # Controls when the workflow will run on: - # Triggers the workflow on new releases - release: - types: [published] + # Triggers the workflow on tag push + push: + tags: + - "v*" # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: @@ -29,3 +30,8 @@ jobs: build: clean sdist -d dist/ # `pip` command to run ("true" is a shortcut for "wheel -w --no-deps .") pip: true + + - name: Github Release + uses: softprops/action-gh-release@v1 + with: + files: dist/* From 9d4f055856483d6a524df56fb29afb7f9fdf936e Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 01:34:06 +0300 Subject: [PATCH 022/133] Ignore tags in tests workflow --- .github/workflows/tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 11105ad..ae780e4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,10 @@ name: Run tests on: # Triggers the workflow on push or pull request events but only for the master branch push: - # Only run if source code or test configurations are changed + # Ignore tags + # https://github.community/t/dont-run-on-tag-creation/137469/7 + branches: + - "**" paths: - "dpath/" - "**.py" @@ -40,4 +43,4 @@ jobs: - name: Run tox with tox-gh-actions uses: ymyzk/run-tox-gh-actions@main with: - tox-args: -vv \ No newline at end of file + tox-args: -vv From 3c5540ca5e2593fd9880c04fe848e032acba96c5 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 01:41:01 +0300 Subject: [PATCH 023/133] Remove problematic trigger --- .github/workflows/tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ae780e4..502fa18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,10 +4,6 @@ name: Run tests on: # Triggers the workflow on push or pull request events but only for the master branch push: - # Ignore tags - # https://github.community/t/dont-run-on-tag-creation/137469/7 - branches: - - "**" paths: - "dpath/" - "**.py" From 7b2d07a69b9e2c1e7bf4253fcbf74f48334674b7 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 01:45:07 +0300 Subject: [PATCH 024/133] Rename workflow --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6ddaadf..0a1b0f6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Deploy to PyPI +name: Deploy and Release # Controls when the workflow will run on: From 902ac60de282899004c46b971e2a38403fbf6478 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 01:48:36 +0300 Subject: [PATCH 025/133] Fix argument name --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0a1b0f6..266eaef 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -25,7 +25,7 @@ jobs: # PyPI username user: ${{ secrets.PYPI_USER }} # PyPI password or API token - password: ${{ secrets.TEST_PYPI_PASS }} + password: ${{ secrets.PYPI_PASS }} # `setup.py` command to run ("true" is a shortcut for "clean sdist -d bdist_wheel -d ") build: clean sdist -d dist/ # `pip` command to run ("true" is a shortcut for "wheel -w --no-deps .") From 85a983b85b1d1c465f2fc82e36472f8a66c23c18 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 10:45:59 +0300 Subject: [PATCH 026/133] Add Python >3 constraint Fixes #130 --- dpath/version.py | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 311aff2..c129f68 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.2" +VERSION = "2.0.3" diff --git a/setup.py b/setup.py index f85ae47..78f1484 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ scripts=[], packages=["dpath"], data_files=[], + python_requires=">=3", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', From 29e1fc31895f3271d251e49c2ffbacc33e64cd1e Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 3 Sep 2021 10:47:21 +0300 Subject: [PATCH 027/133] Don't run tests on pull requests --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 502fa18..bee489d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,6 @@ on: - "dpath/" - "**.py" - "tox.ini" - pull_request: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 19f92e824e5a43e0dd7f0821ead6d7407fa00f13 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 8 Sep 2021 22:32:19 +0300 Subject: [PATCH 028/133] Fix typo --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 583b884..38be73b 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ envlist = py36, pypy37, py38, py39, flake8 python = 3.6: py36 # TODO(moomoohk): Get pypy to work properly. https://github.com/ymyzk/run-tox-gh-actions/issues/3 - pypi-3.7: pypy37 + pypy-3.7: pypy37 3.8: py38 3.9: py39, flake8 From 5ea2ae2509bc4032c66ecd9ad005544091905485 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 8 Sep 2021 22:33:49 +0300 Subject: [PATCH 029/133] Remove comment --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 38be73b..5e521b2 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,6 @@ envlist = py36, pypy37, py38, py39, flake8 [gh-actions] python = 3.6: py36 - # TODO(moomoohk): Get pypy to work properly. https://github.com/ymyzk/run-tox-gh-actions/issues/3 pypy-3.7: pypy37 3.8: py38 3.9: py39, flake8 From df6fe35c28853bb0a26b481ec1130ef7ab5f956a Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 10 Sep 2021 01:05:49 +0300 Subject: [PATCH 030/133] Upgrade deployment workflow - Trigger on version change - Create semver tag (with graceful fail) - Include changelog in release --- .github/workflows/deploy.yml | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 266eaef..568735d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,10 +2,10 @@ name: Deploy and Release # Controls when the workflow will run on: - # Triggers the workflow on tag push + # Triggers the workflow on version change push: - tags: - - "v*" + paths: + - "dpath/version.py" # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: @@ -19,6 +19,32 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 + - name: Get Version + id: get-version + run: | + python -c "from dpath.version import VERSION; print(f'::set-output name=version::v{VERSION}');" + + - name: Check Tag + uses: mukunku/tag-exists-action@v1.0.0 + id: check-tag + with: + tag: ${{ steps.get-version.outputs.version }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create Tag + if: ${{ steps.check-tag.outputs.exists }} == "false" + uses: negz/create-tag@v1 + with: + version: ${{ steps.get-version.outputs.version }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate Changelog + id: generate-changelog + uses: loopwerk/tag-changelog@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: PyPI Deployment uses: casperdcl/deploy-pypi@v2 with: @@ -30,8 +56,10 @@ jobs: build: clean sdist -d dist/ # `pip` command to run ("true" is a shortcut for "wheel -w --no-deps .") pip: true - + - name: Github Release uses: softprops/action-gh-release@v1 with: + tag_name: ${{ steps.get-version.outputs.version }} + body: ${{ steps.generate-changelog.outputs.changes }} files: dist/* From 37cc6775b5487d9acfa299f6bce8e37c0c9aed2c Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 10 Sep 2021 01:21:07 +0300 Subject: [PATCH 031/133] Update repo docs --- MAINTAINERS.md | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index cc5b817..ee0b1d5 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -87,7 +87,9 @@ We name bugfixes as "bugfix/ISSUENUMBER_shortname"; features are named "feature/ Cutting a New Release ===================== -Releases for dpath occur automatically from travis-ci based on tags on the master branch, or on the version/[0-9].x branches for major version LTS. +Releases for dpath occur automatically from Github Actions based on version changes on the master branch. + +Due to legacy reasons older tag names do not follow a uniform format: akesterson@akesterson:~/dpath-python$ git tag 1.0-0 @@ -108,18 +110,31 @@ Releases for dpath occur automatically from travis-ci based on tags on the maste build,1.5,0 build,2.0,0 -Once upon a time, the version string was automatially computed based on the content of these tags. Now, however, the version string is stored statically in dpath/version.py +Moving forward version numbers and tag names will be identical and follow the standard semver format. + +The version string is stored in `dpath/version.py` and tag names/release versions are generated using this string. akesterson@akesterson:~/dpath-python$ cat dpath/version.py VERSION = "2.0.0" To cut a new release, follow this procedure: -1. Commit a new dpath/version.py on the appropriate branch with the format "MAJOR.MINOR.RELEASE" -2. Add a new tag of the form "build,MAJOR.MINOR,RELEASE" to the appropriate branch. This tag must have the same version number as the one commmited in dpath/version.py or we will fill your desk drawers with cockroaches. -3. Push the new branch version and the associated tag to github. -4. travis-ci SHOULD push the new release to pypi. +1. Commit a new `dpath/version.py` on the appropriate branch with the format "MAJOR.MINOR.RELEASE". +2. Github Actions SHOULD push the new release to PyPI on merge to `master`. -If travis-ci fails to update pypi, follow the instructions on manually creating a release, here: +See `.github/workflows/deploy.yml` for more information. + +If the Github workflow fails to update pypi, follow the instructions on manually creating a release, here: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives + +Running Tests +============= + +Tests are managed using [tox](https://tox.readthedocs.io/en/latest/). + +Environment creation and dependency installation is managed by this tool, all one has to do is install it with `pip` and run `tox` in this repo's root directory. + +Tests can also be run with Github Actions via the [tests.yml](https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml) workflow. + +This workflow will run automatically on pretty much any commit to any branch of this repo but manual runs are also available. From 52156b2c4899898b6f0b34591cbbe9525cbe5421 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 10 Sep 2021 01:23:00 +0300 Subject: [PATCH 032/133] Add link to Travis CI page --- MAINTAINERS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ee0b1d5..0f9fb86 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -128,6 +128,8 @@ If the Github workflow fails to update pypi, follow the instructions on manually https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives +Deployment CI was previously implemented using [Travis CI](https://travis-ci.org/github/akesterson/dpath-python). + Running Tests ============= From c8d9c1a48206ee1f70feefed6db888919853b88c Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 10 Sep 2021 16:00:42 +0300 Subject: [PATCH 033/133] Limit deployment workflow to master branch --- .github/workflows/deploy.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 568735d..8210fac 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,8 +4,10 @@ name: Deploy and Release on: # Triggers the workflow on version change push: + branches: + - master paths: - - "dpath/version.py" + - dpath/version.py # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: @@ -33,7 +35,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Create Tag - if: ${{ steps.check-tag.outputs.exists }} == "false" + if: steps.check-tag.outputs.exists == 'false' uses: negz/create-tag@v1 with: version: ${{ steps.get-version.outputs.version }} From 958cba6cb212f91fbbe4777146991211b6f6df44 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 10 Sep 2021 16:03:26 +0300 Subject: [PATCH 034/133] Remove lingering Python 2 references --- dpath/segments.py | 15 +-------------- dpath/version.py | 2 +- setup.py | 1 - 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 9fd017b..1016a06 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -139,11 +139,7 @@ def leaves(obj): def int_str(segment): ''' If the segment is an integer, return the string conversion. - Otherwise return the segment unchanged. The conversion uses 'str' - which means that: - - * in Python 2.x: 0 -> b'0' - * in Python 3.x: 0 -> u'0' + Otherwise return the segment unchanged. The conversion uses 'str'. int_str(segment) -> str ''' @@ -180,15 +176,6 @@ def match(segments, glob): fnmatch.fnmatchcase returns True. If fnmatchcase returns False or throws an exception the result will be False. - A Word of Caution: - - *** Globs can have different results between Python 2.x and 3.x. *** - - fnmatchcase differs in behavior between Python 2.x and 3.x with - regard to handling comparison between different types (e.g. bytes - and unicode). In 2.x it will attempt the comparison implicitly - converting, but in 3.x it will throw an exception. - match(segments, glob) -> bool ''' segments = tuple(segments) diff --git a/dpath/version.py b/dpath/version.py index c129f68..215732a 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.3" +VERSION = "2.0.4" diff --git a/setup.py b/setup.py index 78f1484..abce1a3 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Natural Language :: English', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Libraries :: Python Modules', ], From a4e6a7a99613d97eed04602ba5c356f46f123987 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 10 Sep 2021 18:02:16 +0300 Subject: [PATCH 035/133] Test workflow improvements - Better triggers - Common hashseed generation for test matrix --- .github/workflows/tests.yml | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bee489d..328cb2a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,8 +2,15 @@ name: Run tests # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the master branch + # Triggers the workflow on push or pull request events but only for important files push: + branches: + - master + paths: + - "dpath/" + - "**.py" + - "tox.ini" + pull_request: paths: - "dpath/" - "**.py" @@ -14,11 +21,29 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "tests" + # Generate a common hashseed for all tests + generate-hashseed: + runs-on: ubuntu-latest + + outputs: + hashseed: ${{ steps.generate.outputs.hashseed }} + + steps: + - name: Generate Hashseed + id: generate + run: | + python -c "from random import randint; + hashseed = randint(0, 4294967295) + print(f'{hashseed=}') + print(f'::set-output name=hashseed::{hashseed}')" + + # Tests job tests: # The type of runner that the job will run on runs-on: ubuntu-latest + needs: generate-hashseed + strategy: matrix: # Match versions specified in tox.ini @@ -38,4 +63,4 @@ jobs: - name: Run tox with tox-gh-actions uses: ymyzk/run-tox-gh-actions@main with: - tox-args: -vv + tox-args: -vv --hashseed=${{ needs.generate-hashseed.outputs.hashseed }} From 5681cdadcc4c3c7fb8cf6a552f5957e25c879e9e Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sat, 11 Sep 2021 20:21:33 +0300 Subject: [PATCH 036/133] Add changelog generator config --- .github/tag-changelog-config.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/tag-changelog-config.js diff --git a/.github/tag-changelog-config.js b/.github/tag-changelog-config.js new file mode 100644 index 0000000..bd30aa0 --- /dev/null +++ b/.github/tag-changelog-config.js @@ -0,0 +1,20 @@ +module.exports = { + types: [ + { types: ["other"], label: "Commits" }, + ], + + renderTypeSection: function (label, commits) { + let text = `\n## ${label}\n`; + + commits.forEach((commit) => { + text += `- ${commit.subject}\n`; + }); + + return text; + }, + + renderChangelog: function (release, changes) { + const now = new Date(); + return `# ${release} - ${now.toISOString().substr(0, 10)}\n` + changes + "\n\n"; + }, +}; From 606c50c7d9d86a6782645bd5b0288f062dafaf1b Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 00:32:27 +0300 Subject: [PATCH 037/133] Convert docstrings to double quotes --- dpath/segments.py | 72 +++++++++++++++++++++--------------------- dpath/util.py | 36 ++++++++++----------- tests/test_segments.py | 60 +++++++++++++++++------------------ 3 files changed, 84 insertions(+), 84 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 1016a06..07efd04 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -5,11 +5,11 @@ def kvs(node): - ''' + """ Return a (key, value) iterator for the node. kvs(node) -> (generator -> (key, value)) - ''' + """ try: return iter(node.items()) except AttributeError: @@ -23,23 +23,23 @@ def kvs(node): def leaf(thing): - ''' + """ Return True if thing is a leaf, otherwise False. leaf(thing) -> bool - ''' + """ leaves = (bytes, str, int, float, bool, type(None)) return isinstance(thing, leaves) def leafy(thing): - ''' + """ Same as leaf(thing), but also treats empty sequences and dictionaries as True. leafy(thing) -> bool - ''' + """ try: return leaf(thing) or len(thing) == 0 @@ -49,12 +49,12 @@ def leafy(thing): def walk(obj, location=()): - ''' + """ Yield all valid (segments, value) pairs (from a breadth-first search, right-to-left on sequences). walk(obj) -> (generator -> (segments, value)) - ''' + """ if not leaf(obj): for k, v in kvs(obj): length = None @@ -75,11 +75,11 @@ def walk(obj, location=()): def get(obj, segments): - ''' + """ Return the value at the path indicated by segments. get(obj, segments) -> value - ''' + """ current = obj for (i, segment) in enumerate(segments): if leaf(current): @@ -90,11 +90,11 @@ def get(obj, segments): def has(obj, segments): - ''' + """ Return True if the path exists in the obj. Otherwise return False. has(obj, segments) -> bool - ''' + """ try: get(obj, segments) return True @@ -103,12 +103,12 @@ def has(obj, segments): def expand(segments): - ''' + """ Yield a tuple of segments for each possible length of segments. Starting from the shortest length of segments and increasing by 1. expand(keys) -> (..., keys[:-2], keys[:-1]) - ''' + """ index = 0 for segment in segments: index += 1 @@ -116,11 +116,11 @@ def expand(segments): def types(obj, segments): - ''' + """ For each segment produce a tuple of (segment, type(value)). types(obj, segments) -> ((segment[0], type0), (segment[1], type1), ...) - ''' + """ result = [] for depth in expand(segments): result.append((depth[-1], type(get(obj, depth)))) @@ -128,31 +128,31 @@ def types(obj, segments): def leaves(obj): - ''' + """ Yield all leaves as (segment, value) pairs. leaves(obj) -> (generator -> (segment, value)) - ''' + """ return filter(lambda p: leafy(p[1]), walk(obj)) def int_str(segment): - ''' + """ If the segment is an integer, return the string conversion. Otherwise return the segment unchanged. The conversion uses 'str'. int_str(segment) -> str - ''' + """ if isinstance(segment, int): return str(segment) return segment class Star(object): - ''' + """ Used to create a global STAR symbol for tracking stars added when expanding star-star globs. - ''' + """ pass @@ -160,7 +160,7 @@ class Star(object): def match(segments, glob): - ''' + """ Return True if the segments match the given glob, otherwise False. For the purposes of matching, integers are converted to their string @@ -177,7 +177,7 @@ def match(segments, glob): throws an exception the result will be False. match(segments, glob) -> bool - ''' + """ segments = tuple(segments) glob = tuple(glob) @@ -238,12 +238,12 @@ def match(segments, glob): def extend(thing, index, value=None): - ''' + """ Extend a sequence like thing such that it contains at least index + 1 many elements. The extension values will be None (default). extend(thing, int) -> [thing..., None, ...] - ''' + """ try: expansion = (type(thing)()) @@ -263,12 +263,12 @@ def extend(thing, index, value=None): def __default_creator__(current, segments, i, hints=()): - ''' + """ Create missing path components. If the segment is an int, then it will create a list. Otherwise a dictionary is created. set(obj, segments, value) -> obj - ''' + """ segment = segments[i] length = len(segments) @@ -293,13 +293,13 @@ def __default_creator__(current, segments, i, hints=()): def set(obj, segments, value, creator=__default_creator__, hints=()): - ''' + """ Set the value in obj at the place indicated by segments. If creator is not None (default __default_creator__), then call the creator function to create any missing path components. set(obj, segments, value) -> obj - ''' + """ current = obj length = len(segments) @@ -331,7 +331,7 @@ def set(obj, segments, value, creator=__default_creator__, hints=()): def fold(obj, f, acc): - ''' + """ Walk obj applying f to each path and returning accumulator acc. The function f will be called, for each result in walk(obj): @@ -343,7 +343,7 @@ def fold(obj, f, acc): retrieved from the walk. fold(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc - ''' + """ for pair in walk(obj): if f(obj, pair, acc) is False: break @@ -351,14 +351,14 @@ def fold(obj, f, acc): def foldm(obj, f, acc): - ''' + """ Same as fold(), but permits mutating obj. This requires all paths in walk(obj) to be loaded into memory (whereas fold does not). foldm(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc - ''' + """ pairs = tuple(walk(obj)) for pair in pairs: (segments, value) = pair @@ -368,13 +368,13 @@ def foldm(obj, f, acc): def view(obj, glob): - ''' + """ Return a view of the object where the glob matches. A view retains the same form as the obj, but is limited to only the paths that matched. Views are new objects (a deepcopy of the matching values). view(obj, glob) -> obj' - ''' + """ def f(obj, pair, result): (segments, value) = pair if match(segments, glob): diff --git a/dpath/util.py b/dpath/util.py index 48c03be..39aaa6e 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -11,7 +11,7 @@ def __safe_path__(path, separator): - ''' + """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -19,7 +19,7 @@ def __safe_path__(path, separator): separator stripped off. If you pass a list path, the separator is ignored, and is assumed to be part of each key glob. It will not be stripped. - ''' + """ if not dpath.segments.leaf(path): segments = path else: @@ -45,7 +45,7 @@ def __safe_path__(path, separator): def new(obj, path, value, separator='/', creator=None): - ''' + """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only change existing keys). @@ -57,7 +57,7 @@ def new(obj, path, value, separator='/', creator=None): creator allows you to pass in a creator method that is responsible for creating missing keys at arbitrary levels of the path (see the help for dpath.path.set) - ''' + """ segments = __safe_path__(path, separator) if creator: return dpath.segments.set(obj, segments, value, creator=creator) @@ -65,12 +65,12 @@ def new(obj, path, value, separator='/', creator=None): def delete(obj, glob, separator='/', afilter=None): - ''' + """ Given a obj, delete all elements that match the glob. Returns the number of deleted objects. Raises PathNotFound if no paths are found to delete. - ''' + """ globlist = __safe_path__(glob, separator) def f(obj, pair, counter): @@ -122,10 +122,10 @@ def f(obj, pair, counter): def set(obj, glob, value, separator='/', afilter=None): - ''' + """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. - ''' + """ globlist = __safe_path__(glob, separator) def f(obj, pair, counter): @@ -147,7 +147,7 @@ def f(obj, pair, counter): def get(obj, glob, separator='/', default=_DEFAULT_SENTINAL): - ''' + """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. If the glob is not found and a default is provided, @@ -155,7 +155,7 @@ def get(obj, glob, separator='/', default=_DEFAULT_SENTINAL): If more than one leaf matches the glob, ValueError is raised. If the glob is not found and a default is not provided, KeyError is raised. - ''' + """ if glob == '/': return obj @@ -183,32 +183,32 @@ def f(obj, pair, results): def values(obj, glob, separator='/', afilter=None, dirs=True): - ''' + """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). - ''' + """ yielded = True return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): - ''' + """ Given a path glob, return a dictionary containing all keys that matched the given glob. If 'yielded' is true, then a dictionary will not be returned. Instead tuples will be yielded in the form of (path, value) for every element in the document that matched the glob. - ''' + """ globlist = __safe_path__(glob, separator) def keeper(segments, found): - ''' + """ Generalized test for use in both yielded and folded cases. Returns True if we want this result. Otherwise returns False. - ''' + """ if not dirs and not dpath.segments.leaf(found): return False @@ -234,7 +234,7 @@ def f(obj, pair, result): def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): - ''' + """ Merge source into destination. Like dict.update() but performs deep merging. @@ -272,7 +272,7 @@ def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): * MERGE_TYPESAFE : When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source replaces the destination in this situation. - ''' + """ filtered_src = search(src, '**', afilter=afilter, separator='/') def are_both_mutable(o1, o2): diff --git a/tests/test_segments.py b/tests/test_segments.py index af9df85..2fdf070 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -39,36 +39,36 @@ def teardown(): @given(random_node) def test_kvs(node): - ''' + """ Given a node, kvs should produce a key that when used to extract from the node renders the exact same value given. - ''' + """ for k, v in api.kvs(node): assert node[k] is v @given(random_leaf) def test_leaf_with_leaf(leaf): - ''' + """ Given a leaf, leaf should return True. - ''' + """ assert api.leaf(leaf) is True @given(random_node) def test_leaf_with_node(node): - ''' + """ Given a node, leaf should return False. - ''' + """ assert api.leaf(node) is False @given(random_thing) def test_walk(thing): - ''' + """ Given a thing to walk, walk should yield key, value pairs where key is a tuple of non-zero length. - ''' + """ for k, v in api.walk(thing): assert isinstance(k, tuple) assert len(k) > 0 @@ -76,19 +76,19 @@ def test_walk(thing): @given(random_node) def test_get(node): - ''' + """ Given a node, get should return the exact value given a key for all key, value pairs in the node. - ''' + """ for k, v in api.walk(node): assert api.get(node, k) is v @given(random_node) def test_has(node): - ''' + """ Given a node, has should return True for all paths, False otherwise. - ''' + """ for k, v in api.walk(node): assert api.has(node, k) is True @@ -100,10 +100,10 @@ def test_has(node): @given(random_segments) def test_expand(segments): - ''' + """ Given segments expand should produce as many results are there were segments and the last result should equal the given segments. - ''' + """ count = len(segments) result = list(api.expand(segments)) @@ -115,10 +115,10 @@ def test_expand(segments): @given(random_node) def test_types(node): - ''' + """ Given a node, types should yield a tuple of key, type pairs and the type indicated should equal the type of the value. - ''' + """ for k, v in api.walk(node): ts = api.types(node, k) ta = () @@ -129,9 +129,9 @@ def test_types(node): @given(random_node) def test_leaves(node): - ''' + """ Given a node, leaves should yield only leaf key, value pairs. - ''' + """ for k, v in api.leaves(node): assert api.leafy(v) @@ -245,18 +245,18 @@ def random_segments_with_nonmatching_glob(draw): @given(random_segments_with_glob()) def test_match(pair): - ''' + """ Given segments and a known good glob, match should be True. - ''' + """ (segments, glob) = pair assert api.match(segments, glob) is True @given(random_segments_with_nonmatching_glob()) def test_match_nonmatching(pair): - ''' + """ Given segments and a known bad glob, match should be False. - ''' + """ print(pair) (segments, glob) = pair assert api.match(segments, glob) is False @@ -280,9 +280,9 @@ def random_leaves(draw): @given(walkable=random_walk(), value=random_thing) def test_set_walkable(walkable, value): - ''' + """ Given a walkable location, set should be able to update any value. - ''' + """ (node, (segments, found)) = walkable api.set(node, segments, value) assert api.get(node, segments) is value @@ -294,10 +294,10 @@ def test_set_walkable(walkable, value): value=random_thing, extension=random_segments) def test_set_create_missing(walkable, kstr, kint, value, extension): - ''' + """ Given a walkable non-leaf, set should be able to create missing nodes and set a new value. - ''' + """ (node, (segments, found)) = walkable assume(api.leaf(found)) @@ -319,9 +319,9 @@ def test_set_create_missing(walkable, kstr, kint, value, extension): @given(thing=random_thing) def test_fold(thing): - ''' + """ Given a thing, count paths with fold. - ''' + """ def f(o, p, a): a[0] += 1 @@ -331,9 +331,9 @@ def f(o, p, a): @given(walkable=random_walk()) def test_view(walkable): - ''' + """ Given a walkable location, view that location. - ''' + """ (node, (segments, found)) = walkable assume(found == found) # Hello, nan! We don't want you here. From 4d5fdf201ec760b63fca4ac17440c811044f38a6 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 01:21:20 +0300 Subject: [PATCH 038/133] Add Python >=3.6 constraint for type hints --- setup.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abce1a3..414702b 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,16 @@ scripts=[], packages=["dpath"], data_files=[], - python_requires=">=3", + + # Type hints are great. + # Function annotations were added in Python 3.0. + # Typing module was added in Python 3.5. + # Variable annotations were added in Python 3.6. + # Python versions that are >=3.6 are more popular. + # (Source: https://github.com/hugovk/pypi-tools/blob/master/README.md) + # + # Conclusion: In order to accommodate type hinting support must be limited to Python versions >=3.6. + python_requires=">=3.6", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', From e3dc192c6e349b11270e70c318295b390b0b7718 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 01:23:19 +0300 Subject: [PATCH 039/133] Organize imports --- dpath/segments.py | 5 +++-- dpath/util.py | 6 +++--- setup.py | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 07efd04..5df536e 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,8 +1,9 @@ from copy import deepcopy -from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath import options from fnmatch import fnmatchcase +from dpath import options +from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound + def kvs(node): """ diff --git a/dpath/util.py b/dpath/util.py index 39aaa6e..9f2d9e5 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,8 +1,8 @@ -from collections.abc import MutableMapping -from collections.abc import MutableSequence +from collections.abc import MutableMapping, MutableSequence + +import dpath.segments from dpath import options from dpath.exceptions import InvalidKeyName -import dpath.segments _DEFAULT_SENTINAL = object() MERGE_REPLACE = (1 << 1) diff --git a/setup.py b/setup.py index 414702b..95c7734 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ -from distutils.core import setup -import dpath.version import os +from distutils.core import setup +import dpath.version long_description = open( os.path.join( From 16f511aa4cf673dc7bf8462a998ffd3da226ab8b Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 01:24:27 +0300 Subject: [PATCH 040/133] Fix typo --- dpath/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index 9f2d9e5..ed1bb63 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidKeyName -_DEFAULT_SENTINAL = object() +_DEFAULT_SENTINEL = object() MERGE_REPLACE = (1 << 1) MERGE_ADDITIVE = (1 << 2) MERGE_TYPESAFE = (1 << 3) @@ -146,7 +146,7 @@ def f(obj, pair, counter): return changed -def get(obj, glob, separator='/', default=_DEFAULT_SENTINAL): +def get(obj, glob, separator='/', default=_DEFAULT_SENTINEL): """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. @@ -172,7 +172,7 @@ def f(obj, pair, results): results = dpath.segments.fold(obj, f, []) if len(results) == 0: - if default is not _DEFAULT_SENTINAL: + if default is not _DEFAULT_SENTINEL: return default raise KeyError(glob) From 6f8a55828e7e22c72ba4bb9765cfffdb19fdc14d Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 19:57:02 +0300 Subject: [PATCH 041/133] Implement merge types as an enum Additional type hints --- README.rst | 9 ++++----- dpath/util.py | 38 +++++++++++++++++++++++--------------- tests/test_types.py | 2 +- tests/test_util_merge.py | 14 +++++++------- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/README.rst b/README.rst index ea88f5a..cb9ead5 100644 --- a/README.rst +++ b/README.rst @@ -279,14 +279,13 @@ does. Merge source into destination. Like dict.update() but performs deep merging. - flags is an OR'ed combination of MERGE_ADDITIVE, MERGE_REPLACE - MERGE_TYPESAFE. - * MERGE_ADDITIVE : List objects are combined onto one long + flags is an OR'ed combination of MergeType enum members. + * ADDITIVE : List objects are combined onto one long list (NOT a set). This is the default flag. - * MERGE_REPLACE : Instead of combining list objects, when + * REPLACE : Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination with the source. - * MERGE_TYPESAFE : When 2 keys at equal levels are of different + * TYPESAFE : When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source replaces the destination in this situation. diff --git a/dpath/util.py b/dpath/util.py index ed1bb63..e9efb26 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,16 +1,25 @@ from collections.abc import MutableMapping, MutableSequence +from enum import Flag, IntFlag, auto +from typing import Union, List, Any, Dict import dpath.segments from dpath import options from dpath.exceptions import InvalidKeyName _DEFAULT_SENTINEL = object() -MERGE_REPLACE = (1 << 1) -MERGE_ADDITIVE = (1 << 2) -MERGE_TYPESAFE = (1 << 3) -def __safe_path__(path, separator): +class MergeType(IntFlag): + REPLACE = auto() + ADDITIVE = auto() + TYPESAFE = auto() + + +# Type alias for dict path segments where integers are explicitly casted +IntAwareSegment = Union[int, Any] + + +def __safe_path__(path: str, separator: str) -> Union[List[IntAwareSegment], IntAwareSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -146,7 +155,7 @@ def f(obj, pair, counter): return changed -def get(obj, glob, separator='/', default=_DEFAULT_SENTINEL): +def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> dict: """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. @@ -156,7 +165,7 @@ def get(obj, glob, separator='/', default=_DEFAULT_SENTINEL): If more than one leaf matches the glob, ValueError is raised. If the glob is not found and a default is not provided, KeyError is raised. """ - if glob == '/': + if glob == "/": return obj globlist = __safe_path__(glob, separator) @@ -233,7 +242,7 @@ def f(obj, pair, result): return dpath.segments.fold(obj, f, {}) -def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): +def merge(dst, src, separator='/', afilter=None, flags=MergeType.ADDITIVE): """ Merge source into destination. Like dict.update() but performs deep merging. @@ -262,14 +271,13 @@ def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): objects that you intend to merge. For further notes see https://github.com/akesterson/dpath-python/issues/58 - flags is an OR'ed combination of MERGE_ADDITIVE, MERGE_REPLACE, - MERGE_TYPESAFE. - * MERGE_ADDITIVE : List objects are combined onto one long + flags is an OR'ed combination of MergeType enum members. + * ADDITIVE : List objects are combined onto one long list (NOT a set). This is the default flag. - * MERGE_REPLACE : Instead of combining list objects, when + * REPLACE : Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination with the source. - * MERGE_TYPESAFE : When 2 keys at equal levels are of different + * TYPESAFE : When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source replaces the destination in this situation. """ @@ -295,7 +303,7 @@ def merger(dst, src, _segments=()): "{}".format(segments)) # Validate src and dst types match. - if flags & MERGE_TYPESAFE: + if flags & MergeType.TYPESAFE: if dpath.segments.has(dst, segments): target = dpath.segments.get(dst, segments) tt = type(target) @@ -332,11 +340,11 @@ def merger(dst, src, _segments=()): # # Pretend we have a sequence and account for the flags. try: - if flags & MERGE_ADDITIVE: + if flags & MergeType.ADDITIVE: target += found continue - if flags & MERGE_REPLACE: + if flags & MergeType.REPLACE: try: target[''] except TypeError: diff --git a/tests/test_types.py b/tests/test_types.py index 82f8c05..39942b9 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -106,7 +106,7 @@ def test_types_merge_simple_list_replace(): "list": TestSequence([0, 1, 2, 3]) }) - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.REPLACE) nose.tools.eq_(dst["list"], TestSequence([7, 8, 9, 10])) diff --git a/tests/test_util_merge.py b/tests/test_util_merge.py index 968a4fa..d5765d2 100644 --- a/tests/test_util_merge.py +++ b/tests/test_util_merge.py @@ -19,7 +19,7 @@ def test_merge_typesafe_and_separator(): } try: - dpath.util.merge(dst, src, flags=(dpath.util.MERGE_ADDITIVE | dpath.util.MERGE_TYPESAFE), separator=";") + dpath.util.merge(dst, src, flags=(dpath.util.MergeType.ADDITIVE | dpath.util.MergeType.TYPESAFE), separator=";") except TypeError as e: assert(str(e).endswith("dict;integer")) @@ -59,7 +59,7 @@ def test_merge_simple_list_additive(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_ADDITIVE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.ADDITIVE) nose.tools.eq_(dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) @@ -71,7 +71,7 @@ def test_merge_simple_list_replace(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.REPLACE) nose.tools.eq_(dst["list"], [7, 8, 9, 10]) @@ -123,7 +123,7 @@ def test_merge_typesafe(): ], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.TYPESAFE) @raises(TypeError) @@ -156,20 +156,20 @@ class tcis(list): assert(dst['ms'][2] == 'c') assert("casserole" in dst["mm"]) - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.TYPESAFE) def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} - dpath.util.merge(dct_a, dct_b, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dct_a, dct_b, flags=dpath.util.MergeType.REPLACE) assert(len(dct_a['a']['b']) == 1) def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} - dpath.util.merge(d1, d2, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(d1, d2, flags=dpath.util.MergeType.REPLACE) assert(len(d1['a']) == 1) assert(d1['a'][0] == 'a') From 0704fde58cc059393c1e4da0aeaecacd957d1dc5 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 20:12:23 +0300 Subject: [PATCH 042/133] Update README - Python 3 print functions - Fix dict indentation --- README.rst | 174 +++++++++++++++++++++++------------------------ dpath/version.py | 2 +- 2 files changed, 88 insertions(+), 88 deletions(-) diff --git a/README.rst b/README.rst index ea88f5a..8e0db20 100644 --- a/README.rst +++ b/README.rst @@ -49,10 +49,10 @@ Suppose we have a dictionary like this: x = { "a": { "b": { - "3": 2, - "43": 30, - "c": [], - "d": ['red', 'buggy', 'bumpers'], + "3": 2, + "43": 30, + "c": [], + "d": ['red', 'buggy', 'bumpers'], } } } @@ -96,26 +96,26 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon >>> result = dpath.util.search(x, "a/b/[cd]") - >>> print json.dumps(result, indent=4, sort_keys=True) + >>> print(json.dumps(result, indent=4, sort_keys=True)) { - "a": { - "b": { - "c": [], - "d": [ - "red", - "buggy", - "bumpers" - ] + "a": { + "b": { + "c": [], + "d": [ + "red", + "buggy", + "bumpers" + ] + } } } - } ... Wow that was easy. What if I want to iterate over the results, and not get a merged view? .. code-block:: pycon - >>> for x in dpath.util.search(x, "a/b/[cd]", yielded=True): print x + >>> for x in dpath.util.search(x, "a/b/[cd]", yielded=True): print(x) ... ('a/b/c', []) ('a/b/d', ['red', 'buggy', 'bumpers']) @@ -154,7 +154,7 @@ value 'Waffles'. >>> dpath.util.set(x, 'a/b/[cd]', 'Waffles') 2 - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { @@ -188,7 +188,7 @@ necessary to get to the terminus. keys >>> dpath.util.new(x, 'a/b/e/f/g', "Roffle") - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { @@ -213,7 +213,7 @@ object with None entries in order to make it big enough: >>> dpath.util.new(x, 'a/b/e/f/h', []) >>> dpath.util.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { @@ -291,59 +291,59 @@ does. replaces the destination in this situation. >>> y = {'a': {'b': { 'e': {'f': {'h': [None, 0, 1, None, 13, 14]}}}, 'c': 'RoffleWaffles'}} - >>> print json.dumps(y, indent=4, sort_keys=True) + >>> print(json.dumps(y, indent=4, sort_keys=True)) { - "a": { - "b": { - "e": { - "f": { - "h": [ - null, - 0, - 1, - null, - 13, - 14 - ] - } + "a": { + "b": { + "e": { + "f": { + "h": [ + null, + 0, + 1, + null, + 13, + 14 + ] + } + } + }, + "c": "RoffleWaffles" } - }, - "c": "RoffleWaffles" - } } >>> dpath.util.merge(x, y) - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { - "a": { - "b": { - "3": 2, - "43": 30, - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle", - "h": [ - null, - 0, - 1, - null, - 13, - 14, - null, - null, - null, - null, - null, - null, - null, - "Wow this is a big array, it sure is lonely in here by myself" - ] - } + "a": { + "b": { + "3": 2, + "43": 30, + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle", + "h": [ + null, + 0, + 1, + null, + 13, + 14, + null, + null, + null, + null, + null, + null, + null, + "Wow this is a big array, it sure is lonely in here by myself" + ] + } + } + }, + "c": "RoffleWaffles" } - }, - "c": "RoffleWaffles" - } } Now that's handy. You shouldn't try to use this as a replacement for the @@ -370,21 +370,21 @@ them: .. code-block:: pycon - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { - "a": { - "b": { - "3": 2, - "43": 30, - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle" + "a": { + "b": { + "3": 2, + "43": 30, + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle" + } + } } } - } - } } >>> def afilter(x): ... if "ffle" in str(x): @@ -392,19 +392,19 @@ them: ... return False ... >>> result = dpath.util.search(x, '**', afilter=afilter) - >>> print json.dumps(result, indent=4, sort_keys=True) + >>> print(json.dumps(result, indent=4, sort_keys=True)) { - "a": { - "b": { - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle" + "a": { + "b": { + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle" + } + } } } - } - } } Obviously filtering functions can perform more advanced tests (regular diff --git a/dpath/version.py b/dpath/version.py index 215732a..664cb5f 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.4" +VERSION = "2.0.5" From 2e392176d79357cfb0ad824132f0efddf7f52e4d Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 20:15:15 +0300 Subject: [PATCH 043/133] Add reference to changelog generator config --- .github/workflows/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8210fac..5c1386e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -46,6 +46,7 @@ jobs: uses: loopwerk/tag-changelog@v1 with: token: ${{ secrets.GITHUB_TOKEN }} + config_file: .github/tag-changelog-config.js - name: PyPI Deployment uses: casperdcl/deploy-pypi@v2 From fa26b3954e0a649c9ff2b3da497ffa5ff483de99 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 12:24:22 +0300 Subject: [PATCH 044/133] Rename __safe_path__ to _split_path --- dpath/util.py | 12 ++++++------ tests/test_util_paths.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index e9efb26..ab3b2f4 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -19,7 +19,7 @@ class MergeType(IntFlag): IntAwareSegment = Union[int, Any] -def __safe_path__(path: str, separator: str) -> Union[List[IntAwareSegment], IntAwareSegment]: +def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAwareSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -67,7 +67,7 @@ def new(obj, path, value, separator='/', creator=None): responsible for creating missing keys at arbitrary levels of the path (see the help for dpath.path.set) """ - segments = __safe_path__(path, separator) + segments = _split_path(path, separator) if creator: return dpath.segments.set(obj, segments, value, creator=creator) return dpath.segments.set(obj, segments, value) @@ -80,7 +80,7 @@ def delete(obj, glob, separator='/', afilter=None): Returns the number of deleted objects. Raises PathNotFound if no paths are found to delete. """ - globlist = __safe_path__(glob, separator) + globlist = _split_path(glob, separator) def f(obj, pair, counter): (segments, value) = pair @@ -135,7 +135,7 @@ def set(obj, glob, value, separator='/', afilter=None): Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. """ - globlist = __safe_path__(glob, separator) + globlist = _split_path(glob, separator) def f(obj, pair, counter): (segments, found) = pair @@ -168,7 +168,7 @@ def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) - if glob == "/": return obj - globlist = __safe_path__(glob, separator) + globlist = _split_path(glob, separator) def f(obj, pair, results): (segments, found) = pair @@ -211,7 +211,7 @@ def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): every element in the document that matched the glob. """ - globlist = __safe_path__(glob, separator) + globlist = _split_path(glob, separator) def keeper(segments, found): """ diff --git a/tests/test_util_paths.py b/tests/test_util_paths.py index 27260fe..6d2c4c9 100644 --- a/tests/test_util_paths.py +++ b/tests/test_util_paths.py @@ -2,7 +2,7 @@ def test_util_safe_path_list(): - res = dpath.util.__safe_path__(["Ignore", "the/separator"], None) + res = dpath.util._split_path(["Ignore", "the/separator"], None) assert(len(res) == 2) assert(res[0] == "Ignore") From d96814be1e92fc6252c33ff4466f7e8f16c3fde2 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 12:43:26 +0300 Subject: [PATCH 045/133] Sort out imports --- dpath/util.py | 113 +++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index ab3b2f4..9a78a1c 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,10 +1,9 @@ from collections.abc import MutableMapping, MutableSequence -from enum import Flag, IntFlag, auto +from enum import IntFlag, auto from typing import Union, List, Any, Dict -import dpath.segments -from dpath import options -from dpath.exceptions import InvalidKeyName +from dpath import options, segments +from dpath.exceptions import InvalidKeyName, PathNotFound _DEFAULT_SENTINEL = object() @@ -29,28 +28,28 @@ def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAw ignored, and is assumed to be part of each key glob. It will not be stripped. """ - if not dpath.segments.leaf(path): - segments = path + if not segments.leaf(path): + split_segments = path else: - segments = path.lstrip(separator).split(separator) + split_segments = path.lstrip(separator).split(separator) # FIXME: This check was in the old internal library, but I can't # see a way it could fail... - for i, segment in enumerate(segments): + for i, segment in enumerate(split_segments): if (separator and (separator in segment)): raise InvalidKeyName("{} at {}[{}] contains the separator '{}'" - "".format(segment, segments, i, separator)) + "".format(segment, split_segments, i, separator)) # Attempt to convert integer segments into actual integers. final = [] - for segment in segments: + for segment in split_segments: try: final.append(int(segment)) except: final.append(segment) - segments = final + split_segments = final - return segments + return split_segments def new(obj, path, value, separator='/', creator=None): @@ -67,10 +66,10 @@ def new(obj, path, value, separator='/', creator=None): responsible for creating missing keys at arbitrary levels of the path (see the help for dpath.path.set) """ - segments = _split_path(path, separator) + split_segments = _split_path(path, separator) if creator: - return dpath.segments.set(obj, segments, value, creator=creator) - return dpath.segments.set(obj, segments, value) + return segments.set(obj, split_segments, value, creator=creator) + return segments.set(obj, split_segments, value) def delete(obj, glob, separator='/', afilter=None): @@ -83,18 +82,18 @@ def delete(obj, glob, separator='/', afilter=None): globlist = _split_path(glob, separator) def f(obj, pair, counter): - (segments, value) = pair + (path_segments, value) = pair # Skip segments if they no longer exist in obj. - if not dpath.segments.has(obj, segments): + if not segments.has(obj, path_segments): return - matched = dpath.segments.match(segments, globlist) - selected = afilter and dpath.segments.leaf(value) and afilter(value) + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(value) and afilter(value) if (matched and not afilter) or selected: - key = segments[-1] - parent = dpath.segments.get(obj, segments[:-1]) + key = path_segments[-1] + parent = segments.get(obj, path_segments[:-1]) try: # Attempt to treat parent like a sequence. @@ -107,7 +106,7 @@ def f(obj, pair, counter): # # Note: In order to achieve proper behavior we are # relying on the reverse iteration of - # non-dictionaries from dpath.segments.kvs(). + # non-dictionaries from segments.kvs(). # Otherwise we'd be unable to delete all the tails # of a list and end up with None values when we # don't need them. @@ -123,9 +122,9 @@ def f(obj, pair, counter): counter[0] += 1 - [deleted] = dpath.segments.foldm(obj, f, [0]) + [deleted] = segments.foldm(obj, f, [0]) if not deleted: - raise dpath.exceptions.PathNotFound("Could not find {0} to delete it".format(glob)) + raise PathNotFound("Could not find {0} to delete it".format(glob)) return deleted @@ -138,20 +137,20 @@ def set(obj, glob, value, separator='/', afilter=None): globlist = _split_path(glob, separator) def f(obj, pair, counter): - (segments, found) = pair + (path_segments, found) = pair # Skip segments if they no longer exist in obj. - if not dpath.segments.has(obj, segments): + if not segments.has(obj, path_segments): return - matched = dpath.segments.match(segments, globlist) - selected = afilter and dpath.segments.leaf(found) and afilter(found) + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(found) and afilter(found) if (matched and not afilter) or (matched and selected): - dpath.segments.set(obj, segments, value, creator=None) + segments.set(obj, path_segments, value, creator=None) counter[0] += 1 - [changed] = dpath.segments.foldm(obj, f, [0]) + [changed] = segments.foldm(obj, f, [0]) return changed @@ -171,14 +170,14 @@ def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) - globlist = _split_path(glob, separator) def f(obj, pair, results): - (segments, found) = pair + (path_segments, found) = pair - if dpath.segments.match(segments, globlist): + if segments.match(path_segments, globlist): results.append(found) if len(results) > 1: return False - results = dpath.segments.fold(obj, f, []) + results = segments.fold(obj, f, []) if len(results) == 0: if default is not _DEFAULT_SENTINEL: @@ -213,33 +212,33 @@ def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): globlist = _split_path(glob, separator) - def keeper(segments, found): + def keeper(path, found): """ Generalized test for use in both yielded and folded cases. Returns True if we want this result. Otherwise returns False. """ - if not dirs and not dpath.segments.leaf(found): + if not dirs and not segments.leaf(found): return False - matched = dpath.segments.match(segments, globlist) + matched = segments.match(path, globlist) selected = afilter and afilter(found) return (matched and not afilter) or (matched and selected) if yielded: def yielder(): - for segments, found in dpath.segments.walk(obj): - if keeper(segments, found): - yield (separator.join(map(dpath.segments.int_str, segments)), found) + for path, found in segments.walk(obj): + if keeper(path, found): + yield (separator.join(map(segments.int_str, path)), found) return yielder() else: def f(obj, pair, result): - (segments, found) = pair + (path, found) = pair - if keeper(segments, found): - dpath.segments.set(result, segments, found, hints=dpath.segments.types(obj, segments)) + if keeper(path, found): + segments.set(result, path, found, hints=segments.types(obj, path)) - return dpath.segments.fold(obj, f, {}) + return segments.fold(obj, f, {}) def merge(dst, src, separator='/', afilter=None, flags=MergeType.ADDITIVE): @@ -293,43 +292,43 @@ def are_both_mutable(o1, o2): return False def merger(dst, src, _segments=()): - for key, found in dpath.segments.kvs(src): + for key, found in segments.kvs(src): # Our current path in the source. - segments = _segments + (key,) + current_path = _segments + (key,) if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(segments)) + "{}".format(current_path)) # Validate src and dst types match. if flags & MergeType.TYPESAFE: - if dpath.segments.has(dst, segments): - target = dpath.segments.get(dst, segments) + if segments.has(dst, current_path): + target = segments.get(dst, current_path) tt = type(target) ft = type(found) if tt != ft: - path = separator.join(segments) + path = separator.join(current_path) raise TypeError("Cannot merge objects of type" "{0} and {1} at {2}" "".format(tt, ft, path)) # Path not present in destination, create it. - if not dpath.segments.has(dst, segments): - dpath.segments.set(dst, segments, found) + if not segments.has(dst, current_path): + segments.set(dst, current_path, found) continue # Retrieve the value in the destination. - target = dpath.segments.get(dst, segments) + target = segments.get(dst, current_path) # If the types don't match, replace it. if ((type(found) != type(target)) and (not are_both_mutable(found, target))): - dpath.segments.set(dst, segments, found) + segments.set(dst, current_path, found) continue # If target is a leaf, the replace it. - if dpath.segments.leaf(target): - dpath.segments.set(dst, segments, found) + if segments.leaf(target): + segments.set(dst, current_path, found) continue # At this point we know: @@ -348,14 +347,14 @@ def merger(dst, src, _segments=()): try: target[''] except TypeError: - dpath.segments.set(dst, segments, found) + segments.set(dst, current_path, found) continue except: raise except: # We have a dictionary like thing and we need to attempt to # recursively merge it. - merger(dst, found, segments) + merger(dst, found, current_path) merger(dst, filtered_src) From 07960492985b7101376101234428a274aa271cb3 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 12:44:10 +0300 Subject: [PATCH 046/133] Remove unnecessary check --- dpath/util.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index 9a78a1c..cd9ed16 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -33,13 +33,6 @@ def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAw else: split_segments = path.lstrip(separator).split(separator) - # FIXME: This check was in the old internal library, but I can't - # see a way it could fail... - for i, segment in enumerate(split_segments): - if (separator and (separator in segment)): - raise InvalidKeyName("{} at {}[{}] contains the separator '{}'" - "".format(segment, split_segments, i, separator)) - # Attempt to convert integer segments into actual integers. final = [] for segment in split_segments: From b6bc7d2d152f5fbfb322e3a0755e75e45f7667cd Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 12:48:44 +0300 Subject: [PATCH 047/133] Rename/remove unused variables --- dpath/segments.py | 7 +++---- dpath/util.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 5df536e..c94507d 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -111,7 +111,7 @@ def expand(segments): expand(keys) -> (..., keys[:-2], keys[:-1]) """ index = 0 - for segment in segments: + for _ in segments: index += 1 yield segments[:index] @@ -185,13 +185,13 @@ def match(segments, glob): path_len = len(segments) glob_len = len(glob) - # Index of the star-star in the glob. - ss = -1 # The star-star normalized glob ('**' has been removed). ss_glob = glob if '**' in glob: + # Index of the star-star in the glob. ss = glob.index('**') + if '**' in glob[ss + 1:]: raise InvalidGlob("Invalid glob. Only one '**' is permitted per glob: {}" "".format(glob)) @@ -362,7 +362,6 @@ def foldm(obj, f, acc): """ pairs = tuple(walk(obj)) for pair in pairs: - (segments, value) = pair if f(obj, pair, acc) is False: break return acc diff --git a/dpath/util.py b/dpath/util.py index cd9ed16..06a12eb 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -162,7 +162,7 @@ def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) - globlist = _split_path(glob, separator) - def f(obj, pair, results): + def f(_, pair, results): (path_segments, found) = pair if segments.match(path_segments, globlist): From 2b73ab3723bcb8b513db898ba9f4dad00629c438 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 13:25:25 +0300 Subject: [PATCH 048/133] Some type hinting --- dpath/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index 06a12eb..f26db63 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -15,10 +15,10 @@ class MergeType(IntFlag): # Type alias for dict path segments where integers are explicitly casted -IntAwareSegment = Union[int, Any] +PathSegment = Union[int, str] -def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAwareSegment]: +def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -45,7 +45,7 @@ def _split_path(path: str, separator: str) -> Union[List[IntAwareSegment], IntAw return split_segments -def new(obj, path, value, separator='/', creator=None): +def new(obj: Dict, path: str, value, separator="/", creator=None): """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only From b16c6ad7ade9bb12cccf00055c39f96caeb94e5a Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 15 Sep 2021 13:45:50 +0300 Subject: [PATCH 049/133] More type hinting --- dpath/segments.py | 4 +++- dpath/util.py | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index c94507d..728e041 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,8 +1,10 @@ from copy import deepcopy from fnmatch import fnmatchcase +from typing import List, Sequence, Tuple from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound +from dpath.util import PathSegment def kvs(node): @@ -263,7 +265,7 @@ def extend(thing, index, value=None): return thing -def __default_creator__(current, segments, i, hints=()): +def __default_creator__(current, segments: List[str], i: int, hints: Sequence[Tuple[PathSegment, type]] = ()): """ Create missing path components. If the segment is an int, then it will create a list. Otherwise a dictionary is created. diff --git a/dpath/util.py b/dpath/util.py index f26db63..a0bb851 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,6 +1,6 @@ from collections.abc import MutableMapping, MutableSequence from enum import IntFlag, auto -from typing import Union, List, Any, Dict +from typing import Union, List, Any, Dict, Callable from dpath import options, segments from dpath.exceptions import InvalidKeyName, PathNotFound @@ -17,6 +17,9 @@ class MergeType(IntFlag): # Type alias for dict path segments where integers are explicitly casted PathSegment = Union[int, str] +# Type alias for filter functions +Filter = Callable[[Any], bool] # (Any) -> bool + def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: """ @@ -45,7 +48,8 @@ def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegme return split_segments -def new(obj: Dict, path: str, value, separator="/", creator=None): +# todo: Type hint creator arg +def new(obj: Dict, path: str, value, separator="/", creator=None) -> Dict: """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only @@ -65,7 +69,7 @@ def new(obj: Dict, path: str, value, separator="/", creator=None): return segments.set(obj, split_segments, value) -def delete(obj, glob, separator='/', afilter=None): +def delete(obj: Dict, glob: str, separator='/', afilter: Filter = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -122,7 +126,7 @@ def f(obj, pair, counter): return deleted -def set(obj, glob, value, separator='/', afilter=None): +def set(obj: Dict, glob: str, value, separator='/', afilter: Filter = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -147,7 +151,7 @@ def f(obj, pair, counter): return changed -def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> dict: +def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. @@ -183,7 +187,7 @@ def f(_, pair, results): return results[0] -def values(obj, glob, separator='/', afilter=None, dirs=True): +def values(obj: Dict, glob: str, separator='/', afilter: Filter = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -193,7 +197,7 @@ def values(obj, glob, separator='/', afilter=None, dirs=True): return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): +def search(obj: Dict, glob: str, yielded=False, separator='/', afilter: Filter = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. @@ -234,7 +238,7 @@ def f(obj, pair, result): return segments.fold(obj, f, {}) -def merge(dst, src, separator='/', afilter=None, flags=MergeType.ADDITIVE): +def merge(dst: Dict, src: Dict, separator='/', afilter: Filter = None, flags=MergeType.ADDITIVE): """ Merge source into destination. Like dict.update() but performs deep merging. From fdac68fc5bdd3fe0525e12d1bc4d25e5bc859764 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 17 Sep 2021 01:41:13 +0300 Subject: [PATCH 050/133] Remove parens --- dpath/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/util.py b/dpath/util.py index a0bb851..54f532b 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -319,7 +319,7 @@ def merger(dst, src, _segments=()): target = segments.get(dst, current_path) # If the types don't match, replace it. - if ((type(found) != type(target)) and (not are_both_mutable(found, target))): + if type(found) != type(target) and not are_both_mutable(found, target): segments.set(dst, current_path, found) continue From fb77a64be205bf4a67ddb5f7b1644b7ca8ac5868 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sat, 18 Sep 2021 21:20:18 +0300 Subject: [PATCH 051/133] Refactoring and type hinting --- dpath/segments.py | 34 ++++++++++++++++++++-------------- dpath/util.py | 10 ++++++---- tests/test_segments.py | 2 +- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 728e041..3b46af3 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,17 +1,21 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import List, Sequence, Tuple +from typing import List, Sequence, Tuple, Iterator, Any, Dict, Union from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound from dpath.util import PathSegment -def kvs(node): +def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: """ - Return a (key, value) iterator for the node. + Returns an iterator which yields tuple pairs of (node index, node value), regardless of node type. - kvs(node) -> (generator -> (key, value)) + * For dict nodes `node.items()` will be returned. + * For sequence nodes (lists/tuples/etc.) a zip between index number and index value will be returned. + * Edge cases will result in an empty iterator being returned. + + make_walkable(node) -> (generator -> (key, value)) """ try: return iter(node.items()) @@ -28,8 +32,6 @@ def kvs(node): def leaf(thing): """ Return True if thing is a leaf, otherwise False. - - leaf(thing) -> bool """ leaves = (bytes, str, int, float, bool, type(None)) @@ -40,8 +42,6 @@ def leafy(thing): """ Same as leaf(thing), but also treats empty sequences and dictionaries as True. - - leafy(thing) -> bool """ try: @@ -59,20 +59,21 @@ def walk(obj, location=()): walk(obj) -> (generator -> (segments, value)) """ if not leaf(obj): - for k, v in kvs(obj): + for k, v in make_walkable(obj): length = None try: length = len(k) - except: + except TypeError: pass if length is not None and length == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " "{}".format(location + (k,))) - yield ((location + (k,)), v) - for k, v in kvs(obj): + yield (location + (k,)), v + + for k, v in make_walkable(obj): for found in walk(v, location + (k,)): yield found @@ -240,7 +241,7 @@ def match(segments, glob): return False -def extend(thing, index, value=None): +def extend(thing: List, index: int, value=None): """ Extend a sequence like thing such that it contains at least index + 1 many elements. The extension values will be None (default). @@ -265,7 +266,12 @@ def extend(thing, index, value=None): return thing -def __default_creator__(current, segments: List[str], i: int, hints: Sequence[Tuple[PathSegment, type]] = ()): +def __default_creator__( + current: Union[Dict, List], + segments: List[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () +): """ Create missing path components. If the segment is an int, then it will create a list. Otherwise a dictionary is created. diff --git a/dpath/util.py b/dpath/util.py index 54f532b..08d7db8 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,6 +1,6 @@ from collections.abc import MutableMapping, MutableSequence from enum import IntFlag, auto -from typing import Union, List, Any, Dict, Callable +from typing import Union, List, Any, Dict, Callable, Sequence, Tuple from dpath import options, segments from dpath.exceptions import InvalidKeyName, PathNotFound @@ -20,6 +20,9 @@ class MergeType(IntFlag): # Type alias for filter functions Filter = Callable[[Any], bool] # (Any) -> bool +# Type alias for creator functions +Creator = Callable[[Union[Dict, List], List[PathSegment], int, Sequence[Tuple[PathSegment, type]]], None] + def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: """ @@ -48,8 +51,7 @@ def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegme return split_segments -# todo: Type hint creator arg -def new(obj: Dict, path: str, value, separator="/", creator=None) -> Dict: +def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only @@ -289,7 +291,7 @@ def are_both_mutable(o1, o2): return False def merger(dst, src, _segments=()): - for key, found in segments.kvs(src): + for key, found in segments.make_walkable(src): # Our current path in the source. current_path = _segments + (key,) diff --git a/tests/test_segments.py b/tests/test_segments.py index 2fdf070..71c11ce 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -43,7 +43,7 @@ def test_kvs(node): Given a node, kvs should produce a key that when used to extract from the node renders the exact same value given. """ - for k, v in api.kvs(node): + for k, v in api.make_walkable(node): assert node[k] is v From 474ba6ddab530aa793d62df486e4d19c4092db24 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 19 Sep 2021 01:09:17 +0300 Subject: [PATCH 052/133] Documentation --- dpath/segments.py | 18 ++++++++++++------ dpath/util.py | 43 ++++++++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 3b46af3..81d218b 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,10 +1,10 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import List, Sequence, Tuple, Iterator, Any, Dict, Union +from typing import List, Sequence, Tuple, Iterator, Any, Dict, Union, Optional from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.util import PathSegment +from dpath.util import PathSegment, Creator, Hints def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: @@ -266,9 +266,9 @@ def extend(thing: List, index: int, value=None): return thing -def __default_creator__( +def _default_creator( current: Union[Dict, List], - segments: List[PathSegment], + segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () ): @@ -301,7 +301,13 @@ def __default_creator__( current[segment] = {} -def set(obj, segments, value, creator=__default_creator__, hints=()): +def set( + obj, + segments: Sequence[PathSegment], + value, + creator: Optional[Creator] = _default_creator, + hints: Hints = () +): """ Set the value in obj at the place indicated by segments. If creator is not None (default __default_creator__), then call the creator function to @@ -323,7 +329,7 @@ def set(obj, segments, value, creator=__default_creator__, hints=()): current[segment] except: if creator is not None: - creator(current, segments, i, hints=hints) + creator(current, segments, i, hints) else: raise diff --git a/dpath/util.py b/dpath/util.py index 08d7db8..6c6f6e8 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,6 +1,6 @@ from collections.abc import MutableMapping, MutableSequence from enum import IntFlag, auto -from typing import Union, List, Any, Dict, Callable, Sequence, Tuple +from typing import Union, List, Any, Dict, Callable, Sequence, Tuple, Optional from dpath import options, segments from dpath.exceptions import InvalidKeyName, PathNotFound @@ -9,19 +9,40 @@ class MergeType(IntFlag): - REPLACE = auto() ADDITIVE = auto() + """List objects are combined onto one long list (NOT a set). This is the default flag.""" + + REPLACE = auto() + """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination + with the source.""" + TYPESAFE = auto() + """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source + replaces the destination in this situation.""" -# Type alias for dict path segments where integers are explicitly casted PathSegment = Union[int, str] +"""Type alias for dict path segments where integers are explicitly casted.""" + +Filter = Callable[[Any], bool] +"""Type alias for filter functions. + +(Any) -> bool""" + +Hints = Sequence[Tuple[PathSegment, type]] +"""Type alias for creator function hint sequences.""" -# Type alias for filter functions -Filter = Callable[[Any], bool] # (Any) -> bool +Creator = Callable[[Union[Dict, List], Sequence[PathSegment], int, Optional[Hints]], None] +"""Type alias for creator functions. -# Type alias for creator functions -Creator = Callable[[Union[Dict, List], List[PathSegment], int, Sequence[Tuple[PathSegment, type]]], None] +Example creator function signature: + + def creator( + current: Union[Dict, List], + segments: Sequence[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () + )""" def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: @@ -270,14 +291,6 @@ def merge(dst: Dict, src: Dict, separator='/', afilter: Filter = None, flags=Mer https://github.com/akesterson/dpath-python/issues/58 flags is an OR'ed combination of MergeType enum members. - * ADDITIVE : List objects are combined onto one long - list (NOT a set). This is the default flag. - * REPLACE : Instead of combining list objects, when - 2 list objects are at an equal depth of merge, replace - the destination with the source. - * TYPESAFE : When 2 keys at equal levels are of different - types, raise a TypeError exception. By default, the source - replaces the destination in this situation. """ filtered_src = search(src, '**', afilter=afilter, separator='/') From 43554c2a30581f6fa7dd5d0434649ae1fd591e6f Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 19 Sep 2021 02:23:36 +0300 Subject: [PATCH 053/133] Use fstrings --- dpath/segments.py | 11 +++++------ dpath/util.py | 24 +++++++++++------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 81d218b..289f3fa 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -70,7 +70,7 @@ def walk(obj, location=()): if length is not None and length == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(location + (k,))) + f"{location + (k,)}") yield (location + (k,)), v for k, v in make_walkable(obj): @@ -87,7 +87,7 @@ def get(obj, segments): current = obj for (i, segment) in enumerate(segments): if leaf(current): - raise PathNotFound('Path: {}[{}]'.format(segments, i)) + raise PathNotFound(f"Path: {segments}[{i}]") current = current[segment] return current @@ -196,8 +196,7 @@ def match(segments, glob): ss = glob.index('**') if '**' in glob[ss + 1:]: - raise InvalidGlob("Invalid glob. Only one '**' is permitted per glob: {}" - "".format(glob)) + raise InvalidGlob(f"Invalid glob. Only one '**' is permitted per glob: {glob}") # Convert '**' segment into multiple '*' segments such that the # lengths of the path and glob match. '**' also can collapse and @@ -249,7 +248,7 @@ def extend(thing: List, index: int, value=None): extend(thing, int) -> [thing..., None, ...] """ try: - expansion = (type(thing)()) + expansion = type(thing)() # Using this rather than the multiply notation in order to support a # wider variety of sequence like things. @@ -335,7 +334,7 @@ def set( current = current[segment] if i != length - 1 and leaf(current): - raise PathNotFound('Path: {}[{}]'.format(segments, i)) + raise PathNotFound(f"Path: {segments}[{i}]") if isinstance(segments[-1], int): extend(current, segments[-1]) diff --git a/dpath/util.py b/dpath/util.py index 6c6f6e8..7c95939 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -92,7 +92,7 @@ def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> return segments.set(obj, split_segments, value) -def delete(obj: Dict, glob: str, separator='/', afilter: Filter = None) -> int: +def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -144,12 +144,12 @@ def f(obj, pair, counter): [deleted] = segments.foldm(obj, f, [0]) if not deleted: - raise PathNotFound("Could not find {0} to delete it".format(glob)) + raise PathNotFound(f"Could not find {glob} to delete it") return deleted -def set(obj: Dict, glob: str, value, separator='/', afilter: Filter = None) -> int: +def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -205,12 +205,12 @@ def f(_, pair, results): raise KeyError(glob) elif len(results) > 1: - raise ValueError("dpath.util.get() globs must match only one leaf : %s" % glob) + raise ValueError(f"dpath.util.get() globs must match only one leaf: {glob}") return results[0] -def values(obj: Dict, glob: str, separator='/', afilter: Filter = None, dirs=True): +def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -220,7 +220,7 @@ def values(obj: Dict, glob: str, separator='/', afilter: Filter = None, dirs=Tru return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj: Dict, glob: str, yielded=False, separator='/', afilter: Filter = None, dirs=True): +def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. @@ -249,7 +249,7 @@ def keeper(path, found): def yielder(): for path, found in segments.walk(obj): if keeper(path, found): - yield (separator.join(map(segments.int_str, path)), found) + yield separator.join(map(segments.int_str, path)), found return yielder() else: def f(obj, pair, result): @@ -261,7 +261,7 @@ def f(obj, pair, result): return segments.fold(obj, f, {}) -def merge(dst: Dict, src: Dict, separator='/', afilter: Filter = None, flags=MergeType.ADDITIVE): +def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): """ Merge source into destination. Like dict.update() but performs deep merging. @@ -311,7 +311,7 @@ def merger(dst, src, _segments=()): if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(current_path)) + f"{current_path}") # Validate src and dst types match. if flags & MergeType.TYPESAFE: @@ -321,9 +321,7 @@ def merger(dst, src, _segments=()): ft = type(found) if tt != ft: path = separator.join(current_path) - raise TypeError("Cannot merge objects of type" - "{0} and {1} at {2}" - "".format(tt, ft, path)) + raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") # Path not present in destination, create it. if not segments.has(dst, current_path): @@ -357,7 +355,7 @@ def merger(dst, src, _segments=()): if flags & MergeType.REPLACE: try: - target[''] + target[""] except TypeError: segments.set(dst, current_path, found) continue From 0218ea671467e027a8cf60bca4d886f5dd5ec7e2 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 28 Sep 2021 21:52:34 +0300 Subject: [PATCH 054/133] Remove redundant parentheses --- dpath/segments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 289f3fa..40efb56 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -85,7 +85,7 @@ def get(obj, segments): get(obj, segments) -> value """ current = obj - for (i, segment) in enumerate(segments): + for i, segment in enumerate(segments): if leaf(current): raise PathNotFound(f"Path: {segments}[{i}]") @@ -214,7 +214,7 @@ def match(segments, glob): # If we were successful in matching up the lengths, then we can # compare them using fnmatch. if path_len == len(ss_glob): - for (s, g) in zip(map(int_str, segments), map(int_str, ss_glob)): + for s, g in zip(map(int_str, segments), map(int_str, ss_glob)): # Match the stars we added to the glob to the type of the # segment itself. if g is STAR: From 74fa52deee769435660e1c437143e8d6bab339d8 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 28 Sep 2021 21:52:45 +0300 Subject: [PATCH 055/133] Remove unicode literal --- dpath/segments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/segments.py b/dpath/segments.py index 40efb56..68b6b71 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -221,7 +221,7 @@ def match(segments, glob): if isinstance(s, bytes): g = b'*' else: - g = u'*' + g = '*' # Let's see if the glob matches. We will turn any kind of # exception while attempting to match into a False for the From b359e457a2241f65481d8e9fb4f349086b20fcd8 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Fri, 19 Nov 2021 19:44:12 +1100 Subject: [PATCH 056/133] docs: fix simple typo, communcate -> communicate There is a small typo in MAINTAINERS.md. Should read `communicate` rather than `communcate`. --- MAINTAINERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 0f9fb86..ee327f6 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -10,7 +10,7 @@ There are several individuals in the community who have taken an active role in Where and How do we communicate =============================== -The dpath maintainers communcate in 3 primary ways: +The dpath maintainers communicate in 3 primary ways: 1. Email, directly to each other. 2. Github via issue and pull request comments From 7eadc11896fd7a75a74012f8f93cc3e65816d0a0 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 1 Feb 2022 21:52:12 +0200 Subject: [PATCH 057/133] Some type hints and name improvements --- dpath/segments.py | 4 ++-- dpath/util.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 68b6b71..4ee0689 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -140,7 +140,7 @@ def leaves(obj): return filter(lambda p: leafy(p[1]), walk(obj)) -def int_str(segment): +def int_str(segment: PathSegment) -> PathSegment: """ If the segment is an integer, return the string conversion. Otherwise return the segment unchanged. The conversion uses 'str'. @@ -163,7 +163,7 @@ class Star(object): STAR = Star() -def match(segments, glob): +def match(segments: Sequence[PathSegment], glob: Sequence[str]): """ Return True if the segments match the given glob, otherwise False. diff --git a/dpath/util.py b/dpath/util.py index 7c95939..5ba1588 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -230,7 +230,7 @@ def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = every element in the document that matched the glob. """ - globlist = _split_path(glob, separator) + split_glob = _split_path(glob, separator) def keeper(path, found): """ @@ -240,7 +240,7 @@ def keeper(path, found): if not dirs and not segments.leaf(found): return False - matched = segments.match(path, globlist) + matched = segments.match(path, split_glob) selected = afilter and afilter(found) return (matched and not afilter) or (matched and selected) From e94884e0425ba7923877fa713fbe9081503f4728 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 1 Feb 2022 23:10:51 +0200 Subject: [PATCH 058/133] Add option to control int-like segment conversion --- dpath/options.py | 1 + dpath/util.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dpath/options.py b/dpath/options.py index 41f35c4..91b4290 100644 --- a/dpath/options.py +++ b/dpath/options.py @@ -1 +1,2 @@ ALLOW_EMPTY_STRING_KEYS = False +CONVERT_INT_LIKE_SEGMENTS = True diff --git a/dpath/util.py b/dpath/util.py index 48c03be..00623c4 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -32,14 +32,15 @@ def __safe_path__(path, separator): raise InvalidKeyName("{} at {}[{}] contains the separator '{}'" "".format(segment, segments, i, separator)) - # Attempt to convert integer segments into actual integers. - final = [] - for segment in segments: - try: - final.append(int(segment)) - except: - final.append(segment) - segments = final + if options.CONVERT_INT_LIKE_SEGMENTS: + # Attempt to convert integer segments into actual integers. + final = [] + for segment in segments: + try: + final.append(int(segment)) + except: + final.append(segment) + segments = final return segments From dfb28d42dcc63c7ceafa6c0d206260f238b70c29 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 1 Feb 2022 23:16:41 +0200 Subject: [PATCH 059/133] Bump version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 664cb5f..7a47894 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.5" +VERSION = "2.0.6" From b2f052e6d296640a763af23ccc6c3efea1b64837 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 2 Feb 2022 22:27:04 +0200 Subject: [PATCH 060/133] Small refactor in deletion code --- dpath/util.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index d6622a8..5350a75 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -116,9 +116,13 @@ def f(obj, pair, counter): key = path_segments[-1] parent = segments.get(obj, path_segments[:-1]) - try: - # Attempt to treat parent like a sequence. - parent[0] + # Deletion behavior depends on parent type + if isinstance(parent, dict): + del parent[key] + + else: + # Handle sequence types + # TODO: Consider cases where type isn't a simple list (e.g. set) if len(parent) - 1 == key: # Removing the last element of a sequence. It can be @@ -132,14 +136,12 @@ def f(obj, pair, counter): # of a list and end up with None values when we # don't need them. del parent[key] + else: # This key can't be removed completely because it # would affect the order of items that remain in our # result. parent[key] = None - except: - # Attempt to treat parent like a dictionary instead. - del parent[key] counter[0] += 1 From f9c142bdbfeb5677ef1e14a316489615b5058741 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 2 Feb 2022 22:35:39 +0200 Subject: [PATCH 061/133] Move custom type definitions to dedicated file --- dpath/segments.py | 2 +- dpath/types.py | 39 +++++++++++++++++++++++++++++++++++++++ dpath/util.py | 41 ++--------------------------------------- 3 files changed, 42 insertions(+), 40 deletions(-) create mode 100644 dpath/types.py diff --git a/dpath/segments.py b/dpath/segments.py index 4ee0689..7c5816c 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.util import PathSegment, Creator, Hints +from dpath.types import PathSegment, Creator, Hints def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: diff --git a/dpath/types.py b/dpath/types.py new file mode 100644 index 0000000..2b44ce6 --- /dev/null +++ b/dpath/types.py @@ -0,0 +1,39 @@ +from enum import IntFlag, auto +from typing import Union, Any, Callable, Sequence, Tuple, Dict, List, Optional + + +class MergeType(IntFlag): + ADDITIVE = auto() + """List objects are combined onto one long list (NOT a set). This is the default flag.""" + + REPLACE = auto() + """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination + with the source.""" + + TYPESAFE = auto() + """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source + replaces the destination in this situation.""" + + +PathSegment = Union[int, str] +"""Type alias for dict path segments where integers are explicitly casted.""" + +Filter = Callable[[Any], bool] +"""Type alias for filter functions. + +(Any) -> bool""" + +Hints = Sequence[Tuple[PathSegment, type]] +"""Type alias for creator function hint sequences.""" + +Creator = Callable[[Union[Dict, List], Sequence[PathSegment], int, Optional[Hints]], None] +"""Type alias for creator functions. + +Example creator function signature: + + def creator( + current: Union[Dict, List], + segments: Sequence[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () + )""" \ No newline at end of file diff --git a/dpath/util.py b/dpath/util.py index 5350a75..2384578 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,50 +1,13 @@ from collections.abc import MutableMapping, MutableSequence -from enum import IntFlag, auto -from typing import Union, List, Any, Dict, Callable, Sequence, Tuple, Optional +from typing import Union, List, Any, Dict from dpath import options, segments from dpath.exceptions import InvalidKeyName, PathNotFound +from dpath.types import PathSegment, Filter, Creator, MergeType _DEFAULT_SENTINEL = object() -class MergeType(IntFlag): - ADDITIVE = auto() - """List objects are combined onto one long list (NOT a set). This is the default flag.""" - - REPLACE = auto() - """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination - with the source.""" - - TYPESAFE = auto() - """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source - replaces the destination in this situation.""" - - -PathSegment = Union[int, str] -"""Type alias for dict path segments where integers are explicitly casted.""" - -Filter = Callable[[Any], bool] -"""Type alias for filter functions. - -(Any) -> bool""" - -Hints = Sequence[Tuple[PathSegment, type]] -"""Type alias for creator function hint sequences.""" - -Creator = Callable[[Union[Dict, List], Sequence[PathSegment], int, Optional[Hints]], None] -"""Type alias for creator functions. - -Example creator function signature: - - def creator( - current: Union[Dict, List], - segments: Sequence[PathSegment], - i: int, - hints: Sequence[Tuple[PathSegment, type]] = () - )""" - - def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: """ Given a path and separator, return a tuple of segments. If path is From 86fce6160ad87a3ea5b693cb7e376ca0f93439f0 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 2 Feb 2022 22:35:59 +0200 Subject: [PATCH 062/133] Use MutableMapping instead of dict --- dpath/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/util.py b/dpath/util.py index 2384578..5200f81 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -80,7 +80,7 @@ def f(obj, pair, counter): parent = segments.get(obj, path_segments[:-1]) # Deletion behavior depends on parent type - if isinstance(parent, dict): + if isinstance(parent, MutableMapping): del parent[key] else: From db94e23da7564980e8d17bf9b44e98f4f77683f3 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sat, 1 Oct 2022 21:54:48 +0300 Subject: [PATCH 063/133] Add meaningful exception --- dpath/util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dpath/util.py b/dpath/util.py index 00623c4..8a03e38 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -286,6 +286,9 @@ def are_both_mutable(o1, o2): return False def merger(dst, src, _segments=()): + if not isinstance(dst, dict) or not isinstance(src, dict): + raise ValueError("Merger function supports dict-like objects only") + for key, found in dpath.segments.kvs(src): # Our current path in the source. segments = _segments + (key,) From 51bb94383bd7d78d48a28f4c8f998e09b87c41a3 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 2 Oct 2022 00:56:16 +0300 Subject: [PATCH 064/133] Better type check --- dpath/util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index 8a03e38..cc55e61 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,4 +1,4 @@ -from collections.abc import MutableMapping +from collections.abc import MutableMapping, Sized from collections.abc import MutableSequence from dpath import options from dpath.exceptions import InvalidKeyName @@ -286,13 +286,13 @@ def are_both_mutable(o1, o2): return False def merger(dst, src, _segments=()): - if not isinstance(dst, dict) or not isinstance(src, dict): - raise ValueError("Merger function supports dict-like objects only") - for key, found in dpath.segments.kvs(src): # Our current path in the source. segments = _segments + (key,) + if not isinstance(key, Sized): + raise ValueError("Merger function supports dict-like objects only") + if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " From 27a46eb626c91bbc4d727be805fa947e11ea8d4b Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 2 Oct 2022 01:18:52 +0300 Subject: [PATCH 065/133] Clean up imports --- dpath/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/util.py b/dpath/util.py index cc55e61..b71ed9f 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,8 +1,8 @@ -from collections.abc import MutableMapping, Sized -from collections.abc import MutableSequence +from collections.abc import MutableMapping, Sized, MutableSequence + +import dpath.segments from dpath import options from dpath.exceptions import InvalidKeyName -import dpath.segments _DEFAULT_SENTINAL = object() MERGE_REPLACE = (1 << 1) From 2c265fce7af9db701390b8b5df235ad520fe8b35 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 2 Oct 2022 01:59:59 +0300 Subject: [PATCH 066/133] Make assert calls comply with PEP8 --- tests/test_broken_afilter.py | 16 +++---- tests/test_path_get.py | 6 +-- tests/test_path_paths.py | 2 +- tests/test_types.py | 16 +++---- tests/test_unicode.py | 22 ++++----- tests/test_util_delete.py | 10 ++--- tests/test_util_get_values.py | 44 +++++++++--------- tests/test_util_merge.py | 20 ++++----- tests/test_util_new.py | 28 ++++++------ tests/test_util_paths.py | 6 +-- tests/test_util_search.py | 84 +++++++++++++++++------------------ tests/test_util_set.py | 28 ++++++------ 12 files changed, 141 insertions(+), 141 deletions(-) diff --git a/tests/test_broken_afilter.py b/tests/test_broken_afilter.py index a59454e..3e3a449 100644 --- a/tests/test_broken_afilter.py +++ b/tests/test_broken_afilter.py @@ -26,14 +26,14 @@ def afilter(x): ] for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c']) + assert path in paths + assert "view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c'] for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c']) + assert path in paths + assert "view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c'] def filter(x): sys.stderr.write(str(x)) @@ -54,5 +54,5 @@ def filter(x): results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', yielded=True)] results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', afilter=filter, yielded=True)] - assert(len(results) == 1) - assert(results[0][1]['type'] == 'correct') + assert len(results) == 1 + assert results[0][1]['type'] == 'correct' diff --git a/tests/test_path_get.py b/tests/test_path_get.py index 96347f2..b2a4657 100644 --- a/tests/test_path_get.py +++ b/tests/test_path_get.py @@ -15,6 +15,6 @@ def test_path_get_list_of_dicts(): segments = ['a', 'b', 0, 0] res = dpath.segments.view(tdict, segments) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 1) - assert(res['a']['b'][0][0] == 0) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 1 + assert res['a']['b'][0][0] == 0 diff --git a/tests/test_path_paths.py b/tests/test_path_paths.py index 3364e6d..0cf0bc3 100644 --- a/tests/test_path_paths.py +++ b/tests/test_path_paths.py @@ -34,4 +34,4 @@ def test_path_paths_empty_key_allowed(): pass dpath.options.ALLOW_EMPTY_STRING_KEYS = False - assert("/".join(segments) == "Empty//Key") + assert "/".join(segments) == "Empty//Key" diff --git a/tests/test_types.py b/tests/test_types.py index 82f8c05..6aeac05 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -72,12 +72,12 @@ def test_types_set(): data = TestMapping({"a": TestSequence([0])}) dpath.util.set(data, '/a/0', 1) - assert(data['a'][0] == 1) + assert data['a'][0] == 1 data['a'][0] = 0 dpath.util.set(data, ['a', '0'], 1) - assert(data['a'][0] == 1) + assert data['a'][0] == 1 def test_types_get_list_of_dicts(): @@ -93,9 +93,9 @@ def test_types_get_list_of_dicts(): res = dpath.segments.view(tdict, ['a', 'b', 0, 0]) - assert(isinstance(res['a']['b'], TestSequence)) - assert(len(res['a']['b']) == 1) - assert(res['a']['b'][0][0] == 0) + assert isinstance(res['a']['b'], TestSequence) + assert len(res['a']['b']) == 1 + assert res['a']['b'][0][0] == 0 def test_types_merge_simple_list_replace(): @@ -149,6 +149,6 @@ def afilter(x): }) dpath.util.delete(data, '/a/*', afilter=afilter) - assert (data['a']['b'] == 0) - assert (data['a']['c'] == 1) - assert ('d' not in data['a']) + assert data['a']['b'] == 0 + assert data['a']['c'] == 1 + assert 'd' not in data['a'] diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 104e108..09bc4db 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -6,18 +6,18 @@ def test_unicode_merge(): b = {'文': 'wen'} dpath.util.merge(a, b) - assert(len(a.keys()) == 2) - assert(a['中'] == 'zhong') - assert(a['文'] == 'wen') + assert len(a.keys()) == 2 + assert a['中'] == 'zhong' + assert a['文'] == 'wen' def test_unicode_search(): a = {'中': 'zhong'} results = [[x[0], x[1]] for x in dpath.util.search(a, '*', yielded=True)] - assert(len(results) == 1) - assert(results[0][0] == '中') - assert(results[0][1] == 'zhong') + assert len(results) == 1 + assert results[0][0] == '中' + assert results[0][1] == 'zhong' def test_unicode_str_hybrid(): @@ -25,8 +25,8 @@ def test_unicode_str_hybrid(): b = {u'second': '2'} dpath.util.merge(a, b) - assert(len(a.keys()) == 2) - assert(a[u'second'] == '2') - assert(a['second'] == u'2') - assert(a[u'first'] == '1') - assert(a['first'] == u'1') + assert len(a.keys()) == 2 + assert a[u'second'] == '2' + assert a['second'] == u'2' + assert a[u'first'] == '1' + assert a['first'] == u'1' diff --git a/tests/test_util_delete.py b/tests/test_util_delete.py index c14acf7..4e531aa 100644 --- a/tests/test_util_delete.py +++ b/tests/test_util_delete.py @@ -11,7 +11,7 @@ def test_delete_separator(): } dpath.util.delete(dict, ';a;b', separator=";") - assert('b' not in dict['a']) + assert 'b' not in dict['a'] def test_delete_existing(): @@ -22,7 +22,7 @@ def test_delete_existing(): } dpath.util.delete(dict, '/a/b') - assert('b' not in dict['a']) + assert 'b' not in dict['a'] @raises(dpath.exceptions.PathNotFound) @@ -50,6 +50,6 @@ def afilter(x): } dpath.util.delete(dict, '/a/*', afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert ('d' not in dict['a']) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert 'd' not in dict['a'] diff --git a/tests/test_util_get_values.py b/tests/test_util_get_values.py index 5a14f1e..a0b6686 100644 --- a/tests/test_util_get_values.py +++ b/tests/test_util_get_values.py @@ -11,10 +11,10 @@ def test_util_get_root(): x = {'p': {'a': {'t': {'h': 'value'}}}} ret = dpath.util.get(x, '/p/a/t/h') - assert(ret == 'value') + assert ret == 'value' ret = dpath.util.get(x, '/') - assert(ret == x) + assert ret == x def test_get_explicit_single(): @@ -30,11 +30,11 @@ def test_get_explicit_single(): }, } - assert(dpath.util.get(ehash, '/a/b/c/f') == 2) - assert(dpath.util.get(ehash, ['a', 'b', 'c', 'f']) == 2) - assert(dpath.util.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2) - assert(dpath.util.get(ehash, ['does', 'not', 'exist'], default=None) is None) - assert(dpath.util.get(ehash, ['doesnt', 'exist'], default=5) == 5) + assert dpath.util.get(ehash, '/a/b/c/f') == 2 + assert dpath.util.get(ehash, ['a', 'b', 'c', 'f']) == 2 + assert dpath.util.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2 + assert dpath.util.get(ehash, ['does', 'not', 'exist'], default=None) is None + assert dpath.util.get(ehash, ['doesnt', 'exist'], default=5) == 5 def test_get_glob_single(): @@ -50,10 +50,10 @@ def test_get_glob_single(): }, } - assert(dpath.util.get(ehash, '/a/b/*/f') == 2) - assert(dpath.util.get(ehash, ['a', 'b', '*', 'f']) == 2) - assert(dpath.util.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2) - assert(dpath.util.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6) + assert dpath.util.get(ehash, '/a/b/*/f') == 2 + assert dpath.util.get(ehash, ['a', 'b', '*', 'f']) == 2 + assert dpath.util.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2 + assert dpath.util.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6 def test_get_glob_multiple(): @@ -96,16 +96,16 @@ def test_values(): } ret = dpath.util.values(ehash, '/a/b/c/*') - assert(isinstance(ret, list)) - assert(0 in ret) - assert(1 in ret) - assert(2 in ret) + assert isinstance(ret, list) + assert 0 in ret + assert 1 in ret + assert 2 in ret ret = dpath.util.values(ehash, ['a', 'b', 'c', '*']) - assert(isinstance(ret, list)) - assert(0 in ret) - assert(1 in ret) - assert(2 in ret) + assert isinstance(ret, list) + assert 0 in ret + assert 1 in ret + assert 2 in ret @mock.patch('dpath.util.search') @@ -126,7 +126,7 @@ def test_none_values(): d = {'p': {'a': {'t': {'h': None}}}} v = dpath.util.get(d, 'p/a/t/h') - assert(v is None) + assert v is None def test_values_list(): @@ -142,8 +142,8 @@ def test_values_list(): } ret = dpath.util.values(a, 'actions/*') - assert(isinstance(ret, list)) - assert(len(ret) == 2) + assert isinstance(ret, list) + assert len(ret) == 2 def test_non_leaf_leaf(): diff --git a/tests/test_util_merge.py b/tests/test_util_merge.py index 968a4fa..17c4c0f 100644 --- a/tests/test_util_merge.py +++ b/tests/test_util_merge.py @@ -21,7 +21,7 @@ def test_merge_typesafe_and_separator(): try: dpath.util.merge(dst, src, flags=(dpath.util.MERGE_ADDITIVE | dpath.util.MERGE_TYPESAFE), separator=";") except TypeError as e: - assert(str(e).endswith("dict;integer")) + assert str(e).endswith("dict;integer") return raise Exception("MERGE_TYPESAFE failed to raise an exception when merging between str and int!") @@ -107,9 +107,9 @@ def afilter(x): dst = {} dpath.util.merge(dst, src, afilter=afilter) - assert ("key2" in dst) - assert ("key" not in dst) - assert ("otherdict" not in dst) + assert "key2" in dst + assert "key" not in dst + assert "otherdict" not in dst @raises(TypeError) @@ -152,9 +152,9 @@ class tcis(list): dpath.util.merge(dst, src) print(dst) - assert(dst["mm"]["a"] == src["mm"]["a"]) - assert(dst['ms'][2] == 'c') - assert("casserole" in dst["mm"]) + assert dst["mm"]["a"] == src["mm"]["a"] + assert dst['ms'][2] == 'c' + assert "casserole" in dst["mm"] dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) @@ -163,15 +163,15 @@ def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} dpath.util.merge(dct_a, dct_b, flags=dpath.util.MERGE_REPLACE) - assert(len(dct_a['a']['b']) == 1) + assert len(dct_a['a']['b']) == 1 def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} dpath.util.merge(d1, d2, flags=dpath.util.MERGE_REPLACE) - assert(len(d1['a']) == 1) - assert(d1['a'][0] == 'a') + assert len(d1['a']) == 1 + assert d1['a'][0] == 'a' def test_merge_list(): diff --git a/tests/test_util_new.py b/tests/test_util_new.py index d04b056..3c2a3c4 100644 --- a/tests/test_util_new.py +++ b/tests/test_util_new.py @@ -8,10 +8,10 @@ def test_set_new_separator(): } dpath.util.new(dict, ';a;b', 1, separator=";") - assert(dict['a']['b'] == 1) + assert dict['a']['b'] == 1 dpath.util.new(dict, ['a', 'b'], 1, separator=";") - assert(dict['a']['b'] == 1) + assert dict['a']['b'] == 1 def test_set_new_dict(): @@ -21,10 +21,10 @@ def test_set_new_dict(): } dpath.util.new(dict, '/a/b', 1) - assert(dict['a']['b'] == 1) + assert dict['a']['b'] == 1 dpath.util.new(dict, ['a', 'b'], 1) - assert(dict['a']['b'] == 1) + assert dict['a']['b'] == 1 def test_set_new_list(): @@ -34,12 +34,12 @@ def test_set_new_list(): } dpath.util.new(dict, '/a/1', 1) - assert(dict['a'][1] == 1) - assert(dict['a'][0] is None) + assert dict['a'][1] == 1 + assert dict['a'][0] is None dpath.util.new(dict, ['a', 1], 1) - assert(dict['a'][1] == 1) - assert(dict['a'][0] is None) + assert dict['a'][1] == 1 + assert dict['a'][0] is None def test_set_new_list_path_with_separator(): @@ -50,9 +50,9 @@ def test_set_new_list_path_with_separator(): } dpath.util.new(dict, ['a', 'b/c/d', 0], 1) - assert(len(dict['a']) == 1) - assert(len(dict['a']['b/c/d']) == 1) - assert(dict['a']['b/c/d'][0] == 1) + assert len(dict['a']) == 1 + assert len(dict['a']['b/c/d']) == 1 + assert dict['a']['b/c/d'][0] == 1 def test_set_new_list_integer_path_with_creator(): @@ -78,6 +78,6 @@ def mycreator(obj, pathcomp, nextpathcomp, hints): dpath.util.new(d, '/a/2', 3, creator=mycreator) print(d) - assert(isinstance(d['a'], list)) - assert(len(d['a']) == 3) - assert(d['a'][2] == 3) + assert isinstance(d['a'], list) + assert len(d['a']) == 3 + assert d['a'][2] == 3 diff --git a/tests/test_util_paths.py b/tests/test_util_paths.py index 27260fe..9f6420f 100644 --- a/tests/test_util_paths.py +++ b/tests/test_util_paths.py @@ -4,6 +4,6 @@ def test_util_safe_path_list(): res = dpath.util.__safe_path__(["Ignore", "the/separator"], None) - assert(len(res) == 2) - assert(res[0] == "Ignore") - assert(res[1] == "the/separator") + assert len(res) == 2 + assert res[0] == "Ignore" + assert res[1] == "the/separator" diff --git a/tests/test_util_search.py b/tests/test_util_search.py index e7a4d43..a974963 100644 --- a/tests/test_util_search.py +++ b/tests/test_util_search.py @@ -23,10 +23,10 @@ def test_search_paths_with_separator(): ] for (path, value) in dpath.util.search(dict, '/**', yielded=True, separator=";"): - assert(path in paths) + assert path in paths for (path, value) in dpath.util.search(dict, ['**'], yielded=True, separator=";"): - assert(path in paths) + assert path in paths def test_search_paths(): @@ -51,10 +51,10 @@ def test_search_paths(): ] for (path, value) in dpath.util.search(dict, '/**', yielded=True): - assert(path in paths) + assert path in paths for (path, value) in dpath.util.search(dict, ['**'], yielded=True): - assert(path in paths) + assert path in paths def test_search_afilter(): @@ -81,14 +81,14 @@ def afilter(x): ] for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c']) + assert path in paths + assert "view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c'] for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c']) + assert path in paths + assert "view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c'] def test_search_globbing(): @@ -109,10 +109,10 @@ def test_search_globbing(): ] for (path, value) in dpath.util.search(dict, '/a/**/[df]', yielded=True): - assert(path in paths) + assert path in paths for (path, value) in dpath.util.search(dict, ['a', '**', '[df]'], yielded=True): - assert(path in paths) + assert path in paths def test_search_return_dict_head(): @@ -126,14 +126,14 @@ def test_search_return_dict_head(): }, } res = dpath.util.search(tdict, '/a/b') - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == {0: 0, 1: 1, 2: 2}) + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == {0: 0, 1: 1, 2: 2} res = dpath.util.search(tdict, ['a', 'b']) - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == {0: 0, 1: 1, 2: 2}) + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == {0: 0, 1: 1, 2: 2} def test_search_return_dict_globbed(): @@ -148,14 +148,14 @@ def test_search_return_dict_globbed(): } res = dpath.util.search(tdict, '/a/b/[02]') - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 2) - assert(res['a']['b'] == {0: 0, 2: 2}) + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 2 + assert res['a']['b'] == {0: 0, 2: 2} res = dpath.util.search(tdict, ['a', 'b', '[02]']) - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 2) - assert(res['a']['b'] == {0: 0, 2: 2}) + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 2 + assert res['a']['b'] == {0: 0, 2: 2} def test_search_return_list_head(): @@ -170,14 +170,14 @@ def test_search_return_list_head(): } res = dpath.util.search(tdict, '/a/b') - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, 1, 2]) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, 1, 2] res = dpath.util.search(tdict, ['a', 'b']) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, 1, 2]) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, 1, 2] def test_search_return_list_globbed(): @@ -192,14 +192,14 @@ def test_search_return_list_globbed(): } res = dpath.util.search(tdict, '/a/b/[02]') - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, None, 2]) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, None, 2] res = dpath.util.search(tdict, ['a', 'b', '[02]']) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, None, 2]) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, None, 2] def test_search_list_key_with_separator(): @@ -213,8 +213,8 @@ def test_search_list_key_with_separator(): } res = dpath.util.search(tdict, ['a', '/b/d']) - assert('b' not in res['a']) - assert(res['a']['/b/d'] == 'success') + assert 'b' not in res['a'] + assert res['a']['/b/d'] == 'success' def test_search_multiple_stars(): @@ -232,7 +232,7 @@ def test_search_multiple_stars(): testpath = 'a/*/b/*/c' res = dpath.util.search(testdata, testpath) - assert(len(res['a'][0]['b']) == 3) - assert(res['a'][0]['b'][0]['c'] == 1) - assert(res['a'][0]['b'][1]['c'] == 2) - assert(res['a'][0]['b'][2]['c'] == 3) + assert len(res['a'][0]['b']) == 3 + assert res['a'][0]['b'][0]['c'] == 1 + assert res['a'][0]['b'][1]['c'] == 2 + assert res['a'][0]['b'][2]['c'] == 3 diff --git a/tests/test_util_set.py b/tests/test_util_set.py index 3684a56..1592590 100644 --- a/tests/test_util_set.py +++ b/tests/test_util_set.py @@ -9,11 +9,11 @@ def test_set_existing_separator(): } dpath.util.set(dict, ';a;b', 1, separator=";") - assert(dict['a']['b'] == 1) + assert dict['a']['b'] == 1 dict['a']['b'] = 0 dpath.util.set(dict, ['a', 'b'], 1, separator=";") - assert(dict['a']['b'] == 1) + assert dict['a']['b'] == 1 def test_set_existing_dict(): @@ -24,11 +24,11 @@ def test_set_existing_dict(): } dpath.util.set(dict, '/a/b', 1) - assert(dict['a']['b'] == 1) + assert dict['a']['b'] == 1 dict['a']['b'] = 0 dpath.util.set(dict, ['a', 'b'], 1) - assert(dict['a']['b'] == 1) + assert dict['a']['b'] == 1 def test_set_existing_list(): @@ -39,11 +39,11 @@ def test_set_existing_list(): } dpath.util.set(dict, '/a/0', 1) - assert(dict['a'][0] == 1) + assert dict['a'][0] == 1 dict['a'][0] = 0 dpath.util.set(dict, ['a', '0'], 1) - assert(dict['a'][0] == 1) + assert dict['a'][0] == 1 def test_set_filter(): @@ -61,9 +61,9 @@ def afilter(x): } dpath.util.set(dict, '/a/*', 31337, afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert (dict['a']['d'] == 31337) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert dict['a']['d'] == 31337 dict = { "a": { @@ -74,9 +74,9 @@ def afilter(x): } dpath.util.set(dict, ['a', '*'], 31337, afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert (dict['a']['d'] == 31337) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert dict['a']['d'] == 31337 def test_set_existing_path_with_separator(): @@ -87,5 +87,5 @@ def test_set_existing_path_with_separator(): } dpath.util.set(dict, ['a', 'b/c/d'], 1) - assert(len(dict['a']) == 1) - assert(dict['a']['b/c/d'] == 1) + assert len(dict['a']) == 1 + assert dict['a']['b/c/d'] == 1 From 4a7deacbb4b0fba87e37636074626b0458b755b6 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 2 Oct 2022 02:12:18 +0300 Subject: [PATCH 067/133] Bump version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 7a47894..c23b690 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.6" +VERSION = "2.0.7" From bb3d8234113aaa9876eb0bd81c8557db96b95895 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 23 Nov 2022 21:24:31 +0200 Subject: [PATCH 068/133] Update tests workflow --- .github/workflows/tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 328cb2a..4466a45 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,10 +32,11 @@ jobs: - name: Generate Hashseed id: generate run: | - python -c "from random import randint; + python -c "import os + from random import randint hashseed = randint(0, 4294967295) print(f'{hashseed=}') - print(f'::set-output name=hashseed::{hashseed}')" + open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" # Tests job tests: @@ -53,7 +54,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@main - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@main From 0d5e994f50ec1beac3715fe5052fa81eafb5db8c Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 23 Nov 2022 21:25:05 +0200 Subject: [PATCH 069/133] Run tests on Python 3.10 --- .github/workflows/tests.yml | 2 +- tox.ini | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4466a45..114b8fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,7 +48,7 @@ jobs: strategy: matrix: # Match versions specified in tox.ini - python-version: [3.6, 3.8, 3.9, pypy-3.7] + python-version: ['3.6', '3.8', '3.9', '3.10', 'pypy-3.7'] # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/tox.ini b/tox.ini index 5e521b2..8143008 100644 --- a/tox.ini +++ b/tox.ini @@ -7,14 +7,15 @@ ignore = E501,E722 [tox] -envlist = py36, pypy37, py38, py39, flake8 +envlist = py36, pypy37, py38, py39, py310, flake8 [gh-actions] python = - 3.6: py36 + 3.6.15: py36 pypy-3.7: pypy37 3.8: py38 - 3.9: py39, flake8 + 3.9: py39 + 3.10: py310, flake8 [testenv] deps = From ec644e27ba88961b534c1fa03a3cb398e86ab9c4 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 23 Nov 2022 22:16:27 +0200 Subject: [PATCH 070/133] Drop Python 3.6 Decision made based on statistics from: https://git.afpy.org/mdk/python-versions --- .github/workflows/tests.yml | 2 +- README.rst | 11 ++++++++--- setup.py | 2 +- tox.ini | 3 +-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 114b8fd..3e7d924 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,7 +48,7 @@ jobs: strategy: matrix: # Match versions specified in tox.ini - python-version: ['3.6', '3.8', '3.9', '3.10', 'pypy-3.7'] + python-version: ['3.8', '3.9', '3.10', 'pypy-3.7'] # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/README.rst b/README.rst index 8e0db20..15f65d8 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,7 @@ dpath-python ============ |PyPI| +|Python Version| |Build Status| |Gitter| @@ -451,12 +452,16 @@ components instead of string globs. :target: https://pypi.python.org/pypi/dpath/ :alt: PyPI: Latest Version +.. |Python Version| image:: https://img.shields.io/pypi/pyversions/dpath?style=flat + :target: https://pypi.python.org/pypi/dpath/ + :alt: Supported Python Version + .. |Build Status| image:: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml/badge.svg - :target: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml + :target: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml .. |Gitter| image:: https://badges.gitter.im/dpath-python/chat.svg - :target: https://gitter.im/dpath-python/chat?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge - :alt: Gitter + :target: https://gitter.im/dpath-python/chat?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge + :alt: Gitter Contributors ============ diff --git a/setup.py b/setup.py index abce1a3..1e3d2e2 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ scripts=[], packages=["dpath"], data_files=[], - python_requires=">=3", + python_requires=">=3.7", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', diff --git a/tox.ini b/tox.ini index 8143008..42d0d2a 100644 --- a/tox.ini +++ b/tox.ini @@ -7,11 +7,10 @@ ignore = E501,E722 [tox] -envlist = py36, pypy37, py38, py39, py310, flake8 +envlist = pypy37, py38, py39, py310, flake8 [gh-actions] python = - 3.6.15: py36 pypy-3.7: pypy37 3.8: py38 3.9: py39 From 35ccee6609fa8f8c1351e43ac6a37f80184b1064 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 23 Nov 2022 22:16:42 +0200 Subject: [PATCH 071/133] Update project URL --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1e3d2e2..193c1c3 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ if __name__ == "__main__": setup( name="dpath", - url="https://www.github.com/akesterson/dpath-python", + url="https://github.com/dpath-maintainers/dpath-python", version=dpath.version.VERSION, description="Filesystem-like pathing and searching for dictionaries", long_description=long_description, From 41579a6b4d2d5994bbd0e718c85160f231517be2 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 23 Nov 2022 23:42:17 +0200 Subject: [PATCH 072/133] Update tests to use nose2 --- dpath/version.py | 2 +- tests/test_path_paths.py | 9 +- tests/test_segments.py | 364 +++++++++++++++++----------------- tests/test_types.py | 25 +-- tests/test_util_delete.py | 7 +- tests/test_util_get_values.py | 19 +- tests/test_util_merge.py | 20 +- tox.ini | 4 +- 8 files changed, 217 insertions(+), 233 deletions(-) diff --git a/dpath/version.py b/dpath/version.py index c23b690..4f7a825 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.7" +VERSION = "2.0.8" diff --git a/tests/test_path_paths.py b/tests/test_path_paths.py index 0cf0bc3..56595d2 100644 --- a/tests/test_path_paths.py +++ b/tests/test_path_paths.py @@ -1,10 +1,10 @@ -from nose.tools import raises +from nose2.tools.such import helper + import dpath.segments import dpath.exceptions import dpath.options -@raises(dpath.exceptions.InvalidKeyName) def test_path_paths_empty_key_disallowed(): tdict = { "Empty": { @@ -14,8 +14,9 @@ def test_path_paths_empty_key_disallowed(): } } - for x in dpath.segments.walk(tdict): - pass + with helper.assertRaises(dpath.exceptions.InvalidKeyName): + for x in dpath.segments.walk(tdict): + pass def test_path_paths_empty_key_allowed(): diff --git a/tests/test_segments.py b/tests/test_segments.py index af9df85..40a9f9d 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -1,8 +1,11 @@ -from dpath import options +import os +from unittest import TestCase + +import hypothesis.strategies as st from hypothesis import given, assume, settings, HealthCheck + import dpath.segments as api -import hypothesis.strategies as st -import os +from dpath import options settings.register_profile("default", suppress_health_check=(HealthCheck.too_slow,)) settings.load_profile(os.getenv(u'HYPOTHESIS_PROFILE', 'default')) @@ -27,115 +30,6 @@ random_mutable_node = random_mutable_thing.filter(lambda thing: isinstance(thing, (list, dict))) -def setup(): - # Allow empty strings in segments. - options.ALLOW_EMPTY_STRING_KEYS = True - - -def teardown(): - # Revert back to default. - options.ALLOW_EMPTY_STRING_KEYS = False - - -@given(random_node) -def test_kvs(node): - ''' - Given a node, kvs should produce a key that when used to extract - from the node renders the exact same value given. - ''' - for k, v in api.kvs(node): - assert node[k] is v - - -@given(random_leaf) -def test_leaf_with_leaf(leaf): - ''' - Given a leaf, leaf should return True. - ''' - assert api.leaf(leaf) is True - - -@given(random_node) -def test_leaf_with_node(node): - ''' - Given a node, leaf should return False. - ''' - assert api.leaf(node) is False - - -@given(random_thing) -def test_walk(thing): - ''' - Given a thing to walk, walk should yield key, value pairs where key - is a tuple of non-zero length. - ''' - for k, v in api.walk(thing): - assert isinstance(k, tuple) - assert len(k) > 0 - - -@given(random_node) -def test_get(node): - ''' - Given a node, get should return the exact value given a key for all - key, value pairs in the node. - ''' - for k, v in api.walk(node): - assert api.get(node, k) is v - - -@given(random_node) -def test_has(node): - ''' - Given a node, has should return True for all paths, False otherwise. - ''' - for k, v in api.walk(node): - assert api.has(node, k) is True - - # If we are at a leaf, then we can create a value that isn't - # present easily. - if api.leaf(v): - assert api.has(node, k + (0,)) is False - - -@given(random_segments) -def test_expand(segments): - ''' - Given segments expand should produce as many results are there were - segments and the last result should equal the given segments. - ''' - count = len(segments) - result = list(api.expand(segments)) - - assert count == len(result) - - if count > 0: - assert segments == result[-1] - - -@given(random_node) -def test_types(node): - ''' - Given a node, types should yield a tuple of key, type pairs and the - type indicated should equal the type of the value. - ''' - for k, v in api.walk(node): - ts = api.types(node, k) - ta = () - for tk, tt in ts: - ta += (tk,) - assert type(api.get(node, ta)) is tt - - -@given(random_node) -def test_leaves(node): - ''' - Given a node, leaves should yield only leaf key, value pairs. - ''' - for k, v in api.leaves(node): - assert api.leafy(v) - - @st.composite def mutate(draw, segment): # Convert number segments. @@ -216,7 +110,7 @@ def random_segments_with_glob(draw): stop = draw(st.integers(start, len(glob))) glob[start:stop] = ['**'] - return (segments, glob) + return segments, glob @st.composite @@ -243,25 +137,6 @@ def random_segments_with_nonmatching_glob(draw): return (segments, glob) -@given(random_segments_with_glob()) -def test_match(pair): - ''' - Given segments and a known good glob, match should be True. - ''' - (segments, glob) = pair - assert api.match(segments, glob) is True - - -@given(random_segments_with_nonmatching_glob()) -def test_match_nonmatching(pair): - ''' - Given segments and a known bad glob, match should be False. - ''' - print(pair) - (segments, glob) = pair - assert api.match(segments, glob) is False - - @st.composite def random_walk(draw): node = draw(random_mutable_node) @@ -278,64 +153,179 @@ def random_leaves(draw): return (node, draw(st.sampled_from(found))) -@given(walkable=random_walk(), value=random_thing) -def test_set_walkable(walkable, value): - ''' - Given a walkable location, set should be able to update any value. - ''' - (node, (segments, found)) = walkable - api.set(node, segments, value) - assert api.get(node, segments) is value - - -@given(walkable=random_leaves(), - kstr=random_key_str, - kint=random_key_int, - value=random_thing, - extension=random_segments) -def test_set_create_missing(walkable, kstr, kint, value, extension): - ''' - Given a walkable non-leaf, set should be able to create missing - nodes and set a new value. - ''' - (node, (segments, found)) = walkable - assume(api.leaf(found)) - - parent_segments = segments[:-1] - parent = api.get(node, parent_segments) - - if isinstance(parent, list): - assume(len(parent) < kint) - destination = parent_segments + (kint,) + tuple(extension) - elif isinstance(parent, dict): - assume(kstr not in parent) - destination = parent_segments + (kstr,) + tuple(extension) - else: - raise Exception('mad mad world') - - api.set(node, destination, value) - assert api.get(node, destination) is value +class TestSegments(TestCase): + @classmethod + def setUpClass(cls): + # Allow empty strings in segments. + options.ALLOW_EMPTY_STRING_KEYS = True + + @classmethod + def tearDownClass(cls): + # Revert back to default. + options.ALLOW_EMPTY_STRING_KEYS = False + + @given(random_node) + def test_kvs(self, node): + ''' + Given a node, kvs should produce a key that when used to extract + from the node renders the exact same value given. + ''' + for k, v in api.kvs(node): + assert node[k] is v + + @given(random_leaf) + def test_leaf_with_leaf(self, leaf): + ''' + Given a leaf, leaf should return True. + ''' + assert api.leaf(leaf) is True + + @given(random_node) + def test_leaf_with_node(self, node): + ''' + Given a node, leaf should return False. + ''' + assert api.leaf(node) is False + + @given(random_thing) + def test_walk(self, thing): + ''' + Given a thing to walk, walk should yield key, value pairs where key + is a tuple of non-zero length. + ''' + for k, v in api.walk(thing): + assert isinstance(k, tuple) + assert len(k) > 0 + + @given(random_node) + def test_get(self, node): + ''' + Given a node, get should return the exact value given a key for all + key, value pairs in the node. + ''' + for k, v in api.walk(node): + assert api.get(node, k) is v + + @given(random_node) + def test_has(self, node): + ''' + Given a node, has should return True for all paths, False otherwise. + ''' + for k, v in api.walk(node): + assert api.has(node, k) is True + + # If we are at a leaf, then we can create a value that isn't + # present easily. + if api.leaf(v): + assert api.has(node, k + (0,)) is False + + @given(random_segments) + def test_expand(self, segments): + ''' + Given segments expand should produce as many results are there were + segments and the last result should equal the given segments. + ''' + count = len(segments) + result = list(api.expand(segments)) + + assert count == len(result) + + if count > 0: + assert segments == result[-1] + + @given(random_node) + def test_types(self, node): + ''' + Given a node, types should yield a tuple of key, type pairs and the + type indicated should equal the type of the value. + ''' + for k, v in api.walk(node): + ts = api.types(node, k) + ta = () + for tk, tt in ts: + ta += (tk,) + assert type(api.get(node, ta)) is tt + + @given(random_node) + def test_leaves(self, node): + ''' + Given a node, leaves should yield only leaf key, value pairs. + ''' + for k, v in api.leaves(node): + assert api.leafy(v) + + @given(random_segments_with_glob()) + def test_match(self, pair): + ''' + Given segments and a known good glob, match should be True. + ''' + (segments, glob) = pair + assert api.match(segments, glob) is True + + @given(random_segments_with_nonmatching_glob()) + def test_match_nonmatching(self, pair): + ''' + Given segments and a known bad glob, match should be False. + ''' + (segments, glob) = pair + assert api.match(segments, glob) is False + + @given(walkable=random_walk(), value=random_thing) + def test_set_walkable(self, walkable, value): + ''' + Given a walkable location, set should be able to update any value. + ''' + (node, (segments, found)) = walkable + api.set(node, segments, value) + assert api.get(node, segments) is value + + @given(walkable=random_leaves(), + kstr=random_key_str, + kint=random_key_int, + value=random_thing, + extension=random_segments) + def test_set_create_missing(self, walkable, kstr, kint, value, extension): + ''' + Given a walkable non-leaf, set should be able to create missing + nodes and set a new value. + ''' + (node, (segments, found)) = walkable + assume(api.leaf(found)) + + parent_segments = segments[:-1] + parent = api.get(node, parent_segments) + + if isinstance(parent, list): + assume(len(parent) < kint) + destination = parent_segments + (kint,) + tuple(extension) + elif isinstance(parent, dict): + assume(kstr not in parent) + destination = parent_segments + (kstr,) + tuple(extension) + else: + raise Exception('mad mad world') + api.set(node, destination, value) + assert api.get(node, destination) is value -@given(thing=random_thing) -def test_fold(thing): - ''' - Given a thing, count paths with fold. - ''' - def f(o, p, a): - a[0] += 1 + @given(thing=random_thing) + def test_fold(self, thing): + ''' + Given a thing, count paths with fold. + ''' - [count] = api.fold(thing, f, [0]) - assert count == len(tuple(api.walk(thing))) + def f(o, p, a): + a[0] += 1 + [count] = api.fold(thing, f, [0]) + assert count == len(tuple(api.walk(thing))) -@given(walkable=random_walk()) -def test_view(walkable): - ''' - Given a walkable location, view that location. - ''' - (node, (segments, found)) = walkable - assume(found == found) # Hello, nan! We don't want you here. + @given(walkable=random_walk()) + def test_view(self, walkable): + ''' + Given a walkable location, view that location. + ''' + (node, (segments, found)) = walkable + assume(found == found) # Hello, nan! We don't want you here. - view = api.view(node, segments) - assert api.get(view, segments) == api.get(node, segments) + view = api.view(node, segments) + assert api.get(view, segments) == api.get(node, segments) diff --git a/tests/test_types.py b/tests/test_types.py index 6aeac05..46b12b6 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -1,15 +1,8 @@ -import nose -import dpath.util -from nose.tools import assert_raises +from collections.abc import MutableSequence, MutableMapping + +from nose2.tools.such import helper -try: - # python3, especially 3.8 - from collections.abc import MutableSequence - from collections.abc import MutableMapping -except ImportError: - # python2 - from collections import MutableSequence - from collections import MutableMapping +import dpath.util class TestMapping(MutableMapping): @@ -107,13 +100,13 @@ def test_types_merge_simple_list_replace(): }) dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) - nose.tools.eq_(dst["list"], TestSequence([7, 8, 9, 10])) + assert dst["list"] == TestSequence([7, 8, 9, 10]), "%r != %r" % (dst["list"], TestSequence([7, 8, 9, 10])) def test_types_get_absent(): ehash = TestMapping() - assert_raises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - assert_raises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_types_get_glob_multiple(): @@ -130,8 +123,8 @@ def test_types_get_glob_multiple(): }), }) - assert_raises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.util.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) def test_delete_filter(): diff --git a/tests/test_util_delete.py b/tests/test_util_delete.py index 4e531aa..ecaa9b2 100644 --- a/tests/test_util_delete.py +++ b/tests/test_util_delete.py @@ -1,4 +1,5 @@ -from nose.tools import raises +from nose2.tools.such import helper + import dpath.util import dpath.exceptions @@ -25,14 +26,14 @@ def test_delete_existing(): assert 'b' not in dict['a'] -@raises(dpath.exceptions.PathNotFound) def test_delete_missing(): dict = { "a": { }, } - dpath.util.delete(dict, '/a/b') + with helper.assertRaises(dpath.exceptions.PathNotFound): + dpath.util.delete(dict, '/a/b') def test_delete_filter(): diff --git a/tests/test_util_get_values.py b/tests/test_util_get_values.py index a0b6686..c72aa8f 100644 --- a/tests/test_util_get_values.py +++ b/tests/test_util_get_values.py @@ -1,11 +1,12 @@ -from nose.tools import assert_raises - import datetime import decimal -import dpath.util -import mock import time +import mock +from nose2.tools.such import helper + +import dpath.util + def test_util_get_root(): x = {'p': {'a': {'t': {'h': 'value'}}}} @@ -70,16 +71,16 @@ def test_get_glob_multiple(): }, } - assert_raises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd'], default=3) + helper.assertRaises(ValueError, dpath.util.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd'], default=3) def test_get_absent(): ehash = {} - assert_raises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - assert_raises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_values(): diff --git a/tests/test_util_merge.py b/tests/test_util_merge.py index 17c4c0f..76fbfb6 100644 --- a/tests/test_util_merge.py +++ b/tests/test_util_merge.py @@ -1,6 +1,6 @@ -import nose import copy -from nose.tools import raises + +from nose2.tools.such import helper import dpath.util @@ -36,7 +36,7 @@ def test_merge_simple_int(): } dpath.util.merge(dst, src) - nose.tools.eq_(dst["integer"], src["integer"]) + assert dst["integer"] == src["integer"], "%r != %r" % (dst["integer"], src["integer"]) def test_merge_simple_string(): @@ -48,7 +48,7 @@ def test_merge_simple_string(): } dpath.util.merge(dst, src) - nose.tools.eq_(dst["string"], src["string"]) + assert dst["string"] == src["string"], "%r != %r" % (dst["string"], src["string"]) def test_merge_simple_list_additive(): @@ -60,7 +60,7 @@ def test_merge_simple_list_additive(): } dpath.util.merge(dst, src, flags=dpath.util.MERGE_ADDITIVE) - nose.tools.eq_(dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) + assert dst["list"] == [0, 1, 2, 3, 7, 8, 9, 10], "%r != %r" % (dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) def test_merge_simple_list_replace(): @@ -72,7 +72,7 @@ def test_merge_simple_list_replace(): } dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) - nose.tools.eq_(dst["list"], [7, 8, 9, 10]) + assert dst["list"] == [7, 8, 9, 10], "%r != %r" % (dst["list"], [7, 8, 9, 10]) def test_merge_simple_dict(): @@ -88,7 +88,7 @@ def test_merge_simple_dict(): } dpath.util.merge(dst, src) - nose.tools.eq_(dst["dict"]["key"], src["dict"]["key"]) + assert dst["dict"]["key"] == src["dict"]["key"], "%r != %r" % (dst["dict"]["key"], src["dict"]["key"]) def test_merge_filter(): @@ -112,7 +112,6 @@ def afilter(x): assert "otherdict" not in dst -@raises(TypeError) def test_merge_typesafe(): src = { "dict": { @@ -123,10 +122,9 @@ def test_merge_typesafe(): ], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MERGE_TYPESAFE) -@raises(TypeError) def test_merge_mutables(): class tcid(dict): pass @@ -156,7 +154,7 @@ class tcis(list): assert dst['ms'][2] == 'c' assert "casserole" in dst["mm"] - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MERGE_TYPESAFE) def test_merge_replace_1(): diff --git a/tox.ini b/tox.ini index 42d0d2a..94c441b 100644 --- a/tox.ini +++ b/tox.ini @@ -20,8 +20,8 @@ python = deps = hypothesis mock - nose -commands = nosetests {posargs} + nose2 +commands = nose2 {posargs} [testenv:flake8] deps = From a00adc3ce59197308f8114ce0f6b3b549704635d Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 00:32:51 +0200 Subject: [PATCH 073/133] Catch specific exceptions --- dpath/segments.py | 2 +- dpath/util.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 7c5816c..f452d12 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -326,7 +326,7 @@ def set( # Unfortunately, for our use, 'x in thing' for lists checks # values, not keys whereas dicts check keys. current[segment] - except: + except (KeyError, IndexError): if creator is not None: creator(current, segments, i, hints) else: diff --git a/dpath/util.py b/dpath/util.py index 5200f81..a3c5450 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -29,7 +29,7 @@ def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegme for segment in split_segments: try: final.append(int(segment)) - except: + except ValueError: final.append(segment) split_segments = final @@ -325,9 +325,9 @@ def merger(dst, src, _segments=()): except TypeError: segments.set(dst, current_path, found) continue - except: + except Exception: raise - except: + except Exception: # We have a dictionary like thing and we need to attempt to # recursively merge it. merger(dst, found, current_path) From 8d3a36f49ab62820488edc089decfe3fb693f5e8 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 00:38:16 +0200 Subject: [PATCH 074/133] Add exports --- dpath/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dpath/__init__.py b/dpath/__init__.py index e69de29..922e139 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -0,0 +1,16 @@ +from dpath.util import new, delete, set, get, values, search, merge + +__all__ = [ + "new", + "delete", + "set", + "get", + "values", + "search", + "merge", + "exceptions", + "options", + "segments", + "types", + "version", +] From 3198c54f19d9b04805cd425ef264d4ffc905fbef Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 01:00:07 +0200 Subject: [PATCH 075/133] Move utils functions to top level --- README.rst | 58 ++++---- dpath/__init__.py | 341 +++++++++++++++++++++++++++++++++++++++++++++- dpath/util.py | 337 ++++----------------------------------------- 3 files changed, 394 insertions(+), 342 deletions(-) diff --git a/README.rst b/README.rst index ccb6608..f88f1ed 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ Using Dpath .. code-block:: python - import dpath.util + import dpath Separators ========== @@ -62,8 +62,8 @@ key '43' in the 'b' hash which is in the 'a' hash". That's easy. .. code-block:: pycon - >>> help(dpath.util.get) - Help on function get in module dpath.util: + >>> help(dpath.get) + Help on function get in module dpath: get(obj, glob, separator='/') Given an object which contains only one possible match for the given glob, @@ -72,7 +72,7 @@ key '43' in the 'b' hash which is in the 'a' hash". That's easy. If more than one leaf matches the glob, ValueError is raised. If the glob is not found, KeyError is raised. - >>> dpath.util.get(x, '/a/b/43') + >>> dpath.get(x, '/a/b/43') 30 Or you could say "Give me a new dictionary with the values of all @@ -80,8 +80,8 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon - >>> help(dpath.util.search) - Help on function search in module dpath.util: + >>> help(dpath.search) + Help on function search in module dpath: search(obj, glob, yielded=False) Given a path glob, return a dictionary containing all keys @@ -95,7 +95,7 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon - >>> result = dpath.util.search(x, "a/b/[cd]") + >>> result = dpath.search(x, "a/b/[cd]") >>> print(json.dumps(result, indent=4, sort_keys=True)) { "a": { @@ -115,7 +115,7 @@ not get a merged view? .. code-block:: pycon - >>> for x in dpath.util.search(x, "a/b/[cd]", yielded=True): print(x) + >>> for x in dpath.search(x, "a/b/[cd]", yielded=True): print(x) ... ('a/b/c', []) ('a/b/d', ['red', 'buggy', 'bumpers']) @@ -125,8 +125,8 @@ don't care about the paths they were found at: .. code-block:: pycon - >>> help(dpath.util.values) - Help on function values in module dpath.util: + >>> help(dpath.values) + Help on function values in module dpath: values(obj, glob, separator='/', afilter=None, dirs=True) Given an object and a path glob, return an array of all values which match @@ -134,7 +134,7 @@ don't care about the paths they were found at: and it is primarily a shorthand for a list comprehension over a yielded search call. - >>> dpath.util.values(x, '/a/b/d/*') + >>> dpath.values(x, '/a/b/d/*') ['red', 'buggy', 'bumpers'] Example: Setting existing keys @@ -145,14 +145,14 @@ value 'Waffles'. .. code-block:: pycon - >>> help(dpath.util.set) - Help on function set in module dpath.util: + >>> help(dpath.set) + Help on function set in module dpath: set(obj, glob, value) Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. - >>> dpath.util.set(x, 'a/b/[cd]', 'Waffles') + >>> dpath.set(x, 'a/b/[cd]', 'Waffles') 2 >>> print(json.dumps(x, indent=4, sort_keys=True)) { @@ -175,8 +175,8 @@ necessary to get to the terminus. .. code-block:: pycon - >>> help(dpath.util.new) - Help on function new in module dpath.util: + >>> help(dpath.new) + Help on function new in module dpath: new(obj, path, value) Set the element at the terminus of path to value, and create @@ -187,7 +187,7 @@ necessary to get to the terminus. characters in it, they will become part of the resulting keys - >>> dpath.util.new(x, 'a/b/e/f/g', "Roffle") + >>> dpath.new(x, 'a/b/e/f/g', "Roffle") >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { @@ -211,8 +211,8 @@ object with None entries in order to make it big enough: .. code-block:: pycon - >>> dpath.util.new(x, 'a/b/e/f/h', []) - >>> dpath.util.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') + >>> dpath.new(x, 'a/b/e/f/h', []) + >>> dpath.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { @@ -251,11 +251,11 @@ Handy! Example: Deleting Existing Keys =============================== -To delete keys in an object, use dpath.util.delete, which accepts the same globbing syntax as the other methods. +To delete keys in an object, use dpath.delete, which accepts the same globbing syntax as the other methods. .. code-block:: pycon - >>> help(dpath.util.delete) + >>> help(dpath.delete) delete(obj, glob, separator='/', afilter=None): Given a path glob, delete all elements that match the glob. @@ -266,14 +266,14 @@ To delete keys in an object, use dpath.util.delete, which accepts the same globb Example: Merging ================ -Also, check out dpath.util.merge. The python dict update() method is +Also, check out dpath.merge. The python dict update() method is great and all but doesn't handle merging dictionaries deeply. This one does. .. code-block:: pycon - >>> help(dpath.util.merge) - Help on function merge in module dpath.util: + >>> help(dpath.merge) + Help on function merge in module dpath: merge(dst, src, afilter=None, flags=4, _path='') Merge source into destination. Like dict.update() but performs @@ -310,7 +310,7 @@ does. "c": "RoffleWaffles" } } - >>> dpath.util.merge(x, y) + >>> dpath.merge(x, y) >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { @@ -390,7 +390,7 @@ them: ... return True ... return False ... - >>> result = dpath.util.search(x, '**', afilter=afilter) + >>> result = dpath.search(x, '**', afilter=afilter) >>> print(json.dumps(result, indent=4, sort_keys=True)) { "a": { @@ -429,18 +429,18 @@ Separator got you down? Use lists as paths The default behavior in dpath is to assume that the path given is a string, which must be tokenized by splitting at the separator to yield a distinct set of path components against which dictionary keys can be individually glob tested. However, this presents a problem when you want to use paths that have a separator in their name; the tokenizer cannot properly understand what you mean by '/a/b/c' if it is possible for '/' to exist as a valid character in a key name. -To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.util.* support the use of a list instead of a string as a path. So for example: +To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.* support the use of a list instead of a string as a path. So for example: .. code-block:: python >>> x = { 'a': {'b/c': 0}} - >>> dpath.util.get(['a', 'b/c']) + >>> dpath.get(['a', 'b/c']) 0 dpath.segments : The Low-Level Backend ====================================== -dpath.util is where you want to spend your time: this library has the friendly +dpath is where you want to spend your time: this library has the friendly functions that will understand simple string globs, afilter functions, etc. dpath.segments is the backend pathing library. It passes around tuples of path diff --git a/dpath/__init__.py b/dpath/__init__.py index 922e139..a3180bd 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -1,5 +1,3 @@ -from dpath.util import new, delete, set, get, values, search, merge - __all__ = [ "new", "delete", @@ -14,3 +12,342 @@ "types", "version", ] + +from collections.abc import MutableMapping, MutableSequence +from typing import Union, List, Dict, Any + +from dpath import segments, options +from dpath.exceptions import InvalidKeyName, PathNotFound +from dpath.types import MergeType, PathSegment, Creator, Filter + +_DEFAULT_SENTINEL = object() + + +def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: + """ + Given a path and separator, return a tuple of segments. If path is + already a non-leaf thing, return it. + + Note that a string path with the separator at index[0] will have the + separator stripped off. If you pass a list path, the separator is + ignored, and is assumed to be part of each key glob. It will not be + stripped. + """ + if not segments.leaf(path): + split_segments = path + else: + split_segments = path.lstrip(separator).split(separator) + + if options.CONVERT_INT_LIKE_SEGMENTS: + # Attempt to convert integer segments into actual integers. + final = [] + for segment in split_segments: + try: + final.append(int(segment)) + except ValueError: + final.append(segment) + split_segments = final + + return split_segments + + +def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: + """ + Set the element at the terminus of path to value, and create + it if it does not exist (as opposed to 'set' that can only + change existing keys). + + path will NOT be treated like a glob. If it has globbing + characters in it, they will become part of the resulting + keys + + creator allows you to pass in a creator method that is + responsible for creating missing keys at arbitrary levels of + the path (see the help for dpath.path.set) + """ + split_segments = _split_path(path, separator) + if creator: + return segments.set(obj, split_segments, value, creator=creator) + return segments.set(obj, split_segments, value) + + +def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: + """ + Given a obj, delete all elements that match the glob. + + Returns the number of deleted objects. Raises PathNotFound if no paths are + found to delete. + """ + globlist = _split_path(glob, separator) + + def f(obj, pair, counter): + (path_segments, value) = pair + + # Skip segments if they no longer exist in obj. + if not segments.has(obj, path_segments): + return + + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(value) and afilter(value) + + if (matched and not afilter) or selected: + key = path_segments[-1] + parent = segments.get(obj, path_segments[:-1]) + + # Deletion behavior depends on parent type + if isinstance(parent, MutableMapping): + del parent[key] + + else: + # Handle sequence types + # TODO: Consider cases where type isn't a simple list (e.g. set) + + if len(parent) - 1 == key: + # Removing the last element of a sequence. It can be + # truly removed without affecting the ordering of + # remaining items. + # + # Note: In order to achieve proper behavior we are + # relying on the reverse iteration of + # non-dictionaries from segments.kvs(). + # Otherwise we'd be unable to delete all the tails + # of a list and end up with None values when we + # don't need them. + del parent[key] + + else: + # This key can't be removed completely because it + # would affect the order of items that remain in our + # result. + parent[key] = None + + counter[0] += 1 + + [deleted] = segments.foldm(obj, f, [0]) + if not deleted: + raise PathNotFound(f"Could not find {glob} to delete it") + + return deleted + + +def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: + """ + Given a path glob, set all existing elements in the document + to the given value. Returns the number of elements changed. + """ + globlist = _split_path(glob, separator) + + def f(obj, pair, counter): + (path_segments, found) = pair + + # Skip segments if they no longer exist in obj. + if not segments.has(obj, path_segments): + return + + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(found) and afilter(found) + + if (matched and not afilter) or (matched and selected): + segments.set(obj, path_segments, value, creator=None) + counter[0] += 1 + + [changed] = segments.foldm(obj, f, [0]) + return changed + + +def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: + """ + Given an object which contains only one possible match for the given glob, + return the value for the leaf matching the given glob. + If the glob is not found and a default is provided, + the default is returned. + + If more than one leaf matches the glob, ValueError is raised. If the glob is + not found and a default is not provided, KeyError is raised. + """ + if glob == "/": + return obj + + globlist = _split_path(glob, separator) + + def f(_, pair, results): + (path_segments, found) = pair + + if segments.match(path_segments, globlist): + results.append(found) + if len(results) > 1: + return False + + results = segments.fold(obj, f, []) + + if len(results) == 0: + if default is not _DEFAULT_SENTINEL: + return default + + raise KeyError(glob) + elif len(results) > 1: + raise ValueError(f"dpath.util.get() globs must match only one leaf: {glob}") + + return results[0] + + +def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): + """ + Given an object and a path glob, return an array of all values which match + the glob. The arguments to this function are identical to those of search(). + """ + yielded = True + + return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] + + +def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): + """ + Given a path glob, return a dictionary containing all keys + that matched the given glob. + + If 'yielded' is true, then a dictionary will not be returned. + Instead tuples will be yielded in the form of (path, value) for + every element in the document that matched the glob. + """ + + split_glob = _split_path(glob, separator) + + def keeper(path, found): + """ + Generalized test for use in both yielded and folded cases. + Returns True if we want this result. Otherwise returns False. + """ + if not dirs and not segments.leaf(found): + return False + + matched = segments.match(path, split_glob) + selected = afilter and afilter(found) + + return (matched and not afilter) or (matched and selected) + + if yielded: + def yielder(): + for path, found in segments.walk(obj): + if keeper(path, found): + yield separator.join(map(segments.int_str, path)), found + + return yielder() + else: + def f(obj, pair, result): + (path, found) = pair + + if keeper(path, found): + segments.set(result, path, found, hints=segments.types(obj, path)) + + return segments.fold(obj, f, {}) + + +def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): + """ + Merge source into destination. Like dict.update() but performs deep + merging. + + NOTE: This does not do a deep copy of the source object. Applying merge + will result in references to src being present in the dst tree. If you do + not want src to potentially be modified by other changes in dst (e.g. more + merge calls), then use a deep copy of src. + + NOTE that merge() does NOT copy objects - it REFERENCES. If you merge + take these two dictionaries: + + >>> a = {'a': [0] } + >>> b = {'a': [1] } + + ... and you merge them into an empty dictionary, like so: + + >>> d = {} + >>> dpath.merge(d, a) + >>> dpath.merge(d, b) + + ... you might be surprised to find that a['a'] now contains [0, 1]. + This is because merge() says (d['a'] = a['a']), and thus creates a reference. + This reference is then modified when b is merged, causing both d and + a to have ['a'][0, 1]. To avoid this, make your own deep copies of source + objects that you intend to merge. For further notes see + https://github.com/akesterson/dpath-python/issues/58 + + flags is an OR'ed combination of MergeType enum members. + """ + filtered_src = search(src, '**', afilter=afilter, separator='/') + + def are_both_mutable(o1, o2): + mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) + seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) + + if mapP or seqP: + return True + + return False + + def merger(dst, src, _segments=()): + for key, found in segments.make_walkable(src): + # Our current path in the source. + current_path = _segments + (key,) + + if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: + raise InvalidKeyName("Empty string keys not allowed without " + "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " + f"{current_path}") + + # Validate src and dst types match. + if flags & MergeType.TYPESAFE: + if segments.has(dst, current_path): + target = segments.get(dst, current_path) + tt = type(target) + ft = type(found) + if tt != ft: + path = separator.join(current_path) + raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") + + # Path not present in destination, create it. + if not segments.has(dst, current_path): + segments.set(dst, current_path, found) + continue + + # Retrieve the value in the destination. + target = segments.get(dst, current_path) + + # If the types don't match, replace it. + if type(found) != type(target) and not are_both_mutable(found, target): + segments.set(dst, current_path, found) + continue + + # If target is a leaf, the replace it. + if segments.leaf(target): + segments.set(dst, current_path, found) + continue + + # At this point we know: + # + # * The target exists. + # * The types match. + # * The target isn't a leaf. + # + # Pretend we have a sequence and account for the flags. + try: + if flags & MergeType.ADDITIVE: + target += found + continue + + if flags & MergeType.REPLACE: + try: + target[""] + except TypeError: + segments.set(dst, current_path, found) + continue + except Exception: + raise + except Exception: + # We have a dictionary like thing and we need to attempt to + # recursively merge it. + merger(dst, found, current_path) + + merger(dst, filtered_src) + + return dst diff --git a/dpath/util.py b/dpath/util.py index a3c5450..c3e99d1 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,337 +1,52 @@ -from collections.abc import MutableMapping, MutableSequence -from typing import Union, List, Any, Dict +import warnings +from typing import Any, Dict -from dpath import options, segments -from dpath.exceptions import InvalidKeyName, PathNotFound -from dpath.types import PathSegment, Filter, Creator, MergeType +import dpath +from dpath import _DEFAULT_SENTINEL +from dpath.types import Filter, Creator, MergeType -_DEFAULT_SENTINEL = object() +def deprecated(func): + message =\ + "The dpath.util package is being deprecated. All util functions have been moved to dpath package top level." -def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: - """ - Given a path and separator, return a tuple of segments. If path is - already a non-leaf thing, return it. + def wrapper(*args, **kwargs): + warnings.warn(message, DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) - Note that a string path with the separator at index[0] will have the - separator stripped off. If you pass a list path, the separator is - ignored, and is assumed to be part of each key glob. It will not be - stripped. - """ - if not segments.leaf(path): - split_segments = path - else: - split_segments = path.lstrip(separator).split(separator) - - if options.CONVERT_INT_LIKE_SEGMENTS: - # Attempt to convert integer segments into actual integers. - final = [] - for segment in split_segments: - try: - final.append(int(segment)) - except ValueError: - final.append(segment) - split_segments = final - - return split_segments + return wrapper +@deprecated def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: - """ - Set the element at the terminus of path to value, and create - it if it does not exist (as opposed to 'set' that can only - change existing keys). - - path will NOT be treated like a glob. If it has globbing - characters in it, they will become part of the resulting - keys - - creator allows you to pass in a creator method that is - responsible for creating missing keys at arbitrary levels of - the path (see the help for dpath.path.set) - """ - split_segments = _split_path(path, separator) - if creator: - return segments.set(obj, split_segments, value, creator=creator) - return segments.set(obj, split_segments, value) + return dpath.new(obj, path, value, separator, creator) +@deprecated def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: - """ - Given a obj, delete all elements that match the glob. - - Returns the number of deleted objects. Raises PathNotFound if no paths are - found to delete. - """ - globlist = _split_path(glob, separator) - - def f(obj, pair, counter): - (path_segments, value) = pair - - # Skip segments if they no longer exist in obj. - if not segments.has(obj, path_segments): - return - - matched = segments.match(path_segments, globlist) - selected = afilter and segments.leaf(value) and afilter(value) - - if (matched and not afilter) or selected: - key = path_segments[-1] - parent = segments.get(obj, path_segments[:-1]) - - # Deletion behavior depends on parent type - if isinstance(parent, MutableMapping): - del parent[key] - - else: - # Handle sequence types - # TODO: Consider cases where type isn't a simple list (e.g. set) - - if len(parent) - 1 == key: - # Removing the last element of a sequence. It can be - # truly removed without affecting the ordering of - # remaining items. - # - # Note: In order to achieve proper behavior we are - # relying on the reverse iteration of - # non-dictionaries from segments.kvs(). - # Otherwise we'd be unable to delete all the tails - # of a list and end up with None values when we - # don't need them. - del parent[key] - - else: - # This key can't be removed completely because it - # would affect the order of items that remain in our - # result. - parent[key] = None - - counter[0] += 1 - - [deleted] = segments.foldm(obj, f, [0]) - if not deleted: - raise PathNotFound(f"Could not find {glob} to delete it") - - return deleted + return dpath.delete(obj, glob, separator, afilter) +@deprecated def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: - """ - Given a path glob, set all existing elements in the document - to the given value. Returns the number of elements changed. - """ - globlist = _split_path(glob, separator) - - def f(obj, pair, counter): - (path_segments, found) = pair - - # Skip segments if they no longer exist in obj. - if not segments.has(obj, path_segments): - return - - matched = segments.match(path_segments, globlist) - selected = afilter and segments.leaf(found) and afilter(found) - - if (matched and not afilter) or (matched and selected): - segments.set(obj, path_segments, value, creator=None) - counter[0] += 1 - - [changed] = segments.foldm(obj, f, [0]) - return changed + return dpath.set(obj, glob, value, separator, afilter) +@deprecated def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: - """ - Given an object which contains only one possible match for the given glob, - return the value for the leaf matching the given glob. - If the glob is not found and a default is provided, - the default is returned. - - If more than one leaf matches the glob, ValueError is raised. If the glob is - not found and a default is not provided, KeyError is raised. - """ - if glob == "/": - return obj - - globlist = _split_path(glob, separator) - - def f(_, pair, results): - (path_segments, found) = pair - - if segments.match(path_segments, globlist): - results.append(found) - if len(results) > 1: - return False - - results = segments.fold(obj, f, []) - - if len(results) == 0: - if default is not _DEFAULT_SENTINEL: - return default - - raise KeyError(glob) - elif len(results) > 1: - raise ValueError(f"dpath.util.get() globs must match only one leaf: {glob}") - - return results[0] + return dpath.get(obj, glob, separator, default) +@deprecated def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): - """ - Given an object and a path glob, return an array of all values which match - the glob. The arguments to this function are identical to those of search(). - """ - yielded = True - - return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] + return dpath.values(obj, glob, separator, afilter, dirs) +@deprecated def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): - """ - Given a path glob, return a dictionary containing all keys - that matched the given glob. - - If 'yielded' is true, then a dictionary will not be returned. - Instead tuples will be yielded in the form of (path, value) for - every element in the document that matched the glob. - """ - - split_glob = _split_path(glob, separator) - - def keeper(path, found): - """ - Generalized test for use in both yielded and folded cases. - Returns True if we want this result. Otherwise returns False. - """ - if not dirs and not segments.leaf(found): - return False - - matched = segments.match(path, split_glob) - selected = afilter and afilter(found) - - return (matched and not afilter) or (matched and selected) - - if yielded: - def yielder(): - for path, found in segments.walk(obj): - if keeper(path, found): - yield separator.join(map(segments.int_str, path)), found - return yielder() - else: - def f(obj, pair, result): - (path, found) = pair - - if keeper(path, found): - segments.set(result, path, found, hints=segments.types(obj, path)) - - return segments.fold(obj, f, {}) + return dpath.search(obj, glob, yielded, separator, afilter, dirs) +@deprecated def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): - """ - Merge source into destination. Like dict.update() but performs deep - merging. - - NOTE: This does not do a deep copy of the source object. Applying merge - will result in references to src being present in the dst tree. If you do - not want src to potentially be modified by other changes in dst (e.g. more - merge calls), then use a deep copy of src. - - NOTE that merge() does NOT copy objects - it REFERENCES. If you merge - take these two dictionaries: - - >>> a = {'a': [0] } - >>> b = {'a': [1] } - - ... and you merge them into an empty dictionary, like so: - - >>> d = {} - >>> dpath.util.merge(d, a) - >>> dpath.util.merge(d, b) - - ... you might be surprised to find that a['a'] now contains [0, 1]. - This is because merge() says (d['a'] = a['a']), and thus creates a reference. - This reference is then modified when b is merged, causing both d and - a to have ['a'][0, 1]. To avoid this, make your own deep copies of source - objects that you intend to merge. For further notes see - https://github.com/akesterson/dpath-python/issues/58 - - flags is an OR'ed combination of MergeType enum members. - """ - filtered_src = search(src, '**', afilter=afilter, separator='/') - - def are_both_mutable(o1, o2): - mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) - seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) - - if mapP or seqP: - return True - - return False - - def merger(dst, src, _segments=()): - for key, found in segments.make_walkable(src): - # Our current path in the source. - current_path = _segments + (key,) - - if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: - raise InvalidKeyName("Empty string keys not allowed without " - "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - f"{current_path}") - - # Validate src and dst types match. - if flags & MergeType.TYPESAFE: - if segments.has(dst, current_path): - target = segments.get(dst, current_path) - tt = type(target) - ft = type(found) - if tt != ft: - path = separator.join(current_path) - raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") - - # Path not present in destination, create it. - if not segments.has(dst, current_path): - segments.set(dst, current_path, found) - continue - - # Retrieve the value in the destination. - target = segments.get(dst, current_path) - - # If the types don't match, replace it. - if type(found) != type(target) and not are_both_mutable(found, target): - segments.set(dst, current_path, found) - continue - - # If target is a leaf, the replace it. - if segments.leaf(target): - segments.set(dst, current_path, found) - continue - - # At this point we know: - # - # * The target exists. - # * The types match. - # * The target isn't a leaf. - # - # Pretend we have a sequence and account for the flags. - try: - if flags & MergeType.ADDITIVE: - target += found - continue - - if flags & MergeType.REPLACE: - try: - target[""] - except TypeError: - segments.set(dst, current_path, found) - continue - except Exception: - raise - except Exception: - # We have a dictionary like thing and we need to attempt to - # recursively merge it. - merger(dst, found, current_path) - - merger(dst, filtered_src) - - return dst + return dpath.merge(dst, src, separator, afilter, flags), From 605a7567e385ffe4f261c284e71623fa91f58035 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 01:21:27 +0200 Subject: [PATCH 076/133] Fix tests --- tests/test_segments.py | 2 +- tests/test_types.py | 3 ++- tests/test_util_merge.py | 13 +++++++------ tests/test_util_paths.py | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/test_segments.py b/tests/test_segments.py index 40a9f9d..fb6a8bc 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -170,7 +170,7 @@ def test_kvs(self, node): Given a node, kvs should produce a key that when used to extract from the node renders the exact same value given. ''' - for k, v in api.kvs(node): + for k, v in api.make_walkable(node): assert node[k] is v @given(random_leaf) diff --git a/tests/test_types.py b/tests/test_types.py index 46b12b6..56eb98f 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -3,6 +3,7 @@ from nose2.tools.such import helper import dpath.util +from dpath import MergeType class TestMapping(MutableMapping): @@ -99,7 +100,7 @@ def test_types_merge_simple_list_replace(): "list": TestSequence([0, 1, 2, 3]) }) - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dst, src, flags=MergeType.REPLACE) assert dst["list"] == TestSequence([7, 8, 9, 10]), "%r != %r" % (dst["list"], TestSequence([7, 8, 9, 10])) diff --git a/tests/test_util_merge.py b/tests/test_util_merge.py index 9ea9e84..695e58f 100644 --- a/tests/test_util_merge.py +++ b/tests/test_util_merge.py @@ -4,6 +4,7 @@ import dpath.util +from dpath import MergeType def test_merge_typesafe_and_separator(): @@ -59,7 +60,7 @@ def test_merge_simple_list_additive(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_ADDITIVE) + dpath.util.merge(dst, src, flags=MergeType.ADDITIVE) assert dst["list"] == [0, 1, 2, 3, 7, 8, 9, 10], "%r != %r" % (dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) @@ -71,7 +72,7 @@ def test_merge_simple_list_replace(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dst, src, flags=dpath.util.MergeType.REPLACE) assert dst["list"] == [7, 8, 9, 10], "%r != %r" % (dst["list"], [7, 8, 9, 10]) @@ -122,7 +123,7 @@ def test_merge_typesafe(): ], } - helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MergeType.TYPESAFE) def test_merge_mutables(): @@ -154,20 +155,20 @@ class tcis(list): assert dst['ms'][2] == 'c' assert "casserole" in dst["mm"] - helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MergeType.TYPESAFE) def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} - dpath.util.merge(dct_a, dct_b, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(dct_a, dct_b, flags=dpath.util.MergeType.REPLACE) assert len(dct_a['a']['b']) == 1 def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} - dpath.util.merge(d1, d2, flags=dpath.util.MERGE_REPLACE) + dpath.util.merge(d1, d2, flags=dpath.util.MergeType.REPLACE) assert len(d1['a']) == 1 assert d1['a'][0] == 'a' diff --git a/tests/test_util_paths.py b/tests/test_util_paths.py index f84e43f..49bec1b 100644 --- a/tests/test_util_paths.py +++ b/tests/test_util_paths.py @@ -2,7 +2,7 @@ def test_util_safe_path_list(): - res = dpath.util._split_path(["Ignore", "the/separator"], None) + res = dpath._split_path(["Ignore", "the/separator"], None) assert len(res) == 2 assert res[0] == "Ignore" From 9ef4bdef49c98053d11c25d663603410a96e71cd Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 14:50:54 +0200 Subject: [PATCH 077/133] Mock correct function --- tests/test_util_get_values.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_util_get_values.py b/tests/test_util_get_values.py index c72aa8f..a71b441 100644 --- a/tests/test_util_get_values.py +++ b/tests/test_util_get_values.py @@ -109,12 +109,12 @@ def test_values(): assert 2 in ret -@mock.patch('dpath.util.search') +@mock.patch('dpath.search') def test_values_passes_through(searchfunc): searchfunc.return_value = [] def y(): - pass + return False dpath.util.values({}, '/a/b', ':', y, False) searchfunc.assert_called_with({}, '/a/b', True, ':', y, False) From 9f06f0246696e055dc68ad4a619549ad5e233e46 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 15:10:34 +0200 Subject: [PATCH 078/133] PEP8 change --- dpath/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/types.py b/dpath/types.py index 2b44ce6..ed2c910 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -36,4 +36,4 @@ def creator( segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () - )""" \ No newline at end of file + )""" From b0681205b7a60018c468a88456347d91dde3a9cc Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 18:16:58 +0200 Subject: [PATCH 079/133] Enable DeprecationWarnings in tests --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e188ff3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +import warnings + +warnings.simplefilter("always", DeprecationWarning) From eceb8cb3b2cb625da63deb71bbe6ecc3b2f762b6 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 18:37:12 +0200 Subject: [PATCH 080/133] Run flake8 separately from tests --- .github/workflows/tests.yml | 20 +++++++++++++++++++- flake8.ini | 5 +++++ tox.ini | 9 ++------- 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 flake8.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e7d924..3fa0f78 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,12 +38,30 @@ jobs: print(f'{hashseed=}') open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" + flake8: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@main + + - name: Set up Python 3.10 + uses: actions/setup-python@main + + - name: Run Flake8 + uses: julianwachholz/flake8-action@v2 + with: + checkName: flake8 + config: flake8.ini + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Tests job tests: # The type of runner that the job will run on runs-on: ubuntu-latest - needs: generate-hashseed + needs: [generate-hashseed, flake8] strategy: matrix: diff --git a/flake8.ini b/flake8.ini new file mode 100644 index 0000000..92830d3 --- /dev/null +++ b/flake8.ini @@ -0,0 +1,5 @@ +[flake8] +filename= + setup.py, + dpath/, + tests/ diff --git a/tox.ini b/tox.ini index 94c441b..d613837 100644 --- a/tox.ini +++ b/tox.ini @@ -7,14 +7,14 @@ ignore = E501,E722 [tox] -envlist = pypy37, py38, py39, py310, flake8 +envlist = pypy37, py38, py39, py310 [gh-actions] python = pypy-3.7: pypy37 3.8: py38 3.9: py39 - 3.10: py310, flake8 + 3.10: py310 [testenv] deps = @@ -22,8 +22,3 @@ deps = mock nose2 commands = nose2 {posargs} - -[testenv:flake8] -deps = - flake8 -commands = flake8 setup.py dpath/ tests/ From ddf81a829041a019eeb912427d43091e15990b67 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 19:01:39 +0200 Subject: [PATCH 081/133] Working flake8 with annotations --- .github/workflows/tests.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3fa0f78..348a7fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,14 +47,16 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@main - - - name: Run Flake8 - uses: julianwachholz/flake8-action@v2 with: - checkName: flake8 - config: flake8.ini - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + python-version: "3.10" + + - name: Setup flake8 annotations + uses: rbialon/flake8-annotations@v1.1 + + - name: Lint with flake8 + run: | + pip install flake8 + flake8 setup.py dpath/ tests/ # Tests job tests: From 29d344ebee3c17d6849522bea65eebd98e1dbfa2 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 19:10:15 +0200 Subject: [PATCH 082/133] Style fixes --- dpath/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dpath/types.py b/dpath/types.py index ed2c910..f56da31 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -7,11 +7,11 @@ class MergeType(IntFlag): """List objects are combined onto one long list (NOT a set). This is the default flag.""" REPLACE = auto() - """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination + """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination \ with the source.""" TYPESAFE = auto() - """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source + """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source \ replaces the destination in this situation.""" From b49e823dd07cf081f346fb47b0e1614e4019a825 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 19:14:51 +0200 Subject: [PATCH 083/133] Reorder tasks in workflow --- .github/workflows/tests.yml | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 348a7fd..ba8e2bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,23 +21,8 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # Generate a common hashseed for all tests - generate-hashseed: - runs-on: ubuntu-latest - - outputs: - hashseed: ${{ steps.generate.outputs.hashseed }} - - steps: - - name: Generate Hashseed - id: generate - run: | - python -c "import os - from random import randint - hashseed = randint(0, 4294967295) - print(f'{hashseed=}') - open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" + # Run flake8 linter flake8: runs-on: ubuntu-latest @@ -58,6 +43,23 @@ jobs: pip install flake8 flake8 setup.py dpath/ tests/ + # Generate a common hashseed for all tests + generate-hashseed: + runs-on: ubuntu-latest + + outputs: + hashseed: ${{ steps.generate.outputs.hashseed }} + + steps: + - name: Generate Hashseed + id: generate + run: | + python -c "import os + from random import randint + hashseed = randint(0, 4294967295) + print(f'{hashseed=}') + open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" + # Tests job tests: # The type of runner that the job will run on From 6460b645fcf30e8f8a3f88f73fe87cf2fd295ad8 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 24 Nov 2022 19:15:02 +0200 Subject: [PATCH 084/133] Set minor version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 4f7a825..127c148 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.8" +VERSION = "2.1.0" From 1c8fa849b1a0862643455f1751b699e23612198a Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 25 Nov 2022 15:22:01 +0200 Subject: [PATCH 085/133] Remove references to dpath.util in tests --- dpath/__init__.py | 2 +- dpath/util.py | 17 +++-- tests/test_broken_afilter.py | 18 +++--- tests/{test_util_delete.py => test_delete.py} | 10 +-- ..._util_get_values.py => test_get_values.py} | 64 +++++++++---------- tests/{test_util_merge.py => test_merge.py} | 34 +++++----- tests/{test_util_new.py => test_new.py} | 18 +++--- tests/{test_util_paths.py => test_paths.py} | 2 +- tests/{test_util_search.py => test_search.py} | 46 ++++++------- tests/{test_util_set.py => test_set.py} | 20 +++--- tests/test_types.py | 18 +++--- tests/test_unicode.py | 8 +-- 12 files changed, 128 insertions(+), 129 deletions(-) rename tests/{test_util_delete.py => test_delete.py} (79%) rename tests/{test_util_get_values.py => test_get_values.py} (62%) rename tests/{test_util_merge.py => test_merge.py} (78%) rename tests/{test_util_new.py => test_new.py} (80%) rename tests/{test_util_paths.py => test_paths.py} (91%) rename tests/{test_util_search.py => test_search.py} (72%) rename tests/{test_util_set.py => test_set.py} (74%) diff --git a/dpath/__init__.py b/dpath/__init__.py index a3180bd..79d33cf 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -186,7 +186,7 @@ def f(_, pair, results): raise KeyError(glob) elif len(results) > 1: - raise ValueError(f"dpath.util.get() globs must match only one leaf: {glob}") + raise ValueError(f"dpath.get() globs must match only one leaf: {glob}") return results[0] diff --git a/dpath/util.py b/dpath/util.py index 8d4a6b8..61e5580 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,9 +1,8 @@ import warnings -from typing import Any, Dict import dpath from dpath import _DEFAULT_SENTINEL -from dpath.types import Filter, Creator, MergeType +from dpath.types import MergeType def deprecated(func): @@ -18,35 +17,35 @@ def wrapper(*args, **kwargs): @deprecated -def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: +def new(obj, path, value, separator="/", creator=None): return dpath.new(obj, path, value, separator, creator) @deprecated -def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: +def delete(obj, glob, separator="/", afilter=None): return dpath.delete(obj, glob, separator, afilter) @deprecated -def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: +def set(obj, glob, value, separator="/", afilter=None): return dpath.set(obj, glob, value, separator, afilter) @deprecated -def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: +def get(obj, glob, separator="/", default=_DEFAULT_SENTINEL): return dpath.get(obj, glob, separator, default) @deprecated -def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): +def values(obj, glob, separator="/", afilter=None, dirs=True): return dpath.values(obj, glob, separator, afilter, dirs) @deprecated -def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): +def search(obj, glob, yielded=False, separator="/", afilter = None, dirs=True): return dpath.search(obj, glob, yielded, separator, afilter, dirs) @deprecated -def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): +def merge(dst, src, separator="/", afilter=None, flags=MergeType.ADDITIVE): return dpath.merge(dst, src, separator, afilter, flags), diff --git a/tests/test_broken_afilter.py b/tests/test_broken_afilter.py index 3e3a449..683c727 100644 --- a/tests/test_broken_afilter.py +++ b/tests/test_broken_afilter.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath import sys @@ -25,15 +25,15 @@ def afilter(x): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): + for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): assert path in paths - assert "view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a'] - assert "d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c'] + assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): + for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): assert path in paths - assert "view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a'] - assert "d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c'] + assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] def filter(x): sys.stderr.write(str(x)) @@ -52,7 +52,7 @@ def filter(x): ], } - results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', yielded=True)] - results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', afilter=filter, yielded=True)] + results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', yielded=True)] + results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', afilter=filter, yielded=True)] assert len(results) == 1 assert results[0][1]['type'] == 'correct' diff --git a/tests/test_util_delete.py b/tests/test_delete.py similarity index 79% rename from tests/test_util_delete.py rename to tests/test_delete.py index ecaa9b2..c7879b0 100644 --- a/tests/test_util_delete.py +++ b/tests/test_delete.py @@ -1,6 +1,6 @@ from nose2.tools.such import helper -import dpath.util +import dpath import dpath.exceptions @@ -11,7 +11,7 @@ def test_delete_separator(): }, } - dpath.util.delete(dict, ';a;b', separator=";") + dpath.delete(dict, ';a;b', separator=";") assert 'b' not in dict['a'] @@ -22,7 +22,7 @@ def test_delete_existing(): }, } - dpath.util.delete(dict, '/a/b') + dpath.delete(dict, '/a/b') assert 'b' not in dict['a'] @@ -33,7 +33,7 @@ def test_delete_missing(): } with helper.assertRaises(dpath.exceptions.PathNotFound): - dpath.util.delete(dict, '/a/b') + dpath.delete(dict, '/a/b') def test_delete_filter(): @@ -50,7 +50,7 @@ def afilter(x): }, } - dpath.util.delete(dict, '/a/*', afilter=afilter) + dpath.delete(dict, '/a/*', afilter=afilter) assert dict['a']['b'] == 0 assert dict['a']['c'] == 1 assert 'd' not in dict['a'] diff --git a/tests/test_util_get_values.py b/tests/test_get_values.py similarity index 62% rename from tests/test_util_get_values.py rename to tests/test_get_values.py index a71b441..9eeef82 100644 --- a/tests/test_util_get_values.py +++ b/tests/test_get_values.py @@ -5,16 +5,16 @@ import mock from nose2.tools.such import helper -import dpath.util +import dpath def test_util_get_root(): x = {'p': {'a': {'t': {'h': 'value'}}}} - ret = dpath.util.get(x, '/p/a/t/h') + ret = dpath.get(x, '/p/a/t/h') assert ret == 'value' - ret = dpath.util.get(x, '/') + ret = dpath.get(x, '/') assert ret == x @@ -31,11 +31,11 @@ def test_get_explicit_single(): }, } - assert dpath.util.get(ehash, '/a/b/c/f') == 2 - assert dpath.util.get(ehash, ['a', 'b', 'c', 'f']) == 2 - assert dpath.util.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2 - assert dpath.util.get(ehash, ['does', 'not', 'exist'], default=None) is None - assert dpath.util.get(ehash, ['doesnt', 'exist'], default=5) == 5 + assert dpath.get(ehash, '/a/b/c/f') == 2 + assert dpath.get(ehash, ['a', 'b', 'c', 'f']) == 2 + assert dpath.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2 + assert dpath.get(ehash, ['does', 'not', 'exist'], default=None) is None + assert dpath.get(ehash, ['doesnt', 'exist'], default=5) == 5 def test_get_glob_single(): @@ -51,10 +51,10 @@ def test_get_glob_single(): }, } - assert dpath.util.get(ehash, '/a/b/*/f') == 2 - assert dpath.util.get(ehash, ['a', 'b', '*', 'f']) == 2 - assert dpath.util.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2 - assert dpath.util.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6 + assert dpath.get(ehash, '/a/b/*/f') == 2 + assert dpath.get(ehash, ['a', 'b', '*', 'f']) == 2 + assert dpath.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2 + assert dpath.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6 def test_get_glob_multiple(): @@ -71,16 +71,16 @@ def test_get_glob_multiple(): }, } - helper.assertRaises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) - helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd'], default=3) + helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd'], default=3) def test_get_absent(): ehash = {} - helper.assertRaises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - helper.assertRaises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_values(): @@ -96,13 +96,13 @@ def test_values(): }, } - ret = dpath.util.values(ehash, '/a/b/c/*') + ret = dpath.values(ehash, '/a/b/c/*') assert isinstance(ret, list) assert 0 in ret assert 1 in ret assert 2 in ret - ret = dpath.util.values(ehash, ['a', 'b', 'c', '*']) + ret = dpath.values(ehash, ['a', 'b', 'c', '*']) assert isinstance(ret, list) assert 0 in ret assert 1 in ret @@ -116,17 +116,17 @@ def test_values_passes_through(searchfunc): def y(): return False - dpath.util.values({}, '/a/b', ':', y, False) + dpath.values({}, '/a/b', ':', y, False) searchfunc.assert_called_with({}, '/a/b', True, ':', y, False) - dpath.util.values({}, ['a', 'b'], ':', y, False) + dpath.values({}, ['a', 'b'], ':', y, False) searchfunc.assert_called_with({}, ['a', 'b'], True, ':', y, False) def test_none_values(): d = {'p': {'a': {'t': {'h': None}}}} - v = dpath.util.get(d, 'p/a/t/h') + v = dpath.get(d, 'p/a/t/h') assert v is None @@ -142,7 +142,7 @@ def test_values_list(): ], } - ret = dpath.util.values(a, 'actions/*') + ret = dpath.values(a, 'actions/*') assert isinstance(ret, list) assert len(ret) == 2 @@ -174,18 +174,18 @@ def func(x): } # It should be possible to get the callables: - assert dpath.util.get(testdict, 'a') == func - assert dpath.util.get(testdict, 'b')(42) == 42 + assert dpath.get(testdict, 'a') == func + assert dpath.get(testdict, 'b')(42) == 42 # It should be possible to get other values: - assert dpath.util.get(testdict, 'c/0') == testdict['c'][0] - assert dpath.util.get(testdict, 'd')[0] == testdict['d'][0] - assert dpath.util.get(testdict, 'd/0') == testdict['d'][0] - assert dpath.util.get(testdict, 'd/1') == testdict['d'][1] - assert dpath.util.get(testdict, 'e') == testdict['e'] + assert dpath.get(testdict, 'c/0') == testdict['c'][0] + assert dpath.get(testdict, 'd')[0] == testdict['d'][0] + assert dpath.get(testdict, 'd/0') == testdict['d'][0] + assert dpath.get(testdict, 'd/1') == testdict['d'][1] + assert dpath.get(testdict, 'e') == testdict['e'] # Values should also still work: - assert dpath.util.values(testdict, 'f/config') == ['something'] + assert dpath.values(testdict, 'f/config') == ['something'] # Data classes should also be retrievable: try: @@ -207,4 +207,4 @@ class Connection: ), } - assert dpath.util.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] + assert dpath.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] diff --git a/tests/test_util_merge.py b/tests/test_merge.py similarity index 78% rename from tests/test_util_merge.py rename to tests/test_merge.py index 695e58f..a8b638c 100644 --- a/tests/test_util_merge.py +++ b/tests/test_merge.py @@ -3,7 +3,7 @@ from nose2.tools.such import helper -import dpath.util +import dpath from dpath import MergeType @@ -20,7 +20,7 @@ def test_merge_typesafe_and_separator(): } try: - dpath.util.merge(dst, src, flags=(dpath.util.MergeType.ADDITIVE | dpath.util.MergeType.TYPESAFE), separator=";") + dpath.merge(dst, src, flags=(dpath.MergeType.ADDITIVE | dpath.MergeType.TYPESAFE), separator=";") except TypeError as e: assert str(e).endswith("dict;integer") @@ -36,7 +36,7 @@ def test_merge_simple_int(): "integer": 3, } - dpath.util.merge(dst, src) + dpath.merge(dst, src) assert dst["integer"] == src["integer"], "%r != %r" % (dst["integer"], src["integer"]) @@ -48,7 +48,7 @@ def test_merge_simple_string(): "string": "lol I am a string", } - dpath.util.merge(dst, src) + dpath.merge(dst, src) assert dst["string"] == src["string"], "%r != %r" % (dst["string"], src["string"]) @@ -60,7 +60,7 @@ def test_merge_simple_list_additive(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=MergeType.ADDITIVE) + dpath.merge(dst, src, flags=MergeType.ADDITIVE) assert dst["list"] == [0, 1, 2, 3, 7, 8, 9, 10], "%r != %r" % (dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) @@ -72,7 +72,7 @@ def test_merge_simple_list_replace(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MergeType.REPLACE) + dpath.merge(dst, src, flags=dpath.MergeType.REPLACE) assert dst["list"] == [7, 8, 9, 10], "%r != %r" % (dst["list"], [7, 8, 9, 10]) @@ -88,7 +88,7 @@ def test_merge_simple_dict(): }, } - dpath.util.merge(dst, src) + dpath.merge(dst, src) assert dst["dict"]["key"] == src["dict"]["key"], "%r != %r" % (dst["dict"]["key"], src["dict"]["key"]) @@ -107,7 +107,7 @@ def afilter(x): } dst = {} - dpath.util.merge(dst, src, afilter=afilter) + dpath.merge(dst, src, afilter=afilter) assert "key2" in dst assert "key" not in dst assert "otherdict" not in dst @@ -123,7 +123,7 @@ def test_merge_typesafe(): ], } - helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MergeType.TYPESAFE) + helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) def test_merge_mutables(): @@ -149,26 +149,26 @@ class tcis(list): "ms": tcis(['a', 'b', 'c']), } - dpath.util.merge(dst, src) + dpath.merge(dst, src) print(dst) assert dst["mm"]["a"] == src["mm"]["a"] assert dst['ms'][2] == 'c' assert "casserole" in dst["mm"] - helper.assertRaises(TypeError, dpath.util.merge, dst, src, flags=dpath.util.MergeType.TYPESAFE) + helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} - dpath.util.merge(dct_a, dct_b, flags=dpath.util.MergeType.REPLACE) + dpath.merge(dct_a, dct_b, flags=dpath.MergeType.REPLACE) assert len(dct_a['a']['b']) == 1 def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} - dpath.util.merge(d1, d2, flags=dpath.util.MergeType.REPLACE) + dpath.merge(d1, d2, flags=dpath.MergeType.REPLACE) assert len(d1['a']) == 1 assert d1['a'][0] == 'a' @@ -180,18 +180,18 @@ def test_merge_list(): dst1 = {} for d in [copy.deepcopy(src), copy.deepcopy(p1)]: - dpath.util.merge(dst1, d) + dpath.merge(dst1, d) dst2 = {} for d in [copy.deepcopy(src), copy.deepcopy(p2)]: - dpath.util.merge(dst2, d) + dpath.merge(dst2, d) assert dst1["l"] == [1, 2] assert dst2["l"] == [1] dst1 = {} for d in [src, p1]: - dpath.util.merge(dst1, d) + dpath.merge(dst1, d) dst2 = {} for d in [src, p2]: - dpath.util.merge(dst2, d) + dpath.merge(dst2, d) assert dst1["l"] == [1, 2] assert dst2["l"] == [1, 2] diff --git a/tests/test_util_new.py b/tests/test_new.py similarity index 80% rename from tests/test_util_new.py rename to tests/test_new.py index 3c2a3c4..6da31e7 100644 --- a/tests/test_util_new.py +++ b/tests/test_new.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_set_new_separator(): @@ -7,10 +7,10 @@ def test_set_new_separator(): }, } - dpath.util.new(dict, ';a;b', 1, separator=";") + dpath.new(dict, ';a;b', 1, separator=";") assert dict['a']['b'] == 1 - dpath.util.new(dict, ['a', 'b'], 1, separator=";") + dpath.new(dict, ['a', 'b'], 1, separator=";") assert dict['a']['b'] == 1 @@ -20,10 +20,10 @@ def test_set_new_dict(): }, } - dpath.util.new(dict, '/a/b', 1) + dpath.new(dict, '/a/b', 1) assert dict['a']['b'] == 1 - dpath.util.new(dict, ['a', 'b'], 1) + dpath.new(dict, ['a', 'b'], 1) assert dict['a']['b'] == 1 @@ -33,11 +33,11 @@ def test_set_new_list(): ], } - dpath.util.new(dict, '/a/1', 1) + dpath.new(dict, '/a/1', 1) assert dict['a'][1] == 1 assert dict['a'][0] is None - dpath.util.new(dict, ['a', 1], 1) + dpath.new(dict, ['a', 1], 1) assert dict['a'][1] == 1 assert dict['a'][0] is None @@ -49,7 +49,7 @@ def test_set_new_list_path_with_separator(): }, } - dpath.util.new(dict, ['a', 'b/c/d', 0], 1) + dpath.new(dict, ['a', 'b/c/d', 0], 1) assert len(dict['a']) == 1 assert len(dict['a']['b/c/d']) == 1 assert dict['a']['b/c/d'][0] == 1 @@ -76,7 +76,7 @@ def mycreator(obj, pathcomp, nextpathcomp, hints): obj[target] = {} print(obj) - dpath.util.new(d, '/a/2', 3, creator=mycreator) + dpath.new(d, '/a/2', 3, creator=mycreator) print(d) assert isinstance(d['a'], list) assert len(d['a']) == 3 diff --git a/tests/test_util_paths.py b/tests/test_paths.py similarity index 91% rename from tests/test_util_paths.py rename to tests/test_paths.py index 49bec1b..c63d728 100644 --- a/tests/test_util_paths.py +++ b/tests/test_paths.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_util_safe_path_list(): diff --git a/tests/test_util_search.py b/tests/test_search.py similarity index 72% rename from tests/test_util_search.py rename to tests/test_search.py index a974963..5830088 100644 --- a/tests/test_util_search.py +++ b/tests/test_search.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_search_paths_with_separator(): @@ -22,10 +22,10 @@ def test_search_paths_with_separator(): 'a;b;c;f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True, separator=";"): + for (path, value) in dpath.search(dict, '/**', yielded=True, separator=";"): assert path in paths - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, separator=";"): + for (path, value) in dpath.search(dict, ['**'], yielded=True, separator=";"): assert path in paths @@ -50,10 +50,10 @@ def test_search_paths(): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True): + for (path, value) in dpath.search(dict, '/**', yielded=True): assert path in paths - for (path, value) in dpath.util.search(dict, ['**'], yielded=True): + for (path, value) in dpath.search(dict, ['**'], yielded=True): assert path in paths @@ -80,15 +80,15 @@ def afilter(x): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): + for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): assert path in paths - assert "view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a'] - assert "d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c'] + assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): + for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): assert path in paths - assert "view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a'] - assert "d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c'] + assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] def test_search_globbing(): @@ -108,10 +108,10 @@ def test_search_globbing(): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/a/**/[df]', yielded=True): + for (path, value) in dpath.search(dict, '/a/**/[df]', yielded=True): assert path in paths - for (path, value) in dpath.util.search(dict, ['a', '**', '[df]'], yielded=True): + for (path, value) in dpath.search(dict, ['a', '**', '[df]'], yielded=True): assert path in paths @@ -125,12 +125,12 @@ def test_search_return_dict_head(): }, }, } - res = dpath.util.search(tdict, '/a/b') + res = dpath.search(tdict, '/a/b') assert isinstance(res['a']['b'], dict) assert len(res['a']['b']) == 3 assert res['a']['b'] == {0: 0, 1: 1, 2: 2} - res = dpath.util.search(tdict, ['a', 'b']) + res = dpath.search(tdict, ['a', 'b']) assert isinstance(res['a']['b'], dict) assert len(res['a']['b']) == 3 assert res['a']['b'] == {0: 0, 1: 1, 2: 2} @@ -147,12 +147,12 @@ def test_search_return_dict_globbed(): }, } - res = dpath.util.search(tdict, '/a/b/[02]') + res = dpath.search(tdict, '/a/b/[02]') assert isinstance(res['a']['b'], dict) assert len(res['a']['b']) == 2 assert res['a']['b'] == {0: 0, 2: 2} - res = dpath.util.search(tdict, ['a', 'b', '[02]']) + res = dpath.search(tdict, ['a', 'b', '[02]']) assert isinstance(res['a']['b'], dict) assert len(res['a']['b']) == 2 assert res['a']['b'] == {0: 0, 2: 2} @@ -169,12 +169,12 @@ def test_search_return_list_head(): }, } - res = dpath.util.search(tdict, '/a/b') + res = dpath.search(tdict, '/a/b') assert isinstance(res['a']['b'], list) assert len(res['a']['b']) == 3 assert res['a']['b'] == [0, 1, 2] - res = dpath.util.search(tdict, ['a', 'b']) + res = dpath.search(tdict, ['a', 'b']) assert isinstance(res['a']['b'], list) assert len(res['a']['b']) == 3 assert res['a']['b'] == [0, 1, 2] @@ -191,12 +191,12 @@ def test_search_return_list_globbed(): } } - res = dpath.util.search(tdict, '/a/b/[02]') + res = dpath.search(tdict, '/a/b/[02]') assert isinstance(res['a']['b'], list) assert len(res['a']['b']) == 3 assert res['a']['b'] == [0, None, 2] - res = dpath.util.search(tdict, ['a', 'b', '[02]']) + res = dpath.search(tdict, ['a', 'b', '[02]']) assert isinstance(res['a']['b'], list) assert len(res['a']['b']) == 3 assert res['a']['b'] == [0, None, 2] @@ -212,7 +212,7 @@ def test_search_list_key_with_separator(): }, } - res = dpath.util.search(tdict, ['a', '/b/d']) + res = dpath.search(tdict, ['a', '/b/d']) assert 'b' not in res['a'] assert res['a']['/b/d'] == 'success' @@ -231,7 +231,7 @@ def test_search_multiple_stars(): } testpath = 'a/*/b/*/c' - res = dpath.util.search(testdata, testpath) + res = dpath.search(testdata, testpath) assert len(res['a'][0]['b']) == 3 assert res['a'][0]['b'][0]['c'] == 1 assert res['a'][0]['b'][1]['c'] == 2 diff --git a/tests/test_util_set.py b/tests/test_set.py similarity index 74% rename from tests/test_util_set.py rename to tests/test_set.py index 1592590..ef2dd96 100644 --- a/tests/test_util_set.py +++ b/tests/test_set.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_set_existing_separator(): @@ -8,11 +8,11 @@ def test_set_existing_separator(): }, } - dpath.util.set(dict, ';a;b', 1, separator=";") + dpath.set(dict, ';a;b', 1, separator=";") assert dict['a']['b'] == 1 dict['a']['b'] = 0 - dpath.util.set(dict, ['a', 'b'], 1, separator=";") + dpath.set(dict, ['a', 'b'], 1, separator=";") assert dict['a']['b'] == 1 @@ -23,11 +23,11 @@ def test_set_existing_dict(): }, } - dpath.util.set(dict, '/a/b', 1) + dpath.set(dict, '/a/b', 1) assert dict['a']['b'] == 1 dict['a']['b'] = 0 - dpath.util.set(dict, ['a', 'b'], 1) + dpath.set(dict, ['a', 'b'], 1) assert dict['a']['b'] == 1 @@ -38,11 +38,11 @@ def test_set_existing_list(): ], } - dpath.util.set(dict, '/a/0', 1) + dpath.set(dict, '/a/0', 1) assert dict['a'][0] == 1 dict['a'][0] = 0 - dpath.util.set(dict, ['a', '0'], 1) + dpath.set(dict, ['a', '0'], 1) assert dict['a'][0] == 1 @@ -60,7 +60,7 @@ def afilter(x): } } - dpath.util.set(dict, '/a/*', 31337, afilter=afilter) + dpath.set(dict, '/a/*', 31337, afilter=afilter) assert dict['a']['b'] == 0 assert dict['a']['c'] == 1 assert dict['a']['d'] == 31337 @@ -73,7 +73,7 @@ def afilter(x): } } - dpath.util.set(dict, ['a', '*'], 31337, afilter=afilter) + dpath.set(dict, ['a', '*'], 31337, afilter=afilter) assert dict['a']['b'] == 0 assert dict['a']['c'] == 1 assert dict['a']['d'] == 31337 @@ -86,6 +86,6 @@ def test_set_existing_path_with_separator(): }, } - dpath.util.set(dict, ['a', 'b/c/d'], 1) + dpath.set(dict, ['a', 'b/c/d'], 1) assert len(dict['a']) == 1 assert dict['a']['b/c/d'] == 1 diff --git a/tests/test_types.py b/tests/test_types.py index 56eb98f..aa613ed 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -2,7 +2,7 @@ from nose2.tools.such import helper -import dpath.util +import dpath from dpath import MergeType @@ -65,12 +65,12 @@ def append(self, value): def test_types_set(): data = TestMapping({"a": TestSequence([0])}) - dpath.util.set(data, '/a/0', 1) + dpath.set(data, '/a/0', 1) assert data['a'][0] == 1 data['a'][0] = 0 - dpath.util.set(data, ['a', '0'], 1) + dpath.set(data, ['a', '0'], 1) assert data['a'][0] == 1 @@ -100,14 +100,14 @@ def test_types_merge_simple_list_replace(): "list": TestSequence([0, 1, 2, 3]) }) - dpath.util.merge(dst, src, flags=MergeType.REPLACE) + dpath.merge(dst, src, flags=MergeType.REPLACE) assert dst["list"] == TestSequence([7, 8, 9, 10]), "%r != %r" % (dst["list"], TestSequence([7, 8, 9, 10])) def test_types_get_absent(): ehash = TestMapping() - helper.assertRaises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - helper.assertRaises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_types_get_glob_multiple(): @@ -124,8 +124,8 @@ def test_types_get_glob_multiple(): }), }) - helper.assertRaises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - helper.assertRaises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) def test_delete_filter(): @@ -142,7 +142,7 @@ def afilter(x): }), }) - dpath.util.delete(data, '/a/*', afilter=afilter) + dpath.delete(data, '/a/*', afilter=afilter) assert data['a']['b'] == 0 assert data['a']['c'] == 1 assert 'd' not in data['a'] diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 09bc4db..d4e8033 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,11 +1,11 @@ -import dpath.util +import dpath def test_unicode_merge(): a = {'中': 'zhong'} b = {'文': 'wen'} - dpath.util.merge(a, b) + dpath.merge(a, b) assert len(a.keys()) == 2 assert a['中'] == 'zhong' assert a['文'] == 'wen' @@ -14,7 +14,7 @@ def test_unicode_merge(): def test_unicode_search(): a = {'中': 'zhong'} - results = [[x[0], x[1]] for x in dpath.util.search(a, '*', yielded=True)] + results = [[x[0], x[1]] for x in dpath.search(a, '*', yielded=True)] assert len(results) == 1 assert results[0][0] == '中' assert results[0][1] == 'zhong' @@ -24,7 +24,7 @@ def test_unicode_str_hybrid(): a = {'first': u'1'} b = {u'second': '2'} - dpath.util.merge(a, b) + dpath.merge(a, b) assert len(a.keys()) == 2 assert a[u'second'] == '2' assert a['second'] == u'2' From 17eab50391d6882eb1692ea30729facd55d11ae7 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 25 Nov 2022 15:25:21 +0200 Subject: [PATCH 086/133] PEP8 fix --- dpath/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/util.py b/dpath/util.py index 61e5580..60d0319 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -42,7 +42,7 @@ def values(obj, glob, separator="/", afilter=None, dirs=True): @deprecated -def search(obj, glob, yielded=False, separator="/", afilter = None, dirs=True): +def search(obj, glob, yielded=False, separator="/", afilter=None, dirs=True): return dpath.search(obj, glob, yielded, separator, afilter, dirs) From 36a6f5f4e802282e6a0f30a8c99525bf08ccc654 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Fri, 25 Nov 2022 15:37:11 +0200 Subject: [PATCH 087/133] Add note regarding 3.6 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 4b0f386..a6891d6 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ # (Source: https://github.com/hugovk/pypi-tools/blob/master/README.md) # # Conclusion: In order to accommodate type hinting support must be limited to Python versions >=3.6. + # 3.6 was dropped because of EOL and this issue: https://github.com/actions/setup-python/issues/544 python_requires=">=3.7", classifiers=[ 'Development Status :: 5 - Production/Stable', From 7a565ee55ae02a476e8df0ce1fc69d50ac27297e Mon Sep 17 00:00:00 2001 From: moomoohk Date: Mon, 28 Nov 2022 01:11:32 +0200 Subject: [PATCH 088/133] Type hint improvements and exports --- dpath/__init__.py | 32 ++++++++++++++++++++++---------- dpath/segments.py | 8 ++++---- dpath/types.py | 12 +++++++++--- tests/test_types.py | 10 ++++++++-- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index 79d33cf..c717314 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -11,19 +11,26 @@ "segments", "types", "version", + "MergeType", + "PathSegment", + "Filter", + "Glob", + "Path", + "Hints", + "Creator", ] from collections.abc import MutableMapping, MutableSequence -from typing import Union, List, Dict, Any +from typing import Union, List, Any, Callable, Optional from dpath import segments, options from dpath.exceptions import InvalidKeyName, PathNotFound -from dpath.types import MergeType, PathSegment, Creator, Filter +from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, Path, Hints _DEFAULT_SENTINEL = object() -def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegment]: +def _split_path(path: Path, separator: Optional[str]) -> Union[List[PathSegment], PathSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -51,7 +58,7 @@ def _split_path(path: str, separator: str) -> Union[List[PathSegment], PathSegme return split_segments -def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> Dict: +def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator = None) -> MutableMapping: """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only @@ -71,7 +78,7 @@ def new(obj: Dict, path: str, value, separator="/", creator: Creator = None) -> return segments.set(obj, split_segments, value) -def delete(obj: Dict, glob: str, separator="/", afilter: Filter = None) -> int: +def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -130,7 +137,7 @@ def f(obj, pair, counter): return deleted -def set(obj: Dict, glob: str, value, separator="/", afilter: Filter = None) -> int: +def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -155,7 +162,12 @@ def f(obj, pair, counter): return changed -def get(obj: Dict, glob: str, separator="/", default: Any = _DEFAULT_SENTINEL) -> Dict: +def get( + obj: MutableMapping, + glob: Glob, + separator="/", + default: Any = _DEFAULT_SENTINEL +) -> Union[MutableMapping, object, Callable]: """ Given an object which contains only one possible match for the given glob, return the value for the leaf matching the given glob. @@ -191,7 +203,7 @@ def f(_, pair, results): return results[0] -def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=True): +def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -201,7 +213,7 @@ def values(obj: Dict, glob: str, separator="/", afilter: Filter = None, dirs=Tru return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj: Dict, glob: str, yielded=False, separator="/", afilter: Filter = None, dirs=True): +def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. @@ -243,7 +255,7 @@ def f(obj, pair, result): return segments.fold(obj, f, {}) -def merge(dst: Dict, src: Dict, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): +def merge(dst: MutableMapping, src: MutableMapping, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): """ Merge source into destination. Like dict.update() but performs deep merging. diff --git a/dpath/segments.py b/dpath/segments.py index f452d12..d87a7b2 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,6 +1,6 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import List, Sequence, Tuple, Iterator, Any, Dict, Union, Optional +from typing import List, Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound @@ -266,7 +266,7 @@ def extend(thing: List, index: int, value=None): def _default_creator( - current: Union[Dict, List], + current: Union[MutableMapping, List], segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () @@ -301,12 +301,12 @@ def _default_creator( def set( - obj, + obj: MutableMapping, segments: Sequence[PathSegment], value, creator: Optional[Creator] = _default_creator, hints: Hints = () -): +) -> MutableMapping: """ Set the value in obj at the place indicated by segments. If creator is not None (default __default_creator__), then call the creator function to diff --git a/dpath/types.py b/dpath/types.py index f56da31..b876e6a 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -1,5 +1,5 @@ from enum import IntFlag, auto -from typing import Union, Any, Callable, Sequence, Tuple, Dict, List, Optional +from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping class MergeType(IntFlag): @@ -23,16 +23,22 @@ class MergeType(IntFlag): (Any) -> bool""" +Glob = Union[str, Sequence[str]] +"""Type alias for glob parameters.""" + +Path = Union[str, Sequence[PathSegment]] +"""Type alias for path parameters.""" + Hints = Sequence[Tuple[PathSegment, type]] """Type alias for creator function hint sequences.""" -Creator = Callable[[Union[Dict, List], Sequence[PathSegment], int, Optional[Hints]], None] +Creator = Callable[[Union[MutableMapping, List], Path, int, Optional[Hints]], None] """Type alias for creator functions. Example creator function signature: def creator( - current: Union[Dict, List], + current: Union[MutableMapping, List], segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () diff --git a/tests/test_types.py b/tests/test_types.py index aa613ed..39993f3 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -7,7 +7,10 @@ class TestMapping(MutableMapping): - def __init__(self, data={}): + def __init__(self, data=None): + if data is None: + data = {} + self._mapping = {} self._mapping.update(data) @@ -31,7 +34,10 @@ def __delitem__(self, key): class TestSequence(MutableSequence): - def __init__(self, data=list()): + def __init__(self, data=None): + if data is None: + data = list() + self._list = [] + data def __len__(self): From 3a79ed0dd9f0dba9674f4fe1c870fc58a9a4f4c6 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Mon, 28 Nov 2022 23:03:49 +0200 Subject: [PATCH 089/133] Catch all exceptions in type check --- dpath/segments.py | 4 ++-- dpath/version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index d87a7b2..ccf10e7 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -309,7 +309,7 @@ def set( ) -> MutableMapping: """ Set the value in obj at the place indicated by segments. If creator is not - None (default __default_creator__), then call the creator function to + None (default _default_creator), then call the creator function to create any missing path components. set(obj, segments, value) -> obj @@ -326,7 +326,7 @@ def set( # Unfortunately, for our use, 'x in thing' for lists checks # values, not keys whereas dicts check keys. current[segment] - except (KeyError, IndexError): + except: if creator is not None: creator(current, segments, i, hints) else: diff --git a/dpath/version.py b/dpath/version.py index 127c148..5b0431e 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.0" +VERSION = "2.1.1" From 3e6565be15f956ce2c714ed4e082cd6f61a814ac Mon Sep 17 00:00:00 2001 From: moomoohk Date: Tue, 29 Nov 2022 23:55:05 +0200 Subject: [PATCH 090/133] Cast path segment to int if it's supposed to be an index --- dpath/__init__.py | 14 ++------------ dpath/options.py | 1 - dpath/segments.py | 5 +++++ dpath/types.py | 2 +- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index c717314..c4c5c8f 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -21,7 +21,7 @@ ] from collections.abc import MutableMapping, MutableSequence -from typing import Union, List, Any, Callable, Optional +from typing import Union, List, Any, Callable, Optional, Sequence from dpath import segments, options from dpath.exceptions import InvalidKeyName, PathNotFound @@ -30,7 +30,7 @@ _DEFAULT_SENTINEL = object() -def _split_path(path: Path, separator: Optional[str]) -> Union[List[PathSegment], PathSegment]: +def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: """ Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it. @@ -45,16 +45,6 @@ def _split_path(path: Path, separator: Optional[str]) -> Union[List[PathSegment] else: split_segments = path.lstrip(separator).split(separator) - if options.CONVERT_INT_LIKE_SEGMENTS: - # Attempt to convert integer segments into actual integers. - final = [] - for segment in split_segments: - try: - final.append(int(segment)) - except ValueError: - final.append(segment) - split_segments = final - return split_segments diff --git a/dpath/options.py b/dpath/options.py index 91b4290..41f35c4 100644 --- a/dpath/options.py +++ b/dpath/options.py @@ -1,2 +1 @@ ALLOW_EMPTY_STRING_KEYS = False -CONVERT_INT_LIKE_SEGMENTS = True diff --git a/dpath/segments.py b/dpath/segments.py index ccf10e7..ed368bd 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -320,6 +320,11 @@ def set( # For everything except the last value, walk down the path and # create if creator is set. for (i, segment) in enumerate(segments[:-1]): + + # If segment is non-int but supposed to be a sequence index + if not isinstance(segment, int) and segment.isdigit() and isinstance(current, Sequence): + segment = int(segment) + try: # Optimistically try to get the next value. This makes the # code agnostic to whether current is a list or a dict. diff --git a/dpath/types.py b/dpath/types.py index b876e6a..d54a2e6 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -42,4 +42,4 @@ def creator( segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () - )""" + ) -> PathSegment""" From 056bf0de2fd06f4e4cafad40955ec5558d36c0f0 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 30 Nov 2022 00:00:31 +0200 Subject: [PATCH 091/133] Remove redundant import --- dpath/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index c4c5c8f..9f56e6b 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -21,7 +21,7 @@ ] from collections.abc import MutableMapping, MutableSequence -from typing import Union, List, Any, Callable, Optional, Sequence +from typing import Union, List, Any, Callable, Optional from dpath import segments, options from dpath.exceptions import InvalidKeyName, PathNotFound From 5b937f4e19e287ad8868a22cfb70c7a4dd470160 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 30 Nov 2022 00:12:49 +0200 Subject: [PATCH 092/133] Remove ambiguity of last path segment --- dpath/segments.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index ed368bd..faa763f 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -322,7 +322,7 @@ def set( for (i, segment) in enumerate(segments[:-1]): # If segment is non-int but supposed to be a sequence index - if not isinstance(segment, int) and segment.isdigit() and isinstance(current, Sequence): + if isinstance(segment, str) and isinstance(current, Sequence) and segment.isdigit(): segment = int(segment) try: @@ -341,10 +341,16 @@ def set( if i != length - 1 and leaf(current): raise PathNotFound(f"Path: {segments}[{i}]") - if isinstance(segments[-1], int): - extend(current, segments[-1]) + last_segment = segments[-1] - current[segments[-1]] = value + # Resolve ambiguity of last segment + if isinstance(last_segment, str) and isinstance(current, Sequence) and last_segment.isdigit(): + last_segment = int(last_segment) + + if isinstance(last_segment, int): + extend(current, last_segment) + + current[last_segment] = value return obj @@ -393,9 +399,11 @@ def view(obj, glob): view(obj, glob) -> obj' """ + def f(obj, pair, result): (segments, value) = pair if match(segments, glob): if not has(result, segments): set(result, segments, deepcopy(value), hints=types(obj, segments)) + return fold(obj, f, type(obj)()) From 023a0341ebb2c5d161175c14abe77dc1f46ab2ee Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 30 Nov 2022 00:17:46 +0200 Subject: [PATCH 093/133] Remove bad documentation --- dpath/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/types.py b/dpath/types.py index d54a2e6..b876e6a 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -42,4 +42,4 @@ def creator( segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () - ) -> PathSegment""" + )""" From 38007dff851e54743095490689fe7546dd23b408 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Wed, 30 Nov 2022 17:07:39 +0200 Subject: [PATCH 094/133] Add test for int ambiguity --- tests/test_new.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_new.py b/tests/test_new.py index 6da31e7..15b21c6 100644 --- a/tests/test_new.py +++ b/tests/test_new.py @@ -42,6 +42,16 @@ def test_set_new_list(): assert dict['a'][0] is None +def test_set_list_with_dict_int_ambiguity(): + d = {"list": [{"root": {"1": {"k": None}}}]} + + dpath.new(d, "list/0/root/1/k", "new") + + expected = {"list": [{"root": {"1": {"k": "new"}}}]} + + assert d == expected + + def test_set_new_list_path_with_separator(): # This test kills many birds with one stone, forgive me dict = { From 4a28d52e140a1a33bf322f543ad0dfba9acb1584 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 17:18:32 +0200 Subject: [PATCH 095/133] Support negative indexes --- dpath/segments.py | 24 +++++++++++++++++------- dpath/types.py | 22 ++++++++++++++++++++++ tests/test_search.py | 17 +++++++++++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index faa763f..baa07c8 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.types import PathSegment, Creator, Hints +from dpath.types import PathSegment, Creator, Hints, Glob, Path, CyclicInt def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: @@ -21,7 +21,10 @@ def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: return iter(node.items()) except AttributeError: try: - return zip(range(len(node)), node) + indices = range(len(node)) + # Make all list indices cyclic so negative (wraparound) indexes are supported + indices = map(lambda i: CyclicInt(i, len(node)), indices) + return zip(indices, node) except TypeError: # This can happen in cases where the node isn't leaf(node) == True, # but also isn't actually iterable. Instead of this being an error @@ -163,7 +166,7 @@ class Star(object): STAR = Star() -def match(segments: Sequence[PathSegment], glob: Sequence[str]): +def match(segments: Path, glob: Glob): """ Return True if the segments match the given glob, otherwise False. @@ -214,7 +217,9 @@ def match(segments: Sequence[PathSegment], glob: Sequence[str]): # If we were successful in matching up the lengths, then we can # compare them using fnmatch. if path_len == len(ss_glob): - for s, g in zip(map(int_str, segments), map(int_str, ss_glob)): + # TODO: Delete if not needed (previous code) - i = zip(map(int_str, segments), map(int_str, ss_glob)) + i = zip(segments, ss_glob) + for s, g in i: # Match the stars we added to the glob to the type of the # segment itself. if g is STAR: @@ -223,10 +228,15 @@ def match(segments: Sequence[PathSegment], glob: Sequence[str]): else: g = '*' - # Let's see if the glob matches. We will turn any kind of - # exception while attempting to match into a False for the - # match. try: + # If search path segment (s) is an int and the current evaluated index (g) is int-like, + # then g is surely a sequence index. Convert it to int and compare. + if isinstance(s, int) and isinstance(g, str) and (g.count("-") == 0 or g.lstrip("-").isdigit()): + return s == int(g) + + # Let's see if the glob matches. We will turn any kind of + # exception while attempting to match into a False for the + # match. if not fnmatchcase(s, g): return False except: diff --git a/dpath/types.py b/dpath/types.py index b876e6a..1890a03 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -2,6 +2,28 @@ from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping +class CyclicInt(int): + """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number)""" + + def __new__(cls, value, max_value, *args, **kwargs): + if value >= max_value: + raise TypeError( + f"Tried to initiate a CyclicInt with a value ({value}) " + f"greater than the provided max value ({max_value})" + ) + + obj = super().__new__(cls, value) + obj.max_value = max_value + + return obj + + def __eq__(self, other): + return int(self) == (self.max_value + other) % self.max_value + + def __repr__(self): + return f"" + + class MergeType(IntFlag): ADDITIVE = auto() """List objects are combined onto one long list (NOT a set). This is the default flag.""" diff --git a/tests/test_search.py b/tests/test_search.py index 5830088..d274d3d 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -236,3 +236,20 @@ def test_search_multiple_stars(): assert res['a'][0]['b'][0]['c'] == 1 assert res['a'][0]['b'][1]['c'] == 2 assert res['a'][0]['b'][2]['c'] == 3 + + +def test_search_glob_list(): + d = {'a': {'b': []}} + res = dpath.search(d, 'a/b/*') + assert res == {'a': {'b': []}} + + d = {'a': {'b': [1, 2, 3]}} + dpath.search(d, 'a/b/*', afilter=lambda x: x > 3 if isinstance(x, int) else True) + assert res == {'a': {'b': []}} + + +def test_search_negative_index(): + d = {'a': {'b': [1, 2, 3]}} + res = dpath.search(d, 'a/b/-1') + + assert res == dpath.search(d, "a/b/2") From 308b9779e668df31d80a3060d451b90d7e022551 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 17:43:19 +0200 Subject: [PATCH 096/133] Minor improvements --- dpath/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dpath/types.py b/dpath/types.py index 1890a03..1cd7242 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -3,9 +3,9 @@ class CyclicInt(int): - """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number)""" + """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number).""" - def __new__(cls, value, max_value, *args, **kwargs): + def __new__(cls, value: int, max_value: int, *args, **kwargs): if value >= max_value: raise TypeError( f"Tried to initiate a CyclicInt with a value ({value}) " From cb6b94b87a5e7cdc733e1882825aa5a3bced3af6 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 17:48:27 +0200 Subject: [PATCH 097/133] Improve negative number check --- dpath/segments.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index baa07c8..ece8230 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -230,9 +230,12 @@ def match(segments: Path, glob: Glob): try: # If search path segment (s) is an int and the current evaluated index (g) is int-like, - # then g is surely a sequence index. Convert it to int and compare. - if isinstance(s, int) and isinstance(g, str) and (g.count("-") == 0 or g.lstrip("-").isdigit()): - return s == int(g) + # then g is surely a sequence index as well. Convert it to int and compare. + if isinstance(s, int) and isinstance(g, str): + neg_c = g.count("-") + + if (neg_c == 0 and g.isdigit()) or (neg_c == 1 and g.lstrip("-").isdigit()): + return s == int(g) # Let's see if the glob matches. We will turn any kind of # exception while attempting to match into a False for the From 42ad0a1a0e75987d6fbdb50671ddd51140d712bc Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 17:58:49 +0200 Subject: [PATCH 098/133] Remove unnecessary negative number check --- dpath/segments.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index ece8230..6907a85 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -232,11 +232,12 @@ def match(segments: Path, glob: Glob): # If search path segment (s) is an int and the current evaluated index (g) is int-like, # then g is surely a sequence index as well. Convert it to int and compare. if isinstance(s, int) and isinstance(g, str): - neg_c = g.count("-") - - if (neg_c == 0 and g.isdigit()) or (neg_c == 1 and g.lstrip("-").isdigit()): - return s == int(g) + return s == int(g) + except: + # Will reach this point if g can't be converted to an int... + pass + try: # Let's see if the glob matches. We will turn any kind of # exception while attempting to match into a False for the # match. From 869fe2d6c5fc10397f054796438251bbf0ee130f Mon Sep 17 00:00:00 2001 From: moomoohk Date: Thu, 1 Dec 2022 18:05:06 +0200 Subject: [PATCH 099/133] Fix values to work with fnmatchcase --- dpath/segments.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 6907a85..1e5dc04 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -230,12 +230,13 @@ def match(segments: Path, glob: Glob): try: # If search path segment (s) is an int and the current evaluated index (g) is int-like, - # then g is surely a sequence index as well. Convert it to int and compare. + # then g might be a sequence index as well. Try converting it to an int. if isinstance(s, int) and isinstance(g, str): return s == int(g) except: - # Will reach this point if g can't be converted to an int... - pass + # Will reach this point if g can't be converted to an int (e.g. when g is a RegEx pattern). + # In this case convert s to a str so fnmatch can work on it. + s = str(s) try: # Let's see if the glob matches. We will turn any kind of From f15da4faa1c6618bf259b58d75dade0a3306efcc Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 11:52:23 +0200 Subject: [PATCH 100/133] Add str overload to CyclicInt --- dpath/types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dpath/types.py b/dpath/types.py index 1cd7242..1975dc1 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -23,6 +23,9 @@ def __eq__(self, other): def __repr__(self): return f"" + def __str__(self): + return str(int(self)) + class MergeType(IntFlag): ADDITIVE = auto() From 49e06d92718bba5f6f5f9136edace7051ec714f8 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:13:25 +0200 Subject: [PATCH 101/133] Simplify int handling in matching code --- dpath/segments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 1e5dc04..55914c5 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -229,9 +229,9 @@ def match(segments: Path, glob: Glob): g = '*' try: - # If search path segment (s) is an int and the current evaluated index (g) is int-like, - # then g might be a sequence index as well. Try converting it to an int. - if isinstance(s, int) and isinstance(g, str): + # If search path segment (s) is an int then assume currently evaluated index (g) might be a sequenc + # index as well. Try converting it to an int. + if isinstance(s, int): return s == int(g) except: # Will reach this point if g can't be converted to an int (e.g. when g is a RegEx pattern). From 6572e591b14dd0831e45dfeb8c24c186d96727de Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:13:30 +0200 Subject: [PATCH 102/133] Remove test case --- tests/test_search.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index d274d3d..7219d2b 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -238,16 +238,6 @@ def test_search_multiple_stars(): assert res['a'][0]['b'][2]['c'] == 3 -def test_search_glob_list(): - d = {'a': {'b': []}} - res = dpath.search(d, 'a/b/*') - assert res == {'a': {'b': []}} - - d = {'a': {'b': [1, 2, 3]}} - dpath.search(d, 'a/b/*', afilter=lambda x: x > 3 if isinstance(x, int) else True) - assert res == {'a': {'b': []}} - - def test_search_negative_index(): d = {'a': {'b': [1, 2, 3]}} res = dpath.search(d, 'a/b/-1') From 6c513fa673883630da9591a5a61786903f86f993 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:15:36 +0200 Subject: [PATCH 103/133] Bump version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 5b0431e..b777579 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.1" +VERSION = "2.1.2" From e37fdad711d1c3b720a331b1279171a37f612977 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:28:20 +0200 Subject: [PATCH 104/133] Continue evaluating entire path when handling int --- dpath/segments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 55914c5..ac7d390 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -229,10 +229,10 @@ def match(segments: Path, glob: Glob): g = '*' try: - # If search path segment (s) is an int then assume currently evaluated index (g) might be a sequenc + # If search path segment (s) is an int then assume currently evaluated index (g) might be a sequence # index as well. Try converting it to an int. - if isinstance(s, int): - return s == int(g) + if isinstance(s, int) and s == int(g): + continue except: # Will reach this point if g can't be converted to an int (e.g. when g is a RegEx pattern). # In this case convert s to a str so fnmatch can work on it. From 58407dbda7fcd915df80dc206f48432ac2503846 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 14:34:51 +0200 Subject: [PATCH 105/133] Add type hints --- dpath/segments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/segments.py b/dpath/segments.py index ac7d390..b2f55fd 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -406,7 +406,7 @@ def foldm(obj, f, acc): return acc -def view(obj, glob): +def view(obj: MutableMapping, glob: Glob): """ Return a view of the object where the glob matches. A view retains the same form as the obj, but is limited to only the paths that From 317e3cd5f680b1543d8b511fad1a40db922caf4c Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 2 Dec 2022 14:35:01 +0200 Subject: [PATCH 106/133] Improve CyclicInt type --- dpath/types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dpath/types.py b/dpath/types.py index 1975dc1..6906224 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -18,10 +18,13 @@ def __new__(cls, value: int, max_value: int, *args, **kwargs): return obj def __eq__(self, other): + if not isinstance(other, int): + return False + return int(self) == (self.max_value + other) % self.max_value def __repr__(self): - return f"" + return f"" def __str__(self): return str(int(self)) From 66a64a390544ae9014884524941844410970ecbe Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sat, 3 Dec 2022 22:30:15 +0200 Subject: [PATCH 107/133] Rename CyclicInt to SymmetricInt --- dpath/segments.py | 7 +++---- dpath/types.py | 9 ++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index b2f55fd..7a48817 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.types import PathSegment, Creator, Hints, Glob, Path, CyclicInt +from dpath.types import PathSegment, Creator, Hints, Glob, Path, SymmetricInt def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: @@ -22,8 +22,8 @@ def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: except AttributeError: try: indices = range(len(node)) - # Make all list indices cyclic so negative (wraparound) indexes are supported - indices = map(lambda i: CyclicInt(i, len(node)), indices) + # Convert all list indices to object so negative indexes are supported. + indices = map(lambda i: SymmetricInt(i, len(node)), indices) return zip(indices, node) except TypeError: # This can happen in cases where the node isn't leaf(node) == True, @@ -217,7 +217,6 @@ def match(segments: Path, glob: Glob): # If we were successful in matching up the lengths, then we can # compare them using fnmatch. if path_len == len(ss_glob): - # TODO: Delete if not needed (previous code) - i = zip(map(int_str, segments), map(int_str, ss_glob)) i = zip(segments, ss_glob) for s, g in i: # Match the stars we added to the glob to the type of the diff --git a/dpath/types.py b/dpath/types.py index 6906224..210b24a 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -2,13 +2,13 @@ from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping -class CyclicInt(int): +class SymmetricInt(int): """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number).""" def __new__(cls, value: int, max_value: int, *args, **kwargs): if value >= max_value: raise TypeError( - f"Tried to initiate a CyclicInt with a value ({value}) " + f"Tried to initiate a {cls.__name__} with a value ({value}) " f"greater than the provided max value ({max_value})" ) @@ -21,10 +21,13 @@ def __eq__(self, other): if not isinstance(other, int): return False + if other >= self.max_value or other >= -self.max_value: + return False + return int(self) == (self.max_value + other) % self.max_value def __repr__(self): - return f"" + return f"<{self.__class__.__name__} {int(self)}%{self.max_value}>" def __str__(self): return str(int(self)) From eb1b2e2f9d18bae145fb47819f9b1af8a1a8bc3f Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sat, 3 Dec 2022 22:36:02 +0200 Subject: [PATCH 108/133] Fix sign --- dpath/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/types.py b/dpath/types.py index 210b24a..7bf3d2d 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -21,7 +21,7 @@ def __eq__(self, other): if not isinstance(other, int): return False - if other >= self.max_value or other >= -self.max_value: + if other >= self.max_value or other <= -self.max_value: return False return int(self) == (self.max_value + other) % self.max_value From 41c2652a88a0a663f1995461e76bb07b3fe03978 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Tue, 13 Dec 2022 09:20:41 +0200 Subject: [PATCH 109/133] Remove trailing comma --- dpath/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/util.py b/dpath/util.py index 60d0319..138a5e5 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -48,4 +48,4 @@ def search(obj, glob, yielded=False, separator="/", afilter=None, dirs=True): @deprecated def merge(dst, src, separator="/", afilter=None, flags=MergeType.ADDITIVE): - return dpath.merge(dst, src, separator, afilter, flags), + return dpath.merge(dst, src, separator, afilter, flags) From f3303eb81d0ed06eb98346bf041a9c4c7ed7f2af Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Tue, 13 Dec 2022 09:22:01 +0200 Subject: [PATCH 110/133] Bump version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index b777579..4260069 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.2" +VERSION = "2.1.3" From f871299e36626d7d3ab4ed8328eab2e283024a51 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 8 Jan 2023 18:33:03 +0200 Subject: [PATCH 111/133] Better int ambiguity resolution in default creator --- dpath/segments.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 7a48817..b2df380 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,6 +1,6 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import List, Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping +from typing import List, Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping, MutableSequence from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound @@ -254,7 +254,7 @@ def match(segments: Path, glob: Glob): return False -def extend(thing: List, index: int, value=None): +def extend(thing: MutableSequence, index: int, value=None): """ Extend a sequence like thing such that it contains at least index + 1 many elements. The extension values will be None (default). @@ -280,7 +280,7 @@ def extend(thing: List, index: int, value=None): def _default_creator( - current: Union[MutableMapping, List], + current: Union[MutableMapping, Sequence], segments: Sequence[PathSegment], i: int, hints: Sequence[Tuple[PathSegment, type]] = () @@ -294,7 +294,10 @@ def _default_creator( segment = segments[i] length = len(segments) - if isinstance(segment, int): + if isinstance(current, Sequence): + segment = int(segment) + + if isinstance(current, MutableSequence): extend(current, segment) # Infer the type from the hints provided. @@ -308,7 +311,7 @@ def _default_creator( else: segment_next = None - if isinstance(segment_next, int): + if isinstance(segment_next, int) or segment_next.isdigit(): current[segment] = [] else: current[segment] = {} From 436dac44bc33558743c29acce2e2bdc2b34307ec Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 8 Jan 2023 18:34:37 +0200 Subject: [PATCH 112/133] Remove unused import --- dpath/segments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/segments.py b/dpath/segments.py index b2df380..fa18c8f 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,6 +1,6 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import List, Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping, MutableSequence +from typing import Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping, MutableSequence from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound From b07cc6a9776981e627d224f216c385196e7ace76 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 8 Jan 2023 18:59:24 +0200 Subject: [PATCH 113/133] Resolve int ambiguity in get function --- dpath/segments.py | 9 ++++++--- dpath/types.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index fa18c8f..56ac0ed 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -81,7 +81,7 @@ def walk(obj, location=()): yield found -def get(obj, segments): +def get(obj, segments: Path): """ Return the value at the path indicated by segments. @@ -92,6 +92,9 @@ def get(obj, segments): if leaf(current): raise PathNotFound(f"Path: {segments}[{i}]") + if isinstance(current, Sequence) and isinstance(segment, (str, bytes)) and segment.isdigit(): + segment = int(segment) + current = current[segment] return current @@ -339,7 +342,7 @@ def set( for (i, segment) in enumerate(segments[:-1]): # If segment is non-int but supposed to be a sequence index - if isinstance(segment, str) and isinstance(current, Sequence) and segment.isdigit(): + if isinstance(segment, (str, bytes)) and isinstance(current, Sequence) and segment.isdigit(): segment = int(segment) try: @@ -361,7 +364,7 @@ def set( last_segment = segments[-1] # Resolve ambiguity of last segment - if isinstance(last_segment, str) and isinstance(current, Sequence) and last_segment.isdigit(): + if isinstance(last_segment, (str, bytes)) and isinstance(current, Sequence) and last_segment.isdigit(): last_segment = int(last_segment) if isinstance(last_segment, int): diff --git a/dpath/types.py b/dpath/types.py index 7bf3d2d..c4a4a56 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -46,7 +46,7 @@ class MergeType(IntFlag): replaces the destination in this situation.""" -PathSegment = Union[int, str] +PathSegment = Union[int, str, bytes] """Type alias for dict path segments where integers are explicitly casted.""" Filter = Callable[[Any], bool] From 58b34df1721392006e05e89460f7e33ff4351833 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 8 Jan 2023 19:33:15 +0200 Subject: [PATCH 114/133] Use isdecimal in favor of isdigit --- dpath/segments.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index 56ac0ed..c3c9846 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -92,7 +92,7 @@ def get(obj, segments: Path): if leaf(current): raise PathNotFound(f"Path: {segments}[{i}]") - if isinstance(current, Sequence) and isinstance(segment, (str, bytes)) and segment.isdigit(): + if isinstance(current, Sequence) and isinstance(segment, str) and segment.isdecimal(): segment = int(segment) current = current[segment] @@ -314,7 +314,7 @@ def _default_creator( else: segment_next = None - if isinstance(segment_next, int) or segment_next.isdigit(): + if isinstance(segment_next, int) or (isinstance(segment_next, str) and segment_next.isdecimal()): current[segment] = [] else: current[segment] = {} @@ -342,7 +342,7 @@ def set( for (i, segment) in enumerate(segments[:-1]): # If segment is non-int but supposed to be a sequence index - if isinstance(segment, (str, bytes)) and isinstance(current, Sequence) and segment.isdigit(): + if isinstance(segment, str) and isinstance(current, Sequence) and segment.isdecimal(): segment = int(segment) try: @@ -364,7 +364,7 @@ def set( last_segment = segments[-1] # Resolve ambiguity of last segment - if isinstance(last_segment, (str, bytes)) and isinstance(current, Sequence) and last_segment.isdigit(): + if isinstance(last_segment, str) and isinstance(current, Sequence) and last_segment.isdecimal(): last_segment = int(last_segment) if isinstance(last_segment, int): From cbed5f143fda3c1c4f9cacdf6bd8af2d2a0383e2 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 8 Jan 2023 20:49:35 +0200 Subject: [PATCH 115/133] Add type check tests Thanks to @harel --- tests/test_new.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_new.py b/tests/test_new.py index 15b21c6..ac47e7d 100644 --- a/tests/test_new.py +++ b/tests/test_new.py @@ -52,6 +52,27 @@ def test_set_list_with_dict_int_ambiguity(): assert d == expected +def test_int_segment_list_type_check(): + d = {} + dpath.new(d, "a/b/0/c/0", "hello") + assert 'b' in d.get("a", {}) + assert isinstance(d["a"]["b"], list) + assert len(d["a"]["b"]) == 1 + assert 'c' in d["a"]["b"][0] + assert isinstance(d["a"]["b"][0]["c"], list) + assert len(d["a"]["b"][0]["c"]) == 1 + + +def test_int_segment_dict_type_check(): + d = {"a": {"b": {"0": {}}}} + dpath.new(d, "a/b/0/c/0", "hello") + assert "b" in d.get("a", {}) + assert isinstance(d["a"]["b"], dict) + assert '0' in d["a"]["b"] + assert 'c' in d["a"]["b"]["0"] + assert isinstance(d["a"]["b"]["0"]["c"], list) + + def test_set_new_list_path_with_separator(): # This test kills many birds with one stone, forgive me dict = { From 013b3a793c7a81f4577939c95808717f5a570fb1 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Mon, 9 Jan 2023 08:55:27 +0200 Subject: [PATCH 116/133] Update version.py --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 4260069..5dfae46 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.3" +VERSION = "2.1.4" From 35c371d05cca02b268df2cf0bc2a9e6537576e61 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 17 Mar 2023 09:52:24 +0100 Subject: [PATCH 117/133] add py.typed file --- dpath/py.typed | 0 setup.py | 8 +++++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 dpath/py.typed diff --git a/dpath/py.typed b/dpath/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index a6891d6..d34748e 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ import os -from distutils.core import setup +from setuptools import setup import dpath.version @@ -25,6 +25,7 @@ scripts=[], packages=["dpath"], data_files=[], + package_data={"dpath": ["py.typed"]}, # Type hints are great. # Function annotations were added in Python 3.0. @@ -43,6 +44,11 @@ 'License :: OSI Approved :: MIT License', 'Natural Language :: English', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Software Development :: Libraries :: Python Modules', + 'Typing :: Typed', ], ) From 995b7d97a2eb8b87d2a10c43e23cf3c74b1bf9ee Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Fri, 24 Mar 2023 17:17:56 +0300 Subject: [PATCH 118/133] Update version.py --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 5dfae46..b82fac5 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.4" +VERSION = "2.1.5" From 7684ad6bef5fb67d194f52a3867fbf71538f06bc Mon Sep 17 00:00:00 2001 From: Reuben Gardos Reid <5456207+ReubenJ@users.noreply.github.com> Date: Tue, 9 May 2023 13:34:44 +0200 Subject: [PATCH 119/133] Add testing for Python 3.11 --- .github/workflows/tests.yml | 6 +++--- setup.py | 1 + tox.ini | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba8e2bf..40e44d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,10 +30,10 @@ jobs: - name: Check out code uses: actions/checkout@main - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@main with: - python-version: "3.10" + python-version: "3.11" - name: Setup flake8 annotations uses: rbialon/flake8-annotations@v1.1 @@ -70,7 +70,7 @@ jobs: strategy: matrix: # Match versions specified in tox.ini - python-version: ['3.8', '3.9', '3.10', 'pypy-3.7'] + python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.7'] # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/setup.py b/setup.py index d34748e..2f79c58 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Software Development :: Libraries :: Python Modules', 'Typing :: Typed', ], diff --git a/tox.ini b/tox.ini index d613837..a3ca270 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ ignore = E501,E722 [tox] -envlist = pypy37, py38, py39, py310 +envlist = pypy37, py38, py39, py310, py311 [gh-actions] python = @@ -15,6 +15,7 @@ python = 3.8: py38 3.9: py39 3.10: py310 + 3.11: py311 [testenv] deps = From f68a87cd7264c5646f99da19757783aa95902e17 Mon Sep 17 00:00:00 2001 From: Reuben Gardos Reid <5456207+ReubenJ@users.noreply.github.com> Date: Thu, 11 May 2023 14:52:09 +0200 Subject: [PATCH 120/133] Bump version.py --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index b82fac5..73bee60 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.5" +VERSION = "2.1.6" From 9ffb6df5f34398e6748e9d4182542d8abb527187 Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Sun, 28 Apr 2024 00:43:16 +0200 Subject: [PATCH 121/133] use newer unittest.mock when available --- tests/test_get_values.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_get_values.py b/tests/test_get_values.py index 9eeef82..8fd5107 100644 --- a/tests/test_get_values.py +++ b/tests/test_get_values.py @@ -2,7 +2,11 @@ import decimal import time -import mock +try: + from unittest import mock +except ImportError: + import mock + from nose2.tools.such import helper import dpath From a673d24e727df8beaac9ff08f97ded8fa4471d2d Mon Sep 17 00:00:00 2001 From: squatched Date: Fri, 31 May 2024 14:08:47 -0500 Subject: [PATCH 122/133] Support Empty List Path In get Solves Issue #193. Currently, dpath.get() works with a path of '/' by returning the value of the object passed in. This behavior is not reflected with an empty list. Now, passing an empty list to dpath.get() will behave the same way (returning the root object value). --- dpath/__init__.py | 2 +- tests/test_get_values.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index 9f56e6b..faa62c7 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -167,7 +167,7 @@ def get( If more than one leaf matches the glob, ValueError is raised. If the glob is not found and a default is not provided, KeyError is raised. """ - if glob == "/": + if glob == "/" or (type(glob) is not str and len(glob) == 0): return obj globlist = _split_path(glob, separator) diff --git a/tests/test_get_values.py b/tests/test_get_values.py index 9eeef82..2ae506b 100644 --- a/tests/test_get_values.py +++ b/tests/test_get_values.py @@ -17,6 +17,9 @@ def test_util_get_root(): ret = dpath.get(x, '/') assert ret == x + ret = dpath.get(x, []) + assert ret == x + def test_get_explicit_single(): ehash = { From e137b9f2e091b6bdf2d043b355242a97329f9c07 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:39:13 +0300 Subject: [PATCH 123/133] Minor flake8 change --- dpath/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index 9f56e6b..be61ddb 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -316,7 +316,7 @@ def merger(dst, src, _segments=()): target = segments.get(dst, current_path) # If the types don't match, replace it. - if type(found) != type(target) and not are_both_mutable(found, target): + if type(found) is not type(target) and not are_both_mutable(found, target): segments.set(dst, current_path, found) continue From dc3f5d9d70f492af9b61226dcf95603003801b35 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Tue, 11 Jun 2024 12:19:24 +0300 Subject: [PATCH 124/133] Simplify condition --- dpath/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index 8bb6027..5c9faca 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -167,7 +167,7 @@ def get( If more than one leaf matches the glob, ValueError is raised. If the glob is not found and a default is not provided, KeyError is raised. """ - if glob == "/" or (type(glob) is not str and len(glob) == 0): + if isinstance(glob, str) and glob == "/" or len(glob) == 0: return obj globlist = _split_path(glob, separator) From 34f68ae80397875cc373e24da6c59229dacd6b99 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Tue, 11 Jun 2024 12:43:28 +0300 Subject: [PATCH 125/133] Always import mock from unittest --- tests/test_get_values.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_get_values.py b/tests/test_get_values.py index 4b7f4fe..68d5d3b 100644 --- a/tests/test_get_values.py +++ b/tests/test_get_values.py @@ -2,10 +2,7 @@ import decimal import time -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock from nose2.tools.such import helper From ac48a075e2e209df7250eafa8c0575dd7ddfedb4 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Wed, 12 Jun 2024 22:47:29 +0300 Subject: [PATCH 126/133] Minor list index improvements --- dpath/segments.py | 6 +++--- dpath/types.py | 19 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/dpath/segments.py b/dpath/segments.py index c3c9846..f1443f0 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -4,7 +4,7 @@ from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath.types import PathSegment, Creator, Hints, Glob, Path, SymmetricInt +from dpath.types import PathSegment, Creator, Hints, Glob, Path, ListIndex def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: @@ -22,8 +22,8 @@ def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: except AttributeError: try: indices = range(len(node)) - # Convert all list indices to object so negative indexes are supported. - indices = map(lambda i: SymmetricInt(i, len(node)), indices) + # Convert all list indices to objects so negative indices are supported. + indices = map(lambda i: ListIndex(i, len(node)), indices) return zip(indices, node) except TypeError: # This can happen in cases where the node isn't leaf(node) == True, diff --git a/dpath/types.py b/dpath/types.py index c4a4a56..dc89614 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -2,18 +2,18 @@ from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping -class SymmetricInt(int): - """Same as a normal int but mimicks the behavior of list indexes (can be compared to a negative number).""" +class ListIndex(int): + """Same as a normal int but mimics the behavior of list indices (can be compared to a negative number).""" - def __new__(cls, value: int, max_value: int, *args, **kwargs): - if value >= max_value: + def __new__(cls, value: int, list_length: int, *args, **kwargs): + if value >= list_length: raise TypeError( f"Tried to initiate a {cls.__name__} with a value ({value}) " - f"greater than the provided max value ({max_value})" + f"greater than the provided max value ({list_length})" ) obj = super().__new__(cls, value) - obj.max_value = max_value + obj.list_length = list_length return obj @@ -21,13 +21,10 @@ def __eq__(self, other): if not isinstance(other, int): return False - if other >= self.max_value or other <= -self.max_value: - return False - - return int(self) == (self.max_value + other) % self.max_value + return other == int(self) or self.list_length + other == int(self) def __repr__(self): - return f"<{self.__class__.__name__} {int(self)}%{self.max_value}>" + return f"<{self.__class__.__name__} {int(self)}/{self.list_length}>" def __str__(self): return str(int(self)) From 3d78a18a5f8a8fdf2075c588dada243ebd090f65 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Wed, 12 Jun 2024 22:55:37 +0300 Subject: [PATCH 127/133] Add comment --- dpath/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dpath/types.py b/dpath/types.py index dc89614..4f10162 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -21,6 +21,7 @@ def __eq__(self, other): if not isinstance(other, int): return False + # Based on how Python sequences handle negative indices as described in footnote (3) of https://docs.python.org/3/library/stdtypes.html#common-sequence-operations return other == int(self) or self.list_length + other == int(self) def __repr__(self): From a93d415b7caa9bf6fce60c5daf21a0f880d45d8c Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:16:25 +0300 Subject: [PATCH 128/133] Add 3.12 support --- setup.py | 1 + tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2f79c58..28eb69c 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Software Development :: Libraries :: Python Modules', 'Typing :: Typed', ], diff --git a/tox.ini b/tox.ini index a3ca270..ff9a86d 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ ignore = E501,E722 [tox] -envlist = pypy37, py38, py39, py310, py311 +envlist = pypy37, py38, py39, py310, py311, py312 [gh-actions] python = @@ -16,10 +16,10 @@ python = 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 [testenv] deps = hypothesis - mock nose2 commands = nose2 {posargs} From 74b4b7e12dc0fb1417a92fdf4401dab7b92369f5 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:27:44 +0300 Subject: [PATCH 129/133] Improve None type hints --- dpath/__init__.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index 5c9faca..2bc9942 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -1,3 +1,6 @@ +# Needed for pre-3.10 versions +from __future__ import annotations + __all__ = [ "new", "delete", @@ -48,7 +51,7 @@ def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSe return split_segments -def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator = None) -> MutableMapping: +def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator | None = None) -> MutableMapping: """ Set the element at the terminus of path to value, and create it if it does not exist (as opposed to 'set' that can only @@ -68,7 +71,7 @@ def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator return segments.set(obj, split_segments, value) -def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None) -> int: +def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter | None = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -127,7 +130,7 @@ def f(obj, pair, counter): return deleted -def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter = None) -> int: +def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter | None = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -193,7 +196,7 @@ def f(_, pair, results): return results[0] -def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None, dirs=True): +def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter | None = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -203,13 +206,13 @@ def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = Non return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter = None, dirs=True): +def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter | None = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. If 'yielded' is true, then a dictionary will not be returned. - Instead tuples will be yielded in the form of (path, value) for + Instead, tuples will be yielded in the form of (path, value) for every element in the document that matched the glob. """ @@ -218,7 +221,7 @@ def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilte def keeper(path, found): """ Generalized test for use in both yielded and folded cases. - Returns True if we want this result. Otherwise returns False. + Returns True if we want this result. Otherwise, returns False. """ if not dirs and not segments.leaf(found): return False @@ -245,7 +248,13 @@ def f(obj, pair, result): return segments.fold(obj, f, {}) -def merge(dst: MutableMapping, src: MutableMapping, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): +def merge( + dst: MutableMapping, + src: MutableMapping, + separator="/", + afilter: Filter | None = None, + flags=MergeType.ADDITIVE +): """ Merge source into destination. Like dict.update() but performs deep merging. From da52b6698929e4bd2617fc5c9145a8b64825673f Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:31:56 +0300 Subject: [PATCH 130/133] Add 3.12 to test matrix --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 40e44d3..1170665 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -70,7 +70,7 @@ jobs: strategy: matrix: # Match versions specified in tox.ini - python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.7'] + python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.7', '3.12'] # Steps represent a sequence of tasks that will be executed as part of the job steps: From 6f34f48cdd2ac0e4d99981f69dd64cb770b019ea Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:43:05 +0300 Subject: [PATCH 131/133] Flake with 3.12 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1170665..4f7044b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,10 +30,10 @@ jobs: - name: Check out code uses: actions/checkout@main - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@main with: - python-version: "3.11" + python-version: "3.12" - name: Setup flake8 annotations uses: rbialon/flake8-annotations@v1.1 From 6e569af55b270bc3306300792c913ac6eb8b179f Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:57:49 +0300 Subject: [PATCH 132/133] Update flake8 action --- .github/workflows/tests.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4f7044b..d350624 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,12 +36,9 @@ jobs: python-version: "3.12" - name: Setup flake8 annotations - uses: rbialon/flake8-annotations@v1.1 - - - name: Lint with flake8 - run: | - pip install flake8 - flake8 setup.py dpath/ tests/ + uses: TrueBrain/actions-flake8@v2.3 + with: + path: setup.py dpath/ tests/ # Generate a common hashseed for all tests generate-hashseed: From c8722e6b815bedf4e6aaeea9ccc7d6ff3e9b4f84 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Thu, 13 Jun 2024 01:07:41 +0300 Subject: [PATCH 133/133] Update version.py --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 73bee60..3c00bb4 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.6" +VERSION = "2.2.0"