From 5986257f9fc978d4a61b6e0001df554f80e565cb Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Wed, 5 Jun 2019 18:01:01 -0700 Subject: [PATCH 01/36] changes for 5.1.1 release --- CHANGES | 5 +++++ announcement.msg | 50 ++++++------------------------------------- lib/yaml/__init__.py | 2 +- lib3/yaml/__init__.py | 2 +- setup.py | 2 +- 5 files changed, 15 insertions(+), 46 deletions(-) diff --git a/CHANGES b/CHANGES index 91f0255bb..ce7ce4900 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,11 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/commits/ * https://bitbucket.org/xi/pyyaml/commits/ +5.1.1 (2019-06-05) +------------------ + +* Re-release of 5.1 with regenerated Cython sources to build properly for Python 3.8 + 5.1 (2019-03-13) ---------------- diff --git a/announcement.msg b/announcement.msg index 99bf92422..d3fa69c67 100644 --- a/announcement.msg +++ b/announcement.msg @@ -1,58 +1,22 @@ From: Ingy döt Net To: python-list@python.org, python-announce@python.org, yaml-core@lists.sourceforge.net -Subject: [ANN] PyYAML-5.1: YAML parser and emitter for Python +Subject: [ANN] PyYAML-5.1.1: YAML parser and emitter for Python ======================= - Announcing PyYAML-5.1 +Announcing PyYAML-5.1.1 ======================= -A new MAJOR RELEASE of PyYAML is now available: +A new minor release of PyYAML is now available: https://pypi.org/project/PyYAML/ -This is the first major release of PyYAML under the new maintenance team. - -Among the many changes listed below, this release specifically addresses the -arbitrary code execution issue raised by: - - https://nvd.nist.gov/vuln/detail/CVE-2017-18342 - -(See https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation -for complete details). - -The PyYAML project is now maintained by the YAML and Python communities. -Planning happens on the #yaml-dev, #pyyaml and #libyaml IRC channels on -irc.freenode.net. - +This is a maintenance re-release of PyYAML 5.1 with re-generated Cython +bindings to allow the extension to build properly for Python 3.8. No code +changes were made in the PyYAML sources. Changes ======= -* https://github.com/yaml/pyyaml/pull/35 -- Some modernization of the test running -* https://github.com/yaml/pyyaml/pull/42 -- Install tox in a virtualenv -* https://github.com/yaml/pyyaml/pull/45 -- Allow colon in a plain scalar in a flow context -* https://github.com/yaml/pyyaml/pull/48 -- Fix typos -* https://github.com/yaml/pyyaml/pull/55 -- Improve RepresenterError creation -* https://github.com/yaml/pyyaml/pull/59 -- Resolves #57, update readme issues link -* https://github.com/yaml/pyyaml/pull/60 -- Document and test Python 3.6 support -* https://github.com/yaml/pyyaml/pull/61 -- Use Travis CI built in pip cache support -* https://github.com/yaml/pyyaml/pull/62 -- Remove tox workaround for Travis CI -* https://github.com/yaml/pyyaml/pull/63 -- Adding support to Unicode characters over codepoint 0xffff -* https://github.com/yaml/pyyaml/pull/65 -- Support unicode literals over codepoint 0xffff -* https://github.com/yaml/pyyaml/pull/75 -- add 3.12 changelog -* https://github.com/yaml/pyyaml/pull/76 -- Fallback to Pure Python if Compilation fails -* https://github.com/yaml/pyyaml/pull/84 -- Drop unsupported Python 3.3 -* https://github.com/yaml/pyyaml/pull/102 -- Include license file in the generated wheel package -* https://github.com/yaml/pyyaml/pull/105 -- Removed Python 2.6 & 3.3 support -* https://github.com/yaml/pyyaml/pull/111 -- Remove commented out Psyco code -* https://github.com/yaml/pyyaml/pull/129 -- Remove call to `ord` in lib3 emitter code -* https://github.com/yaml/pyyaml/pull/143 -- Allow to turn off sorting keys in Dumper -* https://github.com/yaml/pyyaml/pull/149 -- Test on Python 3.7-dev -* https://github.com/yaml/pyyaml/pull/158 -- Support escaped slash in double quotes "\/" -* https://github.com/yaml/pyyaml/pull/181 -- Import Hashable from collections.abc -* https://github.com/yaml/pyyaml/pull/256 -- Make default_flow_style=False -* https://github.com/yaml/pyyaml/pull/257 -- Deprecate yaml.load and add FullLoader and UnsafeLoader classes -* https://github.com/yaml/pyyaml/pull/263 -- Windows Appveyor build - +* None Resources ========= diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index e7a419dd2..0688b7d9b 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.1' +__version__ = '5.1.1' try: from cyaml import * diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 5df0bb5fd..8686c63e6 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.1' +__version__ = '5.1.1' try: from .cyaml import * __with_libyaml__ = True diff --git a/setup.py b/setup.py index e21ce9f73..81f01e471 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.1' +VERSION = '5.1.1' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability From 0f64cbfa54b0b22dc7b776b7b98a7cd657e84d78 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 30 Jul 2019 17:40:33 -0700 Subject: [PATCH 02/36] changes for 5.1.2 release --- .appveyor.yml | 6 +++--- .travis.yml | 9 +++++---- CHANGES | 7 ++++++- announcement.msg | 6 +++--- lib/yaml/__init__.py | 2 +- lib3/yaml/__init__.py | 2 +- packaging/build/appveyor.ps1 | 15 ++++++++------- setup.py | 3 ++- 8 files changed, 29 insertions(+), 21 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index bdad5f319..46bbd7f0a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,9 +4,9 @@ version: '{build}' image: - Visual Studio 2015 -#cache: -#- 'C:\Python38\' -#- 'C:\Python38-x64' +cache: +- 'C:\Python38\' +- 'C:\Python38-x64' environment: libyaml_repo_url: https://github.com/yaml/libyaml.git diff --git a/.travis.yml b/.travis.yml index 3dfe3ad9c..ecce0331b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,11 +16,12 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 3.7-dev + - python: 3.7 env: TOXENV=py37 - # This is broken on travis as of 2019/03/12 - # - python: pypy - # env: TOXENV=pypy + - python: 3.8-dev + env: TOXENV=py38 + - python: pypy + env: TOXENV=pypy # build libyaml before_script: diff --git a/CHANGES b/CHANGES index ce7ce4900..e2eed15ee 100644 --- a/CHANGES +++ b/CHANGES @@ -4,10 +4,15 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/commits/ * https://bitbucket.org/xi/pyyaml/commits/ +5.1.2 (2019-07-30) +------------------ + +* Re-release of 5.1 with regenerated Cython sources to build properly for Python 3.8b2+ + 5.1.1 (2019-06-05) ------------------ -* Re-release of 5.1 with regenerated Cython sources to build properly for Python 3.8 +* Re-release of 5.1 with regenerated Cython sources to build properly for Python 3.8b1 5.1 (2019-03-13) ---------------- diff --git a/announcement.msg b/announcement.msg index d3fa69c67..4cf3944fe 100644 --- a/announcement.msg +++ b/announcement.msg @@ -1,16 +1,16 @@ From: Ingy döt Net To: python-list@python.org, python-announce@python.org, yaml-core@lists.sourceforge.net -Subject: [ANN] PyYAML-5.1.1: YAML parser and emitter for Python +Subject: [ANN] PyYAML-5.1.2: YAML parser and emitter for Python ======================= -Announcing PyYAML-5.1.1 +Announcing PyYAML-5.1.2 ======================= A new minor release of PyYAML is now available: https://pypi.org/project/PyYAML/ This is a maintenance re-release of PyYAML 5.1 with re-generated Cython -bindings to allow the extension to build properly for Python 3.8. No code +bindings to allow the extension to build properly for Python 3.8b2+. No code changes were made in the PyYAML sources. Changes diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 0688b7d9b..5d56d6556 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.1.1' +__version__ = '5.1.2' try: from cyaml import * diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 8686c63e6..3f499d366 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.1.1' +__version__ = '5.1.2' try: from .cyaml import * __with_libyaml__ = True diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index a9e6ad289..fc75b53de 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -6,15 +6,14 @@ # Update-AppveyorBuild -Version $dynamic_version Function Bootstrap() { - # uncomment when we want to start testing on Python 3.8 # ensure py38 is present (current Appveyor VS2015 image doesn't include it) - #If(-not $(Test-Path C:\Python38)) { - # choco.exe install python3 --version=3.8.0-a2 --forcex86 --force #--install-arguments="TargetDir=C:\Python38 PrependPath=0" --no-progress - #} + If(-not $(Test-Path C:\Python38)) { + choco.exe install python3 --version=3.8.0-b3 --forcex86 --force --params="/InstallDir:C:\Python38" --no-progress + } - #If(-not $(Test-Path C:\Python38-x64)) { - # choco.exe install python3 --version=3.8.0-a2 --force #--install-arguments="TargetDir=C:\Python38-x64 PrependPath=0" --no-progress - #} + If(-not $(Test-Path C:\Python38-x64)) { + choco.exe install python3 --version=3.8.0-b3 --force --params="/InstallDir:C:\Python38-x64" --no-progress + } Write-Output "patching Windows SDK bits for distutils" @@ -123,6 +122,8 @@ $pythons = @( "C:\Python36-x64" "C:\Python37" "C:\Python37-x64" +"C:\Python38" +"C:\Python38-x64" ) #$pythons = @("C:\$($env:PYTHON_VER)") diff --git a/setup.py b/setup.py index 81f01e471..5cb0b6d25 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.1.1' +VERSION = '5.1.2' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability @@ -33,6 +33,7 @@ "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", From 31f2279252e9d1a3093b738570c26f44a829f0a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller=20=28tinita=29?= Date: Sun, 31 Mar 2019 16:14:50 +0200 Subject: [PATCH 03/36] Fix logic for quoting special characters (#276) * Fix logic for quoting special characters * Remove has_ucs4 from condition on systems with `sys.maxunicode <= 0xffff` the comparison (u'\U00010000' <= ch < u'\U0010ffff') can't be true anyway I think --- lib/yaml/emitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/yaml/emitter.py b/lib/yaml/emitter.py index 9561a8274..23c25ca80 100644 --- a/lib/yaml/emitter.py +++ b/lib/yaml/emitter.py @@ -706,7 +706,7 @@ def analyze_scalar(self, scalar): if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'): if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF' or u'\uE000' <= ch <= u'\uFFFD' - or ((not has_ucs4) or (u'\U00010000' <= ch < u'\U0010ffff'))) and ch != u'\uFEFF': + or (u'\U00010000' <= ch < u'\U0010ffff')) and ch != u'\uFEFF': unicode_characters = True if not self.allow_unicode: special_characters = True From 8a7d0ed1628dc3b21e391b6cabd002dcadd2eee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller=20=28tinita=29?= Date: Sun, 31 Mar 2019 16:16:27 +0200 Subject: [PATCH 04/36] Update CHANGES for 5.1 (#280) * Update CHANGES Some PRs were forgotten, and some were listed that weren't merged. --- CHANGES | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index e2eed15ee..29abdf976 100644 --- a/CHANGES +++ b/CHANGES @@ -27,7 +27,6 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/pull/61 -- Use Travis CI built in pip cache support * https://github.com/yaml/pyyaml/pull/62 -- Remove tox workaround for Travis CI * https://github.com/yaml/pyyaml/pull/63 -- Adding support to Unicode characters over codepoint 0xffff -* https://github.com/yaml/pyyaml/pull/65 -- Support unicode literals over codepoint 0xffff * https://github.com/yaml/pyyaml/pull/75 -- add 3.12 changelog * https://github.com/yaml/pyyaml/pull/76 -- Fallback to Pure Python if Compilation fails * https://github.com/yaml/pyyaml/pull/84 -- Drop unsupported Python 3.3 @@ -35,12 +34,17 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/pull/105 -- Removed Python 2.6 & 3.3 support * https://github.com/yaml/pyyaml/pull/111 -- Remove commented out Psyco code * https://github.com/yaml/pyyaml/pull/129 -- Remove call to `ord` in lib3 emitter code -* https://github.com/yaml/pyyaml/pull/143 -- Allow to turn off sorting keys in Dumper * https://github.com/yaml/pyyaml/pull/149 -- Test on Python 3.7-dev * https://github.com/yaml/pyyaml/pull/158 -- Support escaped slash in double quotes "\/" +* https://github.com/yaml/pyyaml/pull/175 -- Updated link to pypi in release announcement * https://github.com/yaml/pyyaml/pull/181 -- Import Hashable from collections.abc +* https://github.com/yaml/pyyaml/pull/194 -- Reverting https://github.com/yaml/pyyaml/pull/74 +* https://github.com/yaml/pyyaml/pull/195 -- Build libyaml on travis +* https://github.com/yaml/pyyaml/pull/196 -- Force cython when building sdist +* https://github.com/yaml/pyyaml/pull/254 -- Allow to turn off sorting keys in Dumper (2) * https://github.com/yaml/pyyaml/pull/256 -- Make default_flow_style=False * https://github.com/yaml/pyyaml/pull/257 -- Deprecate yaml.load and add FullLoader and UnsafeLoader classes +* https://github.com/yaml/pyyaml/pull/261 -- Skip certain unicode tests when maxunicode not > 0xffff * https://github.com/yaml/pyyaml/pull/263 -- Windows Appveyor build 3.13 (2018-07-05) From 8d7a78003ae59752cb33e65d54cbc7b37f1fa359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller=20=28tinita=29?= Date: Sun, 31 Mar 2019 16:17:49 +0200 Subject: [PATCH 05/36] Change default loader for yaml.add_constructor (#287) * Change default loader for yaml.add_constructor If the Loader parameter is not given, add constructor to all three loaders --- lib/yaml/__init__.py | 18 ++++++++++++++---- lib3/yaml/__init__.py | 18 ++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 5d56d6556..a9db6a327 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -329,22 +329,32 @@ def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper): Loader.add_path_resolver(tag, path, kind) Dumper.add_path_resolver(tag, path, kind) -def add_constructor(tag, constructor, Loader=Loader): +def add_constructor(tag, constructor, Loader=None): """ Add a constructor for the given tag. Constructor is a function that accepts a Loader instance and a node object and produces the corresponding Python object. """ - Loader.add_constructor(tag, constructor) + if Loader == None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) -def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader): +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): """ Add a multi-constructor for the given tag prefix. Multi-constructor is called for a node if its tag starts with tag_prefix. Multi-constructor accepts a Loader instance, a tag suffix, and a node object and produces the corresponding Python object. """ - Loader.add_multi_constructor(tag_prefix, multi_constructor) + if Loader == None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) def add_representer(data_type, representer, Dumper=Dumper): """ diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 3f499d366..ac0b70284 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -326,22 +326,32 @@ def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper): Loader.add_path_resolver(tag, path, kind) Dumper.add_path_resolver(tag, path, kind) -def add_constructor(tag, constructor, Loader=Loader): +def add_constructor(tag, constructor, Loader=None): """ Add a constructor for the given tag. Constructor is a function that accepts a Loader instance and a node object and produces the corresponding Python object. """ - Loader.add_constructor(tag, constructor) + if Loader == None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) -def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader): +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): """ Add a multi-constructor for the given tag prefix. Multi-constructor is called for a node if its tag starts with tag_prefix. Multi-constructor accepts a Loader instance, a tag suffix, and a node object and produces the corresponding Python object. """ - Loader.add_multi_constructor(tag_prefix, multi_constructor) + if Loader == None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) def add_representer(data_type, representer, Dumper=Dumper): """ From a5394c04a2ef67ee218b49d7d8319f9ed8bcbbe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Tue, 19 Mar 2019 10:34:39 -0700 Subject: [PATCH 06/36] Add custom constructors to multiple loaders When someone writes a subclass of the YAMLObject class, the constructors will now be added to all 3 (non-safe) loaders. Furthermore, we support the class variable `yaml_loader` being a list, offering more control of which loaders are affected. To support safe_load in your custom class you could add this: yaml_loader = yaml.SafeLoader yaml_loader = yaml.YAMLObject.yaml_loader yaml_loader.append(yaml.SafeLoader) --- lib/yaml/__init__.py | 9 +++++++-- lib3/yaml/__init__.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index a9db6a327..b6d3e74f2 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -381,7 +381,12 @@ class YAMLObjectMetaclass(type): def __init__(cls, name, bases, kwds): super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: - cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + cls.yaml_dumper.add_representer(cls, cls.to_yaml) class YAMLObject(object): @@ -393,7 +398,7 @@ class YAMLObject(object): __metaclass__ = YAMLObjectMetaclass __slots__ = () # no direct instantiation, so allow immutable subclasses - yaml_loader = Loader + yaml_loader = [Loader, FullLoader, UnsafeLoader] yaml_dumper = Dumper yaml_tag = None diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index ac0b70284..ef200b7b8 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -378,7 +378,12 @@ class YAMLObjectMetaclass(type): def __init__(cls, name, bases, kwds): super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: - cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + cls.yaml_dumper.add_representer(cls, cls.to_yaml) class YAMLObject(metaclass=YAMLObjectMetaclass): @@ -389,7 +394,7 @@ class YAMLObject(metaclass=YAMLObjectMetaclass): __slots__ = () # no direct instantiation, so allow immutable subclasses - yaml_loader = Loader + yaml_loader = [Loader, FullLoader, UnsafeLoader] yaml_dumper = Dumper yaml_tag = None From 4a31b16b04c08f926f01d3f297140634665f004a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Tue, 11 Jun 2019 22:53:40 +0200 Subject: [PATCH 07/36] Change default loader for add_implicit_resolver, add_path_resolver If the Loader parameter is not given, add constructor to all three loaders --- lib/yaml/__init__.py | 18 ++++++++++++++---- lib3/yaml/__init__.py | 18 ++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index b6d3e74f2..09f82cd1d 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -309,24 +309,34 @@ def safe_dump(data, stream=None, **kwds): return dump_all([data], stream, Dumper=SafeDumper, **kwds) def add_implicit_resolver(tag, regexp, first=None, - Loader=Loader, Dumper=Dumper): + Loader=None, Dumper=Dumper): """ Add an implicit scalar detector. If an implicit scalar value matches the given regexp, the corresponding tag is assigned to the scalar. first is a sequence of possible initial characters or None. """ - Loader.add_implicit_resolver(tag, regexp, first) + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) Dumper.add_implicit_resolver(tag, regexp, first) -def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper): +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): """ Add a path based resolver for the given tag. A path is a list of keys that forms a path to a node in the representation tree. Keys can be string values, integers, or None. """ - Loader.add_path_resolver(tag, path, kind) + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) Dumper.add_path_resolver(tag, path, kind) def add_constructor(tag, constructor, Loader=None): diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index ef200b7b8..a31f279fb 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -306,24 +306,34 @@ def safe_dump(data, stream=None, **kwds): return dump_all([data], stream, Dumper=SafeDumper, **kwds) def add_implicit_resolver(tag, regexp, first=None, - Loader=Loader, Dumper=Dumper): + Loader=None, Dumper=Dumper): """ Add an implicit scalar detector. If an implicit scalar value matches the given regexp, the corresponding tag is assigned to the scalar. first is a sequence of possible initial characters or None. """ - Loader.add_implicit_resolver(tag, regexp, first) + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) Dumper.add_implicit_resolver(tag, regexp, first) -def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper): +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): """ Add a path based resolver for the given tag. A path is a list of keys that forms a path to a node in the representation tree. Keys can be string values, integers, or None. """ - Loader.add_path_resolver(tag, path, kind) + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) Dumper.add_path_resolver(tag, path, kind) def add_constructor(tag, constructor, Loader=None): From 8c5e47fe62d7b9e0282a176a4b79b8b2980dc704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Mon, 18 Nov 2019 15:55:27 +0100 Subject: [PATCH 08/36] Move constructor for object/apply to Unsafe --- lib/yaml/constructor.py | 8 ++++---- lib3/yaml/constructor.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 516dad1ce..859c9494a 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -683,10 +683,6 @@ def construct_python_object_new(self, suffix, node): u'tag:yaml.org,2002:python/object:', FullConstructor.construct_python_object) -FullConstructor.add_multi_constructor( - u'tag:yaml.org,2002:python/object/apply:', - FullConstructor.construct_python_object_apply) - FullConstructor.add_multi_constructor( u'tag:yaml.org,2002:python/object/new:', FullConstructor.construct_python_object_new) @@ -703,6 +699,10 @@ def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False) return super(UnsafeConstructor, self).make_python_instance( suffix, node, args, kwds, newobj, unsafe=True) +UnsafeConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + # Constructor is same as UnsafeConstructor. Need to leave this in place in case # people have extended it directly. class Constructor(UnsafeConstructor): diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index 34fc1ae92..fb4f1e9fd 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -694,10 +694,6 @@ def construct_python_object_new(self, suffix, node): 'tag:yaml.org,2002:python/object:', FullConstructor.construct_python_object) -FullConstructor.add_multi_constructor( - 'tag:yaml.org,2002:python/object/apply:', - FullConstructor.construct_python_object_apply) - FullConstructor.add_multi_constructor( 'tag:yaml.org,2002:python/object/new:', FullConstructor.construct_python_object_new) @@ -714,6 +710,10 @@ def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False) return super(UnsafeConstructor, self).make_python_instance( suffix, node, args, kwds, newobj, unsafe=True) +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + # Constructor is same as UnsafeConstructor. Need to leave this in place in case # people have extended it directly. class Constructor(UnsafeConstructor): From 3f3c373f5027ae2a0d305fc63eb453daaf952413 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Thu, 21 Nov 2019 14:43:45 -0800 Subject: [PATCH 09/36] bump version to 5.2b1 --- lib/yaml/__init__.py | 2 +- lib3/yaml/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 09f82cd1d..85db1db40 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.1.2' +__version__ = '5.2b1' try: from cyaml import * diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index a31f279fb..6116109e0 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.1.2' +__version__ = '5.2b1' try: from .cyaml import * __with_libyaml__ = True diff --git a/setup.py b/setup.py index 5cb0b6d25..8a3e21570 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.1.2' +VERSION = '5.2b1' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability From f813bc00f5d0f6f42984edd89ca11f4b5245d5e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Thu, 21 Nov 2019 14:41:06 +0100 Subject: [PATCH 10/36] Changes for 5.2b1 --- CHANGES | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGES b/CHANGES index 29abdf976..14069a1d1 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,21 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/commits/ * https://bitbucket.org/xi/pyyaml/commits/ +5.2b1 (2019-11-26) +------------------ + +* Repair incompatibilities introduced with 5.1. The default Loader was changed, + but several methods like add_constructor still used the old default + https://github.com/yaml/pyyaml/pull/279 -- A more flexible fix for custom tag constructors + https://github.com/yaml/pyyaml/pull/287 -- Change default loader for yaml.add_constructor + https://github.com/yaml/pyyaml/pull/305 -- Change default loader for add_implicit_resolver, add_path_resolver +* Make FullLoader safer by removing python/object/apply from the default FullLoader + https://github.com/yaml/pyyaml/pull/347 -- Move constructor for object/apply to UnsafeConstructor +* Fix bug introduced in 5.1 where quoting went wrong on systems with sys.maxunicode <= 0xffff + https://github.com/yaml/pyyaml/pull/276 -- Fix logic for quoting special characters +* Other PRs: + https://github.com/yaml/pyyaml/pull/280 -- Update CHANGES for 5.1 + 5.1.2 (2019-07-30) ------------------ From f4fd3fbf874db649112f7f02add8eb42ef183741 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 26 Nov 2019 14:01:02 -0800 Subject: [PATCH 11/36] Windows build tweaks * centralized error handling on native commands * ensure that errors from native commands will fail build * use image-included Python 3.8 * drop Python 3.4 wheel builds --- .appveyor.yml | 8 +++--- packaging/build/appveyor.ps1 | 49 +++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 46bbd7f0a..a24842ee6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,9 +4,9 @@ version: '{build}' image: - Visual Studio 2015 -cache: -- 'C:\Python38\' -- 'C:\Python38-x64' +#cache: +#- 'C:\Python38\' +#- 'C:\Python38-x64' environment: libyaml_repo_url: https://github.com/yaml/libyaml.git @@ -14,8 +14,6 @@ environment: # matrix: # - PYTHON_VER: Python27 # - PYTHON_VER: Python27-x64 -# - PYTHON_VER: Python34 -# - PYTHON_VER: Python34-x64 # - PYTHON_VER: Python35 # - PYTHON_VER: Python35-x64 # - PYTHON_VER: Python36 diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index fc75b53de..a60d0bbb2 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -5,16 +5,25 @@ # TODO: get version number from setup.py and/or lib(3)/__version__ # Update-AppveyorBuild -Version $dynamic_version -Function Bootstrap() { - # ensure py38 is present (current Appveyor VS2015 image doesn't include it) - If(-not $(Test-Path C:\Python38)) { - choco.exe install python3 --version=3.8.0-b3 --forcex86 --force --params="/InstallDir:C:\Python38" --no-progress +Function Invoke-Exe([scriptblock]$sb) { + & $sb + $exitcode = $LASTEXITCODE + If($exitcode -ne 0) { + throw "exe failed with nonzero exit code $exitcode" } +} - If(-not $(Test-Path C:\Python38-x64)) { - choco.exe install python3 --version=3.8.0-b3 --force --params="/InstallDir:C:\Python38-x64" --no-progress +Function Bootstrap() { +<# + # ensure python 3.9 prerelease is present (current Appveyor VS2015 image doesn't include it) + If(-not $(Test-Path C:\Python39)) { + Invoke-Exe { choco.exe install python3 --version=3.9.0-a1 --forcex86 --force --params="/InstallDir:C:\Python39" --no-progress } } + If(-not $(Test-Path C:\Python39-x64)) { + Invoke-Exe { choco.exe install python3 --version=3.9.0-a1 --force --params="/InstallDir:C:\Python39-x64" --no-progress } + } +#> Write-Output "patching Windows SDK bits for distutils" # patch 7.0/7.1 vcvars SDK bits up to work with distutils query @@ -22,11 +31,7 @@ Function Bootstrap() { Set-Content -Path 'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat' '@CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /Release /x64' # patch VS9 x64 CMake config for VS Express, hide `reg.exe` stderr noise - $noise = reg.exe import packaging\build\FixVS9CMake.reg 2>&1 - - If($LASTEXITCODE -ne 0) { - throw "reg failed with error code $LASTEXITCODE" - } + Invoke-Exe { $noise = reg.exe import packaging\build\FixVS9CMake.reg 2>&1 } Copy-Item -Path "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.config" -Destination "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.Express.config" -Force Copy-Item -Path "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.config" -Destination "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.Express.config" -Force @@ -40,7 +45,7 @@ Function Bootstrap() { Write-Output "cloning libyaml from $libyaml_repo_url / $libyaml_refspec" If(-not $(Test-Path .\libyaml)) { - git clone -b $libyaml_refspec $libyaml_repo_url 2>&1 + Invoke-Exe { git clone -b $libyaml_refspec $libyaml_repo_url 2>&1 } } } @@ -52,7 +57,7 @@ Function Build-Wheel($python_path) { Write-Output "building pyyaml wheel for $python_path" # query distutils for the VC version used to build this Python; translate to a VS version to choose the right generator - $python_vs_buildver = & $python -c "from distutils.version import LooseVersion; from distutils.msvc9compiler import get_build_version; print(LooseVersion(str(get_build_version())).version[0])" + $python_vs_buildver = Invoke-Exe { & $python -c "from distutils.version import LooseVersion; from distutils.msvc9compiler import get_build_version; print(LooseVersion(str(get_build_version())).version[0])" } $python_cmake_generator = switch($python_vs_buildver) { "9" { "Visual Studio 9 2008" } @@ -62,7 +67,7 @@ Function Build-Wheel($python_path) { } # query arch this python was built for - $python_arch = & $python -c "from distutils.util import get_platform; print(str(get_platform()))" + $python_arch = Invoke-Exe { & $python -c "from distutils.util import get_platform; print(str(get_platform()))" } if($python_arch -eq 'win-amd64') { $python_cmake_generator += " Win64" @@ -70,7 +75,7 @@ Function Build-Wheel($python_path) { } # snarf VS vars (paths, etc) for the matching VS version and arch that built this Python - $raw_vars_out = & cmd.exe /c "`"C:\Program Files (x86)\Microsoft Visual Studio $($python_vs_buildver).0\VC\vcvarsall.bat`" $vcvars_arch & set" + $raw_vars_out = Invoke-Exe { cmd.exe /c "`"C:\Program Files (x86)\Microsoft Visual Studio $($python_vs_buildver).0\VC\vcvarsall.bat`" $vcvars_arch & set" } foreach($kv in $raw_vars_out) { If($kv -match "=") { $kv = $kv.Split("=", 2) @@ -82,23 +87,23 @@ Function Build-Wheel($python_path) { } # ensure pip is current (some appveyor pips are not) - & $python -W "ignore:DEPRECATION" -m pip install --upgrade pip + Invoke-Exe { & $python -W "ignore:DEPRECATION" -m pip install --upgrade pip } # ensure required-for-build packages are present and up-to-date - & $python -W "ignore:DEPRECATION" -m pip install --upgrade cython wheel setuptools --no-warn-script-location + Invoke-Exe { & $python -W "ignore:DEPRECATION" -m pip install --upgrade cython wheel setuptools --no-warn-script-location } pushd libyaml - git clean -fdx + Invoke-Exe { git clean -fdx } popd mkdir libyaml\build pushd libyaml\build - cmake.exe -G $python_cmake_generator -DYAML_STATIC_LIB_NAME=yaml .. - cmake.exe --build . --config Release + Invoke-Exe { cmake.exe -G $python_cmake_generator -DYAML_STATIC_LIB_NAME=yaml .. } + Invoke-Exe { cmake.exe --build . --config Release } popd - & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel + Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel } } Function Upload-Artifacts() { @@ -114,8 +119,6 @@ Bootstrap $pythons = @( "C:\Python27" "C:\Python27-x64" -"C:\Python34" -"C:\Python34-x64" "C:\Python35" "C:\Python35-x64" "C:\Python36" From a5c2a043a26a6bf2787870eec9006b96ba6bca91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Mon, 2 Dec 2019 20:58:55 +0100 Subject: [PATCH 12/36] Version 5.2 --- CHANGES | 2 +- announcement.msg | 25 +++++++++++++++++-------- lib/yaml/__init__.py | 2 +- lib3/yaml/__init__.py | 2 +- setup.py | 2 +- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index 14069a1d1..f4a9ae577 100644 --- a/CHANGES +++ b/CHANGES @@ -4,7 +4,7 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/commits/ * https://bitbucket.org/xi/pyyaml/commits/ -5.2b1 (2019-11-26) +5.2 (2019-12-02) ------------------ * Repair incompatibilities introduced with 5.1. The default Loader was changed, diff --git a/announcement.msg b/announcement.msg index 4cf3944fe..292d0aca9 100644 --- a/announcement.msg +++ b/announcement.msg @@ -1,22 +1,31 @@ -From: Ingy döt Net +From: Tina Müller To: python-list@python.org, python-announce@python.org, yaml-core@lists.sourceforge.net -Subject: [ANN] PyYAML-5.1.2: YAML parser and emitter for Python +Subject: [ANN] PyYAML-5.2: YAML parser and emitter for Python ======================= -Announcing PyYAML-5.1.2 +Announcing PyYAML-5.2 ======================= -A new minor release of PyYAML is now available: +A new release of PyYAML is now available: https://pypi.org/project/PyYAML/ -This is a maintenance re-release of PyYAML 5.1 with re-generated Cython -bindings to allow the extension to build properly for Python 3.8b2+. No code -changes were made in the PyYAML sources. +This fixes some incompatibilities introduced in version 5.1 and also removes +another possibility of loading arbitrary code. Changes ======= -* None +* Repair incompatibilities introduced with 5.1. The default Loader was changed, + but several methods like add_constructor still used the old default + https://github.com/yaml/pyyaml/pull/279 -- A more flexible fix for custom tag constructors + https://github.com/yaml/pyyaml/pull/287 -- Change default loader for yaml.add_constructor + https://github.com/yaml/pyyaml/pull/305 -- Change default loader for add_implicit_resolver, add_path_resolver +* Make FullLoader safer by removing python/object/apply from the default FullLoader + https://github.com/yaml/pyyaml/pull/347 -- Move constructor for object/apply to UnsafeConstructor +* Fix bug introduced in 5.1 where quoting went wrong on systems with sys.maxunicode <= 0xffff + https://github.com/yaml/pyyaml/pull/276 -- Fix logic for quoting special characters +* Other PRs: + https://github.com/yaml/pyyaml/pull/280 -- Update CHANGES for 5.1 Resources ========= diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 85db1db40..32d848e47 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.2b1' +__version__ = '5.2' try: from cyaml import * diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 6116109e0..d9e3ac253 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.2b1' +__version__ = '5.2' try: from .cyaml import * __with_libyaml__ = True diff --git a/setup.py b/setup.py index 8a3e21570..8634ef0f7 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.2b1' +VERSION = '5.2' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability From 15e0e80d6dee9e8e513caa720c9fc2db0e935f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Tue, 3 Dec 2019 00:11:10 +0100 Subject: [PATCH 13/36] Fix appveyor.yml to use libyaml tag not branch --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index a24842ee6..1ca6e2c2d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -10,7 +10,7 @@ image: environment: libyaml_repo_url: https://github.com/yaml/libyaml.git - libyaml_refspec: release/0.2.2 + libyaml_refspec: 0.2.2 # matrix: # - PYTHON_VER: Python27 # - PYTHON_VER: Python27-x64 From 6d854b1e303e0a7b810d3d9fbb03c75c8dbfa169 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 9 Nov 2019 08:36:44 +0100 Subject: [PATCH 14/36] Travis CI: Test on Python 3.8 production release Also drop the EOL Python 3.4 @hugovk Your review please. --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ecce0331b..a7102d699 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,22 +2,20 @@ language: python -sudo: false - cache: pip matrix: include: - python: 2.7 env: TOXENV=py27 - - python: 3.4 - env: TOXENV=py34 - python: 3.5 env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - python: 3.7 env: TOXENV=py37 + - python: 3.8 + env: TOXENV=py38 - python: 3.8-dev env: TOXENV=py38 - python: pypy From 6544c3091f2fb913b81102a33aaa973a6b1299ee Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 9 Nov 2019 08:38:06 +0100 Subject: [PATCH 15/36] tox.ini: Add py38 and remove py34 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f0df1f688..8d1e42d73 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,pypy,py34,py35,py36,py37 +envlist = py27,pypy,py35,py36,py37,py38 [testenv] deps = From 039c9eb3082de06063fa669cf41a73dbeabb5d84 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 9 Nov 2019 08:39:19 +0100 Subject: [PATCH 16/36] setup.py: Remove support for EOL Python 3.4 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 8634ef0f7..eb24676ad 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", From a7a97871fce027d8c3da0d96117f5b71536b8244 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 9 Nov 2019 08:41:02 +0100 Subject: [PATCH 17/36] fixup! setup.py: python_requires='!=3.4.*', --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb24676ad..f88797528 100644 --- a/setup.py +++ b/setup.py @@ -311,5 +311,5 @@ def run(self): distclass=Distribution, cmdclass=cmdclass, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', ) From de11e43d5247991ee2892df4c9dcba434afb6cb6 Mon Sep 17 00:00:00 2001 From: David Kao Date: Fri, 15 Mar 2019 11:21:40 -0700 Subject: [PATCH 18/36] fix typos and stylistic nit --- lib/yaml/constructor.py | 4 ++-- lib/yaml/loader.py | 2 +- lib3/yaml/constructor.py | 4 ++-- lib3/yaml/loader.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 859c9494a..094b3e016 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -497,7 +497,7 @@ def find_python_module(self, name, mark, unsafe=False): except ImportError, exc: raise ConstructorError("while constructing a Python module", mark, "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark) - if not name in sys.modules: + if name not in sys.modules: raise ConstructorError("while constructing a Python module", mark, "module %r is not imported" % name.encode('utf-8'), mark) return sys.modules[name] @@ -517,7 +517,7 @@ def find_python_name(self, name, mark, unsafe=False): except ImportError, exc: raise ConstructorError("while constructing a Python object", mark, "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark) - if not module_name in sys.modules: + if module_name not in sys.modules: raise ConstructorError("while constructing a Python object", mark, "module %r is not imported" % module_name.encode('utf-8'), mark) module = sys.modules[module_name] diff --git a/lib/yaml/loader.py b/lib/yaml/loader.py index a79182eaf..4d773c3cc 100644 --- a/lib/yaml/loader.py +++ b/lib/yaml/loader.py @@ -51,7 +51,7 @@ def __init__(self, stream): # UnsafeLoader is the same as Loader (which is and was always unsafe on # untrusted input). Use of either Loader or UnsafeLoader should be rare, since # FullLoad should be able to load almost all YAML safely. Loader is left intact -# to ensure backwards compatability. +# to ensure backwards compatibility. class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): def __init__(self, stream): diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index fb4f1e9fd..f8d71a136 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -513,7 +513,7 @@ def find_python_module(self, name, mark, unsafe=False): except ImportError as exc: raise ConstructorError("while constructing a Python module", mark, "cannot find module %r (%s)" % (name, exc), mark) - if not name in sys.modules: + if name not in sys.modules: raise ConstructorError("while constructing a Python module", mark, "module %r is not imported" % name, mark) return sys.modules[name] @@ -533,7 +533,7 @@ def find_python_name(self, name, mark, unsafe=False): except ImportError as exc: raise ConstructorError("while constructing a Python object", mark, "cannot find module %r (%s)" % (module_name, exc), mark) - if not module_name in sys.modules: + if module_name not in sys.modules: raise ConstructorError("while constructing a Python object", mark, "module %r is not imported" % module_name, mark) module = sys.modules[module_name] diff --git a/lib3/yaml/loader.py b/lib3/yaml/loader.py index 414cb2c15..e90c11224 100644 --- a/lib3/yaml/loader.py +++ b/lib3/yaml/loader.py @@ -51,7 +51,7 @@ def __init__(self, stream): # UnsafeLoader is the same as Loader (which is and was always unsafe on # untrusted input). Use of either Loader or UnsafeLoader should be rare, since # FullLoad should be able to load almost all YAML safely. Loader is left intact -# to ensure backwards compatability. +# to ensure backwards compatibility. class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): def __init__(self, stream): From e21af4a0925e25bf5fd8c53f9ae6404a794a676e Mon Sep 17 00:00:00 2001 From: Sergey Fursov Date: Thu, 11 Apr 2019 14:20:14 +0300 Subject: [PATCH 19/36] Use `is` instead of equality for comparing with None --- lib/yaml/__init__.py | 4 ++-- lib3/yaml/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 32d848e47..4703dffb0 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -345,7 +345,7 @@ def add_constructor(tag, constructor, Loader=None): Constructor is a function that accepts a Loader instance and a node object and produces the corresponding Python object. """ - if Loader == None: + if Loader is None: loader.Loader.add_constructor(tag, constructor) loader.FullLoader.add_constructor(tag, constructor) loader.UnsafeLoader.add_constructor(tag, constructor) @@ -359,7 +359,7 @@ def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): Multi-constructor accepts a Loader instance, a tag suffix, and a node object and produces the corresponding Python object. """ - if Loader == None: + if Loader is None: loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index d9e3ac253..b7cb86aa1 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -342,7 +342,7 @@ def add_constructor(tag, constructor, Loader=None): Constructor is a function that accepts a Loader instance and a node object and produces the corresponding Python object. """ - if Loader == None: + if Loader is None: loader.Loader.add_constructor(tag, constructor) loader.FullLoader.add_constructor(tag, constructor) loader.UnsafeLoader.add_constructor(tag, constructor) @@ -356,7 +356,7 @@ def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): Multi-constructor accepts a Loader instance, a tag suffix, and a node object and produces the corresponding Python object. """ - if Loader == None: + if Loader is None: loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) From eb459f842f690f809b649726f0a317909a92a29c Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Mon, 24 Jun 2019 21:06:20 +1000 Subject: [PATCH 20/36] Fix up small typo Replace `intendation` with `indentation`. --- lib/yaml/scanner.py | 8 ++++---- lib3/yaml/scanner.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/yaml/scanner.py b/lib/yaml/scanner.py index 5126cf07b..098ea7be8 100644 --- a/lib/yaml/scanner.py +++ b/lib/yaml/scanner.py @@ -332,7 +332,7 @@ def unwind_indent(self, column): ## } #if self.flow_level and self.indent > column: # raise ScannerError(None, None, - # "invalid intendation or unclosed '[' or '{'", + # "invalid indentation or unclosed '[' or '{'", # self.get_mark()) # In the flow context, indentation is ignored. We make the scanner less @@ -370,7 +370,7 @@ def fetch_stream_start(self): def fetch_stream_end(self): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. @@ -389,7 +389,7 @@ def fetch_stream_end(self): def fetch_directive(self): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. @@ -407,7 +407,7 @@ def fetch_document_end(self): def fetch_document_indicator(self, TokenClass): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. Note that there could not be a block collection diff --git a/lib3/yaml/scanner.py b/lib3/yaml/scanner.py index 775dbcc6d..7437ede1c 100644 --- a/lib3/yaml/scanner.py +++ b/lib3/yaml/scanner.py @@ -332,7 +332,7 @@ def unwind_indent(self, column): ## } #if self.flow_level and self.indent > column: # raise ScannerError(None, None, - # "invalid intendation or unclosed '[' or '{'", + # "invalid indentation or unclosed '[' or '{'", # self.get_mark()) # In the flow context, indentation is ignored. We make the scanner less @@ -370,7 +370,7 @@ def fetch_stream_start(self): def fetch_stream_end(self): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. @@ -389,7 +389,7 @@ def fetch_stream_end(self): def fetch_directive(self): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. @@ -407,7 +407,7 @@ def fetch_document_end(self): def fetch_document_indicator(self, TokenClass): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. Note that there could not be a block collection From 5a0cfab86f6bccd5e5bec5c239ccfb94f8f0fe11 Mon Sep 17 00:00:00 2001 From: Filip Salomonsson Date: Sat, 7 Dec 2019 22:34:23 +0100 Subject: [PATCH 21/36] Fix handling of __slots__ (#161) --- lib/yaml/constructor.py | 2 +- lib3/yaml/constructor.py | 2 +- tests/data/construct-python-object.code | 2 ++ tests/data/construct-python-object.data | 2 ++ tests/lib/test_constructor.py | 13 ++++++++++++- tests/lib3/test_constructor.py | 13 ++++++++++++- 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 094b3e016..0c83e43ec 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -578,7 +578,7 @@ def set_python_instance_state(self, instance, state): elif state: slotstate.update(state) for key, value in slotstate.items(): - setattr(object, key, value) + setattr(instance, key, value) def construct_python_object(self, suffix, node): # Format: diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index f8d71a136..31e931893 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -585,7 +585,7 @@ def set_python_instance_state(self, instance, state): elif state: slotstate.update(state) for key, value in slotstate.items(): - setattr(object, key, value) + setattr(instance, key, value) def construct_python_object(self, suffix, node): # Format: diff --git a/tests/data/construct-python-object.code b/tests/data/construct-python-object.code index 7f1edf125..9e611e43a 100644 --- a/tests/data/construct-python-object.code +++ b/tests/data/construct-python-object.code @@ -17,6 +17,8 @@ NewArgsWithState(1, 'two', [3,3,3]), Reduce(1, 'two', [3,3,3]), ReduceWithState(1, 'two', [3,3,3]), +Slots(1, 'two', [3,3,3]), + MyInt(3), MyList(3), MyDict(3), diff --git a/tests/data/construct-python-object.data b/tests/data/construct-python-object.data index bce8b2ed3..66797e4c9 100644 --- a/tests/data/construct-python-object.data +++ b/tests/data/construct-python-object.data @@ -16,6 +16,8 @@ - !!python/object/apply:test_constructor.Reduce [1, two, [3,3,3]] - !!python/object/apply:test_constructor.ReduceWithState { args: [1, two], state: [3,3,3] } +- !!python/object/new:test_constructor.Slots { state: !!python/tuple [null, { foo: 1, bar: 'two', baz: [3,3,3] } ] } + - !!python/object/new:test_constructor.MyInt [3] - !!python/object/new:test_constructor.MyList { listitems: [~, ~, ~] } - !!python/object/new:test_constructor.MyDict { dictitems: {0, 1, 2} } diff --git a/tests/lib/test_constructor.py b/tests/lib/test_constructor.py index beee7b0a7..9e6c85647 100644 --- a/tests/lib/test_constructor.py +++ b/tests/lib/test_constructor.py @@ -16,7 +16,7 @@ def execute(code): def _make_objects(): global MyLoader, MyDumper, MyTestClass1, MyTestClass2, MyTestClass3, YAMLObject1, YAMLObject2, \ AnObject, AnInstance, AState, ACustomState, InitArgs, InitArgsWithState, \ - NewArgs, NewArgsWithState, Reduce, ReduceWithState, MyInt, MyList, MyDict, \ + NewArgs, NewArgsWithState, Reduce, ReduceWithState, Slots, MyInt, MyList, MyDict, \ FixedOffset, today, execute class MyLoader(yaml.Loader): @@ -185,6 +185,17 @@ def __reduce__(self): def __setstate__(self, state): self.baz = state + class Slots(object): + __slots__ = ("foo", "bar", "baz") + def __init__(self, foo=None, bar=None, baz=None): + self.foo = foo + self.bar = bar + self.baz = baz + + def __eq__(self, other): + return type(self) is type(other) and \ + (self.foo, self.bar, self.baz) == (other.foo, other.bar, other.baz) + class MyInt(int): def __eq__(self, other): return type(self) is type(other) and int(self) == int(other) diff --git a/tests/lib3/test_constructor.py b/tests/lib3/test_constructor.py index 427f53c30..3d696498b 100644 --- a/tests/lib3/test_constructor.py +++ b/tests/lib3/test_constructor.py @@ -13,7 +13,7 @@ def execute(code): def _make_objects(): global MyLoader, MyDumper, MyTestClass1, MyTestClass2, MyTestClass3, YAMLObject1, YAMLObject2, \ AnObject, AnInstance, AState, ACustomState, InitArgs, InitArgsWithState, \ - NewArgs, NewArgsWithState, Reduce, ReduceWithState, MyInt, MyList, MyDict, \ + NewArgs, NewArgsWithState, Reduce, ReduceWithState, Slots, MyInt, MyList, MyDict, \ FixedOffset, today, execute class MyLoader(yaml.Loader): @@ -172,6 +172,17 @@ def __reduce__(self): def __setstate__(self, state): self.baz = state + class Slots: + __slots__ = ("foo", "bar", "baz") + def __init__(self, foo=None, bar=None, baz=None): + self.foo = foo + self.bar = bar + self.baz = baz + + def __eq__(self, other): + return type(self) is type(other) and \ + (self.foo, self.bar, self.baz) == (other.foo, other.bar, other.baz) + class MyInt(int): def __eq__(self, other): return type(self) is type(other) and int(self) == int(other) From 03b378d0390c6e5538741196e60b70f5be27fc5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller=20=28tinita=29?= Date: Sat, 7 Dec 2019 22:40:48 +0100 Subject: [PATCH 22/36] Allow add_multi_constructor with None (#358) Loader.add_multi_constructor(None, myconstructor) Also add test for add_multi_constructor('!', ...) etc. See issue #317 --- lib/yaml/constructor.py | 2 +- lib3/yaml/constructor.py | 2 +- tests/data/multi-constructor.code | 4 ++ tests/data/multi-constructor.multi | 3 ++ tests/lib/test_multi_constructor.py | 63 ++++++++++++++++++++++++++++ tests/lib/test_yaml.py | 1 + tests/lib3/test_multi_constructor.py | 63 ++++++++++++++++++++++++++++ tests/lib3/test_yaml.py | 1 + 8 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 tests/data/multi-constructor.code create mode 100644 tests/data/multi-constructor.multi create mode 100644 tests/lib/test_multi_constructor.py create mode 100644 tests/lib3/test_multi_constructor.py diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 0c83e43ec..08bdabfb1 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -74,7 +74,7 @@ def construct_object(self, node, deep=False): constructor = self.yaml_constructors[node.tag] else: for tag_prefix in self.yaml_multi_constructors: - if node.tag.startswith(tag_prefix): + if tag_prefix is not None and node.tag.startswith(tag_prefix): tag_suffix = node.tag[len(tag_prefix):] constructor = self.yaml_multi_constructors[tag_prefix] break diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index 31e931893..ac15aba87 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -72,7 +72,7 @@ def construct_object(self, node, deep=False): constructor = self.yaml_constructors[node.tag] else: for tag_prefix in self.yaml_multi_constructors: - if node.tag.startswith(tag_prefix): + if tag_prefix is not None and node.tag.startswith(tag_prefix): tag_suffix = node.tag[len(tag_prefix):] constructor = self.yaml_multi_constructors[tag_prefix] break diff --git a/tests/data/multi-constructor.code b/tests/data/multi-constructor.code new file mode 100644 index 000000000..590d852a7 --- /dev/null +++ b/tests/data/multi-constructor.code @@ -0,0 +1,4 @@ +[ + {'Tag1': ['a', 1, 'b', 2]}, + {'Tag2': ['a', 1, 'b', 2]}, +] diff --git a/tests/data/multi-constructor.multi b/tests/data/multi-constructor.multi new file mode 100644 index 000000000..4f5d71f42 --- /dev/null +++ b/tests/data/multi-constructor.multi @@ -0,0 +1,3 @@ +--- +- !Tag1 [a, 1, b, 2] +- !!Tag2 [a, 1, b, 2] diff --git a/tests/lib/test_multi_constructor.py b/tests/lib/test_multi_constructor.py new file mode 100644 index 000000000..f6e28fe0e --- /dev/null +++ b/tests/lib/test_multi_constructor.py @@ -0,0 +1,63 @@ +import yaml +import pprint +import sys + +def _load_code(expression): + return eval(expression) + +def myconstructor1(constructor, tag, node): + seq = constructor.construct_sequence(node) + return {tag: seq } + +def myconstructor2(constructor, tag, node): + seq = constructor.construct_sequence(node) + string = '' + try: + i = tag.index('!') + 1 + except: + try: + i = tag.rindex(':') + 1 + except: + pass + if i >= 0: + tag = tag[i:] + return { tag: seq } + +class Multi1(yaml.FullLoader): + pass +class Multi2(yaml.FullLoader): + pass + +def test_multi_constructor(input_filename, code_filename, verbose=False): + input = open(input_filename, 'rb').read().decode('utf-8') + native = _load_code(open(code_filename, 'rb').read()) + + # default multi constructor for ! and !! tags + Multi1.add_multi_constructor('!', myconstructor1) + Multi1.add_multi_constructor('tag:yaml.org,2002:', myconstructor1) + + data = yaml.load(input, Loader=Multi1) + if verbose: + print('Multi1:') + print(data) + print(native) + assert(data == native) + + + # default multi constructor for all tags + Multi2.add_multi_constructor(None, myconstructor2) + + data = yaml.load(input, Loader=Multi2) + if verbose: + print('Multi2:') + print(data) + print(native) + assert(data == native) + + +test_multi_constructor.unittest = ['.multi', '.code'] + +if __name__ == '__main__': + import test_appliance + test_appliance.run(globals()) + diff --git a/tests/lib/test_yaml.py b/tests/lib/test_yaml.py index e26342d61..352cd8d15 100644 --- a/tests/lib/test_yaml.py +++ b/tests/lib/test_yaml.py @@ -12,6 +12,7 @@ from test_recursive import * from test_input_output import * from test_sort_keys import * +from test_multi_constructor import * if __name__ == '__main__': import test_appliance diff --git a/tests/lib3/test_multi_constructor.py b/tests/lib3/test_multi_constructor.py new file mode 100644 index 000000000..f6e28fe0e --- /dev/null +++ b/tests/lib3/test_multi_constructor.py @@ -0,0 +1,63 @@ +import yaml +import pprint +import sys + +def _load_code(expression): + return eval(expression) + +def myconstructor1(constructor, tag, node): + seq = constructor.construct_sequence(node) + return {tag: seq } + +def myconstructor2(constructor, tag, node): + seq = constructor.construct_sequence(node) + string = '' + try: + i = tag.index('!') + 1 + except: + try: + i = tag.rindex(':') + 1 + except: + pass + if i >= 0: + tag = tag[i:] + return { tag: seq } + +class Multi1(yaml.FullLoader): + pass +class Multi2(yaml.FullLoader): + pass + +def test_multi_constructor(input_filename, code_filename, verbose=False): + input = open(input_filename, 'rb').read().decode('utf-8') + native = _load_code(open(code_filename, 'rb').read()) + + # default multi constructor for ! and !! tags + Multi1.add_multi_constructor('!', myconstructor1) + Multi1.add_multi_constructor('tag:yaml.org,2002:', myconstructor1) + + data = yaml.load(input, Loader=Multi1) + if verbose: + print('Multi1:') + print(data) + print(native) + assert(data == native) + + + # default multi constructor for all tags + Multi2.add_multi_constructor(None, myconstructor2) + + data = yaml.load(input, Loader=Multi2) + if verbose: + print('Multi2:') + print(data) + print(native) + assert(data == native) + + +test_multi_constructor.unittest = ['.multi', '.code'] + +if __name__ == '__main__': + import test_appliance + test_appliance.run(globals()) + diff --git a/tests/lib3/test_yaml.py b/tests/lib3/test_yaml.py index e26342d61..352cd8d15 100644 --- a/tests/lib3/test_yaml.py +++ b/tests/lib3/test_yaml.py @@ -12,6 +12,7 @@ from test_recursive import * from test_input_output import * from test_sort_keys import * +from test_multi_constructor import * if __name__ == '__main__': import test_appliance From 6549385d53d9efd546f4b35f6992e6cc9f552657 Mon Sep 17 00:00:00 2001 From: sblondon Date: Sat, 7 Dec 2019 22:44:29 +0100 Subject: [PATCH 23/36] Add use of safe_load() function in README (#285) It helps people to use `safe_load` if they discover the library. It's more secure if `safe_load()` is used by default, and `load()` is used if it's necessary (and the developer knows what is does). --- README | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README b/README index 361839a05..49c87e764 100644 --- a/README +++ b/README @@ -15,6 +15,10 @@ parser and emitter as follows: >>> yaml.load(stream, Loader=yaml.CLoader) >>> yaml.dump(data, Dumper=yaml.CDumper) +If you don't trust the input stream, you should use: + + >>> yaml.safe_load(stream) + PyYAML includes a comprehensive test suite. To run the tests, type 'python setup.py test'. From 0716ae21a1e7ab6b4ef73428c0c8fff49685d057 Mon Sep 17 00:00:00 2001 From: Anish Athalye Date: Sun, 8 Dec 2019 11:18:04 -0500 Subject: [PATCH 24/36] Fix reader for Unicode code points over 0xFFFF (#351) This patch fixes the handling of inputs with Unicode code points over 0xFFFF when running on a Python 2 that does not have UCS-4 support (which certain distributions still ship, e.g. macOS). When Python is compiled without UCS-4 support, it uses UCS-2. In this situation, non-BMP Unicode characters, which have code points over 0xFFFF, are represented as surrogate pairs. For example, if we take u'\U0001f3d4', it will be represented as the surrogate pair u'\ud83c\udfd4'. This can be seen by running, for example: [i for i in u'\U0001f3d4'] In PyYAML, the reader uses a function `check_printable` to validate inputs, making sure that they only contain printable characters. Prior to this patch, on UCS-2 builds, it incorrectly identified surrogate pairs as non-printable. It would be fairly natural to write a regular expression that captures strings that contain only *printable* characters, as opposed to *non-printable* characters (as identified by the old code, so not excluding surrogate pairs): PRINTABLE = re.compile(u'^[\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]*$') Adding support for surrogate pairs to this would be straightforward, adding the option of having a surrogate high followed by a surrogate low (`[\uD800-\uDBFF][\uDC00-\uDFFF]`): PRINTABLE = re.compile(u'^(?:[\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]|[\uD800-\uDBFF][\uDC00-\uDFFF])*$') Then, this regex could be used as follows: def check_printable(self, data): if not self.PRINTABLE.match(data): raise ReaderError(...) However, matching printable strings, rather than searching for non-printable characters as the code currently does, would have the disadvantage of not identifying the culprit character (we wouldn't get the position and the actual non-printable character from a lack of a regex match). Instead, we can modify the NON_PRINTABLE regex to allow legal surrogate pairs. We do this by removing surrogate pairs from the existing character set and adding the following options for illegal uses of surrogate code points: - Surrogate low that doesn't follow a surrogate high (either a surrogate low at the start of a string, or a surrogate low that follows a character that's not a surrogate high): (?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF] - Surrogate high that isn't followed by a surrogate low (either a surrogate high at the end of a string, or a surrogate high that is followed by a character that's not a surrogate low): [\uD800-\uDBFF](?:[^\uDC00-\uDFFF]|$) The behavior of this modified regex should match the one that is used when Python is built with UCS-4 support. --- lib/yaml/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/yaml/reader.py b/lib/yaml/reader.py index b2f10b091..4b377d61e 100644 --- a/lib/yaml/reader.py +++ b/lib/yaml/reader.py @@ -139,7 +139,7 @@ def determine_encoding(self): if has_ucs4: NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]') else: - NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]') + NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uFFFD]|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?:[^\uDC00-\uDFFF]|$)') def check_printable(self, data): match = self.NON_PRINTABLE.search(data) if match: From a826f546c29a154c3a425abe50a1cb102484ba47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Sun, 8 Dec 2019 17:01:02 +0100 Subject: [PATCH 25/36] Enable certain unicode tests when maxunicode not > 0xffff They were disabled in d6cbff662084dd94bde5421ece495482d1b14454 After #351 the tests are working again --- ...4-py2.code => emitting-unacceptable-unicode-character-bug.code | 0 ...4-py2.data => emitting-unacceptable-unicode-character-bug.data | 0 ...ip-ext => emitting-unacceptable-unicode-character-bug.skip-ext | 0 tests/data/{emoticons-ucs4-.unicode => emoticons.unicode} | 0 tests/data/{emoticons2-ucs4-.unicode => emoticons2.unicode} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.code => emitting-unacceptable-unicode-character-bug.code (100%) rename tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.data => emitting-unacceptable-unicode-character-bug.data (100%) rename tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.skip-ext => emitting-unacceptable-unicode-character-bug.skip-ext (100%) rename tests/data/{emoticons-ucs4-.unicode => emoticons.unicode} (100%) rename tests/data/{emoticons2-ucs4-.unicode => emoticons2.unicode} (100%) diff --git a/tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.code b/emitting-unacceptable-unicode-character-bug.code similarity index 100% rename from tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.code rename to emitting-unacceptable-unicode-character-bug.code diff --git a/tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.data b/emitting-unacceptable-unicode-character-bug.data similarity index 100% rename from tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.data rename to emitting-unacceptable-unicode-character-bug.data diff --git a/tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.skip-ext b/emitting-unacceptable-unicode-character-bug.skip-ext similarity index 100% rename from tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.skip-ext rename to emitting-unacceptable-unicode-character-bug.skip-ext diff --git a/tests/data/emoticons-ucs4-.unicode b/tests/data/emoticons.unicode similarity index 100% rename from tests/data/emoticons-ucs4-.unicode rename to tests/data/emoticons.unicode diff --git a/tests/data/emoticons2-ucs4-.unicode b/tests/data/emoticons2.unicode similarity index 100% rename from tests/data/emoticons2-ucs4-.unicode rename to tests/data/emoticons2.unicode From d137e82ad1bc55c283a33ffaccd93e51bcb4c0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller=20=28tinita=29?= Date: Sun, 8 Dec 2019 18:43:47 +0100 Subject: [PATCH 26/36] Use full_load in yaml-highlight example (#359) --- examples/yaml-highlight/yaml_hl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/yaml-highlight/yaml_hl.py b/examples/yaml-highlight/yaml_hl.py index d6f7bf4eb..96e0ae7b1 100755 --- a/examples/yaml-highlight/yaml_hl.py +++ b/examples/yaml-highlight/yaml_hl.py @@ -37,7 +37,7 @@ def __setstate__(self, state): class YAMLHighlight: def __init__(self, options): - config = yaml.load(file(options.config, 'rb').read()) + config = yaml.full_load(file(options.config, 'rb').read()) self.style = config[options.style] if options.input: self.input = file(options.input, 'rb') From 252b4fe54ed60c19c4ba394f26e365dbab07bd78 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 8 Dec 2019 09:46:19 -0800 Subject: [PATCH 27/36] Document that PyYAML is implemented with Cython (#244) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f88797528..a1a2baa14 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", + "Programming Language :: Cython", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", From f1ab37df447dccad6032485ff08c2d3beb6251f1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 9 Dec 2019 17:32:56 +0200 Subject: [PATCH 28/36] Fix for Python 3.10 (#329) --- tests/lib/test_build.py | 2 +- tests/lib/test_build_ext.py | 2 +- tests/lib3/test_build.py | 2 +- tests/lib3/test_build_ext.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/lib/test_build.py b/tests/lib/test_build.py index 901e8ed6c..df3f943ba 100644 --- a/tests/lib/test_build.py +++ b/tests/lib/test_build.py @@ -2,7 +2,7 @@ if __name__ == '__main__': import sys, os, distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.{}-{}.{}'.format(distutils.util.get_platform(), *sys.version_info)) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) import test_yaml, test_appliance diff --git a/tests/lib/test_build_ext.py b/tests/lib/test_build_ext.py index ff195d519..fa843f6e4 100644 --- a/tests/lib/test_build_ext.py +++ b/tests/lib/test_build_ext.py @@ -3,7 +3,7 @@ if __name__ == '__main__': import sys, os, distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.{}-{}.{}'.format(distutils.util.get_platform(), *sys.version_info)) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) import test_yaml_ext, test_appliance diff --git a/tests/lib3/test_build.py b/tests/lib3/test_build.py index 901e8ed6c..df3f943ba 100644 --- a/tests/lib3/test_build.py +++ b/tests/lib3/test_build.py @@ -2,7 +2,7 @@ if __name__ == '__main__': import sys, os, distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.{}-{}.{}'.format(distutils.util.get_platform(), *sys.version_info)) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) import test_yaml, test_appliance diff --git a/tests/lib3/test_build_ext.py b/tests/lib3/test_build_ext.py index ff195d519..fa843f6e4 100644 --- a/tests/lib3/test_build_ext.py +++ b/tests/lib3/test_build_ext.py @@ -3,7 +3,7 @@ if __name__ == '__main__': import sys, os, distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.{}-{}.{}'.format(distutils.util.get_platform(), *sys.version_info)) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) import test_yaml_ext, test_appliance From e1ffe1afaa1de24dd07283accab6e9ecf91f580f Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Fri, 13 Dec 2019 17:30:52 -0600 Subject: [PATCH 29/36] increase size of index, line, and column fields (#310) * increase size of index, line, and column fields * use size_t instead of unsigned long long * better test infrastructure for test for large file * only run large file test when env var is set * fix review comments regarding env vars * fix missing import on python 3 * force all tests in CI --- .appveyor.yml | 2 ++ .travis.yml | 4 ++++ ext/_yaml.pyx | 8 ++++---- tests/lib/test_yaml_ext.py | 18 +++++++++++++++++- tests/lib3/test_yaml_ext.py | 18 +++++++++++++++++- 5 files changed, 44 insertions(+), 6 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 1ca6e2c2d..8c7527499 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,6 +11,8 @@ image: environment: libyaml_repo_url: https://github.com/yaml/libyaml.git libyaml_refspec: 0.2.2 + PYYAML_TEST_GROUP: all + # matrix: # - PYTHON_VER: Python27 # - PYTHON_VER: Python27-x64 diff --git a/.travis.yml b/.travis.yml index a7102d699..b3185f959 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,10 @@ language: python cache: pip +env: + global: + - PYYAML_TEST_GROUP=all + matrix: include: - python: 2.7 diff --git a/ext/_yaml.pyx b/ext/_yaml.pyx index b2f409cc8..ff4efe80b 100644 --- a/ext/_yaml.pyx +++ b/ext/_yaml.pyx @@ -63,13 +63,13 @@ MappingNode = yaml.nodes.MappingNode cdef class Mark: cdef readonly object name - cdef readonly int index - cdef readonly int line - cdef readonly int column + cdef readonly size_t index + cdef readonly size_t line + cdef readonly size_t column cdef readonly buffer cdef readonly pointer - def __init__(self, object name, int index, int line, int column, + def __init__(self, object name, size_t index, size_t line, size_t column, object buffer, object pointer): self.name = name self.index = index diff --git a/tests/lib/test_yaml_ext.py b/tests/lib/test_yaml_ext.py index bdfda3e74..7f9357f66 100644 --- a/tests/lib/test_yaml_ext.py +++ b/tests/lib/test_yaml_ext.py @@ -1,6 +1,6 @@ import _yaml, yaml -import types, pprint +import types, pprint, tempfile, sys, os yaml.PyBaseLoader = yaml.BaseLoader yaml.PySafeLoader = yaml.SafeLoader @@ -233,6 +233,22 @@ def test_c_emitter(data_filename, canonical_filename, verbose=False): test_c_emitter.unittest = ['.data', '.canonical'] test_c_emitter.skip = ['.skip-ext'] +def test_large_file(verbose=False): + SIZE_LINE = 24 + SIZE_ITERATION = 0 + SIZE_FILE = 31 + if sys.maxsize <= 2**32: + return + if os.environ.get('PYYAML_TEST_GROUP', '') != 'all': + return + with tempfile.TemporaryFile() as temp_file: + for i in range(2**(SIZE_FILE-SIZE_ITERATION-SIZE_LINE) + 1): + temp_file.write(('-' + (' ' * (2**SIZE_LINE-4))+ '{}\n')*(2**SIZE_ITERATION)) + temp_file.seek(0) + yaml.load(temp_file, Loader=yaml.CLoader) + +test_large_file.unittest = None + def wrap_ext_function(function): def wrapper(*args, **kwds): _set_up() diff --git a/tests/lib3/test_yaml_ext.py b/tests/lib3/test_yaml_ext.py index 93d397b85..d902214ea 100644 --- a/tests/lib3/test_yaml_ext.py +++ b/tests/lib3/test_yaml_ext.py @@ -1,6 +1,6 @@ import _yaml, yaml -import types, pprint +import types, pprint, tempfile, sys, os yaml.PyBaseLoader = yaml.BaseLoader yaml.PySafeLoader = yaml.SafeLoader @@ -233,6 +233,22 @@ def test_c_emitter(data_filename, canonical_filename, verbose=False): test_c_emitter.unittest = ['.data', '.canonical'] test_c_emitter.skip = ['.skip-ext'] +def test_large_file(verbose=False): + SIZE_LINE = 24 + SIZE_ITERATION = 0 + SIZE_FILE = 31 + if sys.maxsize <= 2**32: + return + if os.environ.get('PYYAML_TEST_GROUP', '') != 'all': + return + with tempfile.TemporaryFile() as temp_file: + for i in range(2**(SIZE_FILE-SIZE_ITERATION-SIZE_LINE) + 1): + temp_file.write(bytes(('-' + (' ' * (2**SIZE_LINE-4))+ '{}\n')*(2**SIZE_ITERATION), 'utf-8')) + temp_file.seek(0) + yaml.load(temp_file, Loader=yaml.CLoader) + +test_large_file.unittest = None + def wrap_ext_function(function): def wrapper(*args, **kwds): _set_up() From 36fdf0c48604641b45553485d1a38a7e6478d981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Chapoton?= Date: Sat, 14 Dec 2019 00:40:56 +0100 Subject: [PATCH 30/36] remove some unused imports (#260) * remove some unused imports as suggested by lgtm https://lgtm.com/projects/g/yaml/pyyaml/ * add back import * from nodes * remove also sys import * remove mkpath import --- lib/yaml/representer.py | 3 ++- lib3/yaml/representer.py | 2 +- setup.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/yaml/representer.py b/lib/yaml/representer.py index 9dca41af7..93e09b67b 100644 --- a/lib/yaml/representer.py +++ b/lib/yaml/representer.py @@ -3,11 +3,12 @@ 'RepresenterError'] from error import * + from nodes import * import datetime -import sys, copy_reg, types +import copy_reg, types class RepresenterError(YAMLError): pass diff --git a/lib3/yaml/representer.py b/lib3/yaml/representer.py index dd144017b..3b0b192ef 100644 --- a/lib3/yaml/representer.py +++ b/lib3/yaml/representer.py @@ -5,7 +5,7 @@ from .error import * from .nodes import * -import datetime, sys, copyreg, types, base64, collections +import datetime, copyreg, types, base64, collections class RepresenterError(YAMLError): pass diff --git a/setup.py b/setup.py index a1a2baa14..1905fe525 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,6 @@ from distutils.core import setup, Command from distutils.core import Distribution as _Distribution from distutils.core import Extension as _Extension -from distutils.dir_util import mkpath from distutils.command.build_ext import build_ext as _build_ext from distutils.command.bdist_rpm import bdist_rpm as _bdist_rpm from distutils.errors import DistutilsError, CompileError, LinkError, DistutilsPlatformError From 49b354896e7d09afd5cc80e7bd786c0acbbcd47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Sat, 14 Dec 2019 00:59:14 +0100 Subject: [PATCH 31/36] tox.ini: passenv = PYYAML_TEST_GROUP --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 8d1e42d73..867f1251b 100644 --- a/tox.ini +++ b/tox.ini @@ -6,3 +6,4 @@ deps = Cython commands = python setup.py test +passenv = PYYAML_TEST_GROUP From 96d65f3de147cac6d500f8047b5542ff98a53884 Mon Sep 17 00:00:00 2001 From: Mattijs Ugen <144798+akaIDIOT@users.noreply.github.com> Date: Sun, 15 Dec 2019 17:59:22 +0100 Subject: [PATCH 32/36] Create timezone-aware datetimes when parsed as such (#163) * On load, now use aware datetimes if possible On loading data, if timestamps have an ISO "+HH:MM" UTC offset then the resultant datetime is converted to UTC. This change adds that timezone information to the datetime objects. Importantly, this addresses a Django warning (and potential error) that appears when using both YAML fixtures in a timezone-aware project. It was raised as a Django issue (https://code.djangoproject.com/ticket/18867), but subsequently closed because the Django devs felt that this is a PyYAML problem. * Create timezone-aware datetime in timezone from data * Create timezone-aware datetime in timezone from data for python2 * Define better timezone implementation for python2 * Handle timezone "Z" for python 3 * Handle timezone "Z" for python 2 * Fix code structure for Python 3 Call datetime.datetime constructor once at return. * Fix code structure for Python 2 Call datetime.datetime constructor once at return. --- lib/yaml/constructor.py | 36 ++++++++++++++++++++++++++++++------ lib3/yaml/constructor.py | 11 ++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 08bdabfb1..568613b78 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -18,6 +18,29 @@ class ConstructorError(MarkedYAMLError): pass + +class timezone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + seconds = abs(offset).total_seconds() + self._name = '%s%02d:%02d' % ( + '-' if offset.days < 0 else '+', + seconds // 3600, + seconds % 3600 // 60 + ) + + def tzname(self, dt=None): + return self._name + + def utcoffset(self, dt=None): + return self._offset + + def dst(self, dt=None): + return datetime.timedelta(0) + + __repr__ = __str__ = tzname + + class BaseConstructor(object): yaml_constructors = {} @@ -293,7 +316,7 @@ def construct_yaml_binary(self, node): return str(value).decode('base64') except (binascii.Error, UnicodeEncodeError), exc: raise ConstructorError(None, None, - "failed to decode base64 data: %s" % exc, node.start_mark) + "failed to decode base64 data: %s" % exc, node.start_mark) timestamp_regexp = re.compile( ur'''^(?P[0-9][0-9][0-9][0-9]) @@ -320,22 +343,23 @@ def construct_yaml_timestamp(self, node): minute = int(values['minute']) second = int(values['second']) fraction = 0 + tzinfo = None if values['fraction']: fraction = values['fraction'][:6] while len(fraction) < 6: fraction += '0' fraction = int(fraction) - delta = None if values['tz_sign']: tz_hour = int(values['tz_hour']) tz_minute = int(values['tz_minute'] or 0) delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) if values['tz_sign'] == '-': delta = -delta - data = datetime.datetime(year, month, day, hour, minute, second, fraction) - if delta: - data -= delta - return data + tzinfo = timezone(delta) + elif values['tz']: + tzinfo = timezone(datetime.timedelta(0)) + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) def construct_yaml_omap(self, node): # Note: we do not check for duplicate keys, because it's too diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index ac15aba87..cd9167eaf 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -324,22 +324,23 @@ def construct_yaml_timestamp(self, node): minute = int(values['minute']) second = int(values['second']) fraction = 0 + tzinfo = None if values['fraction']: fraction = values['fraction'][:6] while len(fraction) < 6: fraction += '0' fraction = int(fraction) - delta = None if values['tz_sign']: tz_hour = int(values['tz_hour']) tz_minute = int(values['tz_minute'] or 0) delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) if values['tz_sign'] == '-': delta = -delta - data = datetime.datetime(year, month, day, hour, minute, second, fraction) - if delta: - data -= delta - return data + tzinfo = datetime.timezone(delta) + elif values['tz']: + tzinfo = datetime.timezone.utc + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) def construct_yaml_omap(self, node): # Note: we do not check for duplicate keys, because it's too From 4fcdcdbf604f7ce6ba190f6fd19b137df7b22b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller=20=28tinita=29?= Date: Mon, 16 Dec 2019 21:31:04 +0100 Subject: [PATCH 33/36] Add tests for timezone (#363) After #163, this adds some test data to check if the datetime objects return the correct timezone --- lib/yaml/constructor.py | 2 +- tests/data/timestamp-bugs.code | 12 ++++++------ tests/data/timestamp-bugs.data | 18 ++++++++++++------ tests/lib/test_constructor.py | 11 +++++++++++ tests/lib3/test_constructor.py | 11 +++++++++++ 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 568613b78..8ce39722d 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -23,7 +23,7 @@ class timezone(datetime.tzinfo): def __init__(self, offset): self._offset = offset seconds = abs(offset).total_seconds() - self._name = '%s%02d:%02d' % ( + self._name = 'UTC%s%02d:%02d' % ( '-' if offset.days < 0 else '+', seconds // 3600, seconds % 3600 // 60 diff --git a/tests/data/timestamp-bugs.code b/tests/data/timestamp-bugs.code index b1d6e9c4f..e2c84cc3c 100644 --- a/tests/data/timestamp-bugs.code +++ b/tests/data/timestamp-bugs.code @@ -1,8 +1,8 @@ [ - datetime.datetime(2001, 12, 15, 3, 29, 43, 100000), - datetime.datetime(2001, 12, 14, 16, 29, 43, 100000), - datetime.datetime(2001, 12, 14, 21, 59, 43, 1010), - datetime.datetime(2001, 12, 14, 21, 59, 43, 0, FixedOffset(60, "+1")), - datetime.datetime(2001, 12, 14, 21, 59, 43, 0, FixedOffset(-90, "-1:30")), - datetime.datetime(2005, 7, 8, 17, 35, 4, 517600), + [datetime.datetime(2001, 12, 15, 3, 29, 43, 100000), 'UTC-05:30'], + [datetime.datetime(2001, 12, 14, 16, 29, 43, 100000), 'UTC+05:30'], + [datetime.datetime(2001, 12, 14, 21, 59, 43, 1010), None], + [datetime.datetime(2001, 12, 14, 21, 59, 43, 0, FixedOffset(60, "+1")), 'UTC+01:00'], + [datetime.datetime(2001, 12, 14, 21, 59, 43, 0, FixedOffset(-90, "-1:30")), 'UTC-01:30'], + [datetime.datetime(2005, 7, 8, 17, 35, 4, 517600), None], ] diff --git a/tests/data/timestamp-bugs.data b/tests/data/timestamp-bugs.data index 721d29082..b243d0d2c 100644 --- a/tests/data/timestamp-bugs.data +++ b/tests/data/timestamp-bugs.data @@ -1,6 +1,12 @@ -- 2001-12-14 21:59:43.10 -5:30 -- 2001-12-14 21:59:43.10 +5:30 -- 2001-12-14 21:59:43.00101 -- 2001-12-14 21:59:43+1 -- 2001-12-14 21:59:43-1:30 -- 2005-07-08 17:35:04.517600 +- !MyTime + - 2001-12-14 21:59:43.10 -5:30 +- !MyTime + - 2001-12-14 21:59:43.10 +5:30 +- !MyTime + - 2001-12-14 21:59:43.00101 +- !MyTime + - 2001-12-14 21:59:43+1 +- !MyTime + - 2001-12-14 21:59:43-1:30 +- !MyTime + - 2005-07-08 17:35:04.517600 diff --git a/tests/lib/test_constructor.py b/tests/lib/test_constructor.py index 9e6c85647..a22fd1891 100644 --- a/tests/lib/test_constructor.py +++ b/tests/lib/test_constructor.py @@ -41,7 +41,18 @@ def construct1(constructor, node): def represent1(representer, native): return representer.represent_mapping("!tag1", native.__dict__) + def my_time_constructor(constructor, node): + seq = constructor.construct_sequence(node) + dt = seq[0] + tz = None + try: + tz = dt.tzinfo.tzname(dt) + except: + pass + return [dt, tz] + yaml.add_constructor("!tag1", construct1, Loader=MyLoader) + yaml.add_constructor("!MyTime", my_time_constructor, Loader=MyLoader) yaml.add_representer(MyTestClass1, represent1, Dumper=MyDumper) class MyTestClass2(MyTestClass1, yaml.YAMLObject): diff --git a/tests/lib3/test_constructor.py b/tests/lib3/test_constructor.py index 3d696498b..877982db8 100644 --- a/tests/lib3/test_constructor.py +++ b/tests/lib3/test_constructor.py @@ -38,7 +38,18 @@ def construct1(constructor, node): def represent1(representer, native): return representer.represent_mapping("!tag1", native.__dict__) + def my_time_constructor(constructor, node): + seq = constructor.construct_sequence(node) + dt = seq[0] + tz = None + try: + tz = dt.tzinfo.tzname(dt) + except: + pass + return [dt, tz] + yaml.add_constructor("!tag1", construct1, Loader=MyLoader) + yaml.add_constructor("!MyTime", my_time_constructor, Loader=MyLoader) yaml.add_representer(MyTestClass1, represent1, Dumper=MyDumper) class MyTestClass2(MyTestClass1, yaml.YAMLObject): From 69b025a9f32fd864ed2096a444a718b7492eb892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Fri, 20 Dec 2019 20:41:29 +0100 Subject: [PATCH 34/36] Changes for 5.3b1 --- CHANGES | 18 ++++++++++++++++++ lib/yaml/__init__.py | 2 +- lib3/yaml/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index f4a9ae577..84c19f714 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,24 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/commits/ * https://bitbucket.org/xi/pyyaml/commits/ +5.3b1 (2019-12-21) + +* https://github.com/yaml/pyyaml/pull/290 -- Use `is` instead of equality for comparing with `None` +* https://github.com/yaml/pyyaml/pull/270 -- fix typos and stylistic nit +* https://github.com/yaml/pyyaml/pull/309 -- Fix up small typo +* https://github.com/yaml/pyyaml/pull/161 -- Fix handling of __slots__ +* https://github.com/yaml/pyyaml/pull/358 -- Allow calling add_multi_constructor with None +* https://github.com/yaml/pyyaml/pull/285 -- Add use of safe_load() function in README +* https://github.com/yaml/pyyaml/pull/351 -- Fix reader for Unicode code points over 0xFFFF +* https://github.com/yaml/pyyaml/pull/360 -- Enable certain unicode tests when maxunicode not > 0xffff +* https://github.com/yaml/pyyaml/pull/359 -- Use full_load in yaml-highlight example +* https://github.com/yaml/pyyaml/pull/244 -- Document that PyYAML is implemented with Cython +* https://github.com/yaml/pyyaml/pull/329 -- Fix for Python 3.10 +* https://github.com/yaml/pyyaml/pull/310 -- increase size of index, line, and column fields +* https://github.com/yaml/pyyaml/pull/260 -- remove some unused imports +* https://github.com/yaml/pyyaml/pull/163 -- Create timezone-aware datetimes when parsed as such +* https://github.com/yaml/pyyaml/pull/363 -- Add tests for timezone + 5.2 (2019-12-02) ------------------ diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 4703dffb0..91b97ea6a 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.2' +__version__ = '5.3b1' try: from cyaml import * diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index b7cb86aa1..524354f97 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.2' +__version__ = '5.3b1' try: from .cyaml import * __with_libyaml__ = True diff --git a/setup.py b/setup.py index 1905fe525..9f872b945 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.2' +VERSION = '5.3b1' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability From 377092fb2eb2dcddb323a42aac4bc9089161b585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Mon, 6 Jan 2020 20:37:50 +0100 Subject: [PATCH 35/36] Changes for 5.3 --- CHANGES | 2 +- lib/yaml/__init__.py | 2 +- lib3/yaml/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index 84c19f714..e74edc224 100644 --- a/CHANGES +++ b/CHANGES @@ -4,7 +4,7 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/commits/ * https://bitbucket.org/xi/pyyaml/commits/ -5.3b1 (2019-12-21) +5.3 (2020-01-06) * https://github.com/yaml/pyyaml/pull/290 -- Use `is` instead of equality for comparing with `None` * https://github.com/yaml/pyyaml/pull/270 -- fix typos and stylistic nit diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 91b97ea6a..195470479 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.3b1' +__version__ = '5.3' try: from cyaml import * diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 524354f97..658a08a6b 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.3b1' +__version__ = '5.3' try: from .cyaml import * __with_libyaml__ = True diff --git a/setup.py b/setup.py index 9f872b945..1a0413a94 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.3b1' +VERSION = '5.3' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability From 2f463cf5b0e98a52bc20e348d1e69761bf263b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tina=20M=C3=BCller?= Date: Mon, 6 Jan 2020 21:09:53 +0100 Subject: [PATCH 36/36] Update announcement.msg --- announcement.msg | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/announcement.msg b/announcement.msg index 292d0aca9..c52ecd56a 100644 --- a/announcement.msg +++ b/announcement.msg @@ -1,31 +1,37 @@ From: Tina Müller To: python-list@python.org, python-announce@python.org, yaml-core@lists.sourceforge.net -Subject: [ANN] PyYAML-5.2: YAML parser and emitter for Python +Subject: [ANN] PyYAML-5.3: YAML parser and emitter for Python ======================= -Announcing PyYAML-5.2 +Announcing PyYAML-5.3 ======================= A new release of PyYAML is now available: https://pypi.org/project/PyYAML/ -This fixes some incompatibilities introduced in version 5.1 and also removes -another possibility of loading arbitrary code. +This release contains some bugfixes (handling of slots, enable unicode for +maxunicode < 0xffff, enable large files), enhancements (create timezone +aware datetimes) and some other small enhancements. Changes ======= -* Repair incompatibilities introduced with 5.1. The default Loader was changed, - but several methods like add_constructor still used the old default - https://github.com/yaml/pyyaml/pull/279 -- A more flexible fix for custom tag constructors - https://github.com/yaml/pyyaml/pull/287 -- Change default loader for yaml.add_constructor - https://github.com/yaml/pyyaml/pull/305 -- Change default loader for add_implicit_resolver, add_path_resolver -* Make FullLoader safer by removing python/object/apply from the default FullLoader - https://github.com/yaml/pyyaml/pull/347 -- Move constructor for object/apply to UnsafeConstructor -* Fix bug introduced in 5.1 where quoting went wrong on systems with sys.maxunicode <= 0xffff - https://github.com/yaml/pyyaml/pull/276 -- Fix logic for quoting special characters -* Other PRs: - https://github.com/yaml/pyyaml/pull/280 -- Update CHANGES for 5.1 +* https://github.com/yaml/pyyaml/pull/290 -- Use `is` instead of equality for comparing with `None` +* https://github.com/yaml/pyyaml/pull/270 -- fix typos and stylistic nit +* https://github.com/yaml/pyyaml/pull/309 -- Fix up small typo +* https://github.com/yaml/pyyaml/pull/161 -- Fix handling of __slots__ +* https://github.com/yaml/pyyaml/pull/358 -- Allow calling add_multi_constructor with None +* https://github.com/yaml/pyyaml/pull/285 -- Add use of safe_load() function in README +* https://github.com/yaml/pyyaml/pull/351 -- Fix reader for Unicode code points over 0xFFFF +* https://github.com/yaml/pyyaml/pull/360 -- Enable certain unicode tests when maxunicode not > 0xffff +* https://github.com/yaml/pyyaml/pull/359 -- Use full_load in yaml-highlight example +* https://github.com/yaml/pyyaml/pull/244 -- Document that PyYAML is implemented with Cython +* https://github.com/yaml/pyyaml/pull/329 -- Fix for Python 3.10 +* https://github.com/yaml/pyyaml/pull/310 -- increase size of index, line, and column fields +* https://github.com/yaml/pyyaml/pull/260 -- remove some unused imports +* https://github.com/yaml/pyyaml/pull/163 -- Create timezone-aware datetimes when parsed as such +* https://github.com/yaml/pyyaml/pull/363 -- Add tests for timezone + Resources =========