diff --git a/.appveyor.yml b/.appveyor.yml index bdad5f319..8c7527499 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -10,12 +10,12 @@ image: environment: libyaml_repo_url: https://github.com/yaml/libyaml.git - libyaml_refspec: release/0.2.2 + libyaml_refspec: 0.2.2 + PYYAML_TEST_GROUP: all + # matrix: # - PYTHON_VER: Python27 # - PYTHON_VER: Python27-x64 -# - PYTHON_VER: Python34 -# - PYTHON_VER: Python34-x64 # - PYTHON_VER: Python35 # - PYTHON_VER: Python35-x64 # - PYTHON_VER: Python36 diff --git a/.travis.yml b/.travis.yml index 3dfe3ad9c..b3185f959 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,25 +2,28 @@ language: python -sudo: false - cache: pip +env: + global: + - PYYAML_TEST_GROUP=all + matrix: include: - python: 2.7 env: TOXENV=py27 - - python: 3.4 - env: TOXENV=py34 - python: 3.5 env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 3.7-dev + - python: 3.7 env: TOXENV=py37 - # This is broken on travis as of 2019/03/12 - # - python: pypy - # env: TOXENV=pypy + - python: 3.8 + env: TOXENV=py38 + - python: 3.8-dev + env: TOXENV=py38 + - python: pypy + env: TOXENV=pypy # build libyaml before_script: diff --git a/CHANGES b/CHANGES index 91f0255bb..e74edc224 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,49 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/commits/ * https://bitbucket.org/xi/pyyaml/commits/ +5.3 (2020-01-06) + +* https://github.com/yaml/pyyaml/pull/290 -- Use `is` instead of equality for comparing with `None` +* https://github.com/yaml/pyyaml/pull/270 -- fix typos and stylistic nit +* https://github.com/yaml/pyyaml/pull/309 -- Fix up small typo +* https://github.com/yaml/pyyaml/pull/161 -- Fix handling of __slots__ +* https://github.com/yaml/pyyaml/pull/358 -- Allow calling add_multi_constructor with None +* https://github.com/yaml/pyyaml/pull/285 -- Add use of safe_load() function in README +* https://github.com/yaml/pyyaml/pull/351 -- Fix reader for Unicode code points over 0xFFFF +* https://github.com/yaml/pyyaml/pull/360 -- Enable certain unicode tests when maxunicode not > 0xffff +* https://github.com/yaml/pyyaml/pull/359 -- Use full_load in yaml-highlight example +* https://github.com/yaml/pyyaml/pull/244 -- Document that PyYAML is implemented with Cython +* https://github.com/yaml/pyyaml/pull/329 -- Fix for Python 3.10 +* https://github.com/yaml/pyyaml/pull/310 -- increase size of index, line, and column fields +* https://github.com/yaml/pyyaml/pull/260 -- remove some unused imports +* https://github.com/yaml/pyyaml/pull/163 -- Create timezone-aware datetimes when parsed as such +* https://github.com/yaml/pyyaml/pull/363 -- Add tests for timezone + +5.2 (2019-12-02) +------------------ + +* Repair incompatibilities introduced with 5.1. The default Loader was changed, + but several methods like add_constructor still used the old default + https://github.com/yaml/pyyaml/pull/279 -- A more flexible fix for custom tag constructors + https://github.com/yaml/pyyaml/pull/287 -- Change default loader for yaml.add_constructor + https://github.com/yaml/pyyaml/pull/305 -- Change default loader for add_implicit_resolver, add_path_resolver +* Make FullLoader safer by removing python/object/apply from the default FullLoader + https://github.com/yaml/pyyaml/pull/347 -- Move constructor for object/apply to UnsafeConstructor +* Fix bug introduced in 5.1 where quoting went wrong on systems with sys.maxunicode <= 0xffff + https://github.com/yaml/pyyaml/pull/276 -- Fix logic for quoting special characters +* Other PRs: + https://github.com/yaml/pyyaml/pull/280 -- Update CHANGES for 5.1 + +5.1.2 (2019-07-30) +------------------ + +* Re-release of 5.1 with regenerated Cython sources to build properly for Python 3.8b2+ + +5.1.1 (2019-06-05) +------------------ + +* Re-release of 5.1 with regenerated Cython sources to build properly for Python 3.8b1 + 5.1 (2019-03-13) ---------------- @@ -17,7 +60,6 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/pull/61 -- Use Travis CI built in pip cache support * https://github.com/yaml/pyyaml/pull/62 -- Remove tox workaround for Travis CI * https://github.com/yaml/pyyaml/pull/63 -- Adding support to Unicode characters over codepoint 0xffff -* https://github.com/yaml/pyyaml/pull/65 -- Support unicode literals over codepoint 0xffff * https://github.com/yaml/pyyaml/pull/75 -- add 3.12 changelog * https://github.com/yaml/pyyaml/pull/76 -- Fallback to Pure Python if Compilation fails * https://github.com/yaml/pyyaml/pull/84 -- Drop unsupported Python 3.3 @@ -25,12 +67,17 @@ For a complete changelog, see: * https://github.com/yaml/pyyaml/pull/105 -- Removed Python 2.6 & 3.3 support * https://github.com/yaml/pyyaml/pull/111 -- Remove commented out Psyco code * https://github.com/yaml/pyyaml/pull/129 -- Remove call to `ord` in lib3 emitter code -* https://github.com/yaml/pyyaml/pull/143 -- Allow to turn off sorting keys in Dumper * https://github.com/yaml/pyyaml/pull/149 -- Test on Python 3.7-dev * https://github.com/yaml/pyyaml/pull/158 -- Support escaped slash in double quotes "\/" +* https://github.com/yaml/pyyaml/pull/175 -- Updated link to pypi in release announcement * https://github.com/yaml/pyyaml/pull/181 -- Import Hashable from collections.abc +* https://github.com/yaml/pyyaml/pull/194 -- Reverting https://github.com/yaml/pyyaml/pull/74 +* https://github.com/yaml/pyyaml/pull/195 -- Build libyaml on travis +* https://github.com/yaml/pyyaml/pull/196 -- Force cython when building sdist +* https://github.com/yaml/pyyaml/pull/254 -- Allow to turn off sorting keys in Dumper (2) * https://github.com/yaml/pyyaml/pull/256 -- Make default_flow_style=False * https://github.com/yaml/pyyaml/pull/257 -- Deprecate yaml.load and add FullLoader and UnsafeLoader classes +* https://github.com/yaml/pyyaml/pull/261 -- Skip certain unicode tests when maxunicode not > 0xffff * https://github.com/yaml/pyyaml/pull/263 -- Windows Appveyor build 3.13 (2018-07-05) diff --git a/README b/README index 361839a05..49c87e764 100644 --- a/README +++ b/README @@ -15,6 +15,10 @@ parser and emitter as follows: >>> yaml.load(stream, Loader=yaml.CLoader) >>> yaml.dump(data, Dumper=yaml.CDumper) +If you don't trust the input stream, you should use: + + >>> yaml.safe_load(stream) + PyYAML includes a comprehensive test suite. To run the tests, type 'python setup.py test'. diff --git a/announcement.msg b/announcement.msg index 99bf92422..c52ecd56a 100644 --- a/announcement.msg +++ b/announcement.msg @@ -1,57 +1,36 @@ -From: Ingy döt Net +From: Tina Müller To: python-list@python.org, python-announce@python.org, yaml-core@lists.sourceforge.net -Subject: [ANN] PyYAML-5.1: YAML parser and emitter for Python +Subject: [ANN] PyYAML-5.3: YAML parser and emitter for Python ======================= - Announcing PyYAML-5.1 +Announcing PyYAML-5.3 ======================= -A new MAJOR RELEASE of PyYAML is now available: +A new release of PyYAML is now available: https://pypi.org/project/PyYAML/ -This is the first major release of PyYAML under the new maintenance team. - -Among the many changes listed below, this release specifically addresses the -arbitrary code execution issue raised by: - - https://nvd.nist.gov/vuln/detail/CVE-2017-18342 - -(See https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation -for complete details). - -The PyYAML project is now maintained by the YAML and Python communities. -Planning happens on the #yaml-dev, #pyyaml and #libyaml IRC channels on -irc.freenode.net. - +This release contains some bugfixes (handling of slots, enable unicode for +maxunicode < 0xffff, enable large files), enhancements (create timezone +aware datetimes) and some other small enhancements. Changes ======= -* https://github.com/yaml/pyyaml/pull/35 -- Some modernization of the test running -* https://github.com/yaml/pyyaml/pull/42 -- Install tox in a virtualenv -* https://github.com/yaml/pyyaml/pull/45 -- Allow colon in a plain scalar in a flow context -* https://github.com/yaml/pyyaml/pull/48 -- Fix typos -* https://github.com/yaml/pyyaml/pull/55 -- Improve RepresenterError creation -* https://github.com/yaml/pyyaml/pull/59 -- Resolves #57, update readme issues link -* https://github.com/yaml/pyyaml/pull/60 -- Document and test Python 3.6 support -* https://github.com/yaml/pyyaml/pull/61 -- Use Travis CI built in pip cache support -* https://github.com/yaml/pyyaml/pull/62 -- Remove tox workaround for Travis CI -* https://github.com/yaml/pyyaml/pull/63 -- Adding support to Unicode characters over codepoint 0xffff -* https://github.com/yaml/pyyaml/pull/65 -- Support unicode literals over codepoint 0xffff -* https://github.com/yaml/pyyaml/pull/75 -- add 3.12 changelog -* https://github.com/yaml/pyyaml/pull/76 -- Fallback to Pure Python if Compilation fails -* https://github.com/yaml/pyyaml/pull/84 -- Drop unsupported Python 3.3 -* https://github.com/yaml/pyyaml/pull/102 -- Include license file in the generated wheel package -* https://github.com/yaml/pyyaml/pull/105 -- Removed Python 2.6 & 3.3 support -* https://github.com/yaml/pyyaml/pull/111 -- Remove commented out Psyco code -* https://github.com/yaml/pyyaml/pull/129 -- Remove call to `ord` in lib3 emitter code -* https://github.com/yaml/pyyaml/pull/143 -- Allow to turn off sorting keys in Dumper -* https://github.com/yaml/pyyaml/pull/149 -- Test on Python 3.7-dev -* https://github.com/yaml/pyyaml/pull/158 -- Support escaped slash in double quotes "\/" -* https://github.com/yaml/pyyaml/pull/181 -- Import Hashable from collections.abc -* https://github.com/yaml/pyyaml/pull/256 -- Make default_flow_style=False -* https://github.com/yaml/pyyaml/pull/257 -- Deprecate yaml.load and add FullLoader and UnsafeLoader classes -* https://github.com/yaml/pyyaml/pull/263 -- Windows Appveyor build +* https://github.com/yaml/pyyaml/pull/290 -- Use `is` instead of equality for comparing with `None` +* https://github.com/yaml/pyyaml/pull/270 -- fix typos and stylistic nit +* https://github.com/yaml/pyyaml/pull/309 -- Fix up small typo +* https://github.com/yaml/pyyaml/pull/161 -- Fix handling of __slots__ +* https://github.com/yaml/pyyaml/pull/358 -- Allow calling add_multi_constructor with None +* https://github.com/yaml/pyyaml/pull/285 -- Add use of safe_load() function in README +* https://github.com/yaml/pyyaml/pull/351 -- Fix reader for Unicode code points over 0xFFFF +* https://github.com/yaml/pyyaml/pull/360 -- Enable certain unicode tests when maxunicode not > 0xffff +* https://github.com/yaml/pyyaml/pull/359 -- Use full_load in yaml-highlight example +* https://github.com/yaml/pyyaml/pull/244 -- Document that PyYAML is implemented with Cython +* https://github.com/yaml/pyyaml/pull/329 -- Fix for Python 3.10 +* https://github.com/yaml/pyyaml/pull/310 -- increase size of index, line, and column fields +* https://github.com/yaml/pyyaml/pull/260 -- remove some unused imports +* https://github.com/yaml/pyyaml/pull/163 -- Create timezone-aware datetimes when parsed as such +* https://github.com/yaml/pyyaml/pull/363 -- Add tests for timezone Resources diff --git a/tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.code b/emitting-unacceptable-unicode-character-bug.code similarity index 100% rename from tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.code rename to emitting-unacceptable-unicode-character-bug.code diff --git a/tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.data b/emitting-unacceptable-unicode-character-bug.data similarity index 100% rename from tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.data rename to emitting-unacceptable-unicode-character-bug.data diff --git a/tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.skip-ext b/emitting-unacceptable-unicode-character-bug.skip-ext similarity index 100% rename from tests/data/emitting-unacceptable-unicode-character-bug-ucs4-py2.skip-ext rename to emitting-unacceptable-unicode-character-bug.skip-ext diff --git a/examples/yaml-highlight/yaml_hl.py b/examples/yaml-highlight/yaml_hl.py index d6f7bf4eb..96e0ae7b1 100755 --- a/examples/yaml-highlight/yaml_hl.py +++ b/examples/yaml-highlight/yaml_hl.py @@ -37,7 +37,7 @@ def __setstate__(self, state): class YAMLHighlight: def __init__(self, options): - config = yaml.load(file(options.config, 'rb').read()) + config = yaml.full_load(file(options.config, 'rb').read()) self.style = config[options.style] if options.input: self.input = file(options.input, 'rb') diff --git a/ext/_yaml.pyx b/ext/_yaml.pyx index b2f409cc8..ff4efe80b 100644 --- a/ext/_yaml.pyx +++ b/ext/_yaml.pyx @@ -63,13 +63,13 @@ MappingNode = yaml.nodes.MappingNode cdef class Mark: cdef readonly object name - cdef readonly int index - cdef readonly int line - cdef readonly int column + cdef readonly size_t index + cdef readonly size_t line + cdef readonly size_t column cdef readonly buffer cdef readonly pointer - def __init__(self, object name, int index, int line, int column, + def __init__(self, object name, size_t index, size_t line, size_t column, object buffer, object pointer): self.name = name self.index = index diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index e7a419dd2..195470479 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '5.1' +__version__ = '5.3' try: from cyaml import * @@ -309,42 +309,62 @@ def safe_dump(data, stream=None, **kwds): return dump_all([data], stream, Dumper=SafeDumper, **kwds) def add_implicit_resolver(tag, regexp, first=None, - Loader=Loader, Dumper=Dumper): + Loader=None, Dumper=Dumper): """ Add an implicit scalar detector. If an implicit scalar value matches the given regexp, the corresponding tag is assigned to the scalar. first is a sequence of possible initial characters or None. """ - Loader.add_implicit_resolver(tag, regexp, first) + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) Dumper.add_implicit_resolver(tag, regexp, first) -def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper): +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): """ Add a path based resolver for the given tag. A path is a list of keys that forms a path to a node in the representation tree. Keys can be string values, integers, or None. """ - Loader.add_path_resolver(tag, path, kind) + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) Dumper.add_path_resolver(tag, path, kind) -def add_constructor(tag, constructor, Loader=Loader): +def add_constructor(tag, constructor, Loader=None): """ Add a constructor for the given tag. Constructor is a function that accepts a Loader instance and a node object and produces the corresponding Python object. """ - Loader.add_constructor(tag, constructor) + if Loader is None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) -def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader): +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): """ Add a multi-constructor for the given tag prefix. Multi-constructor is called for a node if its tag starts with tag_prefix. Multi-constructor accepts a Loader instance, a tag suffix, and a node object and produces the corresponding Python object. """ - Loader.add_multi_constructor(tag_prefix, multi_constructor) + if Loader is None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) def add_representer(data_type, representer, Dumper=Dumper): """ @@ -371,7 +391,12 @@ class YAMLObjectMetaclass(type): def __init__(cls, name, bases, kwds): super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: - cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + cls.yaml_dumper.add_representer(cls, cls.to_yaml) class YAMLObject(object): @@ -383,7 +408,7 @@ class YAMLObject(object): __metaclass__ = YAMLObjectMetaclass __slots__ = () # no direct instantiation, so allow immutable subclasses - yaml_loader = Loader + yaml_loader = [Loader, FullLoader, UnsafeLoader] yaml_dumper = Dumper yaml_tag = None diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 516dad1ce..8ce39722d 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -18,6 +18,29 @@ class ConstructorError(MarkedYAMLError): pass + +class timezone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + seconds = abs(offset).total_seconds() + self._name = 'UTC%s%02d:%02d' % ( + '-' if offset.days < 0 else '+', + seconds // 3600, + seconds % 3600 // 60 + ) + + def tzname(self, dt=None): + return self._name + + def utcoffset(self, dt=None): + return self._offset + + def dst(self, dt=None): + return datetime.timedelta(0) + + __repr__ = __str__ = tzname + + class BaseConstructor(object): yaml_constructors = {} @@ -74,7 +97,7 @@ def construct_object(self, node, deep=False): constructor = self.yaml_constructors[node.tag] else: for tag_prefix in self.yaml_multi_constructors: - if node.tag.startswith(tag_prefix): + if tag_prefix is not None and node.tag.startswith(tag_prefix): tag_suffix = node.tag[len(tag_prefix):] constructor = self.yaml_multi_constructors[tag_prefix] break @@ -293,7 +316,7 @@ def construct_yaml_binary(self, node): return str(value).decode('base64') except (binascii.Error, UnicodeEncodeError), exc: raise ConstructorError(None, None, - "failed to decode base64 data: %s" % exc, node.start_mark) + "failed to decode base64 data: %s" % exc, node.start_mark) timestamp_regexp = re.compile( ur'''^(?P[0-9][0-9][0-9][0-9]) @@ -320,22 +343,23 @@ def construct_yaml_timestamp(self, node): minute = int(values['minute']) second = int(values['second']) fraction = 0 + tzinfo = None if values['fraction']: fraction = values['fraction'][:6] while len(fraction) < 6: fraction += '0' fraction = int(fraction) - delta = None if values['tz_sign']: tz_hour = int(values['tz_hour']) tz_minute = int(values['tz_minute'] or 0) delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) if values['tz_sign'] == '-': delta = -delta - data = datetime.datetime(year, month, day, hour, minute, second, fraction) - if delta: - data -= delta - return data + tzinfo = timezone(delta) + elif values['tz']: + tzinfo = timezone(datetime.timedelta(0)) + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) def construct_yaml_omap(self, node): # Note: we do not check for duplicate keys, because it's too @@ -497,7 +521,7 @@ def find_python_module(self, name, mark, unsafe=False): except ImportError, exc: raise ConstructorError("while constructing a Python module", mark, "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark) - if not name in sys.modules: + if name not in sys.modules: raise ConstructorError("while constructing a Python module", mark, "module %r is not imported" % name.encode('utf-8'), mark) return sys.modules[name] @@ -517,7 +541,7 @@ def find_python_name(self, name, mark, unsafe=False): except ImportError, exc: raise ConstructorError("while constructing a Python object", mark, "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark) - if not module_name in sys.modules: + if module_name not in sys.modules: raise ConstructorError("while constructing a Python object", mark, "module %r is not imported" % module_name.encode('utf-8'), mark) module = sys.modules[module_name] @@ -578,7 +602,7 @@ def set_python_instance_state(self, instance, state): elif state: slotstate.update(state) for key, value in slotstate.items(): - setattr(object, key, value) + setattr(instance, key, value) def construct_python_object(self, suffix, node): # Format: @@ -683,10 +707,6 @@ def construct_python_object_new(self, suffix, node): u'tag:yaml.org,2002:python/object:', FullConstructor.construct_python_object) -FullConstructor.add_multi_constructor( - u'tag:yaml.org,2002:python/object/apply:', - FullConstructor.construct_python_object_apply) - FullConstructor.add_multi_constructor( u'tag:yaml.org,2002:python/object/new:', FullConstructor.construct_python_object_new) @@ -703,6 +723,10 @@ def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False) return super(UnsafeConstructor, self).make_python_instance( suffix, node, args, kwds, newobj, unsafe=True) +UnsafeConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + # Constructor is same as UnsafeConstructor. Need to leave this in place in case # people have extended it directly. class Constructor(UnsafeConstructor): diff --git a/lib/yaml/emitter.py b/lib/yaml/emitter.py index 9561a8274..23c25ca80 100644 --- a/lib/yaml/emitter.py +++ b/lib/yaml/emitter.py @@ -706,7 +706,7 @@ def analyze_scalar(self, scalar): if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'): if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF' or u'\uE000' <= ch <= u'\uFFFD' - or ((not has_ucs4) or (u'\U00010000' <= ch < u'\U0010ffff'))) and ch != u'\uFEFF': + or (u'\U00010000' <= ch < u'\U0010ffff')) and ch != u'\uFEFF': unicode_characters = True if not self.allow_unicode: special_characters = True diff --git a/lib/yaml/loader.py b/lib/yaml/loader.py index a79182eaf..4d773c3cc 100644 --- a/lib/yaml/loader.py +++ b/lib/yaml/loader.py @@ -51,7 +51,7 @@ def __init__(self, stream): # UnsafeLoader is the same as Loader (which is and was always unsafe on # untrusted input). Use of either Loader or UnsafeLoader should be rare, since # FullLoad should be able to load almost all YAML safely. Loader is left intact -# to ensure backwards compatability. +# to ensure backwards compatibility. class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): def __init__(self, stream): diff --git a/lib/yaml/reader.py b/lib/yaml/reader.py index b2f10b091..4b377d61e 100644 --- a/lib/yaml/reader.py +++ b/lib/yaml/reader.py @@ -139,7 +139,7 @@ def determine_encoding(self): if has_ucs4: NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]') else: - NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]') + NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uFFFD]|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?:[^\uDC00-\uDFFF]|$)') def check_printable(self, data): match = self.NON_PRINTABLE.search(data) if match: diff --git a/lib/yaml/representer.py b/lib/yaml/representer.py index 9dca41af7..93e09b67b 100644 --- a/lib/yaml/representer.py +++ b/lib/yaml/representer.py @@ -3,11 +3,12 @@ 'RepresenterError'] from error import * + from nodes import * import datetime -import sys, copy_reg, types +import copy_reg, types class RepresenterError(YAMLError): pass diff --git a/lib/yaml/scanner.py b/lib/yaml/scanner.py index 5126cf07b..098ea7be8 100644 --- a/lib/yaml/scanner.py +++ b/lib/yaml/scanner.py @@ -332,7 +332,7 @@ def unwind_indent(self, column): ## } #if self.flow_level and self.indent > column: # raise ScannerError(None, None, - # "invalid intendation or unclosed '[' or '{'", + # "invalid indentation or unclosed '[' or '{'", # self.get_mark()) # In the flow context, indentation is ignored. We make the scanner less @@ -370,7 +370,7 @@ def fetch_stream_start(self): def fetch_stream_end(self): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. @@ -389,7 +389,7 @@ def fetch_stream_end(self): def fetch_directive(self): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. @@ -407,7 +407,7 @@ def fetch_document_end(self): def fetch_document_indicator(self, TokenClass): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. Note that there could not be a block collection diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 5df0bb5fd..658a08a6b 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -8,7 +8,7 @@ from .loader import * from .dumper import * -__version__ = '5.1' +__version__ = '5.3' try: from .cyaml import * __with_libyaml__ = True @@ -306,42 +306,62 @@ def safe_dump(data, stream=None, **kwds): return dump_all([data], stream, Dumper=SafeDumper, **kwds) def add_implicit_resolver(tag, regexp, first=None, - Loader=Loader, Dumper=Dumper): + Loader=None, Dumper=Dumper): """ Add an implicit scalar detector. If an implicit scalar value matches the given regexp, the corresponding tag is assigned to the scalar. first is a sequence of possible initial characters or None. """ - Loader.add_implicit_resolver(tag, regexp, first) + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) Dumper.add_implicit_resolver(tag, regexp, first) -def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper): +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): """ Add a path based resolver for the given tag. A path is a list of keys that forms a path to a node in the representation tree. Keys can be string values, integers, or None. """ - Loader.add_path_resolver(tag, path, kind) + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) Dumper.add_path_resolver(tag, path, kind) -def add_constructor(tag, constructor, Loader=Loader): +def add_constructor(tag, constructor, Loader=None): """ Add a constructor for the given tag. Constructor is a function that accepts a Loader instance and a node object and produces the corresponding Python object. """ - Loader.add_constructor(tag, constructor) + if Loader is None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) -def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader): +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): """ Add a multi-constructor for the given tag prefix. Multi-constructor is called for a node if its tag starts with tag_prefix. Multi-constructor accepts a Loader instance, a tag suffix, and a node object and produces the corresponding Python object. """ - Loader.add_multi_constructor(tag_prefix, multi_constructor) + if Loader is None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) def add_representer(data_type, representer, Dumper=Dumper): """ @@ -368,7 +388,12 @@ class YAMLObjectMetaclass(type): def __init__(cls, name, bases, kwds): super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: - cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + cls.yaml_dumper.add_representer(cls, cls.to_yaml) class YAMLObject(metaclass=YAMLObjectMetaclass): @@ -379,7 +404,7 @@ class YAMLObject(metaclass=YAMLObjectMetaclass): __slots__ = () # no direct instantiation, so allow immutable subclasses - yaml_loader = Loader + yaml_loader = [Loader, FullLoader, UnsafeLoader] yaml_dumper = Dumper yaml_tag = None diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index 34fc1ae92..cd9167eaf 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -72,7 +72,7 @@ def construct_object(self, node, deep=False): constructor = self.yaml_constructors[node.tag] else: for tag_prefix in self.yaml_multi_constructors: - if node.tag.startswith(tag_prefix): + if tag_prefix is not None and node.tag.startswith(tag_prefix): tag_suffix = node.tag[len(tag_prefix):] constructor = self.yaml_multi_constructors[tag_prefix] break @@ -324,22 +324,23 @@ def construct_yaml_timestamp(self, node): minute = int(values['minute']) second = int(values['second']) fraction = 0 + tzinfo = None if values['fraction']: fraction = values['fraction'][:6] while len(fraction) < 6: fraction += '0' fraction = int(fraction) - delta = None if values['tz_sign']: tz_hour = int(values['tz_hour']) tz_minute = int(values['tz_minute'] or 0) delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) if values['tz_sign'] == '-': delta = -delta - data = datetime.datetime(year, month, day, hour, minute, second, fraction) - if delta: - data -= delta - return data + tzinfo = datetime.timezone(delta) + elif values['tz']: + tzinfo = datetime.timezone.utc + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) def construct_yaml_omap(self, node): # Note: we do not check for duplicate keys, because it's too @@ -513,7 +514,7 @@ def find_python_module(self, name, mark, unsafe=False): except ImportError as exc: raise ConstructorError("while constructing a Python module", mark, "cannot find module %r (%s)" % (name, exc), mark) - if not name in sys.modules: + if name not in sys.modules: raise ConstructorError("while constructing a Python module", mark, "module %r is not imported" % name, mark) return sys.modules[name] @@ -533,7 +534,7 @@ def find_python_name(self, name, mark, unsafe=False): except ImportError as exc: raise ConstructorError("while constructing a Python object", mark, "cannot find module %r (%s)" % (module_name, exc), mark) - if not module_name in sys.modules: + if module_name not in sys.modules: raise ConstructorError("while constructing a Python object", mark, "module %r is not imported" % module_name, mark) module = sys.modules[module_name] @@ -585,7 +586,7 @@ def set_python_instance_state(self, instance, state): elif state: slotstate.update(state) for key, value in slotstate.items(): - setattr(object, key, value) + setattr(instance, key, value) def construct_python_object(self, suffix, node): # Format: @@ -694,10 +695,6 @@ def construct_python_object_new(self, suffix, node): 'tag:yaml.org,2002:python/object:', FullConstructor.construct_python_object) -FullConstructor.add_multi_constructor( - 'tag:yaml.org,2002:python/object/apply:', - FullConstructor.construct_python_object_apply) - FullConstructor.add_multi_constructor( 'tag:yaml.org,2002:python/object/new:', FullConstructor.construct_python_object_new) @@ -714,6 +711,10 @@ def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False) return super(UnsafeConstructor, self).make_python_instance( suffix, node, args, kwds, newobj, unsafe=True) +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + # Constructor is same as UnsafeConstructor. Need to leave this in place in case # people have extended it directly. class Constructor(UnsafeConstructor): diff --git a/lib3/yaml/loader.py b/lib3/yaml/loader.py index 414cb2c15..e90c11224 100644 --- a/lib3/yaml/loader.py +++ b/lib3/yaml/loader.py @@ -51,7 +51,7 @@ def __init__(self, stream): # UnsafeLoader is the same as Loader (which is and was always unsafe on # untrusted input). Use of either Loader or UnsafeLoader should be rare, since # FullLoad should be able to load almost all YAML safely. Loader is left intact -# to ensure backwards compatability. +# to ensure backwards compatibility. class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): def __init__(self, stream): diff --git a/lib3/yaml/representer.py b/lib3/yaml/representer.py index dd144017b..3b0b192ef 100644 --- a/lib3/yaml/representer.py +++ b/lib3/yaml/representer.py @@ -5,7 +5,7 @@ from .error import * from .nodes import * -import datetime, sys, copyreg, types, base64, collections +import datetime, copyreg, types, base64, collections class RepresenterError(YAMLError): pass diff --git a/lib3/yaml/scanner.py b/lib3/yaml/scanner.py index 775dbcc6d..7437ede1c 100644 --- a/lib3/yaml/scanner.py +++ b/lib3/yaml/scanner.py @@ -332,7 +332,7 @@ def unwind_indent(self, column): ## } #if self.flow_level and self.indent > column: # raise ScannerError(None, None, - # "invalid intendation or unclosed '[' or '{'", + # "invalid indentation or unclosed '[' or '{'", # self.get_mark()) # In the flow context, indentation is ignored. We make the scanner less @@ -370,7 +370,7 @@ def fetch_stream_start(self): def fetch_stream_end(self): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. @@ -389,7 +389,7 @@ def fetch_stream_end(self): def fetch_directive(self): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. @@ -407,7 +407,7 @@ def fetch_document_end(self): def fetch_document_indicator(self, TokenClass): - # Set the current intendation to -1. + # Set the current indentation to -1. self.unwind_indent(-1) # Reset simple keys. Note that there could not be a block collection diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index a9e6ad289..a60d0bbb2 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -5,17 +5,25 @@ # TODO: get version number from setup.py and/or lib(3)/__version__ # Update-AppveyorBuild -Version $dynamic_version -Function Bootstrap() { - # uncomment when we want to start testing on Python 3.8 - # ensure py38 is present (current Appveyor VS2015 image doesn't include it) - #If(-not $(Test-Path C:\Python38)) { - # choco.exe install python3 --version=3.8.0-a2 --forcex86 --force #--install-arguments="TargetDir=C:\Python38 PrependPath=0" --no-progress - #} +Function Invoke-Exe([scriptblock]$sb) { + & $sb + $exitcode = $LASTEXITCODE + If($exitcode -ne 0) { + throw "exe failed with nonzero exit code $exitcode" + } +} - #If(-not $(Test-Path C:\Python38-x64)) { - # choco.exe install python3 --version=3.8.0-a2 --force #--install-arguments="TargetDir=C:\Python38-x64 PrependPath=0" --no-progress - #} +Function Bootstrap() { +<# + # ensure python 3.9 prerelease is present (current Appveyor VS2015 image doesn't include it) + If(-not $(Test-Path C:\Python39)) { + Invoke-Exe { choco.exe install python3 --version=3.9.0-a1 --forcex86 --force --params="/InstallDir:C:\Python39" --no-progress } + } + If(-not $(Test-Path C:\Python39-x64)) { + Invoke-Exe { choco.exe install python3 --version=3.9.0-a1 --force --params="/InstallDir:C:\Python39-x64" --no-progress } + } +#> Write-Output "patching Windows SDK bits for distutils" # patch 7.0/7.1 vcvars SDK bits up to work with distutils query @@ -23,11 +31,7 @@ Function Bootstrap() { Set-Content -Path 'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat' '@CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /Release /x64' # patch VS9 x64 CMake config for VS Express, hide `reg.exe` stderr noise - $noise = reg.exe import packaging\build\FixVS9CMake.reg 2>&1 - - If($LASTEXITCODE -ne 0) { - throw "reg failed with error code $LASTEXITCODE" - } + Invoke-Exe { $noise = reg.exe import packaging\build\FixVS9CMake.reg 2>&1 } Copy-Item -Path "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.config" -Destination "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.Express.config" -Force Copy-Item -Path "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.config" -Destination "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.Express.config" -Force @@ -41,7 +45,7 @@ Function Bootstrap() { Write-Output "cloning libyaml from $libyaml_repo_url / $libyaml_refspec" If(-not $(Test-Path .\libyaml)) { - git clone -b $libyaml_refspec $libyaml_repo_url 2>&1 + Invoke-Exe { git clone -b $libyaml_refspec $libyaml_repo_url 2>&1 } } } @@ -53,7 +57,7 @@ Function Build-Wheel($python_path) { Write-Output "building pyyaml wheel for $python_path" # query distutils for the VC version used to build this Python; translate to a VS version to choose the right generator - $python_vs_buildver = & $python -c "from distutils.version import LooseVersion; from distutils.msvc9compiler import get_build_version; print(LooseVersion(str(get_build_version())).version[0])" + $python_vs_buildver = Invoke-Exe { & $python -c "from distutils.version import LooseVersion; from distutils.msvc9compiler import get_build_version; print(LooseVersion(str(get_build_version())).version[0])" } $python_cmake_generator = switch($python_vs_buildver) { "9" { "Visual Studio 9 2008" } @@ -63,7 +67,7 @@ Function Build-Wheel($python_path) { } # query arch this python was built for - $python_arch = & $python -c "from distutils.util import get_platform; print(str(get_platform()))" + $python_arch = Invoke-Exe { & $python -c "from distutils.util import get_platform; print(str(get_platform()))" } if($python_arch -eq 'win-amd64') { $python_cmake_generator += " Win64" @@ -71,7 +75,7 @@ Function Build-Wheel($python_path) { } # snarf VS vars (paths, etc) for the matching VS version and arch that built this Python - $raw_vars_out = & cmd.exe /c "`"C:\Program Files (x86)\Microsoft Visual Studio $($python_vs_buildver).0\VC\vcvarsall.bat`" $vcvars_arch & set" + $raw_vars_out = Invoke-Exe { cmd.exe /c "`"C:\Program Files (x86)\Microsoft Visual Studio $($python_vs_buildver).0\VC\vcvarsall.bat`" $vcvars_arch & set" } foreach($kv in $raw_vars_out) { If($kv -match "=") { $kv = $kv.Split("=", 2) @@ -83,23 +87,23 @@ Function Build-Wheel($python_path) { } # ensure pip is current (some appveyor pips are not) - & $python -W "ignore:DEPRECATION" -m pip install --upgrade pip + Invoke-Exe { & $python -W "ignore:DEPRECATION" -m pip install --upgrade pip } # ensure required-for-build packages are present and up-to-date - & $python -W "ignore:DEPRECATION" -m pip install --upgrade cython wheel setuptools --no-warn-script-location + Invoke-Exe { & $python -W "ignore:DEPRECATION" -m pip install --upgrade cython wheel setuptools --no-warn-script-location } pushd libyaml - git clean -fdx + Invoke-Exe { git clean -fdx } popd mkdir libyaml\build pushd libyaml\build - cmake.exe -G $python_cmake_generator -DYAML_STATIC_LIB_NAME=yaml .. - cmake.exe --build . --config Release + Invoke-Exe { cmake.exe -G $python_cmake_generator -DYAML_STATIC_LIB_NAME=yaml .. } + Invoke-Exe { cmake.exe --build . --config Release } popd - & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel + Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel } } Function Upload-Artifacts() { @@ -115,14 +119,14 @@ Bootstrap $pythons = @( "C:\Python27" "C:\Python27-x64" -"C:\Python34" -"C:\Python34-x64" "C:\Python35" "C:\Python35-x64" "C:\Python36" "C:\Python36-x64" "C:\Python37" "C:\Python37-x64" +"C:\Python38" +"C:\Python38-x64" ) #$pythons = @("C:\$($env:PYTHON_VER)") diff --git a/setup.py b/setup.py index e21ce9f73..1a0413a94 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '5.1' +VERSION = '5.3' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability @@ -25,14 +25,15 @@ "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", + "Programming Language :: Cython", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", @@ -64,7 +65,6 @@ from distutils.core import setup, Command from distutils.core import Distribution as _Distribution from distutils.core import Extension as _Extension -from distutils.dir_util import mkpath from distutils.command.build_ext import build_ext as _build_ext from distutils.command.bdist_rpm import bdist_rpm as _bdist_rpm from distutils.errors import DistutilsError, CompileError, LinkError, DistutilsPlatformError @@ -311,5 +311,5 @@ def run(self): distclass=Distribution, cmdclass=cmdclass, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', ) diff --git a/tests/data/construct-python-object.code b/tests/data/construct-python-object.code index 7f1edf125..9e611e43a 100644 --- a/tests/data/construct-python-object.code +++ b/tests/data/construct-python-object.code @@ -17,6 +17,8 @@ NewArgsWithState(1, 'two', [3,3,3]), Reduce(1, 'two', [3,3,3]), ReduceWithState(1, 'two', [3,3,3]), +Slots(1, 'two', [3,3,3]), + MyInt(3), MyList(3), MyDict(3), diff --git a/tests/data/construct-python-object.data b/tests/data/construct-python-object.data index bce8b2ed3..66797e4c9 100644 --- a/tests/data/construct-python-object.data +++ b/tests/data/construct-python-object.data @@ -16,6 +16,8 @@ - !!python/object/apply:test_constructor.Reduce [1, two, [3,3,3]] - !!python/object/apply:test_constructor.ReduceWithState { args: [1, two], state: [3,3,3] } +- !!python/object/new:test_constructor.Slots { state: !!python/tuple [null, { foo: 1, bar: 'two', baz: [3,3,3] } ] } + - !!python/object/new:test_constructor.MyInt [3] - !!python/object/new:test_constructor.MyList { listitems: [~, ~, ~] } - !!python/object/new:test_constructor.MyDict { dictitems: {0, 1, 2} } diff --git a/tests/data/emoticons-ucs4-.unicode b/tests/data/emoticons.unicode similarity index 100% rename from tests/data/emoticons-ucs4-.unicode rename to tests/data/emoticons.unicode diff --git a/tests/data/emoticons2-ucs4-.unicode b/tests/data/emoticons2.unicode similarity index 100% rename from tests/data/emoticons2-ucs4-.unicode rename to tests/data/emoticons2.unicode diff --git a/tests/data/multi-constructor.code b/tests/data/multi-constructor.code new file mode 100644 index 000000000..590d852a7 --- /dev/null +++ b/tests/data/multi-constructor.code @@ -0,0 +1,4 @@ +[ + {'Tag1': ['a', 1, 'b', 2]}, + {'Tag2': ['a', 1, 'b', 2]}, +] diff --git a/tests/data/multi-constructor.multi b/tests/data/multi-constructor.multi new file mode 100644 index 000000000..4f5d71f42 --- /dev/null +++ b/tests/data/multi-constructor.multi @@ -0,0 +1,3 @@ +--- +- !Tag1 [a, 1, b, 2] +- !!Tag2 [a, 1, b, 2] diff --git a/tests/data/timestamp-bugs.code b/tests/data/timestamp-bugs.code index b1d6e9c4f..e2c84cc3c 100644 --- a/tests/data/timestamp-bugs.code +++ b/tests/data/timestamp-bugs.code @@ -1,8 +1,8 @@ [ - datetime.datetime(2001, 12, 15, 3, 29, 43, 100000), - datetime.datetime(2001, 12, 14, 16, 29, 43, 100000), - datetime.datetime(2001, 12, 14, 21, 59, 43, 1010), - datetime.datetime(2001, 12, 14, 21, 59, 43, 0, FixedOffset(60, "+1")), - datetime.datetime(2001, 12, 14, 21, 59, 43, 0, FixedOffset(-90, "-1:30")), - datetime.datetime(2005, 7, 8, 17, 35, 4, 517600), + [datetime.datetime(2001, 12, 15, 3, 29, 43, 100000), 'UTC-05:30'], + [datetime.datetime(2001, 12, 14, 16, 29, 43, 100000), 'UTC+05:30'], + [datetime.datetime(2001, 12, 14, 21, 59, 43, 1010), None], + [datetime.datetime(2001, 12, 14, 21, 59, 43, 0, FixedOffset(60, "+1")), 'UTC+01:00'], + [datetime.datetime(2001, 12, 14, 21, 59, 43, 0, FixedOffset(-90, "-1:30")), 'UTC-01:30'], + [datetime.datetime(2005, 7, 8, 17, 35, 4, 517600), None], ] diff --git a/tests/data/timestamp-bugs.data b/tests/data/timestamp-bugs.data index 721d29082..b243d0d2c 100644 --- a/tests/data/timestamp-bugs.data +++ b/tests/data/timestamp-bugs.data @@ -1,6 +1,12 @@ -- 2001-12-14 21:59:43.10 -5:30 -- 2001-12-14 21:59:43.10 +5:30 -- 2001-12-14 21:59:43.00101 -- 2001-12-14 21:59:43+1 -- 2001-12-14 21:59:43-1:30 -- 2005-07-08 17:35:04.517600 +- !MyTime + - 2001-12-14 21:59:43.10 -5:30 +- !MyTime + - 2001-12-14 21:59:43.10 +5:30 +- !MyTime + - 2001-12-14 21:59:43.00101 +- !MyTime + - 2001-12-14 21:59:43+1 +- !MyTime + - 2001-12-14 21:59:43-1:30 +- !MyTime + - 2005-07-08 17:35:04.517600 diff --git a/tests/lib/test_build.py b/tests/lib/test_build.py index 901e8ed6c..df3f943ba 100644 --- a/tests/lib/test_build.py +++ b/tests/lib/test_build.py @@ -2,7 +2,7 @@ if __name__ == '__main__': import sys, os, distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.{}-{}.{}'.format(distutils.util.get_platform(), *sys.version_info)) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) import test_yaml, test_appliance diff --git a/tests/lib/test_build_ext.py b/tests/lib/test_build_ext.py index ff195d519..fa843f6e4 100644 --- a/tests/lib/test_build_ext.py +++ b/tests/lib/test_build_ext.py @@ -3,7 +3,7 @@ if __name__ == '__main__': import sys, os, distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.{}-{}.{}'.format(distutils.util.get_platform(), *sys.version_info)) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) import test_yaml_ext, test_appliance diff --git a/tests/lib/test_constructor.py b/tests/lib/test_constructor.py index beee7b0a7..a22fd1891 100644 --- a/tests/lib/test_constructor.py +++ b/tests/lib/test_constructor.py @@ -16,7 +16,7 @@ def execute(code): def _make_objects(): global MyLoader, MyDumper, MyTestClass1, MyTestClass2, MyTestClass3, YAMLObject1, YAMLObject2, \ AnObject, AnInstance, AState, ACustomState, InitArgs, InitArgsWithState, \ - NewArgs, NewArgsWithState, Reduce, ReduceWithState, MyInt, MyList, MyDict, \ + NewArgs, NewArgsWithState, Reduce, ReduceWithState, Slots, MyInt, MyList, MyDict, \ FixedOffset, today, execute class MyLoader(yaml.Loader): @@ -41,7 +41,18 @@ def construct1(constructor, node): def represent1(representer, native): return representer.represent_mapping("!tag1", native.__dict__) + def my_time_constructor(constructor, node): + seq = constructor.construct_sequence(node) + dt = seq[0] + tz = None + try: + tz = dt.tzinfo.tzname(dt) + except: + pass + return [dt, tz] + yaml.add_constructor("!tag1", construct1, Loader=MyLoader) + yaml.add_constructor("!MyTime", my_time_constructor, Loader=MyLoader) yaml.add_representer(MyTestClass1, represent1, Dumper=MyDumper) class MyTestClass2(MyTestClass1, yaml.YAMLObject): @@ -185,6 +196,17 @@ def __reduce__(self): def __setstate__(self, state): self.baz = state + class Slots(object): + __slots__ = ("foo", "bar", "baz") + def __init__(self, foo=None, bar=None, baz=None): + self.foo = foo + self.bar = bar + self.baz = baz + + def __eq__(self, other): + return type(self) is type(other) and \ + (self.foo, self.bar, self.baz) == (other.foo, other.bar, other.baz) + class MyInt(int): def __eq__(self, other): return type(self) is type(other) and int(self) == int(other) diff --git a/tests/lib/test_multi_constructor.py b/tests/lib/test_multi_constructor.py new file mode 100644 index 000000000..f6e28fe0e --- /dev/null +++ b/tests/lib/test_multi_constructor.py @@ -0,0 +1,63 @@ +import yaml +import pprint +import sys + +def _load_code(expression): + return eval(expression) + +def myconstructor1(constructor, tag, node): + seq = constructor.construct_sequence(node) + return {tag: seq } + +def myconstructor2(constructor, tag, node): + seq = constructor.construct_sequence(node) + string = '' + try: + i = tag.index('!') + 1 + except: + try: + i = tag.rindex(':') + 1 + except: + pass + if i >= 0: + tag = tag[i:] + return { tag: seq } + +class Multi1(yaml.FullLoader): + pass +class Multi2(yaml.FullLoader): + pass + +def test_multi_constructor(input_filename, code_filename, verbose=False): + input = open(input_filename, 'rb').read().decode('utf-8') + native = _load_code(open(code_filename, 'rb').read()) + + # default multi constructor for ! and !! tags + Multi1.add_multi_constructor('!', myconstructor1) + Multi1.add_multi_constructor('tag:yaml.org,2002:', myconstructor1) + + data = yaml.load(input, Loader=Multi1) + if verbose: + print('Multi1:') + print(data) + print(native) + assert(data == native) + + + # default multi constructor for all tags + Multi2.add_multi_constructor(None, myconstructor2) + + data = yaml.load(input, Loader=Multi2) + if verbose: + print('Multi2:') + print(data) + print(native) + assert(data == native) + + +test_multi_constructor.unittest = ['.multi', '.code'] + +if __name__ == '__main__': + import test_appliance + test_appliance.run(globals()) + diff --git a/tests/lib/test_yaml.py b/tests/lib/test_yaml.py index e26342d61..352cd8d15 100644 --- a/tests/lib/test_yaml.py +++ b/tests/lib/test_yaml.py @@ -12,6 +12,7 @@ from test_recursive import * from test_input_output import * from test_sort_keys import * +from test_multi_constructor import * if __name__ == '__main__': import test_appliance diff --git a/tests/lib/test_yaml_ext.py b/tests/lib/test_yaml_ext.py index bdfda3e74..7f9357f66 100644 --- a/tests/lib/test_yaml_ext.py +++ b/tests/lib/test_yaml_ext.py @@ -1,6 +1,6 @@ import _yaml, yaml -import types, pprint +import types, pprint, tempfile, sys, os yaml.PyBaseLoader = yaml.BaseLoader yaml.PySafeLoader = yaml.SafeLoader @@ -233,6 +233,22 @@ def test_c_emitter(data_filename, canonical_filename, verbose=False): test_c_emitter.unittest = ['.data', '.canonical'] test_c_emitter.skip = ['.skip-ext'] +def test_large_file(verbose=False): + SIZE_LINE = 24 + SIZE_ITERATION = 0 + SIZE_FILE = 31 + if sys.maxsize <= 2**32: + return + if os.environ.get('PYYAML_TEST_GROUP', '') != 'all': + return + with tempfile.TemporaryFile() as temp_file: + for i in range(2**(SIZE_FILE-SIZE_ITERATION-SIZE_LINE) + 1): + temp_file.write(('-' + (' ' * (2**SIZE_LINE-4))+ '{}\n')*(2**SIZE_ITERATION)) + temp_file.seek(0) + yaml.load(temp_file, Loader=yaml.CLoader) + +test_large_file.unittest = None + def wrap_ext_function(function): def wrapper(*args, **kwds): _set_up() diff --git a/tests/lib3/test_build.py b/tests/lib3/test_build.py index 901e8ed6c..df3f943ba 100644 --- a/tests/lib3/test_build.py +++ b/tests/lib3/test_build.py @@ -2,7 +2,7 @@ if __name__ == '__main__': import sys, os, distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.{}-{}.{}'.format(distutils.util.get_platform(), *sys.version_info)) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) import test_yaml, test_appliance diff --git a/tests/lib3/test_build_ext.py b/tests/lib3/test_build_ext.py index ff195d519..fa843f6e4 100644 --- a/tests/lib3/test_build_ext.py +++ b/tests/lib3/test_build_ext.py @@ -3,7 +3,7 @@ if __name__ == '__main__': import sys, os, distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.{}-{}.{}'.format(distutils.util.get_platform(), *sys.version_info)) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) import test_yaml_ext, test_appliance diff --git a/tests/lib3/test_constructor.py b/tests/lib3/test_constructor.py index 427f53c30..877982db8 100644 --- a/tests/lib3/test_constructor.py +++ b/tests/lib3/test_constructor.py @@ -13,7 +13,7 @@ def execute(code): def _make_objects(): global MyLoader, MyDumper, MyTestClass1, MyTestClass2, MyTestClass3, YAMLObject1, YAMLObject2, \ AnObject, AnInstance, AState, ACustomState, InitArgs, InitArgsWithState, \ - NewArgs, NewArgsWithState, Reduce, ReduceWithState, MyInt, MyList, MyDict, \ + NewArgs, NewArgsWithState, Reduce, ReduceWithState, Slots, MyInt, MyList, MyDict, \ FixedOffset, today, execute class MyLoader(yaml.Loader): @@ -38,7 +38,18 @@ def construct1(constructor, node): def represent1(representer, native): return representer.represent_mapping("!tag1", native.__dict__) + def my_time_constructor(constructor, node): + seq = constructor.construct_sequence(node) + dt = seq[0] + tz = None + try: + tz = dt.tzinfo.tzname(dt) + except: + pass + return [dt, tz] + yaml.add_constructor("!tag1", construct1, Loader=MyLoader) + yaml.add_constructor("!MyTime", my_time_constructor, Loader=MyLoader) yaml.add_representer(MyTestClass1, represent1, Dumper=MyDumper) class MyTestClass2(MyTestClass1, yaml.YAMLObject): @@ -172,6 +183,17 @@ def __reduce__(self): def __setstate__(self, state): self.baz = state + class Slots: + __slots__ = ("foo", "bar", "baz") + def __init__(self, foo=None, bar=None, baz=None): + self.foo = foo + self.bar = bar + self.baz = baz + + def __eq__(self, other): + return type(self) is type(other) and \ + (self.foo, self.bar, self.baz) == (other.foo, other.bar, other.baz) + class MyInt(int): def __eq__(self, other): return type(self) is type(other) and int(self) == int(other) diff --git a/tests/lib3/test_multi_constructor.py b/tests/lib3/test_multi_constructor.py new file mode 100644 index 000000000..f6e28fe0e --- /dev/null +++ b/tests/lib3/test_multi_constructor.py @@ -0,0 +1,63 @@ +import yaml +import pprint +import sys + +def _load_code(expression): + return eval(expression) + +def myconstructor1(constructor, tag, node): + seq = constructor.construct_sequence(node) + return {tag: seq } + +def myconstructor2(constructor, tag, node): + seq = constructor.construct_sequence(node) + string = '' + try: + i = tag.index('!') + 1 + except: + try: + i = tag.rindex(':') + 1 + except: + pass + if i >= 0: + tag = tag[i:] + return { tag: seq } + +class Multi1(yaml.FullLoader): + pass +class Multi2(yaml.FullLoader): + pass + +def test_multi_constructor(input_filename, code_filename, verbose=False): + input = open(input_filename, 'rb').read().decode('utf-8') + native = _load_code(open(code_filename, 'rb').read()) + + # default multi constructor for ! and !! tags + Multi1.add_multi_constructor('!', myconstructor1) + Multi1.add_multi_constructor('tag:yaml.org,2002:', myconstructor1) + + data = yaml.load(input, Loader=Multi1) + if verbose: + print('Multi1:') + print(data) + print(native) + assert(data == native) + + + # default multi constructor for all tags + Multi2.add_multi_constructor(None, myconstructor2) + + data = yaml.load(input, Loader=Multi2) + if verbose: + print('Multi2:') + print(data) + print(native) + assert(data == native) + + +test_multi_constructor.unittest = ['.multi', '.code'] + +if __name__ == '__main__': + import test_appliance + test_appliance.run(globals()) + diff --git a/tests/lib3/test_yaml.py b/tests/lib3/test_yaml.py index e26342d61..352cd8d15 100644 --- a/tests/lib3/test_yaml.py +++ b/tests/lib3/test_yaml.py @@ -12,6 +12,7 @@ from test_recursive import * from test_input_output import * from test_sort_keys import * +from test_multi_constructor import * if __name__ == '__main__': import test_appliance diff --git a/tests/lib3/test_yaml_ext.py b/tests/lib3/test_yaml_ext.py index 93d397b85..d902214ea 100644 --- a/tests/lib3/test_yaml_ext.py +++ b/tests/lib3/test_yaml_ext.py @@ -1,6 +1,6 @@ import _yaml, yaml -import types, pprint +import types, pprint, tempfile, sys, os yaml.PyBaseLoader = yaml.BaseLoader yaml.PySafeLoader = yaml.SafeLoader @@ -233,6 +233,22 @@ def test_c_emitter(data_filename, canonical_filename, verbose=False): test_c_emitter.unittest = ['.data', '.canonical'] test_c_emitter.skip = ['.skip-ext'] +def test_large_file(verbose=False): + SIZE_LINE = 24 + SIZE_ITERATION = 0 + SIZE_FILE = 31 + if sys.maxsize <= 2**32: + return + if os.environ.get('PYYAML_TEST_GROUP', '') != 'all': + return + with tempfile.TemporaryFile() as temp_file: + for i in range(2**(SIZE_FILE-SIZE_ITERATION-SIZE_LINE) + 1): + temp_file.write(bytes(('-' + (' ' * (2**SIZE_LINE-4))+ '{}\n')*(2**SIZE_ITERATION), 'utf-8')) + temp_file.seek(0) + yaml.load(temp_file, Loader=yaml.CLoader) + +test_large_file.unittest = None + def wrap_ext_function(function): def wrapper(*args, **kwds): _set_up() diff --git a/tox.ini b/tox.ini index f0df1f688..867f1251b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,9 @@ [tox] -envlist = py27,pypy,py34,py35,py36,py37 +envlist = py27,pypy,py35,py36,py37,py38 [testenv] deps = Cython commands = python setup.py test +passenv = PYYAML_TEST_GROUP