From c2328ec0f4ad45d84b3d96c2b3f80017ef30af52 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 12 Feb 2017 04:18:45 -0800 Subject: [PATCH 001/232] [backport to 3.6] Support "bpo-" in Misc/NEWS (#42) (cherry picked from commit 79ab8be05fb4ffb5c258d2ca49be5fc2d4880431) --- Doc/tools/extensions/pyspecific.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 273191bbd3c025..1141d6c5ac1891 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -34,7 +34,7 @@ ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://hg.python.org/cpython/file/3.6/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.6/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body @@ -79,7 +79,7 @@ def new_depart_literal_block(self, node): def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): issue = utils.unescape(text) - text = 'issue ' + issue + text = 'bpo-' + issue refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) return [refnode], [] @@ -225,7 +225,7 @@ def run(self): # Support for including Misc/NEWS -issue_re = re.compile('([Ii])ssue #([0-9]+)') +issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)') whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") @@ -253,7 +253,7 @@ def run(self): text = 'The NEWS file is not available.' node = nodes.strong(text, text) return [node] - content = issue_re.sub(r'`\1ssue #\2 `__', + content = issue_re.sub(r'`bpo-\1 `__', content) content = whatsnew_re.sub(r'\1', content) # remove first 3 lines as they are the main heading From 89b1824e693419b20b6a9113e5293f1c1a78065f Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 12 Feb 2017 19:13:31 +0300 Subject: [PATCH 002/232] bpo-27122: Fix comment to point to correct issue number (#48) It took me quite a bit to figure out what this was referring to, since the given issue number is wrong, and the original commit message I found through git blame lists a different, also wrong issue number... see https://bugs.python.org/issue27122#msg279449 (cherry picked from commit af88e7eda4101f36e904771d3cf59a5f740b3b00) --- Lib/contextlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 7d94a579c872b7..8421968525947e 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -105,7 +105,7 @@ def __exit__(self, type, value, traceback): # raised inside the "with" statement from being suppressed. return exc is not value except RuntimeError as exc: - # Don't re-raise the passed in exception. (issue27112) + # Don't re-raise the passed in exception. (issue27122) if exc is value: return False # Likewise, avoid suppressing if a StopIteration exception From 308f789d00735e6707dbc5c8f60a1c8cf245fb4b Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 12 Feb 2017 08:18:07 -0800 Subject: [PATCH 003/232] bpo-29474: Improve documentation for weakref.WeakValueDictionary (#22) There were some grammatical errors in weakref.WeakValueDictionary documentation. --- Doc/library/weakref.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index e289b971e7c269..b02a006d733b6e 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -166,8 +166,8 @@ Extension types can easily be made to support weak references; see performed by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakKeyDictionary` objects have the following additional methods. These -expose the internal references directly. The references are not guaranteed to +:class:`WeakKeyDictionary` objects have an additional method that +exposes the internal references directly. The references are not guaranteed to be "live" at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the keys around longer @@ -192,9 +192,9 @@ than needed. by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakValueDictionary` objects have the following additional methods. -These method have the same issues as the and :meth:`keyrefs` method of -:class:`WeakKeyDictionary` objects. +:class:`WeakValueDictionary` objects have an additional method that has the +same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` +objects. .. method:: WeakValueDictionary.valuerefs() From f66c81ff499fb431e56bc68f5e39c2f7b9fcb6a7 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 12 Feb 2017 13:08:24 -0800 Subject: [PATCH 004/232] [backport to 3.6] bpo-28929: Link the documentation to its source file on GitHub (#37) * bpo-28929: Link the documentation to its source file on GitHub Change the documentation's `Show Source` link on the left menu to GitHub source file. (cherry picked from commit 23bafa294c75c20cb85ae5d97d7571a3a0ad8dd3) * remove if statement --- Doc/tools/templates/customsourcelink.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 243d8107779361..71d0bba683074e 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -3,8 +3,11 @@

{{ _('This Page') }}

{%- endif %} From 89ddffbe9dcb38b79f99563b0d4d594d1105a192 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 13 Feb 2017 09:19:05 +0900 Subject: [PATCH 005/232] bpo-29438: fixed use-after-free in key sharing dict (#39) --- Misc/NEWS | 2 ++ Objects/dictobject.c | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index d79ff2450defaa..ceea2226561fd6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29438: Fixed use-after-free problem in key sharing dict. + - Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - Issue #29337: Fixed possible BytesWarning when compare the code objects. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7b403bcecc5da..b63b78a337225a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4376,15 +4376,19 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, } if (value == NULL) { res = PyDict_DelItem(dict, key); - if (cached != ((PyDictObject *)dict)->ma_keys) { + // Since key sharing dict doesn't allow deletion, PyDict_DelItem() + // always converts dict to combined form. + if ((cached = CACHED_KEYS(tp)) != NULL) { CACHED_KEYS(tp) = NULL; DK_DECREF(cached); } } else { - int was_shared = cached == ((PyDictObject *)dict)->ma_keys; + int was_shared = (cached == ((PyDictObject *)dict)->ma_keys); res = PyDict_SetItem(dict, key, value); - if (was_shared && cached != ((PyDictObject *)dict)->ma_keys) { + if (was_shared && + (cached = CACHED_KEYS(tp)) != NULL && + cached != ((PyDictObject *)dict)->ma_keys) { /* PyDict_SetItem() may call dictresize and convert split table * into combined table. In such case, convert it to split * table again and update type's shared key only when this is From 2d0c2286f8e19e5d6a45cf16f766382115431c2e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 13 Feb 2017 15:48:40 -0800 Subject: [PATCH 006/232] A few README tweaks (#73) (#79) * Add a paragraph at the top for users, not builders, of Python. * Use nicer rst url syntax to avoid borking paragraphs in the plain text. Contributed by Ned Batchelder @nedbat (cherry picked from commit 3cdbd68ce8230cff1afb67472b96fbfa7f047e32) --- README | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README b/README index 4fc97e6013063e..03c4b03f68a1ff 100644 --- a/README +++ b/README @@ -11,6 +11,14 @@ especially how built-in objects like dictionaries and strings work, have changed considerably, and a lot of deprecated features have finally been removed. +Using Python +------------ + +Installable Python kits, and information about using Python, are available at +`python.org`_. + +.. _python.org: https://www.python.org/ + Build Instructions ------------------ @@ -151,7 +159,7 @@ IMPORTANT: If the tests fail and you decide to mail a bug report, *don't* include the output of "make test". It is useless. Run the failing test manually, as follows: - ./python -m test -v test_whatever + ./python -m test -v test_whatever (substituting the top of the source tree for '.' if you built in a different directory). This runs the test in verbose mode. @@ -198,11 +206,12 @@ Proposals for enhancement ------------------------- If you have a proposal to change Python, you may want to send an email to the -comp.lang.python or python-ideas mailing lists for initial feedback. A Python +comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. All current PEPs, as well as guidelines for submitting a new PEP, are listed at https://www.python.org/dev/peps/. +.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ Release Schedule ---------------- @@ -231,4 +240,3 @@ so it may be used in proprietary projects. There are interfaces to some GNU code but these are entirely optional. All trademarks referenced herein are property of their respective holders. - From 0230e64d2c976ab755c145e97bf86032e0fe3a53 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 14 Feb 2017 06:11:12 -0800 Subject: [PATCH 007/232] bpo-28556: Various updates to typing (#28) (#77) various updates from upstream python/typing repo: - Added typing.Counter and typing.ChainMap generics - More flexible typing.NamedTuple - Improved generic ABC caching - More tests - Bugfixes - Other updates * Add Misc/NEWS entry (cherry picked from commit b692dc8475a032740576129d0990ddc3edccab2b) --- Lib/test/mod_generics_cache.py | 14 ++ Lib/test/test_typing.py | 237 +++++++++++++++++++++++++++++---- Lib/typing.py | 138 ++++++++++++++----- Misc/NEWS | 4 + 4 files changed, 335 insertions(+), 58 deletions(-) create mode 100644 Lib/test/mod_generics_cache.py diff --git a/Lib/test/mod_generics_cache.py b/Lib/test/mod_generics_cache.py new file mode 100644 index 00000000000000..d9a60b4b28c325 --- /dev/null +++ b/Lib/test/mod_generics_cache.py @@ -0,0 +1,14 @@ +"""Module for testing the behavior of generics across different modules.""" + +from typing import TypeVar, Generic + +T = TypeVar('T') + + +class A(Generic[T]): + pass + + +class B(Generic[T]): + class A(Generic[T]): + pass diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fce6b5aaffdf93..64d8276658ee7f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -190,6 +190,10 @@ def test_bound_errors(self): with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) + def test_no_bivariant(self): + with self.assertRaises(ValueError): + TypeVar('T', covariant=True, contravariant=True) + class UnionTests(BaseTestCase): @@ -254,6 +258,11 @@ def test_repr(self): self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) u = Union[int, Employee] self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) + T = TypeVar('T') + u = Union[T, int][int] + self.assertEqual(repr(u), repr(int)) + u = Union[List[int], int] + self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') def test_cannot_subclass(self): with self.assertRaises(TypeError): @@ -304,6 +313,15 @@ def test_union_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, Union[int, str]) + def test_no_eval_union(self): + u = Union[int, str] + def f(x: u): ... + self.assertIs(get_type_hints(f)['x'], u) + + def test_function_repr_union(self): + def fun() -> int: ... + self.assertEqual(repr(Union[fun, int]), 'typing.Union[fun, int]') + def test_union_str_pattern(self): # Shouldn't crash; see http://bugs.python.org/issue25390 A = Union[str, Pattern] @@ -401,6 +419,8 @@ def test_callable_wrong_forms(self): Callable[[()], int] with self.assertRaises(TypeError): Callable[[int, 1], 2] + with self.assertRaises(TypeError): + Callable[int] def test_callable_instance_works(self): def f(): @@ -546,15 +566,27 @@ def test_basics(self): Y[str] with self.assertRaises(TypeError): Y[str, str] + self.assertIsSubclass(SimpleMapping[str, int], SimpleMapping) def test_generic_errors(self): T = TypeVar('T') + S = TypeVar('S') with self.assertRaises(TypeError): Generic[T]() + with self.assertRaises(TypeError): + Generic[T][T] + with self.assertRaises(TypeError): + Generic[T][S] with self.assertRaises(TypeError): isinstance([], List[int]) with self.assertRaises(TypeError): issubclass(list, List[int]) + with self.assertRaises(TypeError): + class NewGeneric(Generic): ... + with self.assertRaises(TypeError): + class MyGeneric(Generic[T], Generic[S]): ... + with self.assertRaises(TypeError): + class MyGeneric(List[T], Generic[S]): ... def test_init(self): T = TypeVar('T') @@ -738,6 +770,53 @@ def test_subscript_meta(self): self.assertEqual(Union[T, int][GenericMeta], Union[GenericMeta, int]) self.assertEqual(Callable[..., GenericMeta].__args__, (Ellipsis, GenericMeta)) + def test_generic_hashes(self): + try: + from test import mod_generics_cache + except ImportError: # for Python 3.4 and previous versions + import mod_generics_cache + class A(Generic[T]): + ... + + class B(Generic[T]): + class A(Generic[T]): + ... + + self.assertEqual(A, A) + self.assertEqual(mod_generics_cache.A[str], mod_generics_cache.A[str]) + self.assertEqual(B.A, B.A) + self.assertEqual(mod_generics_cache.B.A[B.A[str]], + mod_generics_cache.B.A[B.A[str]]) + + self.assertNotEqual(A, B.A) + self.assertNotEqual(A, mod_generics_cache.A) + self.assertNotEqual(A, mod_generics_cache.B.A) + self.assertNotEqual(B.A, mod_generics_cache.A) + self.assertNotEqual(B.A, mod_generics_cache.B.A) + + self.assertNotEqual(A[str], B.A[str]) + self.assertNotEqual(A[List[Any]], B.A[List[Any]]) + self.assertNotEqual(A[str], mod_generics_cache.A[str]) + self.assertNotEqual(A[str], mod_generics_cache.B.A[str]) + self.assertNotEqual(B.A[int], mod_generics_cache.A[int]) + self.assertNotEqual(B.A[List[Any]], mod_generics_cache.B.A[List[Any]]) + + self.assertNotEqual(Tuple[A[str]], Tuple[B.A[str]]) + self.assertNotEqual(Tuple[A[List[Any]]], Tuple[B.A[List[Any]]]) + self.assertNotEqual(Union[str, A[str]], Union[str, mod_generics_cache.A[str]]) + self.assertNotEqual(Union[A[str], A[str]], + Union[A[str], mod_generics_cache.A[str]]) + self.assertNotEqual(typing.FrozenSet[A[str]], + typing.FrozenSet[mod_generics_cache.B.A[str]]) + + if sys.version_info[:2] > (3, 2): + self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) + self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) + .endswith('mod_generics_cache.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) + .endswith('mod_generics_cache.B.A[str]]')) + def test_extended_generic_rules_eq(self): T = TypeVar('T') U = TypeVar('U') @@ -835,6 +914,8 @@ def test_fail_with_bare_generic(self): Tuple[Generic[T]] with self.assertRaises(TypeError): List[typing._Protocol] + with self.assertRaises(TypeError): + isinstance(1, Generic) def test_type_erasure_special(self): T = TypeVar('T') @@ -853,6 +934,11 @@ class MyDict(typing.Dict[T, T]): ... class MyDef(typing.DefaultDict[str, T]): ... self.assertIs(MyDef[int]().__class__, MyDef) self.assertIs(MyDef[int]().__orig_class__, MyDef[int]) + # ChainMap was added in 3.3 + if sys.version_info >= (3, 3): + class MyChain(typing.ChainMap[str, T]): ... + self.assertIs(MyChain[int]().__class__, MyChain) + self.assertIs(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) @@ -1203,6 +1289,19 @@ def test_forwardref_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, fr) + def test_forwardref_subclass_type_error(self): + fr = typing._ForwardRef('int') + with self.assertRaises(TypeError): + issubclass(int, fr) + + def test_forward_equality(self): + fr = typing._ForwardRef('int') + self.assertEqual(fr, typing._ForwardRef('int')) + self.assertNotEqual(List['int'], List[int]) + + def test_forward_repr(self): + self.assertEqual(repr(List['int']), "typing.List[_ForwardRef('int')]") + def test_union_forward(self): def foo(a: Union['T']): @@ -1285,6 +1384,15 @@ def foo(a: 'whatevers') -> {}: ith = get_type_hints(C().foo) self.assertEqual(ith, {}) + def test_no_type_check_no_bases(self): + class C: + def meth(self, x: int): ... + @no_type_check + class D(C): + c = C + # verify that @no_type_check never affects bases + self.assertEqual(get_type_hints(C.meth), {'x': int}) + def test_meta_no_type_check(self): @no_type_check_decorator @@ -1401,11 +1509,16 @@ class A: class B(A): x: ClassVar[Optional['B']] = None y: int + b: int class CSub(B): z: ClassVar['CSub'] = B() class G(Generic[T]): lst: ClassVar[List[T]] = [] +class NoneAndForward: + parent: 'NoneAndForward' + meaning: None + class CoolEmployee(NamedTuple): name: str cool: int @@ -1419,10 +1532,13 @@ class XMeth(NamedTuple): def double(self): return 2 * self.x -class XMethBad(NamedTuple): +class XRepr(NamedTuple): x: int - def _fields(self): - return 'no chance for this' + y: int = 1 + def __str__(self): + return f'{self.x} -> {self.y}' + def __add__(self, other): + return 0 """ if PY36: @@ -1431,7 +1547,7 @@ def _fields(self): # fake names for the sake of static analysis ann_module = ann_module2 = ann_module3 = None A = B = CSub = G = CoolEmployee = CoolEmployeeWithDefault = object - XMeth = XMethBad = object + XMeth = XRepr = NoneAndForward = object gth = get_type_hints @@ -1466,6 +1582,8 @@ def test_get_type_hints_classes(self): {'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.S), {'x': str, 'y': str}) self.assertEqual(gth(ann_module.foo), {'x': int}) + self.assertEqual(gth(NoneAndForward, globals()), + {'parent': NoneAndForward, 'meaning': type(None)}) @skipUnless(PY36, 'Python 3.6 required') def test_respect_no_type_check(self): @@ -1482,17 +1600,22 @@ def meth(x: int): ... class Der(ABase): ... self.assertEqual(gth(ABase.meth), {'x': int}) - def test_get_type_hints_for_builins(self): + def test_get_type_hints_for_builtins(self): # Should not fail for built-in classes and functions. self.assertEqual(gth(int), {}) self.assertEqual(gth(type), {}) self.assertEqual(gth(dir), {}) self.assertEqual(gth(len), {}) + self.assertEqual(gth(object.__str__), {}) + self.assertEqual(gth(object().__str__), {}) + self.assertEqual(gth(str.join), {}) def test_previous_behavior(self): def testf(x, y): ... testf.__annotations__['x'] = 'int' self.assertEqual(gth(testf), {'x': int}) + def testg(x: None): ... + self.assertEqual(gth(testg), {'x': type(None)}) def test_get_type_hints_for_object_with_annotations(self): class A: ... @@ -1506,9 +1629,10 @@ def test_get_type_hints_ClassVar(self): self.assertEqual(gth(ann_module2.CV, ann_module2.__dict__), {'var': typing.ClassVar[ann_module2.CV]}) self.assertEqual(gth(B, globals()), - {'y': int, 'x': ClassVar[Optional[B]]}) + {'y': int, 'x': ClassVar[Optional[B]], 'b': int}) self.assertEqual(gth(CSub, globals()), - {'z': ClassVar[CSub], 'y': int, 'x': ClassVar[Optional[B]]}) + {'z': ClassVar[CSub], 'y': int, 'b': int, + 'x': ClassVar[Optional[B]]}) self.assertEqual(gth(G), {'lst': ClassVar[List[T]]}) @@ -1628,6 +1752,11 @@ def test_list(self): def test_deque(self): self.assertIsSubclass(collections.deque, typing.Deque) + class MyDeque(typing.Deque[int]): ... + self.assertIsInstance(MyDeque(), collections.deque) + + def test_counter(self): + self.assertIsSubclass(collections.Counter, typing.Counter) def test_set(self): self.assertIsSubclass(set, typing.Set) @@ -1680,13 +1809,10 @@ class MyDict(typing.Dict[str, int]): self.assertIsSubclass(MyDict, dict) self.assertNotIsSubclass(dict, MyDict) - def test_no_defaultdict_instantiation(self): - with self.assertRaises(TypeError): - typing.DefaultDict() - with self.assertRaises(TypeError): - typing.DefaultDict[KT, VT]() - with self.assertRaises(TypeError): - typing.DefaultDict[str, int]() + def test_defaultdict_instantiation(self): + self.assertIs(type(typing.DefaultDict()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[KT, VT]()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[str, int]()), collections.defaultdict) def test_defaultdict_subclass(self): @@ -1699,13 +1825,49 @@ class MyDefDict(typing.DefaultDict[str, int]): self.assertIsSubclass(MyDefDict, collections.defaultdict) self.assertNotIsSubclass(collections.defaultdict, MyDefDict) - def test_no_deque_instantiation(self): - with self.assertRaises(TypeError): - typing.Deque() - with self.assertRaises(TypeError): - typing.Deque[T]() - with self.assertRaises(TypeError): - typing.Deque[int]() + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_instantiation(self): + self.assertIs(type(typing.ChainMap()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[KT, VT]()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[str, int]()), collections.ChainMap) + class CM(typing.ChainMap[KT, VT]): ... + self.assertIs(type(CM[int, str]()), CM) + + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_subclass(self): + + class MyChainMap(typing.ChainMap[str, int]): + pass + + cm = MyChainMap() + self.assertIsInstance(cm, MyChainMap) + + self.assertIsSubclass(MyChainMap, collections.ChainMap) + self.assertNotIsSubclass(collections.ChainMap, MyChainMap) + + def test_deque_instantiation(self): + self.assertIs(type(typing.Deque()), collections.deque) + self.assertIs(type(typing.Deque[T]()), collections.deque) + self.assertIs(type(typing.Deque[int]()), collections.deque) + class D(typing.Deque[T]): ... + self.assertIs(type(D[int]()), D) + + def test_counter_instantiation(self): + self.assertIs(type(typing.Counter()), collections.Counter) + self.assertIs(type(typing.Counter[T]()), collections.Counter) + self.assertIs(type(typing.Counter[int]()), collections.Counter) + class C(typing.Counter[T]): ... + self.assertIs(type(C[int]()), C) + + def test_counter_subclass_instantiation(self): + + class MyCounter(typing.Counter[int]): + pass + + d = MyCounter() + self.assertIsInstance(d, MyCounter) + self.assertIsInstance(d, typing.Counter) + self.assertIsInstance(d, collections.Counter) def test_no_set_instantiation(self): with self.assertRaises(TypeError): @@ -2018,6 +2180,14 @@ def test_basics(self): collections.OrderedDict([('name', str), ('id', int)])) self.assertIs(Emp._field_types, Emp.__annotations__) + def test_namedtuple_pyversion(self): + if sys.version_info[:2] < (3, 6): + with self.assertRaises(TypeError): + NamedTuple('Name', one=int, other=str) + with self.assertRaises(TypeError): + class NotYet(NamedTuple): + whatever = 0 + @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage(self): tim = CoolEmployee('Tim', 9000) @@ -2055,10 +2225,18 @@ class NonDefaultAfterDefault(NamedTuple): @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage_with_methods(self): - self.assertEquals(XMeth(1).double(), 2) - self.assertEquals(XMeth(42).x, XMeth(42)[0]) - self.assertEquals(XMethBad(1)._fields, ('x',)) - self.assertEquals(XMethBad(1).__annotations__, {'x': int}) + self.assertEqual(XMeth(1).double(), 2) + self.assertEqual(XMeth(42).x, XMeth(42)[0]) + self.assertEqual(str(XRepr(42)), '42 -> 1') + self.assertEqual(XRepr(1, 2) + XRepr(3), 0) + + with self.assertRaises(AttributeError): + exec(""" +class XMethBad(NamedTuple): + x: int + def _fields(self): + return 'no chance for this' +""") @skipUnless(PY36, 'Python 3.6 required') def test_namedtuple_keyword_usage(self): @@ -2138,6 +2316,12 @@ def test_basics(self): Pattern[Union[str, bytes]] Match[Union[bytes, str]] + def test_alias_equality(self): + self.assertEqual(Pattern[str], Pattern[str]) + self.assertNotEqual(Pattern[str], Pattern[bytes]) + self.assertNotEqual(Pattern[str], Match[str]) + self.assertNotEqual(Pattern[str], str) + def test_errors(self): with self.assertRaises(TypeError): # Doesn't fit AnyStr. @@ -2152,6 +2336,9 @@ def test_errors(self): with self.assertRaises(TypeError): # We don't support isinstance(). isinstance(42, Pattern[str]) + with self.assertRaises(TypeError): + # We don't support issubclass(). + issubclass(Pattern[bytes], Pattern[str]) def test_repr(self): self.assertEqual(repr(Pattern), 'Pattern[~AnyStr]') diff --git a/Lib/typing.py b/Lib/typing.py index c9e341753769e0..efe358faf20988 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -10,6 +10,12 @@ import collections.abc as collections_abc except ImportError: import collections as collections_abc # Fallback for PY3.2. +try: + from types import SlotWrapperType, MethodWrapperType, MethodDescriptorType +except ImportError: + SlotWrapperType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) # Please keep __all__ alphabetized within each category. @@ -62,6 +68,7 @@ 'SupportsRound', # Concrete collection types. + 'Counter', 'Deque', 'Dict', 'DefaultDict', @@ -849,19 +856,6 @@ def _next_in_mro(cls): return next_in_mro -def _valid_for_check(cls): - """An internal helper to prohibit isinstance([1], List[str]) etc.""" - if cls is Generic: - raise TypeError("Class %r cannot be used with class " - "or instance checks" % cls) - if ( - cls.__origin__ is not None and - sys._getframe(3).f_globals['__name__'] not in ['abc', 'functools'] - ): - raise TypeError("Parameterized generics cannot be used with class " - "or instance checks") - - def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates the associated __extra__ class in subclass checks performed @@ -872,7 +866,6 @@ def _make_subclasshook(cls): # Registered classes need not be checked here because # cls and its extra share the same _abc_registry. def __extrahook__(subclass): - _valid_for_check(cls) res = cls.__extra__.__subclasshook__(subclass) if res is not NotImplemented: return res @@ -887,7 +880,6 @@ def __extrahook__(subclass): else: # For non-ABC extras we'll just call issubclass(). def __extrahook__(subclass): - _valid_for_check(cls) if cls.__extra__ and issubclass(subclass, cls.__extra__): return True return NotImplemented @@ -974,6 +966,7 @@ def __new__(cls, name, bases, namespace, # remove bare Generic from bases if there are other generic bases if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): bases = tuple(b for b in bases if b is not Generic) + namespace.update({'__origin__': origin, '__extra__': extra}) self = super().__new__(cls, name, bases, namespace, _root=True) self.__parameters__ = tvars @@ -982,8 +975,6 @@ def __new__(cls, name, bases, namespace, self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in args) if args else None - self.__origin__ = origin - self.__extra__ = extra # Speed hack (https://github.com/python/typing/issues/196). self.__next_in_mro__ = _next_in_mro(self) # Preserve base classes on subclassing (__bases__ are type erased now). @@ -994,20 +985,56 @@ def __new__(cls, name, bases, namespace, # with issubclass() and isinstance() in the same way as their # collections.abc counterparts (e.g., isinstance([], Iterable)). if ( - # allow overriding '__subclasshook__' not in namespace and extra or - hasattr(self.__subclasshook__, '__name__') and - self.__subclasshook__.__name__ == '__extrahook__' + # allow overriding + getattr(self.__subclasshook__, '__name__', '') == '__extrahook__' ): self.__subclasshook__ = _make_subclasshook(self) if isinstance(extra, abc.ABCMeta): self._abc_registry = extra._abc_registry + self._abc_cache = extra._abc_cache + elif origin is not None: + self._abc_registry = origin._abc_registry + self._abc_cache = origin._abc_cache if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. self.__qualname__ = origin.__qualname__ - self.__tree_hash__ = hash(self._subs_tree()) if origin else hash((self.__name__,)) + self.__tree_hash__ = (hash(self._subs_tree()) if origin else + super(GenericMeta, self).__hash__()) return self + # _abc_negative_cache and _abc_negative_cache_version + # realised as descriptors, since GenClass[t1, t2, ...] always + # share subclass info with GenClass. + # This is an important memory optimization. + @property + def _abc_negative_cache(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache + return _gorg(self)._abc_generic_negative_cache + + @_abc_negative_cache.setter + def _abc_negative_cache(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + else: + self._abc_generic_negative_cache = value + + @property + def _abc_negative_cache_version(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache_version + return _gorg(self)._abc_generic_negative_cache_version + + @_abc_negative_cache_version.setter + def _abc_negative_cache_version(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + else: + self._abc_generic_negative_cache_version = value + def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: _get_type_vars(self.__parameters__, tvars) @@ -1095,8 +1122,10 @@ def __getitem__(self, params): _check_generic(self, params) tvars = _type_vars(params) args = params + + prepend = (self,) if self.__origin__ is None else () return self.__class__(self.__name__, - self.__bases__, + prepend + self.__bases__, _no_slots_copy(self.__dict__), tvars=tvars, args=args, @@ -1104,6 +1133,17 @@ def __getitem__(self, params): extra=self.__extra__, orig_bases=self.__orig_bases__) + def __subclasscheck__(self, cls): + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if self is Generic: + raise TypeError("Class %r cannot be used with class " + "or instance checks" % self) + return super().__subclasscheck__(cls) + def __instancecheck__(self, instance): # Since we extend ABC.__subclasscheck__ and # ABC.__instancecheck__ inlines the cache checking done by the @@ -1398,6 +1438,11 @@ def _get_defaults(func): return res +_allowed_types = (types.FunctionType, types.BuiltinFunctionType, + types.MethodType, types.ModuleType, + SlotWrapperType, MethodWrapperType, MethodDescriptorType) + + def get_type_hints(obj, globalns=None, localns=None): """Return type hints for an object. @@ -1452,12 +1497,7 @@ def get_type_hints(obj, globalns=None, localns=None): hints = getattr(obj, '__annotations__', None) if hints is None: # Return empty annotations for something that _could_ have them. - if ( - isinstance(obj, types.FunctionType) or - isinstance(obj, types.BuiltinFunctionType) or - isinstance(obj, types.MethodType) or - isinstance(obj, types.ModuleType) - ): + if isinstance(obj, _allowed_types): return {} else: raise TypeError('{!r} is not a module, class, method, ' @@ -1824,8 +1864,7 @@ class Deque(collections.deque, MutableSequence[T], extra=collections.deque): def __new__(cls, *args, **kwds): if _geqv(cls, Deque): - raise TypeError("Type Deque cannot be instantiated; " - "use deque() instead") + return collections.deque(*args, **kwds) return _generic_new(collections.deque, cls, *args, **kwds) @@ -1894,11 +1933,35 @@ class DefaultDict(collections.defaultdict, MutableMapping[KT, VT], def __new__(cls, *args, **kwds): if _geqv(cls, DefaultDict): - raise TypeError("Type DefaultDict cannot be instantiated; " - "use collections.defaultdict() instead") + return collections.defaultdict(*args, **kwds) return _generic_new(collections.defaultdict, cls, *args, **kwds) +class Counter(collections.Counter, Dict[T, int], extra=collections.Counter): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if _geqv(cls, Counter): + return collections.Counter(*args, **kwds) + return _generic_new(collections.Counter, cls, *args, **kwds) + + +if hasattr(collections, 'ChainMap'): + # ChainMap only exists in 3.3+ + __all__.append('ChainMap') + + class ChainMap(collections.ChainMap, MutableMapping[KT, VT], + extra=collections.ChainMap): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if _geqv(cls, ChainMap): + return collections.ChainMap(*args, **kwds) + return _generic_new(collections.ChainMap, cls, *args, **kwds) + + # Determine what base class to use for Generator. if hasattr(collections_abc, 'Generator'): # Sufficiently recent versions of 3.5 have a Generator ABC. @@ -1975,6 +2038,13 @@ def _make_nmtuple(name, types): _PY36 = sys.version_info[:2] >= (3, 6) +# attributes prohibited to set in NamedTuple class syntax +_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', + '_fields', '_field_defaults', '_field_types', + '_make', '_replace', '_asdict') + +_special = ('__module__', '__name__', '__qualname__', '__annotations__') + class NamedTupleMeta(type): @@ -2002,7 +2072,9 @@ def __new__(cls, typename, bases, ns): nm_tpl._field_defaults = defaults_dict # update from user namespace without overriding special namedtuple attributes for key in ns: - if not hasattr(nm_tpl, key): + if key in _prohibited: + raise AttributeError("Cannot overwrite NamedTuple attribute " + key) + elif key not in _special and key not in nm_tpl._fields: setattr(nm_tpl, key, ns[key]) return nm_tpl diff --git a/Misc/NEWS b/Misc/NEWS index ceea2226561fd6..0fe1f9b9d651bf 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -59,6 +59,10 @@ Extension Modules Library ------- +- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, + improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, + Manuel Krebber, and Ɓukasz Langa. + - Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python 3.6.0: check minimum and maximum years. From 9268855ba6e99bf595e75b75780cf29667371f45 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 14 Feb 2017 06:11:48 -0800 Subject: [PATCH 008/232] Fix some sphinx warnings (#9) (#81) * Fix some deprecation warnings in Doc/conf.py * Fix an rst error in Misc/NEWS Contributed by Ryan Gonzalez @kirbyfan64 (cherry picked from commit e7ffb99f842ebff97cffa0fc90b18be4e5abecf2) --- Doc/Makefile | 2 +- Doc/conf.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index 91f937f985831a..6e1c18304080d6 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -10,7 +10,7 @@ PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) .PHONY: help build html htmlhelp latex text changes linkcheck \ diff --git a/Doc/conf.py b/Doc/conf.py index b1bb6208bb4b8e..b3f26d5a692163 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -88,11 +88,24 @@ # Options for LaTeX output # ------------------------ +# Get LaTeX to handle Unicode correctly +latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} + +# Additional stuff for the LaTeX preamble. +latex_elements['preamble'] = r''' +\authoraddress{ + \strong{Python Software Foundation}\\ + Email: \email{docs@python.org} +} +\let\Verbatim=\OriginalVerbatim +\let\endVerbatim=\endOriginalVerbatim +''' + # The paper size ('letter' or 'a4'). -latex_paper_size = 'a4' +latex_elements['papersize'] = 'a4' # The font size ('10pt', '11pt' or '12pt'). -latex_font_size = '10pt' +latex_elements['font_size'] = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -125,22 +138,9 @@ for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst') -# Additional stuff for the LaTeX preamble. -latex_preamble = r''' -\authoraddress{ - \strong{Python Software Foundation}\\ - Email: \email{docs@python.org} -} -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim -''' - # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] -# Get LaTeX to handle Unicode correctly -latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} - # Options for Epub output # ----------------------- From db3deb98e969d3f43b169cd320abd46b21a10c6d Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 14 Feb 2017 08:54:59 -0800 Subject: [PATCH 009/232] bpo-29521 Fix two minor documentation build warnings (#41) (#83) Much of bpo-29521 was fixed in parallel with commit e7ffb99 . This cleans up the rest. Apply parallel change to Doc/make.bat to read "set SPHINXOPTS=-D latex_elements.papersize=" I don't have a Windows system on which to observe the warning, but it should be necessary. The warning: .../workspace/cpython_github/Doc/faq/windows.rst:303: WARNING: unknown option: -t In the Windows FAQ, `How do I keep editors from inserting tabs into my Python source?`, contained a reference to a Python -t option. In Python 2.x, this caused Python to issue warnings about lines with mixed spaces and tabs, but as of Python 3.6 it does nothing. Per discussion at http://bugs.python.org/issue29387, take their wording. Python [3] raises an IndentationError or TabError. Tabnanny is now a module. (cherry picked from commit 3d707be950b387552585451071928e7b39cdfa53) --- Doc/faq/windows.rst | 7 ++++--- Doc/make.bat | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index d7253436beaf61..b9b7b8d359bdf2 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -300,9 +300,10 @@ this respect, and is easily configured to use spaces: Take :menuselection:`Tools --> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent size" to 4, and select the "Insert spaces" radio button. -If you suspect mixed tabs and spaces are causing problems in leading whitespace, -run Python with the :option:`-t` switch or run ``Tools/Scripts/tabnanny.py`` to -check a directory tree in batch mode. +Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs +and spaces are causing problems in leading whitespace. +You may also run the :mod:`tabnanny` module to check a directory tree +in batch mode. How do I check for a keypress without blocking? diff --git a/Doc/make.bat b/Doc/make.bat index da1f8765a4f358..d0b59618261011 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -86,7 +86,7 @@ goto end :build if NOT "%PAPER%" == "" ( - set SPHINXOPTS=-D latex_paper_size=%PAPER% %SPHINXOPTS% + set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% ) cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* From 7d5587e687ba78563b0f026019e638800497e021 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 15 Feb 2017 18:59:47 +0900 Subject: [PATCH 010/232] Update URL of Mersenne Twister Home Page (#20) (#115) --- Modules/_randommodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0d3282db8f2810..e5dd2c96166384 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -2,7 +2,7 @@ /* ------------------------------------------------------------------ The code in this module was based on a download from: - http://www.math.keio.ac.jp/~matumoto/MT2002/emt19937ar.html + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html It was modified in 2002 by Raymond Hettinger as follows: @@ -60,8 +60,8 @@ Any feedback is very welcome. - http://www.math.keio.ac.jp/matumoto/emt.html - email: matumoto@math.keio.ac.jp + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html + email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) */ /* ---------------------------------------------------------------*/ From 1e8cc88a48a049b064f786bb4b97ea60b70dc504 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 15 Feb 2017 13:29:16 -0600 Subject: [PATCH 011/232] Backport master's README.rst to 3.6 (GH-86) Includes GH-2, GH-70, GH-73 (GH-79), and GH-21. --- README | 242 ----------------------------------------------------- README.rst | 235 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+), 242 deletions(-) delete mode 100644 README create mode 100644 README.rst diff --git a/README b/README deleted file mode 100644 index 03c4b03f68a1ff..00000000000000 --- a/README +++ /dev/null @@ -1,242 +0,0 @@ -This is Python version 3.6.1 release candidate 1 -================================================ - -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation. All rights -reserved. - -Python 3.x is a new version of the language, which is incompatible with the -2.x line of releases. The language is mostly the same, but many details, -especially how built-in objects like dictionaries and strings work, -have changed considerably, and a lot of deprecated features have finally -been removed. - -Using Python ------------- - -Installable Python kits, and information about using Python, are available at -`python.org`_. - -.. _python.org: https://www.python.org/ - - -Build Instructions ------------------- - -On Unix, Linux, BSD, OSX, and Cygwin: - - ./configure - make - make test - sudo make install - -This will install Python as python3. - -You can pass many options to the configure script; run "./configure --help" to -find out more. On OSX and Cygwin, the executable is called python.exe; -elsewhere it's just python. - -On Mac OS X, if you have configured Python with --enable-framework, you should -use "make frameworkinstall" to do the installation. Note that this installs the -Python executable in a place that is not normally on your PATH, you may want to -set up a symlink in /usr/local/bin. - -On Windows, see PCbuild/readme.txt. - -If you wish, you can create a subdirectory and invoke configure from there. -For example: - - mkdir debug - cd debug - ../configure --with-pydebug - make - make test - -(This will fail if you *also* built at the top-level directory. -You should do a "make clean" at the toplevel first.) - -To get an optimized build of Python, "configure --enable-optimizations" before -you run make. This sets the default make targets up to enable Profile Guided -Optimization (PGO) and may be used to auto-enable Link Time Optimization (LTO) -on some platforms. For more details, see the sections bellow. - - -Profile Guided Optimization ---------------------------- - -PGO takes advantage of recent versions of the GCC or Clang compilers. -If ran, "make profile-opt" will do several steps. - -First, the entire Python directory is cleaned of temporary files that -may have resulted in a previous compilation. - -Then, an instrumented version of the interpreter is built, using suitable -compiler flags for each flavour. Note that this is just an intermediary -step and the binary resulted after this step is not good for real life -workloads, as it has profiling instructions embedded inside. - -After this instrumented version of the interpreter is built, the Makefile -will automatically run a training workload. This is necessary in order to -profile the interpreter execution. Note also that any output, both stdout -and stderr, that may appear at this step is suppressed. - -Finally, the last step is to rebuild the interpreter, using the information -collected in the previous one. The end result will be a Python binary -that is optimized and suitable for distribution or production installation. - - -Link Time Optimization ----------------------- - -Enabled via configure's --with-lto flag. LTO takes advantages of recent -compiler toolchains ability to optimize across the otherwise arbitrary .o file -boundary when building final executables or shared libraries for additional -performance gains. - - -What's New ----------- - -We have a comprehensive overview of the changes in the "What's New in -Python 3.6" document, found at - - https://docs.python.org/3.6/whatsnew/3.6.html - -For a more detailed change log, read Misc/NEWS (though this file, too, -is incomplete, and also doesn't list anything merged in from the 2.7 -release under development). - -If you want to install multiple versions of Python see the section below -entitled "Installing multiple versions". - - -Documentation -------------- - -Documentation for Python 3.6 is online, updated daily: - - https://docs.python.org/3.6/ - -It can also be downloaded in many formats for faster access. The documentation -is downloadable in HTML, PDF, and reStructuredText formats; the latter version -is primarily for documentation authors, translators, and people with special -formatting requirements. - -If you would like to contribute to the development of Python, relevant -documentation is available at: - - https://docs.python.org/devguide/ - -For information about building Python's documentation, refer to Doc/README.txt. - - -Converting From Python 2.x to 3.x ---------------------------------- - -Python starting with 2.6 contains features to help locating code that needs to -be changed, such as optional warnings when deprecated features are used, and -backported versions of certain key Python 3.x features. - -A source-to-source translation tool, "2to3", can take care of the mundane task -of converting large amounts of source code. It is not a complete solution but -is complemented by the deprecation warnings in 2.6. See -https://docs.python.org/3.6/library/2to3.html for more information. - - -Testing -------- - -To test the interpreter, type "make test" in the top-level directory. -The test set produces some output. You can generally ignore the messages -about skipped tests due to optional features which can't be imported. -If a message is printed about a failed test or a traceback or core dump -is produced, something is wrong. - -By default, tests are prevented from overusing resources like disk space and -memory. To enable these tests, run "make testall". - -IMPORTANT: If the tests fail and you decide to mail a bug report, *don't* -include the output of "make test". It is useless. Run the failing test -manually, as follows: - - ./python -m test -v test_whatever - -(substituting the top of the source tree for '.' if you built in a different -directory). This runs the test in verbose mode. - - -Installing multiple versions ----------------------------- - -On Unix and Mac systems if you intend to install multiple versions of Python -using the same installation prefix (--prefix argument to the configure script) -you must take care that your primary python executable is not overwritten by the -installation of a different version. All files and directories installed using -"make altinstall" contain the major and minor version and can thus live -side-by-side. "make install" also creates ${prefix}/bin/python3 which refers to -${prefix}/bin/pythonX.Y. If you intend to install multiple versions using the -same prefix you must decide which version (if any) is your "primary" version. -Install that version using "make install". Install all other versions using -"make altinstall". - -For example, if you want to install Python 2.7, 3.5, and 3.6 with 3.6 being the -primary version, you would execute "make install" in your 3.6 build directory -and "make altinstall" in the others. - - -Issue Tracker and Mailing List ------------------------------- - -We're soliciting bug reports about all aspects of the language. Fixes are also -welcome, preferably in unified diff format. Please use the issue tracker: - - https://bugs.python.org/ - -If you're not sure whether you're dealing with a bug or a feature, use the -mailing list: - - python-dev@python.org - -To subscribe to the list, use the mailman form: - - https://mail.python.org/mailman/listinfo/python-dev/ - - -Proposals for enhancement -------------------------- - -If you have a proposal to change Python, you may want to send an email to the -comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A Python -Enhancement Proposal (PEP) may be submitted if your idea gains ground. All -current PEPs, as well as guidelines for submitting a new PEP, are listed at -https://www.python.org/dev/peps/. - -.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ - -Release Schedule ----------------- - -See PEP 494 for release details: https://www.python.org/dev/peps/pep-0494/ - - -Copyright and License Information ---------------------------------- - -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016 Python Software Foundation. All rights reserved. - -Copyright (c) 2000 BeOpen.com. All rights reserved. - -Copyright (c) 1995-2001 Corporation for National Research Initiatives. All -rights reserved. - -Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved. - -See the file "LICENSE" for information on the history of this software, -terms & conditions for usage, and a DISCLAIMER OF ALL WARRANTIES. - -This Python distribution contains *no* GNU General Public License (GPL) code, -so it may be used in proprietary projects. There are interfaces to some GNU -code but these are entirely optional. - -All trademarks referenced herein are property of their respective holders. diff --git a/README.rst b/README.rst new file mode 100644 index 00000000000000..d02ae64b7fc660 --- /dev/null +++ b/README.rst @@ -0,0 +1,235 @@ +This is Python version 3.6.1 release candidate 1 +================================================ + +.. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 + :alt: CPython build status on Travis CI + :target: https://travis-ci.org/python/cpython + +.. image:: https://codecov.io/gh/python/cpython/branch/3.6/graph/badge.svg + :alt: CPython code coverage on Codecov + :target: https://codecov.io/gh/python/cpython + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation. All rights +reserved. + +See the end of this file for further copyright and license information. + + +Using Python +------------ + +Installable Python kits, and information about using Python, are available at +`python.org`_. + +.. _python.org: https://www.python.org/ + + +Build Instructions +------------------ + +On Unix, Linux, BSD, OSX, and Cygwin:: + + ./configure + make + make test + sudo make install + +This will install Python as python3. + +You can pass many options to the configure script; run ``./configure --help`` +to find out more. On OSX and Cygwin, the executable is called ``python.exe``; +elsewhere it's just ``python``. + +On Mac OS X, if you have configured Python with ``--enable-framework``, you +should use ``make frameworkinstall`` to do the installation. Note that this +installs the Python executable in a place that is not normally on your PATH, +you may want to set up a symlink in ``/usr/local/bin``. + +On Windows, see `PCbuild/readme.txt +`_. + +If you wish, you can create a subdirectory and invoke configure from there. +For example:: + + mkdir debug + cd debug + ../configure --with-pydebug + make + make test + +(This will fail if you *also* built at the top-level directory. You should do +a ``make clean`` at the toplevel first.) + +To get an optimized build of Python, ``configure --enable-optimizations`` +before you run ``make``. This sets the default make targets up to enable +Profile Guided Optimization (PGO) and may be used to auto-enable Link Time +Optimization (LTO) on some platforms. For more details, see the sections +below. + + +Profile Guided Optimization +--------------------------- + +PGO takes advantage of recent versions of the GCC or Clang compilers. If ran, +``make profile-opt`` will do several steps. + +First, the entire Python directory is cleaned of temporary files that may have +resulted in a previous compilation. + +Then, an instrumented version of the interpreter is built, using suitable +compiler flags for each flavour. Note that this is just an intermediary step +and the binary resulted after this step is not good for real life workloads, as +it has profiling instructions embedded inside. + +After this instrumented version of the interpreter is built, the Makefile will +automatically run a training workload. This is necessary in order to profile +the interpreter execution. Note also that any output, both stdout and stderr, +that may appear at this step is suppressed. + +Finally, the last step is to rebuild the interpreter, using the information +collected in the previous one. The end result will be a Python binary that is +optimized and suitable for distribution or production installation. + + +Link Time Optimization +---------------------- + +Enabled via configure's ``--with-lto`` flag. LTO takes advantage of the +ability of recent compiler toolchains to optimize across the otherwise +arbitrary ``.o`` file boundary when building final executables or shared +libraries for additional performance gains. + + +What's New +---------- + +We have a comprehensive overview of the changes in the `What's New in Python +3.6 `_ document. For a more +detailed change log, read `Misc/NEWS +`_, but a full +accounting of changes can only be gleaned from the `commit history +`_. + +If you want to install multiple versions of Python see the section below +entitled "Installing multiple versions". + + +Documentation +------------- + +`Documentation for Python 3.6 `_ is online, +updated daily. + +It can also be downloaded in many formats for faster access. The documentation +is downloadable in HTML, PDF, and reStructuredText formats; the latter version +is primarily for documentation authors, translators, and people with special +formatting requirements. + +If you would like to contribute to the development of Python, relevant +documentation is available in the `Python Developer's Guide +`_. + +For information about building Python's documentation, refer to `Doc/README.txt +`_. + + +Converting From Python 2.x to 3.x +--------------------------------- + +Significant backward incompatible changes were made for the release of Python +3.0, which may cause programs written for Python 2 to fail when run with Python +3. For more information about porting your code from Python 2 to Python 3, see +the `Porting HOWTO `_. + + +Testing +------- + +To test the interpreter, type ``make test`` in the top-level directory. The +test set produces some output. You can generally ignore the messages about +skipped tests due to optional features which can't be imported. If a message +is printed about a failed test or a traceback or core dump is produced, +something is wrong. + +By default, tests are prevented from overusing resources like disk space and +memory. To enable these tests, run ``make testall``. + +If any tests fail, you can re-run the failing test(s) in verbose mode:: + + make test TESTOPTS="-v test_that_failed" + +If the failure persists and appears to be a problem with Python rather than +your environment, you can `file a bug report `_ and +include relevant output from that command to show the issue. + + +Installing multiple versions +---------------------------- + +On Unix and Mac systems if you intend to install multiple versions of Python +using the same installation prefix (``--prefix`` argument to the configure +script) you must take care that your primary python executable is not +overwritten by the installation of a different version. All files and +directories installed using ``make altinstall`` contain the major and minor +version and can thus live side-by-side. ``make install`` also creates +``${prefix}/bin/python3`` which refers to ``${prefix}/bin/pythonX.Y``. If you +intend to install multiple versions using the same prefix you must decide which +version (if any) is your "primary" version. Install that version using ``make +install``. Install all other versions using ``make altinstall``. + +For example, if you want to install Python 2.7, 3.5, and 3.6 with 3.6 being the +primary version, you would execute ``make install`` in your 3.6 build directory +and ``make altinstall`` in the others. + + +Issue Tracker and Mailing List +------------------------------ + +Bug reports are welcome! You can use the `issue tracker +`_ to report bugs, and/or submit pull requests `on +GitHub `_. + +You can also follow development discussion on the `python-dev mailing list +`_. + + +Proposals for enhancement +------------------------- + +If you have a proposal to change Python, you may want to send an email to the +comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A +Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. +All current PEPs, as well as guidelines for submitting a new PEP, are listed at +`python.org/dev/peps/ `_. + +.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ + + +Release Schedule +---------------- + +See :pep:`494` for Python 3.6 release details. + + +Copyright and License Information +--------------------------------- + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +2012, 2013, 2014, 2015, 2016 Python Software Foundation. All rights reserved. + +Copyright (c) 2000 BeOpen.com. All rights reserved. + +Copyright (c) 1995-2001 Corporation for National Research Initiatives. All +rights reserved. + +Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved. + +See the file "LICENSE" for information on the history of this software, terms & +conditions for usage, and a DISCLAIMER OF ALL WARRANTIES. + +This Python distribution contains *no* GNU General Public License (GPL) code, +so it may be used in proprietary projects. There are interfaces to some GNU +code but these are entirely optional. + +All trademarks referenced herein are property of their respective holders. From 7224a049b88ed37c510861528147e0db54911bb7 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 15 Feb 2017 11:38:48 -0800 Subject: [PATCH 012/232] bpo-29481: add versionadded 3.6.1 to typing.Deque docs (#108) (cherry picked from commit 7e147f1ddb8233964ff0981e6b64fc12edac99aa) --- Doc/library/typing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 6c8982ba743526..d130e1759d8f59 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -574,6 +574,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.6.1 + .. class:: List(list, MutableSequence[T]) Generic version of :class:`list`. From 02e3357e089f37c23d0f3d1ebee9aa3d7a1492a9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 15 Feb 2017 15:14:41 -0800 Subject: [PATCH 013/232] bpo-29521 update Misc/ACKS (#110) (cherry picked from commit 6420088b924a23e5de40be6623d2a80b12f71d97) --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index a6b3df12ff230c..1d6e773b553c52 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -347,6 +347,7 @@ A. Jesse Jiryu Davis Merlijn van Deen John DeGood Ned Deily +Jim DeLaHunt Vincent Delft Arnaud Delobelle Konrad Delong From b300c660d34d2027d443098ea605a8e0eb51d383 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 16 Feb 2017 00:56:54 +0100 Subject: [PATCH 014/232] Backport36 doc fixes: PR#68 and PR#124 (#125) * Travis CI: run rstlint.py in the docs job (#68) Currently, http://buildbot.python.org/all/buildslaves/ware-docs buildbot is only run as post-commit. For example, bpo-29521 (PR#41) introduced two warnings, unnotified by the Travis CI docs job. Modify the docs job to run toosl/rstlint.py. Fix also the two minor warnings which causes the buildbot slave to fail. (cherry picked from commit 2b501866ed493758e4c4b29f0ce9b24023d910a1) * Doc/Makefile: set PYTHON to python3 (#124) rstlint.py run by "make check" doesn't support Python 2. "make venv" runs "$(PYTHON) -m venv", whereas Python 2 doens't provide the venv module: it's a module of Python 3 standard library. (cherry picked from commit 91b0e7d0ca7c59df28f6a6fc1e8eb86a3925b76c) --- Doc/Makefile | 2 +- Doc/faq/windows.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index 6e1c18304080d6..eb62df5b87cebe 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -4,7 +4,7 @@ # # You can set these variables from the command line. -PYTHON = python +PYTHON = python3 SPHINXBUILD = sphinx-build PAPER = SOURCES = diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index b9b7b8d359bdf2..6ac83e45d2e815 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -300,9 +300,9 @@ this respect, and is easily configured to use spaces: Take :menuselection:`Tools --> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent size" to 4, and select the "Insert spaces" radio button. -Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs +Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs and spaces are causing problems in leading whitespace. -You may also run the :mod:`tabnanny` module to check a directory tree +You may also run the :mod:`tabnanny` module to check a directory tree in batch mode. From d372cda5cd46712c4e59262ec1ab981773b20bff Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sat, 18 Feb 2017 16:12:06 -0800 Subject: [PATCH 015/232] bpo-29579: Removes readme.txt from the installer. (#161) --- Misc/NEWS | 2 ++ Tools/msi/exe/exe_files.wxs | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 0fe1f9b9d651bf..5f682e5d8fb4a4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -160,6 +160,8 @@ Library Windows ------- +- bpo-29579: Removes readme.txt from the installer + - Issue #29326: Ignores blank lines in ._pth files (Patch by Alexey Izbyshev) - Issue #28164: Correctly handle special console filenames (patch by Eryk Sun) diff --git a/Tools/msi/exe/exe_files.wxs b/Tools/msi/exe/exe_files.wxs index 01385874fa90b7..e675c21c8975ef 100644 --- a/Tools/msi/exe/exe_files.wxs +++ b/Tools/msi/exe/exe_files.wxs @@ -8,9 +8,6 @@ - - - From 0683d6889bd4430599d22e12e201b8e9c45be5a2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 19 Feb 2017 10:03:35 +0530 Subject: [PATCH 016/232] [3.6] bpo-29571: Use correct locale encoding in test_re (#149) (#153) ``local.getlocale(locale.LC_CTYPE)`` and ``locale.getpreferredencoding(False)`` may give different answers in some cases (such as the ``en_IN`` locale). ``re.LOCALE`` uses the latter, so update the test case to match. --- Lib/test/test_re.py | 2 +- Misc/NEWS | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index b945cf094e99c1..a506b98be4aeb2 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1402,7 +1402,7 @@ def test_ascii_and_unicode_flag(self): def test_locale_flag(self): import locale - _, enc = locale.getlocale(locale.LC_CTYPE) + enc = locale.getpreferredencoding(False) # Search non-ASCII letter for i in range(128, 256): try: diff --git a/Misc/NEWS b/Misc/NEWS index 5f682e5d8fb4a4..b631fde8e52e0d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -202,6 +202,11 @@ Documentation Tests ----- +- Issue #29571: to match the behaviour of the ``re.LOCALE`` flag, + test_re.test_locale_flag now uses ``locale.getpreferredencoding(False)`` to + determine the candidate encoding for the test regex (allowing it to correctly + skip the test when the default locale encoding is a multi-byte encoding) + - Issue #28950: Disallow -j0 to be combined with -T/-l in regrtest command line arguments. From 3337d33a4518f7ab8a7ab6c9a75b8b92ba348b27 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 Feb 2017 14:11:59 +0900 Subject: [PATCH 017/232] [backport to 3.6] bpo-29529: Add .travis.yml to 3.6 branch (#25) * Add .travis.yml to 3.6 branch * Travis CI: run rstlint.py in the docs job (#68) Currently, http://buildbot.python.org/all/buildslaves/ware-docs buildbot is only run as post-commit. For example, bpo-29521 (PR#41) introduced two warnings, unnotified by the Travis CI docs job. Modify the docs job to run toosl/rstlint.py. Fix also the two minor warnings which causes the buildbot slave to fail. * Only run CI checks when appropriate files have changed (#74) Closes python/core-workflow#14 * Use 'make check' instead of 'python3 tools/rstlint.py' (#96) --- .travis.yml | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000000..dc3a00d1f4b5a3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,104 @@ +language: c +dist: trusty +sudo: false +group: beta + +# To cache doc-building dependencies. +cache: pip + +os: + - linux + # macOS builds are disabled as the machines are under-provisioned on Travis, + # adding up to an extra hour completing a full CI run. + #- osx + +compiler: + - clang + - gcc + +env: + - TESTING=cpython + +matrix: + allow_failures: + - env: + - TESTING=coverage + include: + - os: linux + language: python + python: 3.5 + env: + - TESTING=docs + before_script: + - | + if git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '^Doc/' + then + echo "Docs weren't updated, stopping build process." + exit + fi + cd Doc + make venv PYTHON=python3 + script: + - make html SPHINXBUILD="./venv/bin/python3 -m sphinx" SPHINXOPTS="-q" + - make check + - os: linux + language: c + compiler: clang + env: + - TESTING=coverage + before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure + make -s -j4 + # Need a venv that can parse covered code. + ./python -m venv venv + ./venv/bin/python -m pip install -U coverage + script: + # Skip tests that re-run the entire test suite. + - ./venv/bin/python -m coverage run --pylib -m test -uall -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn + after_script: # Probably should be after_success once test suite updated to run under coverage.py. + # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. + - source ./venv/bin/activate + - bash <(curl -s https://codecov.io/bash) + - os: linux + language: cpp + compiler: clang + env: + - TESTING="C++ header compatibility" + before_script: + - ./configure + script: + - echo '#include "Python.h"' > test.cc && $CXX -c test.cc -o /dev/null -I ./Include -I . + +# Travis provides only 2 cores, so don't overdue the parallelism and waste memory. +before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure --with-pydebug + make -j4 + +script: + # `-r -w` implicitly provided through `make buildbottest`. + - make buildbottest TESTOPTS="-j4" + +notifications: + email: false + irc: + channels: + # This is set to a secure variable to prevent forks from notifying the + # IRC channel whenever they fail a build. This can be removed when travis + # implements https://github.com/travis-ci/travis-ci/issues/1094. + # The actual value here is: irc.freenode.net#python-dev + - secure: "s7kAkpcom2yUJ8XqyjFI0obJmhAGrn1xmoivdaPdgBIA++X47TBp1x4pgDsbEsoalef7bEwa4l07KdT4qa+DOd/c4QxaWom7fbN3BuLVsZuVfODnl79+gYq/TAbGfyH+yDs18DXrUfPgwD7C5aW32ugsqAOd4iWzfGJQ5OrOZzqzGjYdYQUEkJFXgxDEIb4aHvxNDWGO3Po9uKISrhb5saQ0l776yLo1Ur7M4oxl8RTbCdgX0vf5TzPg52BgvZpOgt3DHOUYPeiJLKNjAE6ibg0U95sEvMfHX77nz4aFY4/3UI6FFaRla34rZ+mYKrn0TdxOhera1QOgPmM6HzdO4K44FpfK1DS0Xxk9U9/uApq+cG0bU3W+cVUHDBe5+90lpRBAXHeHCgT7TI8gec614aiT8lEr3+yH8OBRYGzkjNK8E2LJZ/SxnVxDe7aLF6AWcoWLfS6/ziAIBFQ5Nc4U72CT8fGVSkl8ywPiRlvixKdvTODMSZo0jMqlfZSNaAPTsNRx4wu5Uis4qekwe32Fz4aB6KGpsuuVjBi+H6v0RKxNJNGY3JKDiEH2TK0UE2auJ5GvLW48aUVFcQMB7euCWYXlSWVRHh3WLU8QXF29Dw4JduRZqUpOdRgMHU79UHRq+mkE0jAS/nBcS6CvsmxCpTSrfVYuMOu32yt18QQoTyU=" + on_success: change + on_failure: always + skip_join: true From b121d737fe60e56dac5a44535e02089c6c02613d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 19 Feb 2017 14:40:55 +0300 Subject: [PATCH 018/232] Tweak PEP 519 documentation in stdlib (#164) * Drop duplicate word 'object' in lzma docs * Fix typo in os docs: fpr -> for (cherry picked from commit d4d48743ac20854de104e08dd66972471684f676) --- Doc/library/lzma.rst | 4 ++-- Doc/library/os.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 5edb23de83bcff..cce6c23e611e31 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -39,7 +39,7 @@ Reading and writing compressed files object`. The *filename* argument can be either an actual file name (given as a - :class:`str`, :class:`bytes` or :term:`path-like object` object), in + :class:`str`, :class:`bytes` or :term:`path-like ` object), in which case the named file is opened, or it can be an existing file object to read from or write to. @@ -76,7 +76,7 @@ Reading and writing compressed files An :class:`LZMAFile` can wrap an already-open :term:`file object`, or operate directly on a named file. The *filename* argument specifies either the file object to wrap, or the name of the file to open (as a :class:`str`, - :class:`bytes` or :term:`path-like object` object). When wrapping an + :class:`bytes` or :term:`path-like ` object). When wrapping an existing file object, the wrapped file will not be closed when the :class:`LZMAFile` is closed. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 37fa2a2868dac9..974ab2d481e210 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2859,7 +2859,7 @@ These functions are all available on Linux only. :ref:`not following symlinks `. .. versionchanged:: 3.6 - Accepts a :term:`path-like object` fpr *path* and *attribute*. + Accepts a :term:`path-like object` for *path* and *attribute*. .. function:: listxattr(path=None, *, follow_symlinks=True) From 7970cd483346dfd7723da214fb27399ecc574095 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 20 Feb 2017 09:46:46 +0900 Subject: [PATCH 019/232] bpo-29520: doc: fix deprecation warning from 'defindex' template (GH-178) --- Doc/tools/templates/indexcontent.html | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1076c1f51b7d19..d795c0a5586bc8 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -1,5 +1,12 @@ -{% extends "defindex.html" %} -{% block tables %} +{% extends "layout.html" %} +{%- block htmltitle -%} +{{ shorttitle }} +{%- endblock -%} +{% block body %} +

{{ docstitle|e }}

+

+ {% trans %}Welcome! This is the documentation for Python {{ release }}.{% endtrans %} +

{% trans %}Parts of the documentation:{% endtrans %}

From a4afdfcf27c2fa53c60b65867765244361abb1fc Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 19 Feb 2017 18:58:33 -0800 Subject: [PATCH 020/232] Change some mercurial/ hg.python.org references. (#8) (#185) (cherry picked from commit b2ee40ed9c9041dcff9c898aa19aacf9ec60308a) --- Doc/faq/general.rst | 6 +++--- Lib/idlelib/help.html | 2 +- Tools/README | 2 +- Tools/importbench/README | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index f1e33afdabf8a5..8f6a907a8a2fda 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -159,7 +159,7 @@ How do I obtain a copy of the Python source? The latest Python source distribution is always available from python.org, at https://www.python.org/downloads/. The latest development sources can be obtained -via anonymous Mercurial access at https://hg.python.org/cpython. +at https://github.com/python/cpython/. The source distribution is a gzipped tar file containing the complete C source, Sphinx-formatted documentation, Python library modules, example programs, and @@ -222,8 +222,8 @@ releases are announced on the comp.lang.python and comp.lang.python.announce newsgroups and on the Python home page at https://www.python.org/; an RSS feed of news is available. -You can also access the development version of Python through Mercurial. See -https://docs.python.org/devguide/faq.html for details. +You can also access the development version of Python through Git. See +`The Python Developer's Guide `_ for details. How do I submit bug reports and patches for Python? diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index ffc03c4112f073..f10cd345e886c8 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -90,7 +90,7 @@

Navigation

25.5. IDLE¶

-

Source code: Lib/idlelib/

+

Source code: Lib/idlelib/

IDLE is Python’s Integrated Development and Learning Environment.

IDLE has the following features:

diff --git a/Tools/README b/Tools/README index 0d961de23d6cb8..edbf4fb83321ba 100644 --- a/Tools/README +++ b/Tools/README @@ -45,4 +45,4 @@ unittestgui A Tkinter based GUI test runner for unittest, with test discovery. -(*) A generic benchmark suite is maintained separately at http://hg.python.org/benchmarks/ +(*) A generic benchmark suite is maintained separately at https://github.com/python/performance diff --git a/Tools/importbench/README b/Tools/importbench/README index 81a5544a383b4c..6ba386c2b608d5 100644 --- a/Tools/importbench/README +++ b/Tools/importbench/README @@ -3,4 +3,4 @@ Importbench is a set of micro-benchmarks for various import scenarios. It should not be used as an overall benchmark of import performance, but rather an easy way to measure impact of possible code changes. For a real-world benchmark of import, use the normal_startup benchmark from -hg.python.org/benchmarks. +https://github.com/python/performance From 9a4577a4bb23888fed2cf192cf1a4c95ce5c26f8 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 20 Feb 2017 14:33:02 +0800 Subject: [PATCH 021/232] bpo-29347: Fix possibly dereferencing undefined pointers when creating weakref objects (#128) (#186) --- Misc/NEWS | 3 +++ Objects/weakrefobject.c | 2 ++ 2 files changed, 5 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index b631fde8e52e0d..0fed99b5e7808f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29347: Fixed possibly dereferencing undefined pointers + when creating weakref objects. + - bpo-29438: Fixed use-after-free problem in key sharing dict. - Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index ab6b23525552b8..9ca386da2563cb 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -24,6 +24,8 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) { self->hash = -1; self->wr_object = ob; + self->wr_prev = NULL; + self->wr_next = NULL; Py_XINCREF(callback); self->wr_callback = callback; } From ce552e2d5c4ff90218fb41847e8ffb1fd3ba3b2d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 20 Feb 2017 22:58:11 +0900 Subject: [PATCH 022/232] bpo-24274: fix erroneous comment in dictobject.c (GH-200) lookdict_unicode() and lookdict_unicode_nodummy() may raise exception when key is not unicode. --- Objects/dictobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b63b78a337225a..a0c1977037b9e6 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -676,7 +676,8 @@ Christian Tismer. lookdict() is general-purpose, and may return DKIX_ERROR if (and only if) a comparison raises an exception. lookdict_unicode() below is specialized to string keys, comparison of which can -never raise an exception; that function can never return DKIX_ERROR. +never raise an exception; that function can never return DKIX_ERROR when key +is string. Otherwise, it falls back to lookdict(). lookdict_unicode_nodummy is further specialized for string keys that cannot be the value. For both, when the key isn't found a DKIX_EMPTY is returned. hashpos returns From d9b3cdd137239a5913de2252c3ce269e35ac63d2 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Mon, 20 Feb 2017 21:59:30 +0000 Subject: [PATCH 023/232] bpo-29602: fix signed zero handling in complex constructor. (#203) (#206) * Fix incorrect handling of signed zeros for complex-related classes. * Add Misc/NEWS entry. (cherry picked from commit 112ec38c15b388fe025ccb85369a584d218b1160) --- Lib/test/test_complex.py | 23 +++++++++++++++++++++++ Misc/NEWS | 4 ++++ Objects/complexobject.c | 6 +++--- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index c249ca724bf0ce..cee49343e268a2 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -387,6 +387,29 @@ def __complex__(self): self.assertAlmostEqual(complex(complex1(1j)), 2j) self.assertRaises(TypeError, complex, complex2(1j)) + @support.requires_IEEE_754 + def test_constructor_special_numbers(self): + class complex2(complex): + pass + for x in 0.0, -0.0, INF, -INF, NAN: + for y in 0.0, -0.0, INF, -INF, NAN: + with self.subTest(x=x, y=y): + z = complex(x, y) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(x, y) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex(complex2(x, y)) + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(complex(x, y)) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + def test_underscores(self): # check underscores for lit in VALID_UNDERSCORE_LITERALS: diff --git a/Misc/NEWS b/Misc/NEWS index 0fed99b5e7808f..d5bbd835aa21a6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + - bpo-29347: Fixed possibly dereferencing undefined pointers when creating weakref objects. diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 31e12784cc34f9..cfaba688c684f6 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1025,11 +1025,11 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } cr.real = PyFloat_AsDouble(tmp); - cr.imag = 0.0; /* Shut up compiler warning */ + cr.imag = 0.0; Py_DECREF(tmp); } if (i == NULL) { - ci.real = 0.0; + ci.real = cr.imag; } else if (PyComplex_Check(i)) { ci = ((PyComplexObject*)i)->cval; @@ -1051,7 +1051,7 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ci_is_complex) { cr.real -= ci.imag; } - if (cr_is_complex) { + if (cr_is_complex && i != NULL) { ci.real += cr.imag; } return complex_subtype_from_doubles(type, cr.real, ci.real); From 51a477c0d53b09d5e876c23288ad006ad64c1e97 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 21 Feb 2017 23:56:26 +0900 Subject: [PATCH 024/232] doc: fix compile error on "shoddy" example extension (GH-216) (cherry picked from commit fb8fe72fc593438f6a0b934c6ff2d9c4aa28673d) --- Doc/includes/setup.py | 1 + Doc/includes/shoddy.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py index b853d23b170985..a38a39de3e7c86 100644 --- a/Doc/includes/setup.py +++ b/Doc/includes/setup.py @@ -5,4 +5,5 @@ Extension("noddy2", ["noddy2.c"]), Extension("noddy3", ["noddy3.c"]), Extension("noddy4", ["noddy4.c"]), + Extension("shoddy", ["shoddy.c"]), ]) diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c index 07a272124ceaba..0c6d412b3c4cda 100644 --- a/Doc/includes/shoddy.c +++ b/Doc/includes/shoddy.c @@ -31,7 +31,7 @@ Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds) static PyTypeObject ShoddyType = { - PyObject_HEAD_INIT(NULL) + PyVarObject_HEAD_INIT(NULL, 0) "shoddy.Shoddy", /* tp_name */ sizeof(Shoddy), /* tp_basicsize */ 0, /* tp_itemsize */ From e48fd93bbb36c6d80aa4eb6af09f58c69d8cf965 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 21 Feb 2017 18:18:27 +0200 Subject: [PATCH 025/232] bpo-29532: Altering a kwarg dictionary passed to functools.partial() no longer affects a partial object after creation. (#209) --- Lib/test/test_functools.py | 9 +++++++++ Misc/NEWS | 13 +++++++++++++ Modules/_functoolsmodule.c | 5 ++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 824549b80342ed..b7d648d0b15a55 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -89,6 +89,15 @@ def func(a=10, b=20): p(b=7) self.assertEqual(d, {'a':3}) + def test_kwargs_copy(self): + # Issue #29532: Altering a kwarg dictionary passed to a constructor + # should not affect a partial object after creation + d = {'a': 3} + p = self.partial(capture, **d) + self.assertEqual(p(), ((), {'a': 3})) + d['a'] = 5 + self.assertEqual(p(), ((), {'a': 3})) + def test_arg_combinations(self): # exercise special code paths for zero args in either partial # object or the caller diff --git a/Misc/NEWS b/Misc/NEWS index d5bbd835aa21a6..83b316c644d72b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,19 @@ Extension Modules Library ------- +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + +- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from + the platform about whether generated UUIDs are generated with a + multiprocessing safe method. + +- bpo-29576: Improve some deprecations in importlib. Some deprecated methods + now emit DeprecationWarnings and have better descriptive messages. + +- bpo-29534: Fixed different behaviour of Decimal.from_float() + for _decimal and _pydecimal. Thanks Andrew Nester. + - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, Manuel Krebber, and Ɓukasz Langa. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index f785a7260e6a2e..7abc9f464027f7 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -88,10 +88,13 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (kw == NULL) { pto->kw = PyDict_New(); } - else { + else if (Py_REFCNT(kw) == 1) { Py_INCREF(kw); pto->kw = kw; } + else { + pto->kw = PyDict_Copy(kw); + } } else { pto->kw = PyDict_Copy(pkw); From 3ab24bdd47fdd9d45719ad49f93d3878d4442d7e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 22 Feb 2017 02:33:24 +0900 Subject: [PATCH 026/232] bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX (GH-219) (cherry picked from commit 3a9ac827c7c87dffc60c4200323948551bcb6662) --- Misc/NEWS | 3 +++ Python/compile.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 83b316c644d72b..5688883375824e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. + Patch by Matthieu Dartiailh. + - bpo-29602: Fix incorrect handling of signed zeros in complex constructor for complex subclasses and for inputs having a __complex__ method. Patch by Serhiy Storchaka. diff --git a/Python/compile.c b/Python/compile.c index 0e1607585245b9..6255ec7d47f5a2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1043,7 +1043,7 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case CALL_FUNCTION_KW: return -oparg-1; case CALL_FUNCTION_EX: - return - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0); + return -1 - ((oparg & 0x01) != 0); case MAKE_FUNCTION: return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) - ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0); From 9b49133082ec23b67e84d2589e66d7810018e424 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 21 Feb 2017 10:30:07 -0800 Subject: [PATCH 027/232] bpo-29453: Remove reference to undefined dictionary ordering in Tutorial (GH-140) (#208) As of Python 3.6 **kwargs are ordered, thus, remove the paragraph stating that ordering is undefined and change snippet to remove the unecessary sorted call. * Add sentence mentioning guaranteed output order of kwargs (cherry picked from commit 32e8f9bdfd4324f1aa4fbbdf1ed8536f2b00cabb) --- Doc/tutorial/controlflow.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index d43461886e7933..6a9bb4889ff857 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -492,8 +492,7 @@ function like this:: for arg in arguments: print(arg) print("-" * 40) - keys = sorted(keywords.keys()) - for kw in keys: + for kw in keywords: print(kw, ":", keywords[kw]) It could be called like this:: @@ -513,13 +512,13 @@ and of course it would print: It's very runny, sir. It's really very, VERY runny, sir. ---------------------------------------- - client : John Cleese shopkeeper : Michael Palin + client : John Cleese sketch : Cheese Shop Sketch -Note that the list of keyword argument names is created by sorting the result -of the keywords dictionary's ``keys()`` method before printing its contents; -if this is not done, the order in which the arguments are printed is undefined. +Note that the order in which the keyword arguments are printed is guaranteed +to match the order in which they were provided in the function call. + .. _tut-arbitraryargs: From e9c0e5559bbadb164d7c57b5a47b5544746dcb89 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Tue, 21 Feb 2017 22:56:07 +0300 Subject: [PATCH 028/232] [3.6] Add .codecov.yml (#210) --- .codecov.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000000000..fcf9df6a7a698e --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,35 @@ +codecov: + notify: + require_ci_to_pass: true +comment: off +ignore: + - "Doc/**/*" + - "Misc/*" + - "Mac/**/*" + - "PC/**/*" + - "PCbuild/**/*" + - "Tools/**/*" + - "Grammar/*" +coverage: + precision: 2 + range: + - 70.0 + - 100.0 + round: down + status: + changes: off + project: off + patch: + default: + target: 100% + only_pulls: true + threshold: 0.05 +parsers: + gcov: + branch_detection: + conditional: true + loop: true + macro: false + method: false + javascript: + enable_partials: false From b067a5eef7fdf69264d3578654996fc3755df4ea Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 22 Feb 2017 04:55:33 +0300 Subject: [PATCH 029/232] bpo-29554: Improve docs for pstat module and profile. (#88) (#227) Clarify that methods take a string which is interpreted as a regex, not a regex object. Also clarify what the old `-1`, `0`, `1` and `2` options were. (cherry picked from commit 8fb1f6e039cbdeb333d83b7a62f0f37af4ce6e02) --- Doc/library/profile.rst | 7 ++++--- Lib/pstats.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 959d9b98a84691..bd67fe486abf77 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -444,9 +444,10 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. significant entries. Initially, the list is taken to be the complete set of profiled functions. Each restriction is either an integer (to select a count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a regular expression (to pattern match - the standard name that is printed. If several restrictions are provided, - then they are applied sequentially. For example:: + select a percentage of lines), or a string that will interpreted as a + regular expression (to pattern match the standard name that is printed). + If several restrictions are provided, then they are applied sequentially. + For example:: print_stats(.1, 'foo:') diff --git a/Lib/pstats.py b/Lib/pstats.py index d861413d4195f7..2c5bf981b85cf8 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -48,11 +48,14 @@ class Stats: printed. The sort_stats() method now processes some additional options (i.e., in - addition to the old -1, 0, 1, or 2). It takes an arbitrary number of - quoted strings to select the sort order. For example sort_stats('time', - 'name') sorts on the major key of 'internal function time', and on the - minor key of 'the name of the function'. Look at the two tables in - sort_stats() and get_sort_arg_defs(self) for more examples. + addition to the old -1, 0, 1, or 2 that are respectively interpreted as + 'stdname', 'calls', 'time', and 'cumulative'). It takes an arbitrary number + of quoted strings to select the sort order. + + For example sort_stats('time', 'name') sorts on the major key of 'internal + function time', and on the minor key of 'the name of the function'. Look at + the two tables in sort_stats() and get_sort_arg_defs(self) for more + examples. All methods return self, so you can string together commands like: Stats('foo', 'goo').strip_dirs().sort_stats('calls').\ From 3cc5817cfaf5663645f4ee447eaed603d2ad290a Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Wed, 22 Feb 2017 06:21:17 +0000 Subject: [PATCH 030/232] Fixed bpo-29565: Corrected ctypes passing of large structs by value on Windows AMD64. (#168) (#220) Fixed bpo-29565: Corrected ctypes passing of large structs by value. (cherry picked from commit a86339b83fbd0932e0529a3c91935e997a234582) --- Lib/ctypes/test/test_callbacks.py | 11 +++++++++++ Lib/ctypes/test/test_structures.py | 23 +++++++++++++++++++++++ Modules/_ctypes/_ctypes_test.c | 13 +++++++++++++ Modules/_ctypes/libffi_msvc/ffi.c | 10 ++++++++++ 4 files changed, 57 insertions(+) diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/ctypes/test/test_callbacks.py index 8eac58f0262caf..f622093df61da5 100644 --- a/Lib/ctypes/test/test_callbacks.py +++ b/Lib/ctypes/test/test_callbacks.py @@ -244,6 +244,7 @@ def callback(a, b, c, d, e): def test_callback_large_struct(self): class Check: pass + # This should mirror the structure in Modules/_ctypes/_ctypes_test.c class X(Structure): _fields_ = [ ('first', c_ulong), @@ -255,6 +256,11 @@ def callback(check, s): check.first = s.first check.second = s.second check.third = s.third + # See issue #29565. + # The structure should be passed by value, so + # any changes to it should not be reflected in + # the value passed + s.first = s.second = s.third = 0x0badf00d check = Check() s = X() @@ -275,6 +281,11 @@ def callback(check, s): self.assertEqual(check.first, 0xdeadbeef) self.assertEqual(check.second, 0xcafebabe) self.assertEqual(check.third, 0x0bad1dea) + # See issue #29565. + # Ensure that the original struct is unchanged. + self.assertEqual(s.first, check.first) + self.assertEqual(s.second, check.second) + self.assertEqual(s.third, check.third) ################################################################ diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 8f6fe5f25429e5..3eded7749ed95e 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -3,6 +3,7 @@ from ctypes.test import need_symbol from struct import calcsize import _testcapi +import _ctypes_test class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -391,6 +392,28 @@ class Z(Y): (1, 0, 0, 0, 0, 0)) self.assertRaises(TypeError, lambda: Z(1, 2, 3, 4, 5, 6, 7)) + def test_pass_by_value(self): + # This should mirror the structure in Modules/_ctypes/_ctypes_test.c + class X(Structure): + _fields_ = [ + ('first', c_ulong), + ('second', c_ulong), + ('third', c_ulong), + ] + + s = X() + s.first = 0xdeadbeef + s.second = 0xcafebabe + s.third = 0x0bad1dea + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_large_struct_update_value + func.argtypes = (X,) + func.restype = None + func(s) + self.assertEqual(s.first, 0xdeadbeef) + self.assertEqual(s.second, 0xcafebabe) + self.assertEqual(s.third, 0x0bad1dea) + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 629ddf66bc4212..59d56d0d218732 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -44,6 +44,19 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) func(in); } +/* + * See issue 29565. Update a structure passed by value; + * the caller should not see any change. + */ + +EXPORT(void) +_testfunc_large_struct_update_value(Test in) +{ + in.first = 0x0badf00d; + in.second = 0x0badf00d; + in.third = 0x0badf00d; +} + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", diff --git a/Modules/_ctypes/libffi_msvc/ffi.c b/Modules/_ctypes/libffi_msvc/ffi.c index 1d82929f530220..91a27dce3f2575 100644 --- a/Modules/_ctypes/libffi_msvc/ffi.c +++ b/Modules/_ctypes/libffi_msvc/ffi.c @@ -239,6 +239,16 @@ ffi_call(/*@dependent@*/ ffi_cif *cif, break; #else case FFI_SYSV: + /* If a single argument takes more than 8 bytes, + then a copy is passed by reference. */ + for (unsigned i = 0; i < cif->nargs; i++) { + size_t z = cif->arg_types[i]->size; + if (z > 8) { + void *temp = alloca(z); + memcpy(temp, avalue[i], z); + avalue[i] = temp; + } + } /*@-usedef@*/ return ffi_call_AMD64(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, fn); From 55b82e10dc6b75b30a72fa56beb60eaf54a008d4 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Thu, 23 Feb 2017 16:17:11 +0100 Subject: [PATCH 031/232] bpo-28911: Clarify the behaviour of assert_called_once_with. (#252) (cherry picked from commit 9d56b34af2efc4e266bf3ae62da5cd2e422a42be) --- Doc/library/unittest.mock.rst | 11 ++++++----- Lib/unittest/mock.py | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index c6d0ec92b6e9ab..a552cbfc70ad2e 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -303,14 +303,14 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_once_with(*args, **kwargs) - Assert that the mock was called exactly once and with the specified - arguments. + Assert that the mock was called exactly once and that that call was + with the specified arguments. >>> mock = Mock(return_value=None) >>> mock('foo', bar='baz') >>> mock.assert_called_once_with('foo', bar='baz') - >>> mock('foo', bar='baz') - >>> mock.assert_called_once_with('foo', bar='baz') + >>> mock('other', bar='values') + >>> mock.assert_called_once_with('other', bar='values') Traceback (most recent call last): ... AssertionError: Expected 'mock' to be called once. Called 2 times. @@ -322,7 +322,8 @@ the *new_callable* argument to :func:`patch`. The assert passes if the mock has *ever* been called, unlike :meth:`assert_called_with` and :meth:`assert_called_once_with` that - only pass if the call is the most recent one. + only pass if the call is the most recent one, and in the case of + :meth:`assert_called_once_with` it must also be the only call. >>> mock = Mock(return_value=None) >>> mock(1, 2, arg='thing') diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index b6b38362341f16..5f97728de28c59 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -815,8 +815,8 @@ def _error_message(): def assert_called_once_with(_mock_self, *args, **kwargs): - """assert that the mock was called exactly once and with the specified - arguments.""" + """assert that the mock was called exactly once and that that call was + with the specified arguments.""" self = _mock_self if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times." % From 2197eac6104311472f200645bc844adb46444b10 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 23 Feb 2017 18:42:30 +0300 Subject: [PATCH 032/232] bpo-28814: Undeprecate inadvertently deprecated inspect functions. (#122) (#243) Nick Coghlan said on bpo-28814: > inspect.getargvalues() and inspect.formatargvalues() were deprecated > in Python 3.5 as part of implementing bpo-20438 > This is incorrect, as these are *frame* introspection related functions, > not callable introspection ones. The documentation and implementation > layout is confusing though, as they're interleaved with the callable > introspection operation This commit undeprecates these functions and adds a note to ignore previous deprecation notices. (cherry picked from commit 0899b9809547ec2894dcf88cf4bba732c5d47d0d) --- Doc/library/inspect.rst | 12 ++++-------- Doc/whatsnew/3.5.rst | 11 ++++++----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 41a784d982d6c2..3fa44a0c99891c 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -905,10 +905,8 @@ Classes and functions are the names of the ``*`` and ``**`` arguments or ``None``. *locals* is the locals dictionary of the given frame. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) @@ -944,10 +942,8 @@ Classes and functions :func:`getargvalues`. The format\* arguments are the corresponding optional formatting functions that are called to turn names and values into strings. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: getmro(cls) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index edb74f043e0b64..a6ba5bbb720ca2 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2327,11 +2327,12 @@ The :func:`inspect.getargspec` function is deprecated and scheduled to be removed in Python 3.6. (See :issue:`20438` for details.) The :mod:`inspect` :func:`~inspect.getfullargspec`, -:func:`~inspect.getargvalues`, :func:`~inspect.getcallargs`, -:func:`~inspect.getargvalues`, :func:`~inspect.formatargspec`, and -:func:`~inspect.formatargvalues` functions are deprecated in favor of -the :func:`inspect.signature` API. -(Contributed by Yury Selivanov in :issue:`20438`.) +:func:`~inspect.getcallargs`, and :func:`~inspect.formatargspec` functions are +deprecated in favor of the :func:`inspect.signature` API. (Contributed by Yury +Selivanov in :issue:`20438`.) + +:func:`~inspect.getargvalues` and :func:`~inspect.formatargvalues` functions +were inadvertently marked as deprecated with the release of Python 3.5.0. Use of :const:`re.LOCALE` flag with str patterns or :const:`re.ASCII` is now deprecated. (Contributed by Serhiy Storchaka in :issue:`22407`.) From f2beceb7e54fadc507137f86cbe9d74f7e5b8f7c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 23 Feb 2017 15:03:20 -0800 Subject: [PATCH 033/232] bpo-29624: Adds purge step and layout test after uploading files. (#258) (#264) --- Tools/msi/uploadrelease.bat | 31 ++++++++++++++++++++++--------- Tools/msi/uploadrelease.proj | 32 +++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 4e319ce4c98ccb..670836be8c1c8b 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -9,6 +9,8 @@ set USER= set TARGET= set DRYRUN=false set NOGPG= +set PURGE_OPTION=/p:Purge=true +set NOTEST= :CheckOpts if "%1" EQU "-h" goto Help @@ -19,7 +21,11 @@ if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts -if "%1" EQU "--no-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && godo CheckOpts +if "%1" EQU "--skip-test" (set NOTEST=true) && shift && godo CheckOpts +if "%1" EQU "-T" (set NOTEST=true) && shift && godo CheckOpts +if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 if not defined PLINK where plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" if not defined PLINK where /R "%ProgramFiles(x86)%\PuTTY" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" @@ -35,7 +41,7 @@ echo Found pscp.exe at %PSCP% if defined NOGPG ( set GPG= - echo Skipping GPG signature generation because of --no-gpg + echo Skipping GPG signature generation because of --skip-gpg ) else ( if not defined GPG where gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" if not defined GPG where /R "%PCBUILD%..\externals\windows-installer" gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" @@ -45,8 +51,12 @@ if defined NOGPG ( call "%PCBUILD%env.bat" > nul 2> nul pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +if not defined NOTEST ( + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 +) msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false popd @@ -55,9 +65,12 @@ exit /B 0 :Help echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] echo. -echo --host (-o) Specify the upload host (required) -echo --user (-u) Specify the user on the host (required) -echo --target (-t) Specify the target directory on the host -echo --dry-run Display commands and filenames without executing them -echo -h Display this help information +echo --host (-o) Specify the upload host (required) +echo --user (-u) Specify the user on the host (required) +echo --target (-t) Specify the target directory on the host +echo --dry-run Display commands and filenames without executing them +echo --skip-gpg Does not generate GPG signatures before uploading +echo --skip-purge Does not perform CDN purge after uploading +echo --skip-test (-T) Does not perform post-upload tests +echo -h Display this help information echo. diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 0d472dea5428a0..305e84fc2d4b28 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -9,6 +9,7 @@ /srv/www.python.org/ftp/python true false + false @@ -64,7 +65,36 @@ echo. echo." /> - + + + + + + + + + + $(TEMP)\%(Filename)_source + $(TEMP)\%(Filename)_source\%(Filename)%(Extension) + $(TEMP)\%(Filename)_layout + $(OutputPath)\%(Filename)_layoutlog + $(OutputPath)\%(Filename)_layoutlog\%(Filename).log + + + + + + + + + + + + + + + From 0a1b656d8ce3da14f8acf947477b8e998e68ef3b Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 24 Feb 2017 15:44:34 -0500 Subject: [PATCH 034/232] bpo-25008: Deprecate smtpd and point to aiosmtpd (#274) (#278) --- Doc/library/smtpd.rst | 6 ++++++ Misc/NEWS | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index e383201aab1461..85ee8a75cf77a9 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -13,6 +13,12 @@ This module offers several classes to implement SMTP (email) servers. +.. seealso:: + + The `aiosmtpd `_ package is a recommended + replacement for this module. It is based on :mod:`asyncio` and provides a + more straightforward API. :mod:`smtpd` should be considered deprecated. + Several server implementations are present; one is a generic do-nothing implementation, which can be overridden, while the other two offer specific mail-sending strategies. diff --git a/Misc/NEWS b/Misc/NEWS index 5688883375824e..d885ca900217ea 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,7 +10,7 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- -- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. - bpo-29602: Fix incorrect handling of signed zeros in complex constructor for @@ -217,6 +217,9 @@ C API Documentation ------------- +- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to + aiosmtpd, a third-party asyncio-based replacement. + - Issue #26355: Add canonical header link on each page to corresponding major version of the documentation. Patch by Matthias Bussonnier. From f28db60179e59633a99fab27646d22d0bd5976a0 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 24 Feb 2017 16:39:30 -0800 Subject: [PATCH 035/232] [3.6] Fix small typos in expressions.rst (GH-276) (GH-281) (cherry picked from commit 132ac381fe7eb593e3d7c63926f90285bba13cdc) --- Doc/reference/expressions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3a4b80557caf67..f4a82699b0be12 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -190,7 +190,7 @@ Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. A comprehension in an :keyword:`async def` function may consist of either a :keyword:`for` or :keyword:`async for` clause following the leading -expression, may contan additonal :keyword:`for` or :keyword:`async for` +expression, may contain additional :keyword:`for` or :keyword:`async for` clauses, and may also use :keyword:`await` expressions. If a comprehension contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an @@ -1317,7 +1317,7 @@ built-in types. * Sequences (instances of :class:`tuple`, :class:`list`, or :class:`range`) can be compared only within each of their types, with the restriction that ranges do not support order comparison. Equality comparison across these types - results in unequality, and ordering comparison across these types raises + results in inequality, and ordering comparison across these types raises :exc:`TypeError`. Sequences compare lexicographically using comparison of corresponding From bea9d2f6488f01794098d9fd4710b98df1bd9472 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 24 Feb 2017 16:40:50 -0800 Subject: [PATCH 036/232] [3.6] bpo-28556: Update to typing: treat subscripted generics as proxies (GH-265) (GH-268) (cherry picked from commit abb3b8ad94d699c8560d94ee9bac9c917b382abe) (cherry picked from commit 365cb5bb9069273e6970c9d5d17ee2fe5003e7ac) --- Lib/test/test_typing.py | 35 +++++++++++++++++++++++++++++++++++ Lib/typing.py | 10 ++++++++++ 2 files changed, 45 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 64d8276658ee7f..f0070ec975791a 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -674,6 +674,41 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_subscripted_generics_as_proxies(self): + T = TypeVar('T') + class C(Generic[T]): + x = 'def' + self.assertEqual(C[int].x, 'def') + self.assertEqual(C[C[int]].x, 'def') + C[C[int]].x = 'changed' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[str].x, 'changed') + C[List[str]].z = 'new' + self.assertEqual(C.z, 'new') + self.assertEqual(C[Tuple[int]].z, 'new') + + self.assertEqual(C().x, 'changed') + self.assertEqual(C[Tuple[str]]().z, 'new') + + class D(C[T]): + pass + self.assertEqual(D[int].x, 'changed') + self.assertEqual(D.z, 'new') + D.z = 'from derived z' + D[int].x = 'from derived x' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[int].z, 'new') + self.assertEqual(D.x, 'from derived x') + self.assertEqual(D[str].z, 'from derived z') + + def test_abc_registry_kept(self): + T = TypeVar('T') + class C(Generic[T]): ... + C.register(int) + self.assertIsInstance(1, C) + C[int] + self.assertIsInstance(1, C) + def test_false_subclasses(self): class MyMapping(MutableMapping[str, str]): pass self.assertNotIsInstance({}, MyMapping) diff --git a/Lib/typing.py b/Lib/typing.py index efe358faf20988..9a0f49099a3114 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1158,6 +1158,16 @@ def __copy__(self): self.__parameters__, self.__args__, self.__origin__, self.__extra__, self.__orig_bases__) + def __setattr__(self, attr, value): + # We consider all the subscripted genrics as proxies for original class + if ( + attr.startswith('__') and attr.endswith('__') or + attr.startswith('_abc_') + ): + super(GenericMeta, self).__setattr__(attr, value) + else: + super(GenericMeta, _gorg(self)).__setattr__(attr, value) + # Prevent checks for Generic to crash when defining Generic. Generic = None From 814213bdf1b95a5d863311b61c671010291f1197 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 25 Feb 2017 11:56:48 -0800 Subject: [PATCH 037/232] [3.6] Backport README.rst changes (GH-294) (cherry picked from commit b3f1f59cf451d4a25b204e7a24f7be4c95e40be8) (cherry picked from commit 1b3d88eb33085e90af729c4c2f78b5ba1b942b1e) (cherry picked from commit c0866855d54bbbe7bcac299c4af45389efacce0b) (cherry picked from commit e32ec9334b35f897ace8a05128838f92c5e0b2fb) (cherry picked from commit 6329aea78e3627d250009aa5aaaf47cd11587f4d) --- Doc/Makefile | 2 +- Doc/{README.txt => README.rst} | 21 +++++++++++---------- Doc/conf.py | 2 +- Doc/make.bat | 2 +- README.rst | 30 +++++++++++++++++++++--------- 5 files changed, 35 insertions(+), 22 deletions(-) rename Doc/{README.txt => README.rst} (83%) diff --git a/Doc/Makefile b/Doc/Makefile index eb62df5b87cebe..94697f95671894 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -153,7 +153,7 @@ dist: cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub check: - $(PYTHON) tools/rstlint.py -i tools -i venv + $(PYTHON) tools/rstlint.py -i tools -i venv -i README.rst serve: ../Tools/scripts/serve.py build/html diff --git a/Doc/README.txt b/Doc/README.rst similarity index 83% rename from Doc/README.txt rename to Doc/README.rst index 4f8e9f8f1417fb..dcd3d6e80ff3c4 100644 --- a/Doc/README.txt +++ b/Doc/README.rst @@ -2,20 +2,21 @@ Python Documentation README ~~~~~~~~~~~~~~~~~~~~~~~~~~~ This directory contains the reStructuredText (reST) sources to the Python -documentation. You don't need to build them yourself, prebuilt versions are -available at . +documentation. You don't need to build them yourself, `prebuilt versions are +available `_. Documentation on authoring Python documentation, including information about -both style and markup, is available in the "Documenting Python" chapter of the -developers guide . +both style and markup, is available in the "`Documenting Python +`_" chapter of the +developers guide. Building the docs ================= -You need to have Sphinx installed; it is the toolset +You need to have `Sphinx `_ installed; it is the toolset used to build the docs. It is not included in this tree, but maintained -separately and available from PyPI . +separately and `available from PyPI `_. Using make @@ -108,11 +109,11 @@ see the make targets above). Contributing ============ -Bugs in the content should be reported to the Python bug tracker at -https://bugs.python.org. +Bugs in the content should be reported to the +`Python bug tracker `_. -Bugs in the toolset should be reported in the Sphinx bug tracker at -https://www.bitbucket.org/birkenfeld/sphinx/issues/. +Bugs in the toolset should be reported in the +`Sphinx bug tracker `_. You can also send a mail to the Python Documentation Team at docs@python.org, and we will process your request as soon as possible. diff --git a/Doc/conf.py b/Doc/conf.py index b3f26d5a692163..18aebb68a8d8df 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -37,7 +37,7 @@ needs_sphinx = '1.2' # Ignore any .rst files in the venv/ directory. -exclude_patterns = ['venv/*'] +exclude_patterns = ['venv/*', 'README.rst'] # Options for HTML output diff --git a/Doc/make.bat b/Doc/make.bat index d0b59618261011..b1a3738f309d3b 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -74,7 +74,7 @@ echo. Provided by this script: echo. clean, check, serve, htmlview echo. echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.txt in this directory or +echo.filenames to build or are ignored. See README.rst in this directory or echo.the documentation for your version of Sphinx for more exhaustive lists echo.of available targets and descriptions of each. echo. diff --git a/README.rst b/README.rst index d02ae64b7fc660..242572c5e361db 100644 --- a/README.rst +++ b/README.rst @@ -15,6 +15,22 @@ reserved. See the end of this file for further copyright and license information. +General Information +------------------- + +- Website: https://www.python.org +- Source code: https://github.com/python/cpython +- Issue tracker: https://bugs.python.org +- Documentation: https://docs.python.org +- Developer's Guide: https://docs.python.org/devguide/ + +Contributing to CPython +----------------------- + +For more complete instructions on contributing to CPython development, +see the `Developer Guide`_. + +.. _Developer Guide: https://docs.python.org/devguide/ Using Python ------------ @@ -28,7 +44,7 @@ Installable Python kits, and information about using Python, are available at Build Instructions ------------------ -On Unix, Linux, BSD, OSX, and Cygwin:: +On Unix, Linux, BSD, macOS, and Cygwin:: ./configure make @@ -38,10 +54,10 @@ On Unix, Linux, BSD, OSX, and Cygwin:: This will install Python as python3. You can pass many options to the configure script; run ``./configure --help`` -to find out more. On OSX and Cygwin, the executable is called ``python.exe``; +to find out more. On macOS and Cygwin, the executable is called ``python.exe``; elsewhere it's just ``python``. -On Mac OS X, if you have configured Python with ``--enable-framework``, you +On macOS, if you have configured Python with ``--enable-framework``, you should use ``make frameworkinstall`` to do the installation. Note that this installs the Python executable in a place that is not normally on your PATH, you may want to set up a symlink in ``/usr/local/bin``. @@ -126,12 +142,8 @@ is downloadable in HTML, PDF, and reStructuredText formats; the latter version is primarily for documentation authors, translators, and people with special formatting requirements. -If you would like to contribute to the development of Python, relevant -documentation is available in the `Python Developer's Guide -`_. - -For information about building Python's documentation, refer to `Doc/README.txt -`_. +For information about building Python's documentation, refer to `Doc/README.rst +`_. Converting From Python 2.x to 3.x From 1a8dd944d4016593e78a7a6c3c635adee783abfd Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 25 Feb 2017 22:34:06 -0800 Subject: [PATCH 038/232] [3.6] bpo-28929: Add to Misc/NEWS (GH-284) mention bpo-28929 in the Documentation section of What's New in Python 3.6.1 release candidate 1 --- Misc/NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index d885ca900217ea..0f5c27902ba423 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -217,6 +217,8 @@ C API Documentation ------------- +- bpo-28929: Link the documentation to its source file on GitHub. + - bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to aiosmtpd, a third-party asyncio-based replacement. From b00232842c5dd0d41054a02a7800fc5f82a92beb Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 25 Feb 2017 22:34:51 -0800 Subject: [PATCH 039/232] Fix small typos in introduction and datastructures of tutorial (GH-272) (GH-297) (cherry picked from commit 53c1892dc3de1de612b1cf95dc7bf09f82c1babf) --- Doc/tutorial/datastructures.rst | 8 ++++---- Doc/tutorial/introduction.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 953a68b44af7bb..6140ece046b975 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -22,11 +22,11 @@ objects: Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. -.. method:: list.extend(L) +.. method:: list.extend(iterable) :noindex: - Extend the list by appending all the items in the given list. Equivalent to - ``a[len(a):] = L``. + Extend the list by appending all the items from the iterable. Equivalent to + ``a[len(a):] = iterable``. .. method:: list.insert(i, x) @@ -68,7 +68,7 @@ objects: The optional arguments *start* and *end* are interpreted as in the slice notation and are used to limit the search to a particular subsequence of - *x*. The returned index is computed relative to the beginning of the full + the list. The returned index is computed relative to the beginning of the full sequence rather than the *start* argument. diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 7e8ee3e5ea19bc..52120a0b16e75f 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -359,7 +359,7 @@ The built-in function :func:`len` returns the length of a string:: Information about string formatting with :meth:`str.format`. :ref:`old-string-formatting` - The old formatting operations invoked when strings and Unicode strings are + The old formatting operations invoked when strings are the left operand of the ``%`` operator are described in more detail here. From 91c499f8f94b89c08686ac9e7c81ea1d89221cbc Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 26 Feb 2017 00:00:09 -0800 Subject: [PATCH 040/232] mark various test data binary (#233) (#304) --- .gitattributes | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000000..55a3f49922bf19 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +Misc/NEWS merge=union + +*.pck binary +Lib/test/cjkencodings/* binary +Lib/test/decimaltestdata/*.decTest binary +Lib/test/sndhdrdata/sndhdr.* binary +Lib/test/test_email/data/msg_26.txt binary +Lib/test/xmltestdata/* binary +Lib/venv/scripts/nt/* binary +Lib/test/coding20731.py binary From fae59e1aa87ee9598d032e0bd697969a5784025f Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 16:05:30 +0300 Subject: [PATCH 041/232] bpo-28961: Address my comments from earlier code review (#306) (cherry picked from commit dea1536fd3a8424d537794cd53715df0989cbbe1) --- Lib/unittest/test/testmock/testhelpers.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py index d5f9e7c1d69204..d2202a7b4132e9 100644 --- a/Lib/unittest/test/testmock/testhelpers.py +++ b/Lib/unittest/test/testmock/testhelpers.py @@ -307,18 +307,9 @@ def test_two_args_call(self): self.assertEqual(args, other_args) def test_call_with_name(self): - self.assertEqual( - 'foo', - _Call((), 'foo')[0], - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), ), )[0] - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), {'hello': 'world'}), )[0] - ) + self.assertEqual(_Call((), 'foo')[0], 'foo') + self.assertEqual(_Call((('bar', 'barz'),),)[0], '') + self.assertEqual(_Call((('bar', 'barz'), {'hello': 'world'}),)[0], '') class SpecSignatureTest(unittest.TestCase): From 5c7b8f423ab1de6221351036ac4e139861a2f4a6 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 16:07:12 +0300 Subject: [PATCH 042/232] Backport recent .travis.yml changes (#308) Backported changes from master: * b52260d8bf392aa04c48b8c2467a4c034184de86 * 984eef7d6d78e1213d6ea99897343a5059a07c59 * 532519770dea5d353f0b0d718c8881a15c7542df * 91b0e7d0ca7c59df28f6a6fc1e8eb86a3925b76c --- .travis.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc3a00d1f4b5a3..27b63c6c08b376 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,11 @@ group: beta # To cache doc-building dependencies. cache: pip +branches: + only: + - master + - /^\d\.\d$/ + os: - linux # macOS builds are disabled as the machines are under-provisioned on Travis, @@ -20,6 +25,7 @@ env: - TESTING=cpython matrix: + fast_finish: true allow_failures: - env: - TESTING=coverage @@ -30,17 +36,10 @@ matrix: env: - TESTING=docs before_script: - - | - if git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '^Doc/' - then - echo "Docs weren't updated, stopping build process." - exit - fi - cd Doc - make venv PYTHON=python3 + - cd Doc + - make venv script: - - make html SPHINXBUILD="./venv/bin/python3 -m sphinx" SPHINXOPTS="-q" - - make check + - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q" - os: linux language: c compiler: clang From 7333d1760e12cf27bcf63265f72521892285c10a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Feb 2017 07:33:54 -0800 Subject: [PATCH 043/232] bpo-29648: import.rst: Add reference to create_module() (GH-290) (GH-314) (cherry picked from commit 46ce7599af82a929506baeaaee5c149970440c4c) --- Doc/reference/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 5e2c1c8b0758d8..b603d1f978a7a6 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -431,7 +431,7 @@ on the module object. If the method returns ``None``, the import machinery will create the new module itself. .. versionadded:: 3.4 - The create_module() method of loaders. + The :meth:`~importlib.abc.Loader.create_module` method of loaders. .. versionchanged:: 3.4 The :meth:`~importlib.abc.Loader.load_module` method was replaced by From 1457984c003b1e461f8aab93bfc37fd8e42312d5 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Feb 2017 07:44:16 -0800 Subject: [PATCH 044/232] bpo-22594: Add a link to the regex module in re documentation (GH-241) (GH-316) (cherry picked from commit ed6795e46f7653e23b862efad240a93453e7df97) --- Doc/library/re.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index fe5ebcc67c840b..9cced512706554 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -42,6 +42,12 @@ module-level functions and methods on that don't require you to compile a regex object first, but miss some fine-tuning parameters. +.. seealso:: + + The third-party `regex `_ module, + which has an API compatible with the standard library :mod:`re` module, + but offers additional functionality and a more thorough Unicode support. + .. _re-syntax: From 893e86e9d3c0caeb878ccb1120c7259e022f3b68 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 19:04:00 +0300 Subject: [PATCH 045/232] bpo-29121: Remove outdated documentation about transactions (#313) (#319) Patch by Aviv Palivoda. (cherry picked from commit fe70d924bb6106d4c21eb414f4a1ba1324e8f46a) --- Doc/library/sqlite3.rst | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 5635577da0f946..d1f7a6f120620b 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -927,14 +927,6 @@ By default, the :mod:`sqlite3` module opens transactions implicitly before a Data Modification Language (DML) statement (i.e. ``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). -So if you are within a transaction and issue a command like ``CREATE TABLE -...``, ``VACUUM``, ``PRAGMA``, the :mod:`sqlite3` module will commit implicitly -before executing that command. There are two reasons for doing that. The first -is that some of these commands don't work within transactions. The other reason -is that sqlite3 needs to keep track of the transaction state (if a transaction -is active or not). The current transaction state is exposed through the -:attr:`Connection.in_transaction` attribute of the connection object. - You can control which kind of ``BEGIN`` statements sqlite3 implicitly executes (or none at all) via the *isolation_level* parameter to the :func:`connect` call, or via the :attr:`isolation_level` property of connections. @@ -945,6 +937,9 @@ Otherwise leave it at its default, which will result in a plain "BEGIN" statement, or set it to one of SQLite's supported isolation levels: "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". +The current transaction state is exposed through the +:attr:`Connection.in_transaction` attribute of the connection object. + .. versionchanged:: 3.6 :mod:`sqlite3` used to implicitly commit an open transaction before DDL statements. This is no longer the case. From 76995cab69d5ef83d31d8a5754cbad11be4038cb Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 19:09:10 +0300 Subject: [PATCH 046/232] bpo-28518: Start a transaction implicitly before a DML statement (#245) (#318) Patch by Aviv Palivoda. (cherry picked from commit 4a926caf8e5fd8af771b2c34bfb6e91c732331fe) --- Lib/sqlite3/test/transactions.py | 9 +++++++++ Misc/NEWS | 3 +++ Modules/_sqlite/cursor.c | 9 ++++----- Modules/_sqlite/statement.c | 12 +++++++----- Modules/_sqlite/statement.h | 2 +- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Lib/sqlite3/test/transactions.py b/Lib/sqlite3/test/transactions.py index 45f1b04c69648f..b8a13de55bc720 100644 --- a/Lib/sqlite3/test/transactions.py +++ b/Lib/sqlite3/test/transactions.py @@ -179,6 +179,15 @@ def CheckDdlDoesNotAutostartTransaction(self): result = self.con.execute("select * from test").fetchall() self.assertEqual(result, []) + def CheckImmediateTransactionalDDL(self): + # You can achieve transactional DDL by issuing a BEGIN + # statement manually. + self.con.execute("begin immediate") + self.con.execute("create table test(i)") + self.con.rollback() + with self.assertRaises(sqlite.OperationalError): + self.con.execute("select * from test") + def CheckTransactionalDDL(self): # You can achieve transactional DDL by issuing a BEGIN # statement manually. diff --git a/Misc/NEWS b/Misc/NEWS index 0f5c27902ba423..a48debdc20ecff 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,6 +69,9 @@ Extension Modules Library ------- +- bpo-28518: Start a transaction implicitly before a DML statement. + Patch by Aviv Palivoda. + - bpo-29532: Altering a kwarg dictionary passed to functools.partial() no longer affects a partial object after creation. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 39f7a6508c2aef..8341fb8480172c 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -511,10 +511,9 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* pysqlite_statement_reset(self->statement); pysqlite_statement_mark_dirty(self->statement); - /* For backwards compatibility reasons, do not start a transaction if a - DDL statement is encountered. If anybody wants transactional DDL, - they can issue a BEGIN statement manually. */ - if (self->connection->begin_statement && !sqlite3_stmt_readonly(self->statement->st) && !self->statement->is_ddl) { + /* We start a transaction implicitly before a DML statement. + SELECT is the only exception. See #9924. */ + if (self->connection->begin_statement && self->statement->is_dml) { if (sqlite3_get_autocommit(self->connection->db)) { result = _pysqlite_connection_begin(self->connection); if (!result) { @@ -609,7 +608,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* } } - if (!sqlite3_stmt_readonly(self->statement->st)) { + if (self->statement->is_dml) { self->rowcount += (long)sqlite3_changes(self->connection->db); } else { self->rowcount= -1L; diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 0df661b9c7ad31..087375be9b63d8 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -73,8 +73,9 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con Py_INCREF(sql); self->sql = sql; - /* determine if the statement is a DDL statement */ - self->is_ddl = 0; + /* Determine if the statement is a DML statement. + SELECT is the only exception. See #9924. */ + self->is_dml = 0; for (p = sql_cstr; *p != 0; p++) { switch (*p) { case ' ': @@ -84,9 +85,10 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con continue; } - self->is_ddl = (PyOS_strnicmp(p, "create ", 7) == 0) - || (PyOS_strnicmp(p, "drop ", 5) == 0) - || (PyOS_strnicmp(p, "reindex ", 8) == 0); + self->is_dml = (PyOS_strnicmp(p, "insert ", 7) == 0) + || (PyOS_strnicmp(p, "update ", 7) == 0) + || (PyOS_strnicmp(p, "delete ", 7) == 0) + || (PyOS_strnicmp(p, "replace ", 8) == 0); break; } diff --git a/Modules/_sqlite/statement.h b/Modules/_sqlite/statement.h index 6eef16857f7b85..8db10f6649ce80 100644 --- a/Modules/_sqlite/statement.h +++ b/Modules/_sqlite/statement.h @@ -38,7 +38,7 @@ typedef struct sqlite3_stmt* st; PyObject* sql; int in_use; - int is_ddl; + int is_dml; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Statement; From deea29e61e61f0e216bff3f0ca008f5ee231793f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Feb 2017 08:55:23 -0800 Subject: [PATCH 047/232] bpo-26184: import.rst: Improve versionchanged note (GH-277) (#320) (cherry picked from commit 6b4a5f45e2df524174a97832571c82c76a3d424a) --- Doc/reference/import.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index b603d1f978a7a6..484457a8a5bb5c 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -464,8 +464,11 @@ import machinery will create the new module itself. .. versionchanged:: 3.5 A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. Starting in Python 3.6 it will be an error to not - define ``create_module()`` on a loader attached to a ModuleSpec. + ``create_module()`` is not. + +.. versionchanged:: 3.6 + An :exc:`ImportError` is raised when ``exec_module()`` is defined but + ``create_module`` is not. Submodules ---------- From 6b81003bdbd9375886bae54f876650bcdccfe6c7 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 26 Feb 2017 20:38:31 +0300 Subject: [PATCH 048/232] bpo-28624: Add a test that checks that cwd parameter of Popen() accepts PathLike objects (#157) (#323) (cherry picked from commit d5c11f7ace48701bb950c6345deee88c35c66e26) --- Doc/library/subprocess.rst | 10 +++++++--- Lib/test/test_subprocess.py | 10 ++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ad2abe82453622..548e4a68cbf1d1 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -466,9 +466,13 @@ functions. The *pass_fds* parameter was added. If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. In particular, the function looks for - *executable* (or for the first item in *args*) relative to *cwd* if the - executable path is a relative path. + *cwd* before executing the child. *cwd* can be a :class:`str` and + :term:`path-like ` object. In particular, the function + looks for *executable* (or for the first item in *args*) relative to *cwd* + if the executable path is a relative path. + + .. versionchanged:: 3.6 + *cwd* parameter accepts a :term:`path-like object`. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index e63f9f254cd2b2..3c871dde5c532f 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -347,6 +347,16 @@ def test_cwd(self): temp_dir = self._normalize_cwd(temp_dir) self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + def test_cwd_with_pathlike(self): + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + + class _PathLikeObj: + def __fspath__(self): + return temp_dir + + self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): # Check that Popen looks for args[0] relative to cwd if args[0] diff --git a/Misc/ACKS b/Misc/ACKS index 1d6e773b553c52..c3b29a42a34f75 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -269,6 +269,7 @@ Albert Chin-A-Young Adal Chiriliuc Matt Chisholm Lita Cho +Sayan Chowdhury Anders Chrigström Tom Christiansen Renee Chu diff --git a/Misc/NEWS b/Misc/NEWS index a48debdc20ecff..dfafe7b081adae 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,6 +69,9 @@ Extension Modules Library ------- +- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts + PathLike objects. Patch by Sayan Chowdhury. + - bpo-28518: Start a transaction implicitly before a DML statement. Patch by Aviv Palivoda. From c9131b61fa060a51ec181053cade9f0a7ee91e4f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 27 Feb 2017 04:09:32 +0900 Subject: [PATCH 049/232] [3.6] bpo-29110: Fix file object leak in `aifc.open` (#310) (cherry picked from commit 03f68b60e17b57f6f13729ff73245dbb37b30a4c) (GH-162) (cherry picked from commit 5dc33eea538361f8a218255f83db2e9298dd8c53) (GH-293) --- Lib/aifc.py | 37 +++++++++++++++++++++++++------------ Lib/test/test_aifc.py | 18 +++++++++++++++++- Misc/NEWS | 3 +++ 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/Lib/aifc.py b/Lib/aifc.py index 692d0bfd272bf0..13ad7dc5ca3d62 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -303,6 +303,8 @@ class Aifc_read: # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk # _framesize -- size of one frame in the file + _file = None # Set here since __del__ checks it + def initfp(self, file): self._version = 0 self._convert = None @@ -344,9 +346,15 @@ def initfp(self, file): def __init__(self, f): if isinstance(f, str): - f = builtins.open(f, 'rb') - # else, assume it is an open file object already - self.initfp(f) + file_object = builtins.open(f, 'rb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + else: + # assume it is an open file object already + self.initfp(f) def __enter__(self): return self @@ -541,18 +549,23 @@ class Aifc_write: # _datalength -- the size of the audio samples written to the header # _datawritten -- the size of the audio samples actually written + _file = None # Set here since __del__ checks it + def __init__(self, f): if isinstance(f, str): - filename = f - f = builtins.open(f, 'wb') - else: - # else, assume it is an open file object already - filename = '???' - self.initfp(f) - if filename[-5:] == '.aiff': - self._aifc = 0 + file_object = builtins.open(f, 'wb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + + # treat .aiff file extensions as non-compressed audio + if f.endswith('.aiff'): + self._aifc = 0 else: - self._aifc = 1 + # assume it is an open file object already + self.initfp(f) def initfp(self, file): self._file = file diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 1bd1f89c8aa609..a731a5136ba5fd 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -1,5 +1,6 @@ -from test.support import findfile, TESTFN, unlink +from test.support import check_no_resource_warning, findfile, TESTFN, unlink import unittest +from unittest import mock from test import audiotests from audioop import byteswap import io @@ -149,6 +150,21 @@ def test_skipunknown(self): #This file contains chunk types aifc doesn't recognize. self.f = aifc.open(findfile('Sine-1000Hz-300ms.aif')) + def test_close_opened_files_on_error(self): + non_aifc_file = findfile('pluck-pcm8.wav', subdir='audiodata') + with check_no_resource_warning(self): + with self.assertRaises(aifc.Error): + # Try opening a non-AIFC file, with the expectation that + # `aifc.open` will fail (without raising a ResourceWarning) + self.f = aifc.open(non_aifc_file, 'rb') + + # Aifc_write.initfp() won't raise in normal case. But some errors + # (e.g. MemoryError, KeyboardInterrupt, etc..) can happen. + with mock.patch.object(aifc.Aifc_write, 'initfp', + side_effect=RuntimeError): + with self.assertRaises(RuntimeError): + self.fout = aifc.open(TESTFN, 'wb') + def test_params_added(self): f = self.f = aifc.open(TESTFN, 'wb') f.aiff() diff --git a/Misc/NEWS b/Misc/NEWS index dfafe7b081adae..5fad3c82c1a3a0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -78,6 +78,9 @@ Library - bpo-29532: Altering a kwarg dictionary passed to functools.partial() no longer affects a partial object after creation. +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + - bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from the platform about whether generated UUIDs are generated with a multiprocessing safe method. From 7e4897a2fb91e49f131a42ed6de88b5185f7dea8 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Feb 2017 13:30:44 -0800 Subject: [PATCH 050/232] bpo-26184: import.rst: Improve versionchanged note (GH-325) (GH-326) (cherry picked from commit 1f5639c77f736c18fb5a85b4a1850121e25c788e) --- Doc/reference/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 484457a8a5bb5c..4dbd9d88c301a0 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -468,7 +468,7 @@ import machinery will create the new module itself. .. versionchanged:: 3.6 An :exc:`ImportError` is raised when ``exec_module()`` is defined but - ``create_module`` is not. + ``create_module()`` is not. Submodules ---------- From 4b6c41768a15fc85e3069603ef89344bd97f79af Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 27 Feb 2017 11:45:42 +0800 Subject: [PATCH 051/232] bpo-29376: Fix assertion error in threading._DummyThread.is_alive() (GH-330) --- Lib/test/test_threading.py | 3 +++ Lib/threading.py | 4 ++++ Misc/NEWS | 2 ++ 3 files changed, 9 insertions(+) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 2c2914fd6d8969..6b6c4d220a3bd0 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -170,6 +170,9 @@ def f(mutex): mutex.acquire() self.assertIn(tid, threading._active) self.assertIsInstance(threading._active[tid], threading._DummyThread) + #Issue 29376 + self.assertTrue(threading._active[tid].is_alive()) + self.assertRegex(repr(threading._active[tid]), '_DummyThread') del threading._active[tid] # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) diff --git a/Lib/threading.py b/Lib/threading.py index 4829ff426e0bfd..95978d310a2f3a 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1217,6 +1217,10 @@ def __init__(self): def _stop(self): pass + def is_alive(self): + assert not self._is_stopped and self._started.is_set() + return True + def join(self, timeout=None): assert False, "cannot join a dummy thread" diff --git a/Misc/NEWS b/Misc/NEWS index 5fad3c82c1a3a0..c47657fb28357f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,6 +69,8 @@ Extension Modules Library ------- +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + - bpo-28624: Add a test that checks that cwd parameter of Popen() accepts PathLike objects. Patch by Sayan Chowdhury. From 8aa1ad892ff3cf4d8881cdae0e95be1a5bad175e Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 27 Feb 2017 13:55:07 +0800 Subject: [PATCH 052/232] bpo-29662: fix wrong indentation in timeit.Timer's doc (GH-333) --- Doc/library/timeit.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 3b772765aca260..5793c54ae8d1a3 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -134,21 +134,21 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None) - Automatically determine how many times to call :meth:`.timeit`. + Automatically determine how many times to call :meth:`.timeit`. - This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual - (number of loops, time taken for that number of loops). It calls - :meth:`.timeit` with *number* set to successive powers of ten (10, - 100, 1000, ...) up to a maximum of one billion, until the time taken - is at least 0.2 second, or the maximum is reached. + This is a convenience function that calls :meth:`.timeit` repeatedly + so that the total time >= 0.2 second, returning the eventual + (number of loops, time taken for that number of loops). It calls + :meth:`.timeit` with *number* set to successive powers of ten (10, + 100, 1000, ...) up to a maximum of one billion, until the time taken + is at least 0.2 second, or the maximum is reached. - If *callback* is given and is not ``None``, it will be called after - each trial with two arguments: ``callback(number, time_taken)``. + If *callback* is given and is not ``None``, it will be called after + each trial with two arguments: ``callback(number, time_taken)``. - .. versionadded:: 3.6 + .. versionadded:: 3.6 .. method:: Timer.repeat(repeat=3, number=1000000) From bc10e6bd7b6b4a738ce327e31550048d3dd8d8bd Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 27 Feb 2017 06:02:21 -0800 Subject: [PATCH 053/232] Asyncio documentation: remove `self` from method signatures (GH-334) (GH-337) (cherry picked from commit 091b84f23a2ff57e8320ebf6fdf889af39096ab9) --- Doc/library/asyncio-protocol.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3fbf51058673d9..482ffbbed35b1e 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -36,7 +36,7 @@ BaseTransport Base class for transports. - .. method:: close(self) + .. method:: close() Close the transport. If the transport has a buffer for outgoing data, buffered data will be flushed asynchronously. No more data @@ -44,7 +44,7 @@ BaseTransport protocol's :meth:`connection_lost` method will be called with :const:`None` as its argument. - .. method:: is_closing(self) + .. method:: is_closing() Return ``True`` if the transport is closing or is closed. @@ -251,7 +251,7 @@ BaseSubprocessTransport if it hasn't returned, similarly to the :attr:`subprocess.Popen.returncode` attribute. - .. method:: kill(self) + .. method:: kill() Kill the subprocess, as in :meth:`subprocess.Popen.kill`. From 046041e23b3c7e0b4a08705e2792ef3e28d03384 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 27 Feb 2017 17:26:10 +0300 Subject: [PATCH 054/232] Delete accidentally merged entries from Misc/NEWS (#338) --- Misc/NEWS | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index c47657fb28357f..7ec6b1291a3d2d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -83,16 +83,6 @@ Library - bpo-29110: Fix file object leak in aifc.open() when file is given as a filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. -- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from - the platform about whether generated UUIDs are generated with a - multiprocessing safe method. - -- bpo-29576: Improve some deprecations in importlib. Some deprecated methods - now emit DeprecationWarnings and have better descriptive messages. - -- bpo-29534: Fixed different behaviour of Decimal.from_float() - for _decimal and _pydecimal. Thanks Andrew Nester. - - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, Manuel Krebber, and Ɓukasz Langa. From 53039ad3814a8918c5311f37bd654428b9843fcc Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Mon, 27 Feb 2017 16:08:01 +0000 Subject: [PATCH 055/232] bpo-28598: Support __rmod__ for RHS subclasses of str in % string formatting operations (#95) --- Lib/test/test_unicode.py | 9 +++++++++ Misc/NEWS | 3 +++ Python/ceval.c | 12 +++++++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 86ebd45e7b031d..2844bc5540c38b 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1448,6 +1448,15 @@ def test_formatting_huge_precision(self): with self.assertRaises(ValueError): result = format_string % 2.34 + def test_issue28598_strsubclass_rhs(self): + # A subclass of str with an __rmod__ method should be able to hook + # into the % operator + class SubclassedStr(str): + def __rmod__(self, other): + return 'Success, self.__rmod__({!r}) was called'.format(other) + self.assertEqual('lhs %% %r' % SubclassedStr('rhs'), + "Success, self.__rmod__('lhs %% %r') was called") + @support.cpython_only def test_formatting_huge_precision_c_limits(self): from _testcapi import INT_MAX diff --git a/Misc/NEWS b/Misc/NEWS index 7ec6b1291a3d2d..fe420b62f231dd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- Issue #28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. diff --git a/Python/ceval.c b/Python/ceval.c index d5172b9631f43d..c9e9c327f31401 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1409,9 +1409,15 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(BINARY_MODULO) { PyObject *divisor = POP(); PyObject *dividend = TOP(); - PyObject *res = PyUnicode_CheckExact(dividend) ? - PyUnicode_Format(dividend, divisor) : - PyNumber_Remainder(dividend, divisor); + PyObject *res; + if (PyUnicode_CheckExact(dividend) && ( + !PyUnicode_Check(divisor) || PyUnicode_CheckExact(divisor))) { + // fast path; string formatting, but not if the RHS is a str subclass + // (see issue28598) + res = PyUnicode_Format(dividend, divisor); + } else { + res = PyNumber_Remainder(dividend, divisor); + } Py_DECREF(divisor); Py_DECREF(dividend); SET_TOP(res); From bb59d89ceeb1abfb4d73c7fc60b534e4464adf35 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 27 Feb 2017 19:14:11 +0300 Subject: [PATCH 056/232] bpo-27788 : synchronise platform.py version number (#246) (#341) Was bumped in the docstring by b9f4feab1b9c9ffa8ea29af3d82bc536f9f3005a but not in `__version__` (cherry picked from commit 6059ce45aa96f52fa0150e68ea655fbfdc25609a) --- Lib/platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/platform.py b/Lib/platform.py index e48ad0b6e7bdea..062347537531ba 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -110,7 +110,7 @@ """ -__version__ = '1.0.7' +__version__ = '1.0.8' import collections import sys, os, re, subprocess From 7accf2033d03025cc5324f8b9d22582bca3623e9 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 27 Feb 2017 20:41:21 +0300 Subject: [PATCH 057/232] bpo-29655: Fixed possible reference leaks in `import *`. (#301) (#348) Patch by Matthias Bussonnier. (cherry picked from commit 160edb43571311a3785785c1dfa784afc52d87be) --- Python/ceval.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index c9e9c327f31401..83667353af232c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2857,13 +2857,16 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(IMPORT_STAR) { PyObject *from = POP(), *locals; int err; - if (PyFrame_FastToLocalsWithError(f) < 0) + if (PyFrame_FastToLocalsWithError(f) < 0) { + Py_DECREF(from); goto error; + } locals = f->f_locals; if (locals == NULL) { PyErr_SetString(PyExc_SystemError, "no locals found during 'import *'"); + Py_DECREF(from); goto error; } err = import_all_from(locals, from); From c4a786b8ad2a625594551ab3bc991a6831ba7688 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 28 Feb 2017 11:28:44 +0800 Subject: [PATCH 058/232] bpo-29661: fix contradiction in timeit.Timer.autorange's docstring (GH-331) (GH-353) --- Lib/timeit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 Lib/timeit.py diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100644 new mode 100755 index 2770efa35a0160..8eea766b4c344b --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -208,7 +208,7 @@ def repeat(self, repeat=default_repeat, number=default_number): return r def autorange(self, callback=None): - """Return the number of loops so that total time >= 0.2. + """Return the number of loops and time taken so that total time >= 0.2. Calls the timeit method with *number* set to successive powers of ten (10, 100, 1000, ...) up to a maximum of one billion, until From d413aa7893fd8ac64693c22deb2c1eee1840dbd7 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 28 Feb 2017 23:19:58 -0500 Subject: [PATCH 059/232] Disable mention-bot for maintenance branches (GH-365) --- .mention-bot | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .mention-bot diff --git a/.mention-bot b/.mention-bot new file mode 100644 index 00000000000000..cb53b993fb1e8a --- /dev/null +++ b/.mention-bot @@ -0,0 +1,3 @@ +{ + "findPotentialReviewers": false +} From 8458b2b4ef06805d65c9e253433727e60820992e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 28 Feb 2017 21:07:32 -0800 Subject: [PATCH 060/232] correct documentation for enum.html (GH-358) (GH-367) (cherry picked from commit 626584284e74a68fff8157f9afe77b3088ff7be9) --- Doc/library/enum.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 5cd6472f3e2f45..6548adf789da17 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -24,8 +24,8 @@ Module Contents --------------- This module defines four enumeration classes that can be used to define unique -sets of names and values: :class:`Enum`, :class:`IntEnum`, and -:class:`IntFlags`. It also defines one decorator, :func:`unique`, and one +sets of names and values: :class:`Enum`, :class:`IntEnum`, :class:`Flag`, and +:class:`IntFlag`. It also defines one decorator, :func:`unique`, and one helper, :class:`auto`. .. class:: Enum From 1b93ed4f3ea5a4ac9786f15870e9b2812ba87cb1 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 28 Feb 2017 21:08:13 -0800 Subject: [PATCH 061/232] Fixed a handful of typos (GH-343) (GH-364) (cherry picked from commit 1cf2a809b1eb4f9fc8ac3ccc97424586892d1e1a) --- Doc/library/email.compat32-message.rst | 4 ++-- Doc/library/email.message.rst | 2 +- Doc/library/idle.rst | 2 +- Doc/library/ssl.rst | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 2c65079ebdc3f1..afb5df5d428929 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -37,7 +37,7 @@ generating a serialized version of the mssage, and for recursively walking over the object tree. Note that duplicate headers are supported but special methods must be used to access them. -The :class:`Message` psuedo-dictionary is indexed by the header names, which +The :class:`Message` pseudo-dictionary is indexed by the header names, which must be ASCII values. The values of the dictionary are strings that are supposed to contain only ASCII characters; there is some special handling for non-ASCII input, but it doesn't always produce the correct results. Headers @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: This is a legacy method. On the :class:`~email.emailmessage.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the - realted ``make`` and ``add`` methods. + related ``make`` and ``add`` methods. .. method:: get_payload(i=None, decode=False) diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 32852e706986b4..d36e769f763cd8 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -364,7 +364,7 @@ message objects. *header* specifies an alternative header to :mailheader:`Content-Type`. If the value contains non-ASCII characters, the charset and language may - be explicity specified using the optional *charset* and *language* + be explicitly specified using the optional *charset* and *language* parameters. Optional *language* specifies the :rfc:`2231` language, defaulting to the empty string. Both *charset* and *language* should be strings. The default is to use the ``utf8`` *charset* and ``None`` for diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index a629bc50dbc749..07c2a25d6a012c 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -446,7 +446,7 @@ longer or disable the extension. Calltips ^^^^^^^^ -A calltip is shown when one types :kbd:`(` after the name of an *acccessible* +A calltip is shown when one types :kbd:`(` after the name of an *accessible* function. A name expression may include dots and subscripts. A calltip remains until it is clicked, the cursor is moved out of the argument area, or :kbd:`)` is typed. When the cursor is in the argument part of a definition, diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 07b97715a985b5..7b76b2a0064bf8 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -616,7 +616,7 @@ Constants .. data:: PROTOCOL_TLS_CLIENT - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, but only support client-side :class:`SSLSocket` connections. The protocol enables :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` by default. @@ -625,7 +625,7 @@ Constants .. data:: PROTOCOL_TLS_SERVER - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, but only support server-side :class:`SSLSocket` connections. .. versionadded:: 3.6 @@ -948,7 +948,7 @@ SSL Sockets :ref:`notes on non-blocking sockets `. Usually, :class:`SSLSocket` are not created directly, but using the - the :meth:`SSLContext.wrap_socket` method. + :meth:`SSLContext.wrap_socket` method. .. versionchanged:: 3.5 The :meth:`sendfile` method was added. From 023532e558bb0c5bf60195aebbafe63a0bebd85e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 1 Mar 2017 21:14:43 +0900 Subject: [PATCH 062/232] bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. (GH-378) --- Misc/NEWS | 4 ++++ Python/ceval.c | 13 +++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index fe420b62f231dd..33559a7abb8ce1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. + It should raise TypeError when kwargs is not a dict. But it might + cause segv when args=NULL and kwargs is not a dict. + - Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. diff --git a/Python/ceval.c b/Python/ceval.c index 83667353af232c..02bc67ef71090a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4699,11 +4699,7 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) assert(!PyErr_Occurred()); #endif - if (args == NULL) { - return _PyObject_FastCallDict(func, NULL, 0, kwargs); - } - - if (!PyTuple_Check(args)) { + if (args != NULL && !PyTuple_Check(args)) { PyErr_SetString(PyExc_TypeError, "argument list must be a tuple"); return NULL; @@ -4715,7 +4711,12 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) return NULL; } - return PyObject_Call(func, args, kwargs); + if (args == NULL) { + return _PyObject_FastCallDict(func, NULL, 0, kwargs); + } + else { + return PyObject_Call(func, args, kwargs); + } } const char * From b05d48e021bc9c0700939978ab1ba300e0407a0e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 1 Mar 2017 07:35:23 -0800 Subject: [PATCH 063/232] email.compat32-message.rst: Fix typo in the word `message` (GH-379) (GH-380) (cherry picked from commit da62373b0d32c14a4137512ef6f13c24fbcaa2c1) --- Doc/library/email.compat32-message.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index afb5df5d428929..b0707646e64ac6 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -33,9 +33,9 @@ having a MIME type such as :mimetype:`multipart/\*` or The conceptual model provided by a :class:`Message` object is that of an ordered dictionary of headers with additional methods for accessing both specialized information from the headers, for accessing the payload, for -generating a serialized version of the mssage, and for recursively walking over -the object tree. Note that duplicate headers are supported but special methods -must be used to access them. +generating a serialized version of the message, and for recursively walking +over the object tree. Note that duplicate headers are supported but special +methods must be used to access them. The :class:`Message` pseudo-dictionary is indexed by the header names, which must be ASCII values. The values of the dictionary are strings that are From 784ba7c8ad53638c94270011d55d2536ff0cd2dd Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 2 Mar 2017 12:32:13 -0500 Subject: [PATCH 064/232] bpo-29697: Don't use OpenSSL <1.0.2 fallback on 1.1+ (#397) --- Modules/_ssl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b1988570604db1..c0a7b8e1052c33 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2729,12 +2729,12 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) #endif -#ifndef OPENSSL_NO_ECDH +#if !defined(OPENSSL_NO_ECDH) && !defined(OPENSSL_VERSION_1_1) /* Allow automatic ECDH curve selection (on OpenSSL 1.0.2+), or use prime256v1 by default. This is Apache mod_ssl's initialization policy, so we should be safe. OpenSSL 1.1 has it enabled by default. */ -#if defined(SSL_CTX_set_ecdh_auto) && !defined(OPENSSL_VERSION_1_1) +#if defined(SSL_CTX_set_ecdh_auto) SSL_CTX_set_ecdh_auto(self->ctx, 1); #else { From 65bd0bdf3d285e3917d66c600c95cb0842e3b3be Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 2 Mar 2017 12:37:50 -0500 Subject: [PATCH 065/232] In SSL module version examples, don't use a legacy version. (#381) (#400) --- Doc/library/ssl.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 7b76b2a0064bf8..bbb13745b203e6 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -846,7 +846,7 @@ Constants The version string of the OpenSSL library loaded by the interpreter:: >>> ssl.OPENSSL_VERSION - 'OpenSSL 0.9.8k 25 Mar 2009' + 'OpenSSL 1.0.2k 26 Jan 2017' .. versionadded:: 3.2 @@ -856,7 +856,7 @@ Constants OpenSSL library:: >>> ssl.OPENSSL_VERSION_INFO - (0, 9, 8, 11, 15) + (1, 0, 2, 11, 15) .. versionadded:: 3.2 @@ -865,9 +865,9 @@ Constants The raw version number of the OpenSSL library, as a single integer:: >>> ssl.OPENSSL_VERSION_NUMBER - 9470143 + 268443839 >>> hex(ssl.OPENSSL_VERSION_NUMBER) - '0x9080bf' + '0x100020bf' .. versionadded:: 3.2 From a6e84933d204f807e0e81b6a2237193b2e8ab89a Mon Sep 17 00:00:00 2001 From: Brian Coleman Date: Thu, 2 Mar 2017 22:21:53 +0000 Subject: [PATCH 066/232] bpo-29683 - Fixes to _PyCode_SetExtra when co_extra->ce->extras is (#402) allocated. On PyMem_Realloc failure, _PyCode_SetExtra should free co_extra if co_extra->ce_extras could not be allocated. On PyMem_Realloc success, _PyCode_SetExtra should set all unused slots in co_extra->ce_extras to NULL. --- Misc/NEWS | 3 +++ Objects/codeobject.c | 24 ++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 33559a7abb8ce1..a57c476a0b5be1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by + Brian Coleman. + - bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. It should raise TypeError when kwargs is not a dict. But it might cause segv when args=NULL and kwargs is not a dict. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d8a675f9fbfc6..df8b9538fe9ed5 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -856,16 +856,15 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra *) o->co_extra; if (co_extra == NULL) { - o->co_extra = (_PyCodeObjectExtra*) PyMem_Malloc( - sizeof(_PyCodeObjectExtra)); - if (o->co_extra == NULL) { + co_extra = PyMem_Malloc(sizeof(_PyCodeObjectExtra)); + if (co_extra == NULL) { return -1; } - co_extra = (_PyCodeObjectExtra *) o->co_extra; co_extra->ce_extras = PyMem_Malloc( tstate->co_extra_user_count * sizeof(void*)); if (co_extra->ce_extras == NULL) { + PyMem_Free(co_extra); return -1; } @@ -874,20 +873,25 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { co_extra->ce_extras[i] = NULL; } + + o->co_extra = co_extra; } else if (co_extra->ce_size <= index) { - co_extra->ce_extras = PyMem_Realloc( + void** ce_extras = PyMem_Realloc( co_extra->ce_extras, tstate->co_extra_user_count * sizeof(void*)); - if (co_extra->ce_extras == NULL) { + if (ce_extras == NULL) { return -1; } - co_extra->ce_size = tstate->co_extra_user_count; - - for (Py_ssize_t i = co_extra->ce_size; i < co_extra->ce_size; i++) { - co_extra->ce_extras[i] = NULL; + for (Py_ssize_t i = co_extra->ce_size; + i < tstate->co_extra_user_count; + i++) { + ce_extras[i] = NULL; } + + co_extra->ce_extras = ce_extras; + co_extra->ce_size = tstate->co_extra_user_count; } co_extra->ce_extras[index] = extra; From 01e5230ef0b28658cf7311be199363eda98808bd Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 22:06:15 -0500 Subject: [PATCH 067/232] bpo-29703: asyncio: Fix creating new event loops in child processes. (#404) (#410) --- Lib/asyncio/events.py | 8 +++++++- Lib/asyncio/test_utils.py | 5 ++++- Lib/test/test_asyncio/test_events.py | 22 ++++++++++++++++++++++ Misc/NEWS | 3 +++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 28a45fc3cc5aee..7b30b4c84ad9a7 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -11,6 +11,7 @@ import functools import inspect +import os import reprlib import socket import subprocess @@ -611,6 +612,9 @@ def new_event_loop(self): # A TLS for the running event loop, used by _get_running_loop. class _RunningLoop(threading.local): _loop = None + _pid = None + + _running_loop = _RunningLoop() @@ -620,7 +624,8 @@ def _get_running_loop(): This is a low-level function intended to be used by event loops. This function is thread-specific. """ - return _running_loop._loop + if _running_loop._pid == os.getpid(): + return _running_loop._loop def _set_running_loop(loop): @@ -629,6 +634,7 @@ def _set_running_loop(loop): This is a low-level function intended to be used by event loops. This function is thread-specific. """ + _running_loop._pid = os.getpid() _running_loop._loop = loop diff --git a/Lib/asyncio/test_utils.py b/Lib/asyncio/test_utils.py index 99e3839f456858..b12d5db2a9755d 100644 --- a/Lib/asyncio/test_utils.py +++ b/Lib/asyncio/test_utils.py @@ -449,12 +449,15 @@ def new_test_loop(self, gen=None): self.set_event_loop(loop) return loop + def unpatch_get_running_loop(self): + events._get_running_loop = self._get_running_loop + def setUp(self): self._get_running_loop = events._get_running_loop events._get_running_loop = lambda: None def tearDown(self): - events._get_running_loop = self._get_running_loop + self.unpatch_get_running_loop() events.set_event_loop(None) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 28d92a9f4e3eac..802763bd11ff61 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1,6 +1,7 @@ """Tests for events.py.""" import collections.abc +import concurrent.futures import functools import gc import io @@ -57,6 +58,15 @@ def osx_tiger(): return version < (10, 5) +def _test_get_event_loop_new_process__sub_proc(): + async def doit(): + return 'hello' + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop.run_until_complete(doit()) + + ONLYCERT = data_file('ssl_cert.pem') ONLYKEY = data_file('ssl_key.pem') SIGNED_CERTFILE = data_file('keycert3.pem') @@ -2181,6 +2191,18 @@ def tearDown(self): asyncio.set_child_watcher(None) super().tearDown() + def test_get_event_loop_new_process(self): + async def main(): + pool = concurrent.futures.ProcessPoolExecutor() + return await self.loop.run_in_executor( + pool, _test_get_event_loop_new_process__sub_proc) + + self.unpatch_get_running_loop() + + self.assertEqual( + self.loop.run_until_complete(main()), + 'hello') + if hasattr(selectors, 'KqueueSelector'): class KqueueEventLoopTests(UnixEventLoopTestsMixin, SubprocessTestsMixin, diff --git a/Misc/NEWS b/Misc/NEWS index a57c476a0b5be1..8391c6a795a2c7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -79,6 +79,9 @@ Extension Modules Library ------- +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + - bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). - bpo-28624: Add a test that checks that cwd parameter of Popen() accepts From dea5101ae101aefed14de98e6bb1658f4cae8712 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 22:20:00 -0500 Subject: [PATCH 068/232] bpo-28893: Set __cause__ for errors in async iteration protocol (#407) --- Lib/test/test_coroutines.py | 38 +++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Python/ceval.c | 6 +++--- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 78439a2acae2d8..b4c7b5be6e208c 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1680,6 +1680,44 @@ async def foo(): warnings.simplefilter("error") run_async(foo()) + def test_for_11(self): + class F: + def __aiter__(self): + return self + def __anext__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __anext__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + + def test_for_12(self): + class F: + def __aiter__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __aiter__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + def test_for_tuple(self): class Done(Exception): pass diff --git a/Misc/NEWS b/Misc/NEWS index 8391c6a795a2c7..7e355e1b91c041 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-28893: Set correct __cause__ for errors about invalid awaitables + returned from __aiter__ and __anext__. + - bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by Brian Coleman. diff --git a/Python/ceval.c b/Python/ceval.c index 02bc67ef71090a..9cac771abdf65b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1904,13 +1904,13 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) awaitable = _PyCoro_GetAwaitableIter(iter); if (awaitable == NULL) { - SET_TOP(NULL); - PyErr_Format( + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __aiter__: %.100s", Py_TYPE(iter)->tp_name); + SET_TOP(NULL); Py_DECREF(iter); goto error; } else { @@ -1969,7 +1969,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { - PyErr_Format( + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", From 13802a3b11eb5202b16e464cbfb85c144f8581ce Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 22:16:33 -0500 Subject: [PATCH 069/232] bpo-29271: Fix Task.current_task and Task.all_tasks to accept None. (#406) --- Lib/test/test_asyncio/test_tasks.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ Modules/_asynciomodule.c | 16 ++++++++-------- Modules/clinic/_asynciomodule.c.h | 6 +++--- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index a18d49ae374112..4f05319457985a 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1462,6 +1462,14 @@ def test_current_task(self): def coro(loop): self.assertTrue(Task.current_task(loop=loop) is task) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(loop) + try: + self.assertIs(Task.current_task(None), task) + self.assertIs(Task.current_task(), task) + finally: + asyncio.set_event_loop(None) + task = self.new_task(self.loop, coro(self.loop)) self.loop.run_until_complete(task) self.assertIsNone(Task.current_task(loop=self.loop)) @@ -1806,8 +1814,17 @@ def kill_me(loop): # schedule the task coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) + self.assertEqual(Task.all_tasks(loop=self.loop), {task}) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(self.loop) + try: + self.assertEqual(Task.all_tasks(), {task}) + self.assertEqual(Task.all_tasks(None), {task}) + finally: + asyncio.set_event_loop(None) + # execute the task so it waits for future self.loop._run_once() self.assertEqual(len(self.loop._ready), 0) diff --git a/Misc/NEWS b/Misc/NEWS index 7e355e1b91c041..ab5c602c454e93 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -82,6 +82,9 @@ Extension Modules Library ------- +- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C + to accept None argument as their pure Python implementation. + - bpo-29703: Fix asyncio to support instantiation of new event loops in child processes. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index fff90468a5927c..d0e43ae47d9ec8 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1413,7 +1413,7 @@ TaskObj_get_fut_waiter(TaskObj *task) @classmethod _asyncio.Task.current_task - loop: 'O' = NULL + loop: 'O' = None Return the currently running task in an event loop or None. @@ -1424,12 +1424,12 @@ None is returned when called not in the context of a Task. static PyObject * _asyncio_Task_current_task_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=99fbe7332c516e03 input=cd784537f02cf833]*/ +/*[clinic end generated code: output=99fbe7332c516e03 input=a0d6cdf2e3b243e1]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } @@ -1500,7 +1500,7 @@ task_all_tasks(PyObject *loop) @classmethod _asyncio.Task.all_tasks - loop: 'O' = NULL + loop: 'O' = None Return a set of all tasks for an event loop. @@ -1509,12 +1509,12 @@ By default all tasks for the current event loop are returned. static PyObject * _asyncio_Task_all_tasks_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=11f9b20749ccca5d input=cd64aa5f88bd5c49]*/ +/*[clinic end generated code: output=11f9b20749ccca5d input=c6f5b53bd487488f]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 052d252331f6a7..41f11f4cac5494 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -278,7 +278,7 @@ _asyncio_Task_current_task(PyTypeObject *type, PyObject **args, Py_ssize_t nargs PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:current_task", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -310,7 +310,7 @@ _asyncio_Task_all_tasks(PyTypeObject *type, PyObject **args, Py_ssize_t nargs, P PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:all_tasks", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -517,4 +517,4 @@ _asyncio_Task__wakeup(TaskObj *self, PyObject **args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=8f036321bb083066 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=40ca6c9da517da73 input=a9049054013a1b77]*/ From 604faba1db724951ee440337099d111e3ecade49 Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Thu, 2 Mar 2017 22:21:18 -0600 Subject: [PATCH 070/232] bpo-29704: Fix asyncio.SubprocessStreamProtocol closing (#405) --- Lib/asyncio/subprocess.py | 17 +- Lib/test/test_asyncio/test_subprocess.py | 24 + Misc/NEWS | 785 +++++++++++++++++++++++ 3 files changed, 824 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index b2f5304f772121..4c85466859f8f0 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -24,6 +24,8 @@ def __init__(self, limit, loop): self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None + self._process_exited = False + self._pipe_fds = [] def __repr__(self): info = [self.__class__.__name__] @@ -43,12 +45,14 @@ def connection_made(self, transport): self.stdout = streams.StreamReader(limit=self._limit, loop=self._loop) self.stdout.set_transport(stdout_transport) + self._pipe_fds.append(1) stderr_transport = transport.get_pipe_transport(2) if stderr_transport is not None: self.stderr = streams.StreamReader(limit=self._limit, loop=self._loop) self.stderr.set_transport(stderr_transport) + self._pipe_fds.append(2) stdin_transport = transport.get_pipe_transport(0) if stdin_transport is not None: @@ -86,9 +90,18 @@ def pipe_connection_lost(self, fd, exc): else: reader.set_exception(exc) + if fd in self._pipe_fds: + self._pipe_fds.remove(fd) + self._maybe_close_transport() + def process_exited(self): - self._transport.close() - self._transport = None + self._process_exited = True + self._maybe_close_transport() + + def _maybe_close_transport(self): + if len(self._pipe_fds) == 0 and self._process_exited: + self._transport.close() + self._transport = None class Process: diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bba688bb5a53c7..2e14a8a9735535 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -459,6 +459,30 @@ def test_popen_error(self): self.loop.run_until_complete(create) self.assertEqual(warns, []) + def test_read_stdout_after_process_exit(self): + @asyncio.coroutine + def execute(): + code = '\n'.join(['import sys', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + + fut = asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + loop=self.loop) + + process = yield from fut + while True: + data = yield from process.stdout.read(65536) + if data: + yield from asyncio.sleep(0.3, loop=self.loop) + else: + break + + self.loop.run_until_complete(execute()) + if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS b/Misc/NEWS index ab5c602c454e93..ef45957e47e76c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -4,6 +4,791 @@ Python News What's New in Python 3.6.1 release candidate 1? =============================================== +What's New in Python 3.7.0 alpha 1? +=================================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +- bpo-28893: Set correct __cause__ for errors about invalid awaitables + returned from __aiter__ and __anext__. + +- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by + Brian Coleman. + +- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. + It should raise TypeError when kwargs is not a dict. But it might + cause segv when args=NULL and kwargs is not a dict. + +- bpo-28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. + Patch by Matthieu Dartiailh. + +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + +- bpo-29347: Fixed possibly dereferencing undefined pointers + when creating weakref objects. + +- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef, + and AsyncFunctionDef ast nodes. docstring is not first stmt in their body + anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object + for module and class. + +- bpo-29438: Fixed use-after-free problem in key sharing dict. + +- bpo-29546: Set the 'path' and 'name' attribute on ImportError for ``from ... import ...``. + +- bpo-29546: Improve from-import error message with location + +- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. + +- Issue #29337: Fixed possible BytesWarning when compare the code objects. + Warnings could be emitted at compile time. + +- Issue #29327: Fixed a crash when pass the iterable keyword argument to + sorted(). + +- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). + +- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. + +- Issue #29049: Call _PyObject_GC_TRACK() lazily when calling Python function. + Calling function is up to 5% faster. + +- Issue #28927: bytes.fromhex() and bytearray.fromhex() now ignore all ASCII + whitespace, not only spaces. Patch by Robert Xiao. + +- Issue #28932: Do not include if it does not exist. + +- Issue #25677: Correct the positioning of the syntax error caret for + indented blocks. Based on patch by Michael Layzell. + +- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate + form. + +- Issue #18896: Python function can now have more than 255 parameters. + collections.namedtuple() now supports tuples with more than 255 elements. + +- Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by + Chi Hsuan Yen. + +- Issue #26919: On Android, operating system data is now always encoded/decoded + to/from UTF-8, instead of the locale encoding to avoid inconsistencies with + os.fsencode() and os.fsdecode() which are already using UTF-8. + +- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy + bug triggerable by a monkey-patched len() function. + +- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() + must not convert combined table into split table. Patch written by INADA + Naoki. + +- Issue #28739: f-string expressions are no longer accepted as docstrings and + by ast.literal_eval() even if they do not include expressions. + +- Issue #28512: Fixed setting the offset attribute of SyntaxError by + PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). + +- Issue #28918: Fix the cross compilation of xxlimited when Python has been + built with Py_DEBUG defined. + +- Issue #23722: Rather than silently producing a class that doesn't support + zero-argument ``super()`` in methods, failing to pass the new + ``__classcell__`` namespace entry up to ``type.__new__`` now results in a + ``DeprecationWarning`` and a class that supports zero-argument ``super()``. + +- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of + a descriptor that is used inside that class no longer prevents calling the + __set_name__ method of other descriptors. + +- Issue #28799: Remove the ``PyEval_GetCallStats()`` function and deprecate + the untested and undocumented ``sys.callstats()`` function. Remove the + ``CALL_PROFILE`` special build: use the :func:`sys.setprofile` function, + :mod:`cProfile` or :mod:`profile` to profile function calls. + +- Issue #12844: More than 255 arguments can now be passed to a function. + +- Issue #28782: Fix a bug in the implementation ``yield from`` when checking + if the next instruction is YIELD_FROM. Regression introduced by WORDCODE + (issue #26647). + +- Issue #28774: Fix error position of the unicode error in ASCII and Latin1 + encoders when a string returned by the error handler contains multiple + non-encodable characters (non-ASCII for the ASCII codec, characters out + of the U+0000-U+00FF range for Latin1). + +- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. + Improve speed of dict literal with constant keys up to 30%. + +- Issue #28532: Show sys.version when -V option is supplied twice. + +- Issue #27100: The with-statement now checks for __enter__ before it + checks for __exit__. This gives less confusing error messages when + both methods are missing. Patch by Jonathan Ellington. + +- Issue #28746: Fix the set_inheritable() file descriptor method on platforms + that do not have the ioctl FIOCLEX and FIONCLEX commands. + +- Issue #26920: Fix not getting the locale's charset upon initializing the + interpreter, on platforms that do not have langinfo. + +- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X + when decode astral characters. Patch by Xiang Zhang. + +- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. + +- Issue #19398: Extra slash no longer added to sys.path components in case of + empty compile-time PYTHONPATH components. + +- Issue #28621: Sped up converting int to float by reusing faster bits counting + implementation. Patch by Adrian Wielgosik. + +- Issue #28580: Optimize iterating split table values. + Patch by Xiang Zhang. + +- Issue #28583: PyDict_SetDefault didn't combine split table when needed. + Patch by Xiang Zhang. + +- Issue #28128: Deprecation warning for invalid str and byte escape + sequences now prints better information about where the error + occurs. Patch by Serhiy Storchaka and Eric Smith. + +- Issue #28509: dict.update() no longer allocate unnecessary large memory. + +- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug + build. + +- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused + keeping unreachable code. + +- Issue #28214: Improved exception reporting for problematic __set_name__ + attributes. + +- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception + loss in PyTraceBack_Here(). + +- Issue #28183: Optimize and cleanup dict iteration. + +- Issue #26081: Added C implementation of asyncio.Future. + Original patch by Yury Selivanov. + +- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). + Patch by Xiang Zhang. + +- Issue #28376: The type of long range iterator is now registered as Iterator. + Patch by Oren Milman. + +- Issue #28376: Creating instances of range_iterator by calling range_iterator + type now is disallowed. Calling iter() on range instance is the only way. + Patch by Oren Milman. + +- Issue #26906: Resolving special methods of uninitialized type now causes + implicit initialization of the type instead of a fail. + +- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. + Original patch by Niklas Koep. + +- Issue #24098: Fixed possible crash when AST is changed in process of + compiling it. + +- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when + hashes have same lower bits. + +- Issue #28350: String constants with null character no longer interned. + +- Issue #26617: Fix crash when GC runs during weakref callbacks. + +- Issue #27942: String constants now interned recursively in tuples and frozensets. + +- Issue #28289: ImportError.__init__ now resets not specified attributes. + +- Issue #21578: Fixed misleading error message when ImportError called with + invalid keyword args. + +- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. + Patch by Soumya Sharma. + +- Issue #28086: Single var-positional argument of tuple subtype was passed + unscathed to the C-defined function. Now it is converted to exact tuple. + +- Issue #28214: Now __set_name__ is looked up on the class instead of the + instance. + +- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() + syscall fails with EPERM, for example when blocked by SECCOMP. + +- Issue #28192: Don't import readline in isolated mode. + +- Issue #27441: Remove some redundant assignments to ob_size in longobject.c. + Thanks Oren Milman. + +- Issue #27222: Clean up redundant code in long_rshift function. Thanks + Oren Milman. + +- Upgrade internal unicode databases to Unicode version 9.0.0. + +- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport + should use the same optimization level as the interpreter. + +- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly + optimize memcpy(). + +- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a + "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. + +- Issue #26182: Raise DeprecationWarning when async and await keywords are + used as variable/attribute/class/function name. + +- Issue #26182: Fix a refleak in code that raises DeprecationWarning. + +- Issue #28721: Fix asynchronous generators aclose() and athrow() to + handle StopAsyncIteration propagation properly. + +- Issue #26110: Speed-up method calls: add LOAD_METHOD and CALL_METHOD + opcodes. + +Extension Modules +----------------- + +- Issue #29169: Update zlib to 1.2.11. + +Library +------- + +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before + all pipes are closed. + +- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C + to accept None argument as their pure Python implementation. + +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + +- bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher + and its subclasses can now be used as a decorator. + +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + +- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts + PathLike objects. Patch by Sayan Chowdhury. + +- bpo-28518: Start a transaction implicitly before a DML statement. + Patch by Aviv Palivoda. + +- Issue #16285: urrlib.parse.quote is now based on RFC 3986 and hence includes + '~' in the set of characters that is not quoted by default. Patch by + Christian Theune and Ratnadeep Debnath. + +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + +- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from + the platform about whether generated UUIDs are generated with a + multiprocessing safe method. + +- bpo-29576: Improve some deprecations in importlib. Some deprecated methods + now emit DeprecationWarnings and have better descriptive messages. + +- bpo-29534: Fixed different behaviour of Decimal.from_float() + for _decimal and _pydecimal. Thanks Andrew Nester. + +- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, + improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, + Manuel Krebber, and Ɓukasz Langa. + +- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python + 3.6.0: check minimum and maximum years. + +- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir + +- Issue #29444: Fixed out-of-bounds buffer access in the group() method of + the match object. Based on patch by WGH. + +- Issue #29377: Add SlotWrapperType, MethodWrapperType, and + MethodDescriptorType built-in types to types module. + Original patch by Manuel Krebber. + +- Issue #29218: Unused install_misc command is now removed. It has been + documented as unused since 2000. Patch by Eric N. Vander Weele. + +- Issue #29368: The extend() method is now called instead of the append() + method when unpickle collections.deque and other list-like objects. + This can speed up unpickling to 2 times. + +- Issue #29338: The help of a builtin or extension class now includes the + constructor signature if __text_signature__ is provided for the class. + +- Issue #29335: Fix subprocess.Popen.wait() when the child process has + exited to a stopped instead of terminated state (ex: when under ptrace). + +- Issue #29290: Fix a regression in argparse that help messages would wrap at + non-breaking spaces. + +- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. + +- Issue #29197: Removed deprecated function ntpath.splitunc(). + +- Issue #29210: Removed support of deprecated argument "exclude" in + tarfile.TarFile.add(). + +- Issue #29219: Fixed infinite recursion in the repr of uninitialized + ctypes.CDLL instances. + +- Issue #29192: Removed deprecated features in the http.cookies module. + +- Issue #29193: A format string argument for string.Formatter.format() + is now positional-only. + +- Issue #29195: Removed support of deprecated undocumented keyword arguments + in methods of regular expression objects. + +- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. + KeyError could be raised when cached function with full cache was + simultaneously called from differen threads with the same uncached arguments. + +- Issue #20804: The unittest.mock.sentinel attributes now preserve their + identity when they are copied or pickled. + +- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with + leading dots could match related hostnames again (e.g. .b.c matches a.b.c). + Patch by Milan Oberkirch. + +- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter + anymore. Patch written by Jiajun Huang. + +- Issue #15812: inspect.getframeinfo() now correctly shows the first line of + a context. Patch by Sam Breese. + +- Issue #28985: Update authorizer constants in sqlite3 module. + Patch by Dingyuan Wang. + +- Issue #29094: Offsets in a ZIP file created with extern file object and modes + "w" and "x" now are relative to the start of the file. + +- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows + +- Issue #13051: Fixed recursion errors in large or resized + curses.textpad.Textbox. Based on patch by Tycho Andersen. + +- Issue #9770: curses.ascii predicates now work correctly with negative + integers. + +- Issue #28427: old keys should not remove new values from + WeakValueDictionary when collecting from another thread. + +- Issue 28923: Remove editor artifacts from Tix.py. + +- Issue #28871: Fixed a crash when deallocate deep ElementTree. + +- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and + WeakValueDictionary.pop() when a GC collection happens in another + thread. + +- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that + doesn't own its elements as limits. + +- Issue #16255: subprocess.Popen uses /system/bin/sh on Android as the shell, + instead of /bin/sh. + +- Issue #28779: multiprocessing.set_forkserver_preload() would crash the + forkserver process if a preloaded module instantiated some + multiprocessing objects such as locks. + +- Issue #26937: The chown() method of the tarfile.TarFile class does not fail + now when the grp module cannot be imported, as for example on Android + platforms. + +- Issue #28847: dbm.dumb now supports reading read-only files and no longer + writes the index file when it is not changed. A deprecation warning is now + emitted if the index file is missed and recreated in the 'r' and 'w' modes + (will be an error in future Python releases). + +- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in + re.sub() replacement templates regular expressions now are errors. + +- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): + call warnings.showwarning() if it was overriden inside the context manager. + +- Issue #27172: To assist with upgrades from 2.7, the previously documented + deprecation of ``inspect.getfullargspec()`` has been reversed. This decision + may be revisited again after the Python 2.7 branch is no longer officially + supported. + +- Issue #28740: Add sys.getandroidapilevel(): return the build time API version + of Android as an integer. Function only available on Android. + +- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and + :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by + Omar Sandoval. + +- Issue #28752: Restored the __reduce__() methods of datetime objects. + +- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created + by re.compile(), become comparable (only x==y and x!=y operators). This + change should fix the issue #18383: don't duplicate warning filters when the + warnings module is reloaded (thing usually only done in unit tests). + +- Issue #20572: Remove the subprocess.Popen.wait endtime parameter. It was + deprecated in 3.4 and undocumented prior to that. + +- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and + from_buffer_copy() methods on abstract classes like Array. + +- Issue #28548: In the "http.server" module, parse the protocol version if + possible, to avoid using HTTP 0.9 in some error responses. + +- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. + Patch by Vajrasky Kok + +- Issue #28563: Fixed possible DoS and arbitrary code execution when handle + plural form selections in the gettext module. The expression parser now + supports exact syntax supported by GNU gettext. + +- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when + the garbage collector is invoked in other thread. Based on patch by + Sebastian Cufre. + +- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if + given empty data twice. Patch by Benjamin Fogle. + +- Issue #28549: Fixed segfault in curses's addch() with ncurses6. + +- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar + file with compression before trying to open it without compression. Otherwise + it had 50% chance failed with ignore_zeros=True. + +- Issue #23262: The webbrowser module now supports Firefox 36+ and derived + browsers. Based on patch by Oleg Broytman. + +- Issue #24241: The webbrowser in an X environment now prefers using the + default browser directly. Also, the webbrowser register() function now has + a documented 'preferred' argument, to specify browsers to be returned by + get() with no arguments. Patch by David Steele + +- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused + by representing the scale as float value internally in Tk. tkinter.IntVar + now works if float value is set to underlying Tk variable. + +- Issue #28255: calendar.TextCalendar.prweek() no longer prints a space after + a weeks's calendar. calendar.TextCalendar.pryear() no longer prints redundant + newline after a year's calendar. Based on patch by Xiang Zhang. + +- Issue #28255: calendar.TextCalendar.prmonth() no longer prints a space + at the start of new line after printing a month's calendar. Patch by + Xiang Zhang. + +- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. + Based on patch by Kaarle Ritvanen. + +- Issue #28353: os.fwalk() no longer fails on broken links. + +- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept + non-None value is passed to it.send(val). + +- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix + for readability (was "`"). + +- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin + a workaround to Tix library bug. + +- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. + +- Issue #25953: re.sub() now raises an error for invalid numerical group + reference in replacement template even if the pattern is not found in + the string. Error message for invalid group reference now includes the + group index and the position of the reference. + Based on patch by SilentGhost. + +- Issue #28469: timeit now uses the sequence 1, 2, 5, 10, 20, 50,... instead + of 1, 10, 100,... for autoranging. + +- Issue #28115: Command-line interface of the zipfile module now uses argparse. + Added support of long options. + +- Issue #18219: Optimize csv.DictWriter for large number of columns. + Patch by Mariatta Wijaya. + +- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. + +- Issue #23214: In the "io" module, the argument to BufferedReader and + BytesIO's read1() methods is now optional and can be -1, matching the + BufferedIOBase specification. + +- Issue #28480: Fix error building socket module when multithreading is + disabled. + +- Issue #28240: timeit: remove ``-c/--clock`` and ``-t/--time`` command line + options which were deprecated since Python 3.3. + +- Issue #28240: timeit now repeats the benchmarks 5 times instead of only 3 + to make benchmarks more reliable. + +- Issue #28240: timeit autorange now uses a single loop iteration if the + benchmark takes less than 10 seconds, instead of 10 iterations. + "python3 -m timeit -s 'import time' 'time.sleep(1)'" now takes 4 seconds + instead of 40 seconds. + +- Distutils.sdist now looks for README and setup.py files with case + sensitivity. This behavior matches that found in Setuptools 6.0 and + later. See `setuptools 100 + `_ for rationale. + +- Issue #24452: Make webbrowser support Chrome on Mac OS X. Patch by + Ned Batchelder. + +- Issue #20766: Fix references leaked by pdb in the handling of SIGINT + handlers. + +- Issue #27998: Fixed bytes path support in os.scandir() on Windows. + Patch by Eryk Sun. + +- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. + +- Issue #26293: Fixed writing ZIP files that starts not from the start of the + file. Offsets in ZIP file now are relative to the start of the archive in + conforming to the specification. + +- Issue #28380: unittest.mock Mock autospec functions now properly support + assert_called, assert_not_called, and assert_called_once. + +- Issue #28229: lzma module now supports pathlib. + +- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. + +- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. + +- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. + +- Issue #28332: Deprecated silent truncations in socket.htons and socket.ntohs. + Original patch by Oren Milman. + +- Issue #27358: Optimized merging var-keyword arguments and improved error + message when passing a non-mapping as a var-keyword argument. + +- Issue #28257: Improved error message when passing a non-iterable as + a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. + +- Issue #28322: Fixed possible crashes when unpickle itertools objects from + incorrect pickle data. Based on patch by John Leitch. + +- Issue #28228: imghdr now supports pathlib. + +- Issue #28226: compileall now supports pathlib. + +- Issue #28314: Fix function declaration (C flags) for the getiterator() method + of xml.etree.ElementTree.Element. + +- Issue #28148: Stop using localtime() and gmtime() in the time + module. + + Introduced platform independent _PyTime_localtime API that is + similar to POSIX localtime_r, but available on all platforms. Patch + by Ed Schouten. + +- Issue #28253: Fixed calendar functions for extreme months: 0001-01 + and 9999-12. + + Methods itermonthdays() and itermonthdays2() are reimplemented so + that they don't call itermonthdates() which can cause datetime.date + under/overflow. + +- Issue #28275: Fixed possible use after free in the decompress() + methods of the LZMADecompressor and BZ2Decompressor classes. + Original patch by John Leitch. + +- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() + if pass invalid string-like object as a name. Patch by Xiang Zhang. + +- Issue #18844: random.choices() now has k as a keyword-only argument + to improve the readability of common cases and come into line + with the signature used in other languages. + +- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. + Patch by Madison May. + +- Issue #27611: Fixed support of default root window in the tkinter.tix module. + Added the master parameter in the DisplayStyle constructor. + +- Issue #27348: In the traceback module, restore the formatting of exception + messages like "Exception: None". This fixes a regression introduced in + 3.5a2. + +- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). + +- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is + interrupted by a signal and a signal handler raises a Python exception. + +- Issue #28200: Fix memory leak on Windows in the os module (fix + path_converter() function). + +- Issue #25400: RobotFileParser now correctly returns default values for + crawl_delay and request_rate. Initial patch by Peter Wirtz. + +- Issue #27932: Prevent memory leak in win32_ver(). + +- Fix UnboundLocalError in socket._sendfile_use_sendfile. + +- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of + os.stat(). Patch by Eryk Sun. + +- Issue #22493: Warning message emitted by using inline flags in the middle of + regular expression now contains a (truncated) regex pattern. + Patch by Tim Graham. + +- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when + an empty bytestring is passed. + +- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. + +- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. + Patch by Gergely Imreh and Markus Holtermann. + +- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. + Patch by Eryk Sun. + +- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). + +- Issue #27906: Fix socket accept exhaustion during high TCP traffic. + Patch by Kevin Conway. + +- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. + Patch by Seth Michael Larson. + +- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. + Patch by iceboy. + +- Issue #26909: Fix slow pipes IO in asyncio. + Patch by INADA Naoki. + +- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. + +- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. + Patch by Mark Williams. + +- Issue #28325: Remove vestigial MacOS 9 macurl2path module and its tests. + +- Issue #28368: Refuse monitoring processes if the child watcher has + no loop attached. + Patch by Vincent Michel. + +- Issue #28369: Raise RuntimeError when transport's FD is used with + add_reader, add_writer, etc. + +- Issue #28370: Speedup asyncio.StreamReader.readexactly. + Patch by ĐšĐŸŃ€Đ”ĐœĐ±Đ”Ń€Đł МарĐș. + +- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. + +- Issue #28372: Fix asyncio to support formatting of non-python coroutines. + +- Issue #28399: Remove UNIX socket from FS before binding. + Patch by ĐšĐŸŃ€Đ”ĐœĐ±Đ”Ń€Đł МарĐș. + +- Issue #27972: Prohibit Tasks to await on themselves. + +- Issue #24142: Reading a corrupt config file left configparser in an + invalid state. Original patch by Florian Höch. + +Windows +------- + +- bpo-29579: Removes readme.txt from the installer. + +- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) + +- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default + +- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c + +- Issue #28402: Adds signed catalog files for stdlib on Windows. + +- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by + Eryk Sun) + +- Issue #28251: Improvements to help manuals on Windows. + +- Issue #28110: launcher.msi has different product codes between 32-bit and + 64-bit + +- Issue #28161: Opening CON for write access fails + +- Issue #28162: WindowsConsoleIO readall() fails if first line starts with + Ctrl+Z + +- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to + _open_osfhandle + +- Issue #28164: _PyIO_get_console_type fails for various paths + +- Issue #28137: Renames Windows path file to ._pth + +- Issue #28138: Windows ._pth file should allow import site + +C API +----- + +- Issue #27867: Function PySlice_GetIndicesEx() is deprecated and replaced with + a macro if Py_LIMITED_API is not set or set to the value between 0x03050400 + and 0x03060000 (not including) or 0x03060100 or higher. Added functions + PySlice_Unpack() and PySlice_AdjustIndices(). + +- Issue #29083: Fixed the declaration of some public API functions. + PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in + limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and + Py_BuildValue() were not available in limited API of version < 3.3 when + PY_SSIZE_T_CLEAN is defined. + +- Issue #28769: The result of PyUnicode_AsUTF8AndSize() and PyUnicode_AsUTF8() + is now of type ``const char *`` rather of ``char *``. + +- Issue #29058: All stable API extensions added after Python 3.2 are now + available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of + the minimum Python version supporting this API. + +- Issue #28822: The index parameters *start* and *end* of PyUnicode_FindChar() + are now adjusted to behave like ``str[start:end]``. + +- Issue #28808: PyUnicode_CompareWithASCIIString() now never raises exceptions. + +- Issue #28761: The fields name and doc of structures PyMemberDef, PyGetSetDef, + PyStructSequence_Field, PyStructSequence_Desc, and wrapperbase are now of + type ``const char *`` rather of ``char *``. + +- Issue #28748: Private variable _Py_PackageContext is now of type ``const char *`` + rather of ``char *``. + +- Issue #19569: Compiler warnings are now emitted if use most of deprecated + functions. + +- Issue #28426: Deprecated undocumented functions PyUnicode_AsEncodedObject(), + PyUnicode_AsDecodedObject(), PyUnicode_AsDecodedUnicode() and + PyUnicode_AsEncodedUnicode(). + +Documentation +------------- + +- bpo-28929: Link the documentation to its source file on GitHub. + +- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to + aiosmtpd, a third-party asyncio-based replacement. + +- Issue #26355: Add canonical header link on each page to corresponding major + version of the documentation. Patch by Matthias Bussonnier. + +- Issue #29349: Fix Python 2 syntax in code for building the documentation. *Release date: XXXX-XX-XX* From 2ef08d3be780457c444741b67e6181675b044be9 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 23:57:33 -0500 Subject: [PATCH 071/232] asyncio: Optimize _get_running_loop() to call getpid() only when there's a loop --- Lib/asyncio/events.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 7b30b4c84ad9a7..e85634e588f5a5 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -624,8 +624,9 @@ def _get_running_loop(): This is a low-level function intended to be used by event loops. This function is thread-specific. """ - if _running_loop._pid == os.getpid(): - return _running_loop._loop + running_loop = _running_loop._loop + if running_loop is not None and _running_loop._pid == os.getpid(): + return running_loop def _set_running_loop(loop): From d8b72e4a0673c414120b029065dbe77055f12e82 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 2 Mar 2017 23:46:56 -0500 Subject: [PATCH 072/232] bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback/C (#408) --- Lib/test/test_asyncio/test_futures.py | 29 +++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_asynciomodule.c | 2 +- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 89afdcaff28665..99336f86ab824e 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -569,6 +569,35 @@ def test_remove_done_callback(self): self.assertEqual(bag, [2]) self.assertEqual(f.result(), 'foo') + def test_remove_done_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_schedule_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + def mut(f): + f.remove_done_callback(str) + + fut = self._new_future() + fut.add_done_callback(mut) + fut.add_done_callback(str) + fut.add_done_callback(str) + fut.set_result(1) + test_utils.run_briefly(self.loop) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Misc/NEWS b/Misc/NEWS index ef45957e47e76c..a13d8c74f23228 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -261,6 +261,9 @@ Extension Modules Library ------- +- bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback + implemented in C. + - bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before all pipes are closed. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index d0e43ae47d9ec8..a77ff9689e4705 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -521,7 +521,7 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) return NULL; } - for (i = 0; i < len; i++) { + for (i = 0; i < PyList_GET_SIZE(self->fut_callbacks); i++) { int ret; PyObject *item = PyList_GET_ITEM(self->fut_callbacks, i); From b8f5d07158f171a18be4d42f9769d3c9c073e748 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 2 Mar 2017 21:50:10 -0800 Subject: [PATCH 073/232] bpo-29026: Clarify documentation of time.time (GH-34) (GH-417) (cherry picked from commit 23557d59b819f57800ddef0b1373acef8e024670) --- Doc/library/time.rst | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index ae17f6f4f0d236..d2e2ac4410aba3 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -17,11 +17,23 @@ semantics of these functions varies among platforms. An explanation of some terminology and conventions is in order. +.. _epoch: + .. index:: single: epoch -* The :dfn:`epoch` is the point where the time starts. On January 1st of that - year, at 0 hours, the "time since the epoch" is zero. For Unix, the epoch is - 1970. To find out what the epoch is, look at ``gmtime(0)``. +* The :dfn:`epoch` is the point where the time starts, and is platform + dependent. For Unix, the epoch is January 1, 1970, 00:00:00 (UTC). + To find out what the epoch is on a given platform, look at + ``time.gmtime(0)``. + +.. _leap seconds: https://en.wikipedia.org/wiki/Leap_second + +.. index:: seconds since the epoch + +* The term :dfn:`seconds since the epoch` refers to the total number + of elapsed seconds since the epoch, typically excluding + `leap seconds`_. Leap seconds are excluded from this total on all + POSIX-compliant platforms. .. index:: single: Year 2038 @@ -467,7 +479,7 @@ The module defines the following functions and data items: (2) The range really is ``0`` to ``61``; value ``60`` is valid in - timestamps representing leap seconds and value ``61`` is supported + timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. (3) @@ -572,12 +584,28 @@ The module defines the following functions and data items: .. function:: time() - Return the time in seconds since the epoch as a floating point number. + Return the time in seconds since the epoch_ as a floating point + number. The specific date of the epoch and the handling of + `leap seconds`_ is platform dependent. + On Windows and most Unix systems, the epoch is January 1, 1970, + 00:00:00 (UTC) and leap seconds are not counted towards the time + in seconds since the epoch. This is commonly referred to as + `Unix time `_. + To find out what the epoch is on a given platform, look at + ``gmtime(0)``. + Note that even though the time is always returned as a floating point number, not all systems provide time with a better precision than 1 second. While this function normally returns non-decreasing values, it can return a - lower value than a previous call if the system clock has been set back between - the two calls. + lower value than a previous call if the system clock has been set back + between the two calls. + + The number returned by :func:`.time` may be converted into a more common + time format (i.e. year, month, day, hour, etc...) in UTC by passing it to + :func:`gmtime` function or in local time by passing it to the + :func:`localtime` function. In both cases a + :class:`struct_time` object is returned, from which the components + of the calendar date may be accessed as attributes. .. data:: timezone From 93692b1a7cfd70be2aed33d77da05518d3a41f51 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 2 Mar 2017 21:56:06 -0800 Subject: [PATCH 074/232] getpass: update docstrings (GH-49) (GH-419) (cherry picked from commit baf7bb30a02aabde260143136bdf5b3738a1d409) --- Lib/getpass.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/getpass.py b/Lib/getpass.py index be511211585a48..36e17e4cb6965d 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -7,7 +7,6 @@ echoing of the password contents while reading. On Windows, the msvcrt module will be used. -On the Mac EasyDialogs.AskPassword is used, if available. """ From 2adc66848135be8e86ba08690a858737a65ae40d Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 3 Mar 2017 01:05:44 -0500 Subject: [PATCH 075/232] Restore NEWS file after incorrect git automerge --- Misc/NEWS | 788 +----------------------------------------------------- 1 file changed, 3 insertions(+), 785 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index a13d8c74f23228..2692043b6cbab6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -4,8 +4,6 @@ Python News What's New in Python 3.6.1 release candidate 1? =============================================== -What's New in Python 3.7.0 alpha 1? -=================================== *Release date: XXXX-XX-XX* @@ -22,7 +20,7 @@ Core and Builtins It should raise TypeError when kwargs is not a dict. But it might cause segv when args=NULL and kwargs is not a dict. -- bpo-28598: Support __rmod__ for subclasses of str being called before +- Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. @@ -35,17 +33,8 @@ Core and Builtins - bpo-29347: Fixed possibly dereferencing undefined pointers when creating weakref objects. -- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef, - and AsyncFunctionDef ast nodes. docstring is not first stmt in their body - anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object - for module and class. - - bpo-29438: Fixed use-after-free problem in key sharing dict. -- bpo-29546: Set the 'path' and 'name' attribute on ImportError for ``from ... import ...``. - -- bpo-29546: Improve from-import error message with location - - Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - Issue #29337: Fixed possible BytesWarning when compare the code objects. @@ -58,12 +47,6 @@ Core and Builtins - Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. -- Issue #29049: Call _PyObject_GC_TRACK() lazily when calling Python function. - Calling function is up to 5% faster. - -- Issue #28927: bytes.fromhex() and bytearray.fromhex() now ignore all ASCII - whitespace, not only spaces. Patch by Robert Xiao. - - Issue #28932: Do not include if it does not exist. - Issue #25677: Correct the positioning of the syntax error caret for @@ -72,12 +55,6 @@ Core and Builtins - Issue #29000: Fixed bytes formatting of octals with zero padding in alternate form. -- Issue #18896: Python function can now have more than 255 parameters. - collections.namedtuple() now supports tuples with more than 255 elements. - -- Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by - Chi Hsuan Yen. - - Issue #26919: On Android, operating system data is now always encoded/decoded to/from UTF-8, instead of the locale encoding to avoid inconsistencies with os.fsencode() and os.fsdecode() which are already using UTF-8. @@ -85,10 +62,6 @@ Core and Builtins - Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy bug triggerable by a monkey-patched len() function. -- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() - must not convert combined table into split table. Patch written by INADA - Naoki. - - Issue #28739: f-string expressions are no longer accepted as docstrings and by ast.literal_eval() even if they do not include expressions. @@ -98,161 +71,9 @@ Core and Builtins - Issue #28918: Fix the cross compilation of xxlimited when Python has been built with Py_DEBUG defined. -- Issue #23722: Rather than silently producing a class that doesn't support - zero-argument ``super()`` in methods, failing to pass the new - ``__classcell__`` namespace entry up to ``type.__new__`` now results in a - ``DeprecationWarning`` and a class that supports zero-argument ``super()``. - -- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of - a descriptor that is used inside that class no longer prevents calling the - __set_name__ method of other descriptors. - -- Issue #28799: Remove the ``PyEval_GetCallStats()`` function and deprecate - the untested and undocumented ``sys.callstats()`` function. Remove the - ``CALL_PROFILE`` special build: use the :func:`sys.setprofile` function, - :mod:`cProfile` or :mod:`profile` to profile function calls. - -- Issue #12844: More than 255 arguments can now be passed to a function. - -- Issue #28782: Fix a bug in the implementation ``yield from`` when checking - if the next instruction is YIELD_FROM. Regression introduced by WORDCODE - (issue #26647). - -- Issue #28774: Fix error position of the unicode error in ASCII and Latin1 - encoders when a string returned by the error handler contains multiple - non-encodable characters (non-ASCII for the ASCII codec, characters out - of the U+0000-U+00FF range for Latin1). - - Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. Improve speed of dict literal with constant keys up to 30%. -- Issue #28532: Show sys.version when -V option is supplied twice. - -- Issue #27100: The with-statement now checks for __enter__ before it - checks for __exit__. This gives less confusing error messages when - both methods are missing. Patch by Jonathan Ellington. - -- Issue #28746: Fix the set_inheritable() file descriptor method on platforms - that do not have the ioctl FIOCLEX and FIONCLEX commands. - -- Issue #26920: Fix not getting the locale's charset upon initializing the - interpreter, on platforms that do not have langinfo. - -- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X - when decode astral characters. Patch by Xiang Zhang. - -- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. - -- Issue #19398: Extra slash no longer added to sys.path components in case of - empty compile-time PYTHONPATH components. - -- Issue #28621: Sped up converting int to float by reusing faster bits counting - implementation. Patch by Adrian Wielgosik. - -- Issue #28580: Optimize iterating split table values. - Patch by Xiang Zhang. - -- Issue #28583: PyDict_SetDefault didn't combine split table when needed. - Patch by Xiang Zhang. - -- Issue #28128: Deprecation warning for invalid str and byte escape - sequences now prints better information about where the error - occurs. Patch by Serhiy Storchaka and Eric Smith. - -- Issue #28509: dict.update() no longer allocate unnecessary large memory. - -- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug - build. - -- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused - keeping unreachable code. - -- Issue #28214: Improved exception reporting for problematic __set_name__ - attributes. - -- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception - loss in PyTraceBack_Here(). - -- Issue #28183: Optimize and cleanup dict iteration. - -- Issue #26081: Added C implementation of asyncio.Future. - Original patch by Yury Selivanov. - -- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). - Patch by Xiang Zhang. - -- Issue #28376: The type of long range iterator is now registered as Iterator. - Patch by Oren Milman. - -- Issue #28376: Creating instances of range_iterator by calling range_iterator - type now is disallowed. Calling iter() on range instance is the only way. - Patch by Oren Milman. - -- Issue #26906: Resolving special methods of uninitialized type now causes - implicit initialization of the type instead of a fail. - -- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. - Original patch by Niklas Koep. - -- Issue #24098: Fixed possible crash when AST is changed in process of - compiling it. - -- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when - hashes have same lower bits. - -- Issue #28350: String constants with null character no longer interned. - -- Issue #26617: Fix crash when GC runs during weakref callbacks. - -- Issue #27942: String constants now interned recursively in tuples and frozensets. - -- Issue #28289: ImportError.__init__ now resets not specified attributes. - -- Issue #21578: Fixed misleading error message when ImportError called with - invalid keyword args. - -- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. - Patch by Soumya Sharma. - -- Issue #28086: Single var-positional argument of tuple subtype was passed - unscathed to the C-defined function. Now it is converted to exact tuple. - -- Issue #28214: Now __set_name__ is looked up on the class instead of the - instance. - -- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() - syscall fails with EPERM, for example when blocked by SECCOMP. - -- Issue #28192: Don't import readline in isolated mode. - -- Issue #27441: Remove some redundant assignments to ob_size in longobject.c. - Thanks Oren Milman. - -- Issue #27222: Clean up redundant code in long_rshift function. Thanks - Oren Milman. - -- Upgrade internal unicode databases to Unicode version 9.0.0. - -- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport - should use the same optimization level as the interpreter. - -- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly - optimize memcpy(). - -- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a - "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. - -- Issue #26182: Raise DeprecationWarning when async and await keywords are - used as variable/attribute/class/function name. - -- Issue #26182: Fix a refleak in code that raises DeprecationWarning. - -- Issue #28721: Fix asynchronous generators aclose() and athrow() to - handle StopAsyncIteration propagation properly. - -- Issue #26110: Speed-up method calls: add LOAD_METHOD and CALL_METHOD - opcodes. - Extension Modules ----------------- @@ -264,611 +85,8 @@ Library - bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback implemented in C. -- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before - all pipes are closed. - -- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C - to accept None argument as their pure Python implementation. - -- bpo-29703: Fix asyncio to support instantiation of new event loops - in child processes. - -- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other - exception) to exception(s) raised in the dispatched methods. - Patch by Petr Motejlek. - -- bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher - and its subclasses can now be used as a decorator. - -- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). - -- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts - PathLike objects. Patch by Sayan Chowdhury. - -- bpo-28518: Start a transaction implicitly before a DML statement. - Patch by Aviv Palivoda. - -- Issue #16285: urrlib.parse.quote is now based on RFC 3986 and hence includes - '~' in the set of characters that is not quoted by default. Patch by - Christian Theune and Ratnadeep Debnath. - -- bpo-29532: Altering a kwarg dictionary passed to functools.partial() - no longer affects a partial object after creation. - -- bpo-29110: Fix file object leak in aifc.open() when file is given as a - filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. - -- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from - the platform about whether generated UUIDs are generated with a - multiprocessing safe method. - -- bpo-29576: Improve some deprecations in importlib. Some deprecated methods - now emit DeprecationWarnings and have better descriptive messages. - -- bpo-29534: Fixed different behaviour of Decimal.from_float() - for _decimal and _pydecimal. Thanks Andrew Nester. - -- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, - improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, - Manuel Krebber, and Ɓukasz Langa. - -- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python - 3.6.0: check minimum and maximum years. - -- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir - -- Issue #29444: Fixed out-of-bounds buffer access in the group() method of - the match object. Based on patch by WGH. - -- Issue #29377: Add SlotWrapperType, MethodWrapperType, and - MethodDescriptorType built-in types to types module. - Original patch by Manuel Krebber. - -- Issue #29218: Unused install_misc command is now removed. It has been - documented as unused since 2000. Patch by Eric N. Vander Weele. - -- Issue #29368: The extend() method is now called instead of the append() - method when unpickle collections.deque and other list-like objects. - This can speed up unpickling to 2 times. - -- Issue #29338: The help of a builtin or extension class now includes the - constructor signature if __text_signature__ is provided for the class. - -- Issue #29335: Fix subprocess.Popen.wait() when the child process has - exited to a stopped instead of terminated state (ex: when under ptrace). - -- Issue #29290: Fix a regression in argparse that help messages would wrap at - non-breaking spaces. - -- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. - -- Issue #29197: Removed deprecated function ntpath.splitunc(). - -- Issue #29210: Removed support of deprecated argument "exclude" in - tarfile.TarFile.add(). - -- Issue #29219: Fixed infinite recursion in the repr of uninitialized - ctypes.CDLL instances. - -- Issue #29192: Removed deprecated features in the http.cookies module. - -- Issue #29193: A format string argument for string.Formatter.format() - is now positional-only. - -- Issue #29195: Removed support of deprecated undocumented keyword arguments - in methods of regular expression objects. - -- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. - KeyError could be raised when cached function with full cache was - simultaneously called from differen threads with the same uncached arguments. - -- Issue #20804: The unittest.mock.sentinel attributes now preserve their - identity when they are copied or pickled. - -- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with - leading dots could match related hostnames again (e.g. .b.c matches a.b.c). - Patch by Milan Oberkirch. - -- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter - anymore. Patch written by Jiajun Huang. - -- Issue #15812: inspect.getframeinfo() now correctly shows the first line of - a context. Patch by Sam Breese. - -- Issue #28985: Update authorizer constants in sqlite3 module. - Patch by Dingyuan Wang. - -- Issue #29094: Offsets in a ZIP file created with extern file object and modes - "w" and "x" now are relative to the start of the file. - -- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows - -- Issue #13051: Fixed recursion errors in large or resized - curses.textpad.Textbox. Based on patch by Tycho Andersen. - -- Issue #9770: curses.ascii predicates now work correctly with negative - integers. - -- Issue #28427: old keys should not remove new values from - WeakValueDictionary when collecting from another thread. - -- Issue 28923: Remove editor artifacts from Tix.py. - -- Issue #28871: Fixed a crash when deallocate deep ElementTree. - -- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and - WeakValueDictionary.pop() when a GC collection happens in another - thread. - -- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that - doesn't own its elements as limits. - -- Issue #16255: subprocess.Popen uses /system/bin/sh on Android as the shell, - instead of /bin/sh. - -- Issue #28779: multiprocessing.set_forkserver_preload() would crash the - forkserver process if a preloaded module instantiated some - multiprocessing objects such as locks. - -- Issue #26937: The chown() method of the tarfile.TarFile class does not fail - now when the grp module cannot be imported, as for example on Android - platforms. - -- Issue #28847: dbm.dumb now supports reading read-only files and no longer - writes the index file when it is not changed. A deprecation warning is now - emitted if the index file is missed and recreated in the 'r' and 'w' modes - (will be an error in future Python releases). - -- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in - re.sub() replacement templates regular expressions now are errors. - -- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): - call warnings.showwarning() if it was overriden inside the context manager. - -- Issue #27172: To assist with upgrades from 2.7, the previously documented - deprecation of ``inspect.getfullargspec()`` has been reversed. This decision - may be revisited again after the Python 2.7 branch is no longer officially - supported. - -- Issue #28740: Add sys.getandroidapilevel(): return the build time API version - of Android as an integer. Function only available on Android. - -- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and - :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by - Omar Sandoval. - -- Issue #28752: Restored the __reduce__() methods of datetime objects. - -- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created - by re.compile(), become comparable (only x==y and x!=y operators). This - change should fix the issue #18383: don't duplicate warning filters when the - warnings module is reloaded (thing usually only done in unit tests). - -- Issue #20572: Remove the subprocess.Popen.wait endtime parameter. It was - deprecated in 3.4 and undocumented prior to that. - -- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and - from_buffer_copy() methods on abstract classes like Array. - -- Issue #28548: In the "http.server" module, parse the protocol version if - possible, to avoid using HTTP 0.9 in some error responses. - -- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. - Patch by Vajrasky Kok - -- Issue #28563: Fixed possible DoS and arbitrary code execution when handle - plural form selections in the gettext module. The expression parser now - supports exact syntax supported by GNU gettext. - -- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when - the garbage collector is invoked in other thread. Based on patch by - Sebastian Cufre. - -- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if - given empty data twice. Patch by Benjamin Fogle. - -- Issue #28549: Fixed segfault in curses's addch() with ncurses6. - -- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar - file with compression before trying to open it without compression. Otherwise - it had 50% chance failed with ignore_zeros=True. - -- Issue #23262: The webbrowser module now supports Firefox 36+ and derived - browsers. Based on patch by Oleg Broytman. - -- Issue #24241: The webbrowser in an X environment now prefers using the - default browser directly. Also, the webbrowser register() function now has - a documented 'preferred' argument, to specify browsers to be returned by - get() with no arguments. Patch by David Steele - -- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused - by representing the scale as float value internally in Tk. tkinter.IntVar - now works if float value is set to underlying Tk variable. - -- Issue #28255: calendar.TextCalendar.prweek() no longer prints a space after - a weeks's calendar. calendar.TextCalendar.pryear() no longer prints redundant - newline after a year's calendar. Based on patch by Xiang Zhang. - -- Issue #28255: calendar.TextCalendar.prmonth() no longer prints a space - at the start of new line after printing a month's calendar. Patch by - Xiang Zhang. - -- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. - Based on patch by Kaarle Ritvanen. - -- Issue #28353: os.fwalk() no longer fails on broken links. - -- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept - non-None value is passed to it.send(val). - -- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix - for readability (was "`"). - -- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin - a workaround to Tix library bug. - -- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. - -- Issue #25953: re.sub() now raises an error for invalid numerical group - reference in replacement template even if the pattern is not found in - the string. Error message for invalid group reference now includes the - group index and the position of the reference. - Based on patch by SilentGhost. - -- Issue #28469: timeit now uses the sequence 1, 2, 5, 10, 20, 50,... instead - of 1, 10, 100,... for autoranging. - -- Issue #28115: Command-line interface of the zipfile module now uses argparse. - Added support of long options. - -- Issue #18219: Optimize csv.DictWriter for large number of columns. - Patch by Mariatta Wijaya. - -- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. - -- Issue #23214: In the "io" module, the argument to BufferedReader and - BytesIO's read1() methods is now optional and can be -1, matching the - BufferedIOBase specification. - -- Issue #28480: Fix error building socket module when multithreading is - disabled. - -- Issue #28240: timeit: remove ``-c/--clock`` and ``-t/--time`` command line - options which were deprecated since Python 3.3. - -- Issue #28240: timeit now repeats the benchmarks 5 times instead of only 3 - to make benchmarks more reliable. - -- Issue #28240: timeit autorange now uses a single loop iteration if the - benchmark takes less than 10 seconds, instead of 10 iterations. - "python3 -m timeit -s 'import time' 'time.sleep(1)'" now takes 4 seconds - instead of 40 seconds. - -- Distutils.sdist now looks for README and setup.py files with case - sensitivity. This behavior matches that found in Setuptools 6.0 and - later. See `setuptools 100 - `_ for rationale. - -- Issue #24452: Make webbrowser support Chrome on Mac OS X. Patch by - Ned Batchelder. - -- Issue #20766: Fix references leaked by pdb in the handling of SIGINT - handlers. - -- Issue #27998: Fixed bytes path support in os.scandir() on Windows. - Patch by Eryk Sun. - -- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. - -- Issue #26293: Fixed writing ZIP files that starts not from the start of the - file. Offsets in ZIP file now are relative to the start of the archive in - conforming to the specification. - -- Issue #28380: unittest.mock Mock autospec functions now properly support - assert_called, assert_not_called, and assert_called_once. - -- Issue #28229: lzma module now supports pathlib. - -- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. - -- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. - -- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. - -- Issue #28332: Deprecated silent truncations in socket.htons and socket.ntohs. - Original patch by Oren Milman. - -- Issue #27358: Optimized merging var-keyword arguments and improved error - message when passing a non-mapping as a var-keyword argument. - -- Issue #28257: Improved error message when passing a non-iterable as - a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. - -- Issue #28322: Fixed possible crashes when unpickle itertools objects from - incorrect pickle data. Based on patch by John Leitch. - -- Issue #28228: imghdr now supports pathlib. - -- Issue #28226: compileall now supports pathlib. - -- Issue #28314: Fix function declaration (C flags) for the getiterator() method - of xml.etree.ElementTree.Element. - -- Issue #28148: Stop using localtime() and gmtime() in the time - module. - - Introduced platform independent _PyTime_localtime API that is - similar to POSIX localtime_r, but available on all platforms. Patch - by Ed Schouten. - -- Issue #28253: Fixed calendar functions for extreme months: 0001-01 - and 9999-12. - - Methods itermonthdays() and itermonthdays2() are reimplemented so - that they don't call itermonthdates() which can cause datetime.date - under/overflow. - -- Issue #28275: Fixed possible use after free in the decompress() - methods of the LZMADecompressor and BZ2Decompressor classes. - Original patch by John Leitch. - -- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() - if pass invalid string-like object as a name. Patch by Xiang Zhang. - -- Issue #18844: random.choices() now has k as a keyword-only argument - to improve the readability of common cases and come into line - with the signature used in other languages. - -- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. - Patch by Madison May. - -- Issue #27611: Fixed support of default root window in the tkinter.tix module. - Added the master parameter in the DisplayStyle constructor. - -- Issue #27348: In the traceback module, restore the formatting of exception - messages like "Exception: None". This fixes a regression introduced in - 3.5a2. - -- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). - -- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is - interrupted by a signal and a signal handler raises a Python exception. - -- Issue #28200: Fix memory leak on Windows in the os module (fix - path_converter() function). - -- Issue #25400: RobotFileParser now correctly returns default values for - crawl_delay and request_rate. Initial patch by Peter Wirtz. - -- Issue #27932: Prevent memory leak in win32_ver(). - -- Fix UnboundLocalError in socket._sendfile_use_sendfile. - -- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of - os.stat(). Patch by Eryk Sun. - -- Issue #22493: Warning message emitted by using inline flags in the middle of - regular expression now contains a (truncated) regex pattern. - Patch by Tim Graham. - -- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when - an empty bytestring is passed. - -- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. - -- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. - Patch by Gergely Imreh and Markus Holtermann. - -- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. - Patch by Eryk Sun. - -- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). - -- Issue #27906: Fix socket accept exhaustion during high TCP traffic. - Patch by Kevin Conway. - -- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. - Patch by Seth Michael Larson. - -- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. - Patch by iceboy. - -- Issue #26909: Fix slow pipes IO in asyncio. - Patch by INADA Naoki. - -- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. - -- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. - Patch by Mark Williams. - -- Issue #28325: Remove vestigial MacOS 9 macurl2path module and its tests. - -- Issue #28368: Refuse monitoring processes if the child watcher has - no loop attached. - Patch by Vincent Michel. - -- Issue #28369: Raise RuntimeError when transport's FD is used with - add_reader, add_writer, etc. - -- Issue #28370: Speedup asyncio.StreamReader.readexactly. - Patch by ĐšĐŸŃ€Đ”ĐœĐ±Đ”Ń€Đł МарĐș. - -- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. - -- Issue #28372: Fix asyncio to support formatting of non-python coroutines. - -- Issue #28399: Remove UNIX socket from FS before binding. - Patch by ĐšĐŸŃ€Đ”ĐœĐ±Đ”Ń€Đł МарĐș. - -- Issue #27972: Prohibit Tasks to await on themselves. - -- Issue #24142: Reading a corrupt config file left configparser in an - invalid state. Original patch by Florian Höch. - -Windows -------- - -- bpo-29579: Removes readme.txt from the installer. - -- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) - -- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default - -- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c - -- Issue #28402: Adds signed catalog files for stdlib on Windows. - -- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by - Eryk Sun) - -- Issue #28251: Improvements to help manuals on Windows. - -- Issue #28110: launcher.msi has different product codes between 32-bit and - 64-bit - -- Issue #28161: Opening CON for write access fails - -- Issue #28162: WindowsConsoleIO readall() fails if first line starts with - Ctrl+Z - -- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to - _open_osfhandle - -- Issue #28164: _PyIO_get_console_type fails for various paths - -- Issue #28137: Renames Windows path file to ._pth - -- Issue #28138: Windows ._pth file should allow import site - -C API ------ - -- Issue #27867: Function PySlice_GetIndicesEx() is deprecated and replaced with - a macro if Py_LIMITED_API is not set or set to the value between 0x03050400 - and 0x03060000 (not including) or 0x03060100 or higher. Added functions - PySlice_Unpack() and PySlice_AdjustIndices(). - -- Issue #29083: Fixed the declaration of some public API functions. - PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in - limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and - Py_BuildValue() were not available in limited API of version < 3.3 when - PY_SSIZE_T_CLEAN is defined. - -- Issue #28769: The result of PyUnicode_AsUTF8AndSize() and PyUnicode_AsUTF8() - is now of type ``const char *`` rather of ``char *``. - -- Issue #29058: All stable API extensions added after Python 3.2 are now - available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of - the minimum Python version supporting this API. - -- Issue #28822: The index parameters *start* and *end* of PyUnicode_FindChar() - are now adjusted to behave like ``str[start:end]``. - -- Issue #28808: PyUnicode_CompareWithASCIIString() now never raises exceptions. - -- Issue #28761: The fields name and doc of structures PyMemberDef, PyGetSetDef, - PyStructSequence_Field, PyStructSequence_Desc, and wrapperbase are now of - type ``const char *`` rather of ``char *``. - -- Issue #28748: Private variable _Py_PackageContext is now of type ``const char *`` - rather of ``char *``. - -- Issue #19569: Compiler warnings are now emitted if use most of deprecated - functions. - -- Issue #28426: Deprecated undocumented functions PyUnicode_AsEncodedObject(), - PyUnicode_AsDecodedObject(), PyUnicode_AsDecodedUnicode() and - PyUnicode_AsEncodedUnicode(). - -Documentation -------------- - -- bpo-28929: Link the documentation to its source file on GitHub. - -- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to - aiosmtpd, a third-party asyncio-based replacement. - -- Issue #26355: Add canonical header link on each page to corresponding major - version of the documentation. Patch by Matthias Bussonnier. - -- Issue #29349: Fix Python 2 syntax in code for building the documentation. - -*Release date: XXXX-XX-XX* - -Core and Builtins ------------------ - -- bpo-28893: Set correct __cause__ for errors about invalid awaitables - returned from __aiter__ and __anext__. - -- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by - Brian Coleman. - -- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. - It should raise TypeError when kwargs is not a dict. But it might - cause segv when args=NULL and kwargs is not a dict. - -- Issue #28598: Support __rmod__ for subclasses of str being called before - str.__mod__. Patch by Martijn Pieters. - -- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. - Patch by Matthieu Dartiailh. - -- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for - complex subclasses and for inputs having a __complex__ method. Patch - by Serhiy Storchaka. - -- bpo-29347: Fixed possibly dereferencing undefined pointers - when creating weakref objects. - -- bpo-29438: Fixed use-after-free problem in key sharing dict. - -- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - -- Issue #29337: Fixed possible BytesWarning when compare the code objects. - Warnings could be emitted at compile time. - -- Issue #29327: Fixed a crash when pass the iterable keyword argument to - sorted(). - -- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). - -- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. - -- Issue #28932: Do not include if it does not exist. - -- Issue #25677: Correct the positioning of the syntax error caret for - indented blocks. Based on patch by Michael Layzell. - -- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate - form. - -- Issue #26919: On Android, operating system data is now always encoded/decoded - to/from UTF-8, instead of the locale encoding to avoid inconsistencies with - os.fsencode() and os.fsdecode() which are already using UTF-8. - -- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy - bug triggerable by a monkey-patched len() function. - -- Issue #28739: f-string expressions are no longer accepted as docstrings and - by ast.literal_eval() even if they do not include expressions. - -- Issue #28512: Fixed setting the offset attribute of SyntaxError by - PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). - -- Issue #28918: Fix the cross compilation of xxlimited when Python has been - built with Py_DEBUG defined. - -- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. - Improve speed of dict literal with constant keys up to 30%. - -Extension Modules ------------------ - -- Issue #29169: Update zlib to 1.2.11. - -Library -------- +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes + before all pipes are closed. - bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C to accept None argument as their pure Python implementation. From c7ff163abf428d73d4054754db3c99d06487e3ad Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 3 Mar 2017 18:07:18 +0300 Subject: [PATCH 076/232] Add Python version since deprecation in base64 methods. (#33) (#429) Allow developers to not have to either test on N Python versions or looked through multiple versions of the docs to know whether they can easily update. (cherry picked from commit c643a967dd7d33ccefa5b61b38caf40b448057ce) --- Doc/library/base64.rst | 17 +++++++++++++---- Lib/base64.py | 6 ++++-- Lib/test/test_base64.py | 8 ++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 080d9d77ec984e..ceecf17cba23e3 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -237,14 +237,18 @@ The legacy interface: .. function:: decodebytes(s) - decodestring(s) Decode the :term:`bytes-like object` *s*, which must contain one or more lines of base64 encoded data, and return the decoded :class:`bytes`. - ``decodestring`` is a deprecated alias. .. versionadded:: 3.1 +.. function:: decodestring(s) + + Deprecated alias of :func:`decodebytes`. + + .. deprecated:: 3.1 + .. function:: encode(input, output) @@ -257,14 +261,19 @@ The legacy interface: .. function:: encodebytes(s) - encodestring(s) Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary data, and return :class:`bytes` containing the base64-encoded data, with newlines (``b'\n'``) inserted after every 76 bytes of output, and ensuring that there is a trailing newline, as per :rfc:`2045` (MIME). - ``encodestring`` is a deprecated alias. + .. versionadded:: 3.1 + +.. function:: encodestring(s) + + Deprecated alias of :func:`encodebytes`. + + .. deprecated:: 3.1 An example usage of the module: diff --git a/Lib/base64.py b/Lib/base64.py index 58f6ad6816ee2f..eb8f258a2d1977 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -541,7 +541,8 @@ def encodebytes(s): def encodestring(s): """Legacy alias of encodebytes().""" import warnings - warnings.warn("encodestring() is a deprecated alias, use encodebytes()", + warnings.warn("encodestring() is a deprecated alias since 3.1, " + "use encodebytes()", DeprecationWarning, 2) return encodebytes(s) @@ -554,7 +555,8 @@ def decodebytes(s): def decodestring(s): """Legacy alias of decodebytes().""" import warnings - warnings.warn("decodestring() is a deprecated alias, use decodebytes()", + warnings.warn("decodestring() is a deprecated alias since Python 3.1, " + "use decodebytes()", DeprecationWarning, 2) return decodebytes(s) diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 4f86aaa0c0eb49..47547396b8cb54 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -18,6 +18,14 @@ def check_type_errors(self, f): int_data = memoryview(b"1234").cast('I') self.assertRaises(TypeError, f, int_data) + def test_encodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.encodestring(b"www.python.org") + + def test_decodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.decodestring(b"d3d3LnB5dGhvbi5vcmc=\n") + def test_encodebytes(self): eq = self.assertEqual eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n") From 21ce65aa67f0dc63002ab0a5fb21ef921cf5279e Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 3 Mar 2017 20:48:37 +0300 Subject: [PATCH 077/232] [3.6] bpo-29623: Make PathLike objects work with ConfigParser.read() (#242) (#432) (cherry picked from commit 85b8d01c916b482dac937b93ede1e53b1db0361c) Conflicts: Lib/test/test_configparser.py --- Doc/library/configparser.rst | 20 +++++++++++++------- Lib/configparser.py | 5 ++++- Lib/test/test_configparser.py | 12 +++++++++++- Misc/NEWS | 3 +++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index e09562dc9046c7..af57cba2b5661d 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -983,13 +983,16 @@ ConfigParser Objects .. method:: read(filenames, encoding=None) Attempt to read and parse a list of filenames, returning a list of - filenames which were successfully parsed. If *filenames* is a string, it - is treated as a single filename. If a file named in *filenames* cannot - be opened, that file will be ignored. This is designed so that you can - specify a list of potential configuration file locations (for example, - the current directory, the user's home directory, and some system-wide - directory), and all existing configuration files in the list will be - read. If none of the named files exist, the :class:`ConfigParser` + filenames which were successfully parsed. + + If *filenames* is a string or :term:`path-like object`, it is treated as + a single filename. If a file named in *filenames* cannot be opened, that + file will be ignored. This is designed so that you can specify a list of + potential configuration file locations (for example, the current + directory, the user's home directory, and some system-wide directory), + and all existing configuration files in the list will be read. + + If none of the named files exist, the :class:`ConfigParser` instance will contain an empty dataset. An application which requires initial values to be loaded from a file should load the required file or files using :meth:`read_file` before calling :meth:`read` for any @@ -1006,6 +1009,9 @@ ConfigParser Objects The *encoding* parameter. Previously, all files were read using the default encoding for :func:`open`. + .. versionadded:: 3.6.1 + The *filenames* parameter accepts a :term:`path-like object`. + .. method:: read_file(f, source=None) diff --git a/Lib/configparser.py b/Lib/configparser.py index af5aca1feae34a..230ab2b017eade 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -143,6 +143,7 @@ import functools import io import itertools +import os import re import sys import warnings @@ -687,7 +688,7 @@ def read(self, filenames, encoding=None): Return list of successfully read files. """ - if isinstance(filenames, str): + if isinstance(filenames, (str, os.PathLike)): filenames = [filenames] read_ok = [] for filename in filenames: @@ -696,6 +697,8 @@ def read(self, filenames, encoding=None): self._read(fp, filename) except OSError: continue + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) read_ok.append(filename) return read_ok diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 0d06080da33c52..72c3f19fb41f56 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,7 @@ import configparser import io import os -import sys +import pathlib import textwrap import unittest import warnings @@ -721,6 +721,16 @@ def test_read_returns_file_list(self): parsed_files = cf.read(file1) self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we pass only a Path object: + cf = self.newconfig() + parsed_files = cf.read(pathlib.Path(file1)) + self.assertEqual(parsed_files, [file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we passed both a filename and a Path object: + cf = self.newconfig() + parsed_files = cf.read([pathlib.Path(file1), file1]) + self.assertEqual(parsed_files, [file1, file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: cf = self.newconfig() parsed_files = cf.read(["nonexistent-file"]) diff --git a/Misc/NEWS b/Misc/NEWS index 2692043b6cbab6..692dc189f4f1fd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -82,6 +82,9 @@ Extension Modules Library ------- +- bpo-29623: Allow use of path-like object as a single argument in + ConfigParser.read(). Patch by David Ellis. + - bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback implemented in C. From 9d07aceedabcdc9826489f8b9baffff056283bb3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 3 Mar 2017 12:58:17 -0800 Subject: [PATCH 078/232] bpo-29455: Mention coverage.py in trace module documentation (GH-435) (cherry picked from commit 5dfccb06dc513ae67fac5fee66356ad58a4de170) --- Doc/library/trace.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index b0ac81271c5c18..5cb7029adf5e9e 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -13,6 +13,12 @@ annotated statement coverage listings, print caller/callee relationships and list functions executed during a program run. It can be used in another program or from the command line. +.. seealso:: + + `Coverage.py `_ + A popular third-party coverage tool that provides HTML + output along with advanced features such as branch coverage. + .. _trace-cli: Command-Line Usage From 1936ba93c900d47d0c8c915184ac2fa7773c952e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 3 Mar 2017 13:24:13 -0800 Subject: [PATCH 079/232] bpo-29709: Improve Boolean Operations documentation (#433) (#437) (cherry picked from commit 8eb531d9db0861e14222445fcaebe1a373bba170) --- Doc/library/stdtypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9a4f42caa408a5..6729a20a9c2ecc 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -108,11 +108,11 @@ Notes: (1) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`False`. + argument if the first one is false. (2) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`True`. + argument if the first one is true. (3) ``not`` has a lower priority than non-Boolean operators, so ``not a == b`` is From 03f7cb060414b17d88cb406f3e09f26c902d8658 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Fri, 3 Mar 2017 16:07:25 -0600 Subject: [PATCH 080/232] bpo-29572: Update Windows build to OpenSSL 1.0.2k (GH-442) --- Misc/NEWS | 2 ++ PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 2 +- PCbuild/readme.txt | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 692dc189f4f1fd..2bdf5403c3e1be 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -276,6 +276,8 @@ Tests Build ----- +- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. + - Issue #26851: Set Android compilation and link flags. - Issue #28768: Fix implicit declaration of function _setmode. Patch by diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index a5185becee5a96..98a755d870ad90 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2j +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k set libraries=%libraries% sqlite-3.14.2.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 diff --git a/PCbuild/python.props b/PCbuild/python.props index dde94f733a13ae..d6bfd0877e09df 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -45,7 +45,7 @@ $(ExternalsDir)sqlite-3.14.2.0\ $(ExternalsDir)bzip2-1.0.6\ $(ExternalsDir)xz-5.2.2\ - $(ExternalsDir)openssl-1.0.2j\ + $(ExternalsDir)openssl-1.0.2k\ $(opensslDir)include32 $(opensslDir)include64 $(ExternalsDir)\nasm-2.11.06\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c04ba4e46b9894..8924b3fcfa2e80 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -169,7 +169,7 @@ _lzma Homepage: http://tukaani.org/xz/ _ssl - Python wrapper for version 1.0.2j of the OpenSSL secure sockets + Python wrapper for version 1.0.2k of the OpenSSL secure sockets library, which is built by ssl.vcxproj Homepage: http://www.openssl.org/ From 226af23e858e2914cf78dfa6fd441c7b90a4cc91 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 3 Mar 2017 14:47:06 -0800 Subject: [PATCH 081/232] bpo-26213: Document _UNPACK bytecodes and BUILD_MAP changes (GH-440) (cherry picked from commit 0705f66eb369aa6a6cdb699e24ff61e1ab2e0c56) --- Doc/library/dis.rst | 55 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a15690ba489a66..1c1e1a2aa5d4b3 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -772,8 +772,13 @@ All of the following opcodes use their arguments. .. opcode:: BUILD_MAP (count) - Pushes a new dictionary object onto the stack. The dictionary is pre-sized - to hold *count* entries. + Pushes a new dictionary object onto the stack. Pops ``2 * count`` items + so that the dictionary holds *count* entries: + ``{..., TOS3: TOS2, TOS1: TOS}``. + + .. versionchanged:: 3.5 + The dictionary is created from stack items instead of creating an + empty dictionary pre-sized to hold *count* items. .. opcode:: BUILD_CONST_KEY_MAP (count) @@ -793,6 +798,52 @@ All of the following opcodes use their arguments. .. versionadded:: 3.6 +.. opcode:: BUILD_TUPLE_UNPACK (count) + + Pops *count* iterables from the stack, joins them in a single tuple, + and pushes the result. Implements iterable unpacking in tuple + displays ``(*x, *y, *z)``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_LIST_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list + instead of tuple. Implements iterable unpacking in list + displays ``[*x, *y, *z]``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_SET_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a set + instead of tuple. Implements iterable unpacking in set + displays ``{*x, *y, *z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK (count) + + Pops *count* mappings from the stack, merges them into a single dictionary, + and pushes the result. Implements dictionary unpacking in dictionary + displays ``{**x, **y, **z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (oparg) + + This is similar to :opcode:`BUILD_MAP_UNPACK`, + but is used for ``f(**x, **y, **z)`` call syntax. The lowest byte of + *oparg* is the count of mappings, the relative position of the + corresponding callable ``f`` is encoded in the second byte of *oparg*. + + .. versionadded:: 3.5 + + .. opcode:: LOAD_ATTR (namei) Replaces TOS with ``getattr(TOS, co_names[namei])``. From d3e1e9df724d97ab83113c2d5fa15179d1dcd560 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 3 Mar 2017 21:32:17 -0800 Subject: [PATCH 082/232] bpo-27593: Updates Windows build to use information from git (#262) (#450) * bpo-27593: Updates Windows build to use information from git --- PCbuild/build.bat | 8 ++++---- PCbuild/pythoncore.vcxproj | 24 ++++++++++++------------ Tools/msi/buildrelease.bat | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 70ab340905e4c9..f7f2858d7d9d99 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -105,9 +105,9 @@ if "%platf%"=="x64" ( ) ) -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if exist "%HG%" set HGProperty=/p:HG="%HG%" -if not exist "%HG%" echo Cannot find Mercurial on PATH & set HGProperty= +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" +if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= rem Setup the environment call "%dir%env.bat" %vs_platf% >nul @@ -145,7 +145,7 @@ msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:IncludeExternals=%IncludeExternals%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ - /p:UseTestMarker=%UseTestMarker% %HGProperty%^ + /p:UseTestMarker=%UseTestMarker% %GITProperty%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 6b23d8ecacce3b..8f8bede8a8818e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -407,24 +407,24 @@ - hg - <_HG>$(HG) - <_HG Condition="$(HG.Contains(` `))">"$(HG)" + git + <_GIT>$(GIT) + <_GIT Condition="$(GIT.Contains(` `))">"$(GIT)" - + - - - + + + - $([System.IO.File]::ReadAllText('$(IntDir)hgbranch.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)hgversion.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)hgtag.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gittag.txt').Trim()) - + - HGVERSION="$(HgVersion)";HGTAG="$(HgTag)";HGBRANCH="$(HgBranch)";%(PreprocessorDefinitions) + GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions) diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 4659a32b838cc1..81a3f86cab36b0 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -65,8 +65,8 @@ if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1 if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if not exist "%HG%" echo Cannot find Mercurial on PATH && exit /B 1 +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 call "%D%get_externals.bat" From 793f822068335cff68f25304a915c726f3f3a599 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 3 Mar 2017 21:57:47 -0800 Subject: [PATCH 083/232] Fixes git command (#451) (#452) --- PCbuild/pythoncore.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8f8bede8a8818e..d7e9473638cbb2 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -415,7 +415,7 @@ - + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) From 95c50e5aed9e5683676e18349dd94b11901a66b3 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 01:05:06 -0500 Subject: [PATCH 084/232] [3.6] bpo-27593: Get SCM build info from git instead of hg. (#446) (#454) * bpo-27593: Get SCM build info from git instead of hg. (#446) sys.version and the platform module python_build(), python_branch(), and python_revision() functions now use git information rather than hg when building from a repo. Based on original patches by Brett Cannon and Steve Dower. (cherry picked from commit 5c4b0d063aba0a68c325073f5f312a2c9f40d178) --- Include/pylifecycle.h | 4 ++-- Lib/platform.py | 4 +++- Lib/test/test_platform.py | 12 +++++----- Makefile.pre.in | 12 +++++----- Misc/NEWS | 6 +++++ Modules/getbuildinfo.c | 46 ++++++++++++++++++------------------- Python/sysmodule.c | 6 ++--- configure | 48 +++++++++++++++++++-------------------- configure.ac | 26 ++++++++++----------- 9 files changed, 86 insertions(+), 78 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 5a67666874da94..01abfa9fcd6fb7 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -70,8 +70,8 @@ PyAPI_FUNC(const char *) Py_GetCopyright(void); PyAPI_FUNC(const char *) Py_GetCompiler(void); PyAPI_FUNC(const char *) Py_GetBuildInfo(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(const char *) _Py_hgidentifier(void); -PyAPI_FUNC(const char *) _Py_hgversion(void); +PyAPI_FUNC(const char *) _Py_gitidentifier(void); +PyAPI_FUNC(const char *) _Py_gitversion(void); #endif /* Internal -- various one-time initializations */ diff --git a/Lib/platform.py b/Lib/platform.py index 062347537531ba..cc2db9870d84b6 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1198,7 +1198,9 @@ def _sys_version(sys_version=None): elif buildtime: builddate = builddate + ' ' + buildtime - if hasattr(sys, '_mercurial'): + if hasattr(sys, '_git'): + _, branch, revision = sys._git + elif hasattr(sys, '_mercurial'): _, branch, revision = sys._mercurial elif hasattr(sys, 'subversion'): # sys.subversion was added in Python 2.5 diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 5f875eff6c7ede..ea6752b61d8f77 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -67,12 +67,12 @@ def test_processor(self): def setUp(self): self.save_version = sys.version - self.save_mercurial = sys._mercurial + self.save_git = sys._git self.save_platform = sys.platform def tearDown(self): sys.version = self.save_version - sys._mercurial = self.save_mercurial + sys._git = self.save_git sys.platform = self.save_platform def test_sys_version(self): @@ -102,7 +102,7 @@ def test_sys_version(self): ('CPython', '2.4.3', '', '', 'truncation', '', 'GCC')), ): # branch and revision are not "parsed", but fetched - # from sys._mercurial. Ignore them + # from sys._git. Ignore them (name, version, branch, revision, buildno, builddate, compiler) \ = platform._sys_version(input) self.assertEqual( @@ -149,10 +149,10 @@ def test_sys_version(self): sys_versions.items(): sys.version = version_tag if subversion is None: - if hasattr(sys, "_mercurial"): - del sys._mercurial + if hasattr(sys, "_git"): + del sys._git else: - sys._mercurial = subversion + sys._git = subversion if sys_platform is not None: sys.platform = sys_platform self.assertEqual(platform.python_implementation(), info[0]) diff --git a/Makefile.pre.in b/Makefile.pre.in index 93aae9124726bd..8f27d7320c1d69 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,9 +41,9 @@ RANLIB= @RANLIB@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ -HGVERSION= @HGVERSION@ -HGTAG= @HGTAG@ -HGBRANCH= @HGBRANCH@ +GITVERSION= @GITVERSION@ +GITTAG= @GITTAG@ +GITBRANCH= @GITBRANCH@ PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ @@ -740,9 +740,9 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ $(MODOBJS) \ $(srcdir)/Modules/getbuildinfo.c $(CC) -c $(PY_CORE_CFLAGS) \ - -DHGVERSION="\"`LC_ALL=C $(HGVERSION)`\"" \ - -DHGTAG="\"`LC_ALL=C $(HGTAG)`\"" \ - -DHGBRANCH="\"`LC_ALL=C $(HGBRANCH)`\"" \ + -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ + -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ -o $@ $(srcdir)/Modules/getbuildinfo.c Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile diff --git a/Misc/NEWS b/Misc/NEWS index 2bdf5403c3e1be..debadc1e54c141 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -23,6 +23,8 @@ Core and Builtins - Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. +- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. + - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. @@ -276,6 +278,10 @@ Tests Build ----- +- bpo-27593: sys.version and the platform module python_build(), + python_branch(), and python_revision() functions now use + git information rather than hg when building from a repo. + - bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. - Issue #26851: Set Android compilation and link flags. diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 0971a64fccbadc..5f941a26e1d54c 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -21,47 +21,47 @@ #endif /* XXX Only unix build process has been tested */ -#ifndef HGVERSION -#define HGVERSION "" +#ifndef GITVERSION +#define GITVERSION "" #endif -#ifndef HGTAG -#define HGTAG "" +#ifndef GITTAG +#define GITTAG "" #endif -#ifndef HGBRANCH -#define HGBRANCH "" +#ifndef GITBRANCH +#define GITBRANCH "" #endif const char * Py_GetBuildInfo(void) { - static char buildinfo[50 + sizeof(HGVERSION) + - ((sizeof(HGTAG) > sizeof(HGBRANCH)) ? - sizeof(HGTAG) : sizeof(HGBRANCH))]; - const char *revision = _Py_hgversion(); + static char buildinfo[50 + sizeof(GITVERSION) + + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ? + sizeof(GITTAG) : sizeof(GITBRANCH))]; + const char *revision = _Py_gitversion(); const char *sep = *revision ? ":" : ""; - const char *hgid = _Py_hgidentifier(); - if (!(*hgid)) - hgid = "default"; + const char *gitid = _Py_gitidentifier(); + if (!(*gitid)) + gitid = "default"; PyOS_snprintf(buildinfo, sizeof(buildinfo), - "%s%s%s, %.20s, %.9s", hgid, sep, revision, + "%s%s%s, %.20s, %.9s", gitid, sep, revision, DATE, TIME); return buildinfo; } const char * -_Py_hgversion(void) +_Py_gitversion(void) { - return HGVERSION; + return GITVERSION; } const char * -_Py_hgidentifier(void) +_Py_gitidentifier(void) { - const char *hgtag, *hgid; - hgtag = HGTAG; - if ((*hgtag) && strcmp(hgtag, "tip") != 0) - hgid = hgtag; + const char *gittag, *gitid; + gittag = GITTAG; + if ((*gittag) && strcmp(gittag, "undefined") != 0) + gitid = gittag; else - hgid = HGBRANCH; - return hgid; + gitid = GITBRANCH; + return gitid; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 52034ff7fb5ddf..99e6b5ea018ea1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1942,9 +1942,9 @@ _PySys_Init(void) PyUnicode_FromString(Py_GetVersion())); SET_SYS_FROM_STRING("hexversion", PyLong_FromLong(PY_VERSION_HEX)); - SET_SYS_FROM_STRING("_mercurial", - Py_BuildValue("(szz)", "CPython", _Py_hgidentifier(), - _Py_hgversion())); + SET_SYS_FROM_STRING("_git", + Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(), + _Py_gitversion())); SET_SYS_FROM_STRING("dont_write_bytecode", PyBool_FromLong(Py_DontWriteBytecodeFlag)); SET_SYS_FROM_STRING("api_version", diff --git a/configure b/configure index 1500cea4a8292b..d4eccb1d68d856 100755 --- a/configure +++ b/configure @@ -760,10 +760,10 @@ build_os build_vendor build_cpu build -HAS_HG -HGBRANCH -HGTAG -HGVERSION +HAS_GIT +GITBRANCH +GITTAG +GITVERSION BASECPPFLAGS target_alias host_alias @@ -2698,17 +2698,17 @@ fi -if test -e $srcdir/.hg/dirstate +if test -e $srcdir/.git/HEAD then -# Extract the first word of "hg", so it can be a program name with args. -set dummy hg; ac_word=$2 +# Extract the first word of "git", so it can be a program name with args. +set dummy git; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_HAS_HG+:} false; then : +if ${ac_cv_prog_HAS_GIT+:} false; then : $as_echo_n "(cached) " >&6 else - if test -n "$HAS_HG"; then - ac_cv_prog_HAS_HG="$HAS_HG" # Let the user override the test. + if test -n "$HAS_GIT"; then + ac_cv_prog_HAS_GIT="$HAS_GIT" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH @@ -2717,7 +2717,7 @@ do test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_HAS_HG="found" + ac_cv_prog_HAS_GIT="found" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -2725,13 +2725,13 @@ done done IFS=$as_save_IFS - test -z "$ac_cv_prog_HAS_HG" && ac_cv_prog_HAS_HG="not-found" + test -z "$ac_cv_prog_HAS_GIT" && ac_cv_prog_HAS_GIT="not-found" fi fi -HAS_HG=$ac_cv_prog_HAS_HG -if test -n "$HAS_HG"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_HG" >&5 -$as_echo "$HAS_HG" >&6; } +HAS_GIT=$ac_cv_prog_HAS_GIT +if test -n "$HAS_GIT"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_GIT" >&5 +$as_echo "$HAS_GIT" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } @@ -2739,17 +2739,17 @@ fi else -HAS_HG=no-repository +HAS_GIT=no-repository fi -if test $HAS_HG = found +if test $HAS_GIT = found then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" + GITVERSION="git -C \$(srcdir) rev-parse HEAD" + GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else - HGVERSION="" - HGTAG="" - HGBRANCH="" + GITVERSION="" + GITTAG="" + GITBRANCH="" fi diff --git a/configure.ac b/configure.ac index 64f12f28790a7d..f00a2a6fa498f7 100644 --- a/configure.ac +++ b/configure.ac @@ -25,25 +25,25 @@ else BASECPPFLAGS="" fi -AC_SUBST(HGVERSION) -AC_SUBST(HGTAG) -AC_SUBST(HGBRANCH) +AC_SUBST(GITVERSION) +AC_SUBST(GITTAG) +AC_SUBST(GITBRANCH) -if test -e $srcdir/.hg/dirstate +if test -e $srcdir/.git/HEAD then -AC_CHECK_PROG(HAS_HG, hg, found, not-found) +AC_CHECK_PROG(HAS_GIT, git, found, not-found) else -HAS_HG=no-repository +HAS_GIT=no-repository fi -if test $HAS_HG = found +if test $HAS_GIT = found then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" + GITVERSION="git -C \$(srcdir) rev-parse HEAD" + GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else - HGVERSION="" - HGTAG="" - HGBRANCH="" + GITVERSION="" + GITTAG="" + GITBRANCH="" fi AC_CONFIG_SRCDIR([Include/object.h]) From 5ddddb166d9979161624c17cce3bdc11e1d9f3b8 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 02:55:09 -0500 Subject: [PATCH 085/232] bpo-29550: Temporarily skip "make touch" in Mac installer build. (#456) (#458) (cherry picked from commit 7cd9b22e98a5b7a99a7539500ea946a7a58e1c8d) --- Mac/BuildScript/build-installer.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 8dfd092d08b512..5649911c24c505 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -13,7 +13,7 @@ Python 2.6. In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed version of hg and a third-party version of +requires an installed version of git and a third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to @@ -23,7 +23,7 @@ 32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates), hg, and for the documentation +(2.5 is needed for Python parser updates), git, and for the documentation build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). Usage: see USAGE variable in the script. @@ -635,9 +635,9 @@ def checkEnvironment(): base_path = base_path + ':' + OLD_DEVELOPER_TOOLS os.environ['PATH'] = base_path print("Setting default PATH: %s"%(os.environ['PATH'])) - # Ensure ws have access to hg and to sphinx-build. + # Ensure ws have access to git and to sphinx-build. # You may have to create links in /usr/bin for them. - runCommand('hg --version') + runCommand('git --version') runCommand('sphinx-build --version') def parseOptions(args=None): @@ -1142,8 +1142,9 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) - print("Running make touch") - runCommand("make touch") + # bpo-29550: avoid using make touch until it is fixed for git + # print("Running make touch") + # runCommand("make touch") print("Running make") runCommand("make") From 343d48379c31f31cb0bcc843c537990c4283703d Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 03:04:48 -0500 Subject: [PATCH 086/232] bpo-29572: Update macOS installer build to OpenSSL 1.0.2k (#457) (#459) (cherry picked from commit cfcd76777e35c83d548d8736f5d7dc92fe56d806) --- Mac/BuildScript/build-installer.py | 6 +++--- Mac/BuildScript/openssl_sdk_makedepend.patch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 5649911c24c505..f59ae34df2549a 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -213,9 +213,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2j", - url="https://www.openssl.org/source/openssl-1.0.2j.tar.gz", - checksum='96322138f0b69e61b7212bc53d5e912b', + name="OpenSSL 1.0.2k", + url="https://www.openssl.org/source/openssl-1.0.2k.tar.gz", + checksum='f965fc0bf01bf882b31314b61391ae65', patches=[ "openssl_sdk_makedepend.patch", ], diff --git a/Mac/BuildScript/openssl_sdk_makedepend.patch b/Mac/BuildScript/openssl_sdk_makedepend.patch index e22d67e4e45143..0caac0a64c1ec6 100644 --- a/Mac/BuildScript/openssl_sdk_makedepend.patch +++ b/Mac/BuildScript/openssl_sdk_makedepend.patch @@ -1,6 +1,6 @@ # HG changeset patch # -# using openssl 1.0.2j +# using openssl 1.0.2k # # - support building with an OS X SDK From 1d391f926b37484b8d4b326003a72c0084db19ec Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 06:19:05 -0500 Subject: [PATCH 087/232] [3.6] bpo-28087: Skip test_asyncore and test_eintr poll failures on macOS (#463) Skip some tests of select.poll when running on macOS due to unresolved issues with the underlying system poll function on some macOS versions. (cherry picked from commit de04644627f82d9dc48b3423def7ff5b4aa1926a) --- Lib/test/eintrdata/eintr_tester.py | 2 ++ Lib/test/test_asyncore.py | 3 +++ Misc/NEWS | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index d194e775eaeb8f..c619b3d6c38a59 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -437,6 +437,8 @@ def test_select(self): self.stop_alarm() self.assertGreaterEqual(dt, self.sleep_time) + @unittest.skipIf(sys.platform == "darwin", + "poll may fail on macOS; see issue #28087") @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') def test_poll(self): poller = select.poll() diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 51c65737a441de..d05462b7efce35 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -661,6 +661,9 @@ def test_handle_expt(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") + if sys.platform == "darwin" and self.use_poll: + self.skipTest("poll may fail on macOS; see issue #28087") + class TestClient(BaseClient): def handle_expt(self): self.socket.recv(1024, socket.MSG_OOB) diff --git a/Misc/NEWS b/Misc/NEWS index debadc1e54c141..9aee4974344e10 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -261,6 +261,10 @@ Documentation Tests ----- +- bpo-28087: Skip test_asyncore and test_eintr poll failures on macOS. + Skip some tests of select.poll when running on macOS due to unresolved + issues with the underlying system poll function on some macOS versions. + - Issue #29571: to match the behaviour of the ``re.LOCALE`` flag, test_re.test_locale_flag now uses ``locale.getpreferredencoding(False)`` to determine the candidate encoding for the test regex (allowing it to correctly From 5976b9a88b8510789ed6e1679cc3578f7b345eb1 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 07:04:41 -0500 Subject: [PATCH 088/232] Refresh Python/importlib.h and Python/importlib_external.h (#464) --- Python/importlib.h | 2 +- Python/importlib_external.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/importlib.h b/Python/importlib.h index 195fbfd13b6227..4607c40de60e7c 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -392,7 +392,7 @@ const unsigned char _Py_M__importlib[] = { 111,118,101,100,197,0,0,0,115,2,0,0,0,0,8,114, 58,0,0,0,114,33,0,0,0,41,1,218,9,118,101,114, 98,111,115,105,116,121,99,1,0,0,0,1,0,0,0,3, - 0,0,0,5,0,0,0,71,0,0,0,115,54,0,0,0, + 0,0,0,4,0,0,0,71,0,0,0,115,54,0,0,0, 116,0,106,1,106,2,124,1,107,5,114,50,124,0,106,3, 100,6,131,1,115,30,100,3,124,0,23,0,125,0,116,4, 124,0,106,5,124,2,142,0,116,0,106,6,100,4,141,2, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 2f40978fb78e13..886aa68009ec03 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -578,7 +578,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,17,95,102,105,110,100,95,109,111,100,117,108,101,95,115, 104,105,109,157,1,0,0,115,10,0,0,0,0,10,14,1, 16,1,4,1,22,1,114,127,0,0,0,99,4,0,0,0, - 0,0,0,0,11,0,0,0,22,0,0,0,67,0,0,0, + 0,0,0,0,11,0,0,0,19,0,0,0,67,0,0,0, 115,136,1,0,0,105,0,125,4,124,2,100,1,107,9,114, 22,124,2,124,4,100,2,60,0,110,4,100,3,125,2,124, 3,100,1,107,9,114,42,124,3,124,4,100,4,60,0,124, From cefca3d7b5ee3f8421c082ccf24758fcb641fb6c Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 12:18:43 -0500 Subject: [PATCH 089/232] Update pydoc topics for 3.6.0rc1 --- Lib/pydoc_data/topics.py | 370 ++++++++++++++++++--------------------- 1 file changed, 171 insertions(+), 199 deletions(-) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index c7fac3395b3be0..2caab630978fa0 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Dec 6 18:51:51 2016 -topics = {'assert': '\n' - 'The "assert" statement\n' +# Autogenerated by Sphinx on Sat Mar 4 12:14:44 2017 +topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' 'Assert statements are a convenient way to insert debugging ' @@ -39,8 +38,7 @@ 'Assignments to "__debug__" are illegal. The value for the ' 'built-in\n' 'variable is determined when the interpreter starts.\n', - 'assignment': '\n' - 'Assignment statements\n' + 'assignment': 'Assignment statements\n' '*********************\n' '\n' 'Assignment statements are used to (re)bind names to values and ' @@ -405,8 +403,7 @@ 'See also: **PEP 526** - Variable and attribute annotation ' 'syntax\n' ' **PEP 484** - Type hints\n', - 'atom-identifiers': '\n' - 'Identifiers (Names)\n' + 'atom-identifiers': 'Identifiers (Names)\n' '*******************\n' '\n' 'An identifier occurring as an atom is a name. See ' @@ -446,8 +443,7 @@ 'happen. If the class name consists only of underscores, ' 'no\n' 'transformation is done.\n', - 'atom-literals': '\n' - 'Literals\n' + 'atom-literals': 'Literals\n' '********\n' '\n' 'Python supports string and bytes literals and various ' @@ -476,8 +472,7 @@ 'may obtain\n' 'the same object or a different object with the same ' 'value.\n', - 'attribute-access': '\n' - 'Customizing attribute access\n' + 'attribute-access': 'Customizing attribute access\n' '****************************\n' '\n' 'The following methods can be defined to customize the ' @@ -851,8 +846,7 @@ '* *__class__* assignment works only if both classes have ' 'the same\n' ' *__slots__*.\n', - 'attribute-references': '\n' - 'Attribute references\n' + 'attribute-references': 'Attribute references\n' '********************\n' '\n' 'An attribute reference is a primary followed by a ' @@ -875,8 +869,7 @@ 'determined by the object. Multiple evaluations of ' 'the same attribute\n' 'reference may yield different objects.\n', - 'augassign': '\n' - 'Augmented assignment statements\n' + 'augassign': 'Augmented assignment statements\n' '*******************************\n' '\n' 'Augmented assignment is the combination, in a single statement, ' @@ -940,8 +933,7 @@ 'about\n' 'class and instance attributes applies as for regular ' 'assignments.\n', - 'binary': '\n' - 'Binary arithmetic operations\n' + 'binary': 'Binary arithmetic operations\n' '****************************\n' '\n' 'The binary arithmetic operations have the conventional priority\n' @@ -1029,8 +1021,7 @@ 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' 'The numeric arguments are first converted to a common type.\n', - 'bitwise': '\n' - 'Binary bitwise operations\n' + 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' 'Each of the three bitwise operations has a different priority ' @@ -1050,8 +1041,7 @@ 'The "|" operator yields the bitwise (inclusive) OR of its ' 'arguments,\n' 'which must be integers.\n', - 'bltin-code-objects': '\n' - 'Code Objects\n' + 'bltin-code-objects': 'Code Objects\n' '************\n' '\n' 'Code objects are used by the implementation to ' @@ -1074,8 +1064,7 @@ '\n' 'See The standard type hierarchy for more ' 'information.\n', - 'bltin-ellipsis-object': '\n' - 'The Ellipsis Object\n' + 'bltin-ellipsis-object': 'The Ellipsis Object\n' '*******************\n' '\n' 'This object is commonly used by slicing (see ' @@ -1087,8 +1076,7 @@ '"Ellipsis" singleton.\n' '\n' 'It is written as "Ellipsis" or "...".\n', - 'bltin-null-object': '\n' - 'The Null Object\n' + 'bltin-null-object': 'The Null Object\n' '***************\n' '\n' "This object is returned by functions that don't " @@ -1100,8 +1088,7 @@ 'same singleton.\n' '\n' 'It is written as "None".\n', - 'bltin-type-objects': '\n' - 'Type Objects\n' + 'bltin-type-objects': 'Type Objects\n' '************\n' '\n' 'Type objects represent the various object types. An ' @@ -1113,8 +1100,7 @@ 'all standard built-in types.\n' '\n' 'Types are written like this: "".\n', - 'booleans': '\n' - 'Boolean operations\n' + 'booleans': 'Boolean operations\n' '******************\n' '\n' ' or_test ::= and_test | or_test "or" and_test\n' @@ -1163,8 +1149,7 @@ 'its\n' 'argument (for example, "not \'foo\'" produces "False" rather ' 'than "\'\'".)\n', - 'break': '\n' - 'The "break" statement\n' + 'break': 'The "break" statement\n' '*********************\n' '\n' ' break_stmt ::= "break"\n' @@ -1185,8 +1170,7 @@ 'clause, that "finally" clause is executed before really leaving ' 'the\n' 'loop.\n', - 'callable-types': '\n' - 'Emulating callable objects\n' + 'callable-types': 'Emulating callable objects\n' '**************************\n' '\n' 'object.__call__(self[, args...])\n' @@ -1195,8 +1179,7 @@ 'this method\n' ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' ' "x.__call__(arg1, arg2, ...)".\n', - 'calls': '\n' - 'Calls\n' + 'calls': 'Calls\n' '*****\n' '\n' 'A call calls a callable object (e.g., a *function*) with a ' @@ -1217,7 +1200,8 @@ ' ("," "*" expression | "," ' 'keyword_item)*\n' ' keywords_arguments ::= (keyword_item | "**" expression)\n' - ' ("," keyword_item | "**" expression)*\n' + ' ("," keyword_item | "," "**" ' + 'expression)*\n' ' keyword_item ::= identifier "=" expression\n' '\n' 'An optional trailing comma may be present after the positional and\n' @@ -1382,8 +1366,7 @@ ' The class must define a "__call__()" method; the effect is then ' 'the\n' ' same as if that method was called.\n', - 'class': '\n' - 'Class definitions\n' + 'class': 'Class definitions\n' '*****************\n' '\n' 'A class definition defines a class object (see section The ' @@ -1469,8 +1452,7 @@ '\n' 'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n' ' Class Decorators\n', - 'comparisons': '\n' - 'Comparisons\n' + 'comparisons': 'Comparisons\n' '***********\n' '\n' 'Unlike C, all comparison operations in Python have the same ' @@ -1623,7 +1605,7 @@ 'restriction that\n' ' ranges do not support order comparison. Equality ' 'comparison across\n' - ' these types results in unequality, and ordering comparison ' + ' these types results in inequality, and ordering comparison ' 'across\n' ' these types raises "TypeError".\n' '\n' @@ -1762,6 +1744,12 @@ ' to sequences, but not to sets or mappings). See also the\n' ' "total_ordering()" decorator.\n' '\n' + '* The "hash()" result should be consistent with equality. ' + 'Objects\n' + ' that are equal should either have the same hash value, or ' + 'be marked\n' + ' as unhashable.\n' + '\n' 'Python does not enforce these consistency rules. In fact, ' 'the\n' 'not-a-number values are an example for not following these ' @@ -1833,8 +1821,7 @@ 'is determined using the "id()" function. "x is not y" yields ' 'the\n' 'inverse truth value. [4]\n', - 'compound': '\n' - 'Compound statements\n' + 'compound': 'Compound statements\n' '*******************\n' '\n' 'Compound statements contain (groups of) other statements; they ' @@ -2613,7 +2600,8 @@ 'functions, even if they do not contain "await" or "async" ' 'keywords.\n' '\n' - 'It is a "SyntaxError" to use "yield" expressions in "async def"\n' + 'It is a "SyntaxError" to use "yield from" expressions in "async ' + 'def"\n' 'coroutines.\n' '\n' 'An example of a coroutine function:\n' @@ -2724,8 +2712,7 @@ ' body is transformed into the namespace\'s "__doc__" item ' 'and\n' " therefore the class's *docstring*.\n", - 'context-managers': '\n' - 'With Statement Context Managers\n' + 'context-managers': 'With Statement Context Managers\n' '*******************************\n' '\n' 'A *context manager* is an object that defines the ' @@ -2787,8 +2774,7 @@ ' The specification, background, and examples for the ' 'Python "with"\n' ' statement.\n', - 'continue': '\n' - 'The "continue" statement\n' + 'continue': 'The "continue" statement\n' '************************\n' '\n' ' continue_stmt ::= "continue"\n' @@ -2805,8 +2791,7 @@ '"finally" clause, that "finally" clause is executed before ' 'really\n' 'starting the next loop cycle.\n', - 'conversions': '\n' - 'Arithmetic conversions\n' + 'conversions': 'Arithmetic conversions\n' '**********************\n' '\n' 'When a description of an arithmetic operator below uses the ' @@ -2832,8 +2817,7 @@ "left argument to the '%' operator). Extensions must define " 'their own\n' 'conversion behavior.\n', - 'customization': '\n' - 'Basic customization\n' + 'customization': 'Basic customization\n' '*******************\n' '\n' 'object.__new__(cls[, ...])\n' @@ -3152,15 +3136,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only ' - 'required property\n' + ' "__hash__()" should return an integer. The only required ' + 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive ' - 'or) the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the ' + 'components of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -3272,8 +3259,7 @@ ' neither "__len__()" nor "__bool__()", all its instances ' 'are\n' ' considered true.\n', - 'debugger': '\n' - '"pdb" --- The Python Debugger\n' + 'debugger': '"pdb" --- The Python Debugger\n' '*****************************\n' '\n' '**Source code:** Lib/pdb.py\n' @@ -3938,8 +3924,7 @@ '[1] Whether a frame is considered to originate in a certain ' 'module\n' ' is determined by the "__name__" in the frame globals.\n', - 'del': '\n' - 'The "del" statement\n' + 'del': 'The "del" statement\n' '*******************\n' '\n' ' del_stmt ::= "del" target_list\n' @@ -3968,8 +3953,7 @@ 'Changed in version 3.2: Previously it was illegal to delete a name\n' 'from the local namespace if it occurs as a free variable in a nested\n' 'block.\n', - 'dict': '\n' - 'Dictionary displays\n' + 'dict': 'Dictionary displays\n' '*******************\n' '\n' 'A dictionary display is a possibly empty series of key/datum pairs\n' @@ -4013,8 +3997,7 @@ 'should be *hashable*, which excludes all mutable objects.) Clashes\n' 'between duplicate keys are not detected; the last datum (textually\n' 'rightmost in the display) stored for a given key value prevails.\n', - 'dynamic-features': '\n' - 'Interaction with dynamic features\n' + 'dynamic-features': 'Interaction with dynamic features\n' '*********************************\n' '\n' 'Name resolution of free variables occurs at runtime, not ' @@ -4050,8 +4033,7 @@ 'override the global and local namespace. If only one ' 'namespace is\n' 'specified, it is used for both.\n', - 'else': '\n' - 'The "if" statement\n' + 'else': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -4068,8 +4050,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'exceptions': '\n' - 'Exceptions\n' + 'exceptions': 'Exceptions\n' '**********\n' '\n' 'Exceptions are a means of breaking out of the normal flow of ' @@ -4145,8 +4126,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'execmodel': '\n' - 'Execution model\n' + 'execmodel': 'Execution model\n' '***************\n' '\n' '\n' @@ -4477,8 +4457,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'exprlists': '\n' - 'Expression lists\n' + 'exprlists': 'Expression lists\n' '****************\n' '\n' ' expression_list ::= expression ( "," expression )* [","]\n' @@ -4515,8 +4494,7 @@ 'value of that expression. (To create an empty tuple, use an ' 'empty pair\n' 'of parentheses: "()".)\n', - 'floating': '\n' - 'Floating point literals\n' + 'floating': 'Floating point literals\n' '***********************\n' '\n' 'Floating point literals are described by the following lexical\n' @@ -4552,8 +4530,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'for': '\n' - 'The "for" statement\n' + 'for': 'The "for" statement\n' '*******************\n' '\n' 'The "for" statement is used to iterate over the elements of a ' @@ -4625,8 +4602,7 @@ '\n' ' for x in a[:]:\n' ' if x < 0: a.remove(x)\n', - 'formatstrings': '\n' - 'Format String Syntax\n' + 'formatstrings': 'Format String Syntax\n' '********************\n' '\n' 'The "str.format()" method and the "Formatter" class share ' @@ -5345,8 +5321,7 @@ ' 9 9 11 1001\n' ' 10 A 12 1010\n' ' 11 B 13 1011\n', - 'function': '\n' - 'Function definitions\n' + 'function': 'Function definitions\n' '********************\n' '\n' 'A function definition defines a user-defined function object ' @@ -5515,8 +5490,7 @@ '\n' ' **PEP 3107** - Function Annotations\n' ' The original specification for function annotations.\n', - 'global': '\n' - 'The "global" statement\n' + 'global': 'The "global" statement\n' '**********************\n' '\n' ' global_stmt ::= "global" identifier ("," identifier)*\n' @@ -5560,8 +5534,7 @@ 'code containing the function call. The same applies to the ' '"eval()"\n' 'and "compile()" functions.\n', - 'id-classes': '\n' - 'Reserved classes of identifiers\n' + 'id-classes': 'Reserved classes of identifiers\n' '*******************************\n' '\n' 'Certain classes of identifiers (besides keywords) have ' @@ -5609,8 +5582,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'identifiers': '\n' - 'Identifiers and keywords\n' + 'identifiers': 'Identifiers and keywords\n' '************************\n' '\n' 'Identifiers (also referred to as *names*) are described by ' @@ -5758,8 +5730,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'if': '\n' - 'The "if" statement\n' + 'if': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -5775,8 +5746,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'imaginary': '\n' - 'Imaginary literals\n' + 'imaginary': 'Imaginary literals\n' '******************\n' '\n' 'Imaginary literals are described by the following lexical ' @@ -5796,8 +5766,7 @@ '\n' ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j ' '3.14_15_93j\n', - 'import': '\n' - 'The "import" statement\n' + 'import': 'The "import" statement\n' '**********************\n' '\n' ' import_stmt ::= "import" module ["as" name] ( "," module ' @@ -6058,8 +6027,7 @@ '\n' ' **PEP 236** - Back to the __future__\n' ' The original proposal for the __future__ mechanism.\n', - 'in': '\n' - 'Membership test operations\n' + 'in': 'Membership test operations\n' '**************************\n' '\n' 'The operators "in" and "not in" test for membership. "x in s"\n' @@ -6094,8 +6062,7 @@ '\n' 'The operator "not in" is defined to have the inverse true value of\n' '"in".\n', - 'integers': '\n' - 'Integer literals\n' + 'integers': 'Integer literals\n' '****************\n' '\n' 'Integer literals are described by the following lexical ' @@ -6141,8 +6108,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'lambda': '\n' - 'Lambdas\n' + 'lambda': 'Lambdas\n' '*******\n' '\n' ' lambda_expr ::= "lambda" [parameter_list]: expression\n' @@ -6165,8 +6131,7 @@ 'Note that functions created with lambda expressions cannot ' 'contain\n' 'statements or annotations.\n', - 'lists': '\n' - 'List displays\n' + 'lists': 'List displays\n' '*************\n' '\n' 'A list display is a possibly empty series of expressions enclosed ' @@ -6183,8 +6148,7 @@ 'from left to right and placed into the list object in that order.\n' 'When a comprehension is supplied, the list is constructed from the\n' 'elements resulting from the comprehension.\n', - 'naming': '\n' - 'Naming and binding\n' + 'naming': 'Naming and binding\n' '******************\n' '\n' '\n' @@ -6397,8 +6361,7 @@ 'override the global and local namespace. If only one namespace ' 'is\n' 'specified, it is used for both.\n', - 'nonlocal': '\n' - 'The "nonlocal" statement\n' + 'nonlocal': 'The "nonlocal" statement\n' '************************\n' '\n' ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' @@ -6429,8 +6392,7 @@ '\n' ' **PEP 3104** - Access to Names in Outer Scopes\n' ' The specification for the "nonlocal" statement.\n', - 'numbers': '\n' - 'Numeric literals\n' + 'numbers': 'Numeric literals\n' '****************\n' '\n' 'There are three types of numeric literals: integers, floating ' @@ -6444,8 +6406,7 @@ 'is actually an expression composed of the unary operator \'"-"\' ' 'and the\n' 'literal "1".\n', - 'numeric-types': '\n' - 'Emulating numeric types\n' + 'numeric-types': 'Emulating numeric types\n' '***********************\n' '\n' 'The following methods can be defined to emulate numeric ' @@ -6621,8 +6582,7 @@ ' "__index__()" is defined "__int__()" should also be ' 'defined, and\n' ' both should return the same value.\n', - 'objects': '\n' - 'Objects, values and types\n' + 'objects': 'Objects, values and types\n' '*************************\n' '\n' "*Objects* are Python's abstraction for data. All data in a " @@ -6750,8 +6710,7 @@ 'created empty lists. (Note that "c = d = []" assigns the same ' 'object\n' 'to both "c" and "d".)\n', - 'operator-summary': '\n' - 'Operator precedence\n' + 'operator-summary': 'Operator precedence\n' '*******************\n' '\n' 'The following table summarizes the operator precedence ' @@ -6924,8 +6883,7 @@ 'arithmetic\n' ' or bitwise unary operator on its right, that is, ' '"2**-1" is "0.5".\n', - 'pass': '\n' - 'The "pass" statement\n' + 'pass': 'The "pass" statement\n' '********************\n' '\n' ' pass_stmt ::= "pass"\n' @@ -6938,8 +6896,7 @@ ' def f(arg): pass # a function that does nothing (yet)\n' '\n' ' class C: pass # a class with no methods (yet)\n', - 'power': '\n' - 'The power operator\n' + 'power': 'The power operator\n' '******************\n' '\n' 'The power operator binds more tightly than unary operators on its\n' @@ -6973,8 +6930,7 @@ 'Raising a negative number to a fractional power results in a ' '"complex"\n' 'number. (In earlier versions it raised a "ValueError".)\n', - 'raise': '\n' - 'The "raise" statement\n' + 'raise': 'The "raise" statement\n' '*********************\n' '\n' ' raise_stmt ::= "raise" [expression ["from" expression]]\n' @@ -7059,8 +7015,7 @@ 'Exceptions, and information about handling exceptions is in ' 'section\n' 'The try statement.\n', - 'return': '\n' - 'The "return" statement\n' + 'return': 'The "return" statement\n' '**********************\n' '\n' ' return_stmt ::= "return" [expression_list]\n' @@ -7087,9 +7042,15 @@ 'generator is done and will cause "StopIteration" to be raised. ' 'The\n' 'returned value (if any) is used as an argument to construct\n' - '"StopIteration" and becomes the "StopIteration.value" attribute.\n', - 'sequence-types': '\n' - 'Emulating container types\n' + '"StopIteration" and becomes the "StopIteration.value" attribute.\n' + '\n' + 'In an asynchronous generator function, an empty "return" ' + 'statement\n' + 'indicates that the asynchronous generator is done and will cause\n' + '"StopAsyncIteration" to be raised. A non-empty "return" statement ' + 'is\n' + 'a syntax error in an asynchronous generator function.\n', + 'sequence-types': 'Emulating container types\n' '*************************\n' '\n' 'The following methods can be defined to implement ' @@ -7310,8 +7271,7 @@ ' iteration protocol via "__getitem__()", see this ' 'section in the\n' ' language reference.\n', - 'shifting': '\n' - 'Shifting operations\n' + 'shifting': 'Shifting operations\n' '*******************\n' '\n' 'The shifting operations have lower priority than the arithmetic\n' @@ -7335,8 +7295,7 @@ 'operand is\n' ' larger than "sys.maxsize" an "OverflowError" exception is ' 'raised.\n', - 'slicings': '\n' - 'Slicings\n' + 'slicings': 'Slicings\n' '********\n' '\n' 'A slicing selects a range of items in a sequence object (e.g., ' @@ -7387,8 +7346,7 @@ 'as lower bound, upper bound and stride, respectively, ' 'substituting\n' '"None" for missing expressions.\n', - 'specialattrs': '\n' - 'Special Attributes\n' + 'specialattrs': 'Special Attributes\n' '******************\n' '\n' 'The implementation adds a few special read-only attributes ' @@ -7473,8 +7431,7 @@ '[5] To format only a tuple you should therefore provide a\n' ' singleton tuple whose only element is the tuple to be ' 'formatted.\n', - 'specialnames': '\n' - 'Special method names\n' + 'specialnames': 'Special method names\n' '********************\n' '\n' 'A class can implement certain operations that are invoked by ' @@ -7835,15 +7792,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only required ' + ' "__hash__()" should return an integer. The only required ' 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive or) ' - 'the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the components ' + 'of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -9262,8 +9222,7 @@ 'special method *must* be set on the class object itself in ' 'order to be\n' 'consistently invoked by the interpreter).\n', - 'string-methods': '\n' - 'String Methods\n' + 'string-methods': 'String Methods\n' '**************\n' '\n' 'Strings implement all of the common sequence operations, ' @@ -9500,12 +9459,11 @@ 'characters\n' ' and there is at least one character, false otherwise. ' 'Decimal\n' - ' characters are those from general category "Nd". This ' - 'category\n' - ' includes digit characters, and all characters that can ' - 'be used to\n' - ' form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC ' - 'DIGIT ZERO.\n' + ' characters are those that can be used to form numbers ' + 'in base 10,\n' + ' e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a ' + 'decimal character\n' + ' is a character in the Unicode General Category "Nd".\n' '\n' 'str.isdigit()\n' '\n' @@ -9515,10 +9473,13 @@ 'include decimal\n' ' characters and digits that need special handling, such ' 'as the\n' - ' compatibility superscript digits. Formally, a digit is ' - 'a character\n' - ' that has the property value Numeric_Type=Digit or\n' - ' Numeric_Type=Decimal.\n' + ' compatibility superscript digits. This covers digits ' + 'which cannot\n' + ' be used to form numbers in base 10, like the Kharosthi ' + 'numbers.\n' + ' Formally, a digit is a character that has the property ' + 'value\n' + ' Numeric_Type=Digit or Numeric_Type=Decimal.\n' '\n' 'str.isidentifier()\n' '\n' @@ -10064,8 +10025,7 @@ " '00042'\n" ' >>> "-42".zfill(5)\n' " '-0042'\n", - 'strings': '\n' - 'String and Bytes literals\n' + 'strings': 'String and Bytes literals\n' '*************************\n' '\n' 'String literals are described by the following lexical ' @@ -10299,8 +10259,7 @@ 'followed by a newline is interpreted as those two characters as ' 'part\n' 'of the literal, *not* as a line continuation.\n', - 'subscriptions': '\n' - 'Subscriptions\n' + 'subscriptions': 'Subscriptions\n' '*************\n' '\n' 'A subscription selects an item of a sequence (string, tuple ' @@ -10357,8 +10316,7 @@ "A string's items are characters. A character is not a " 'separate data\n' 'type but a string of exactly one character.\n', - 'truth': '\n' - 'Truth Value Testing\n' + 'truth': 'Truth Value Testing\n' '*******************\n' '\n' 'Any object can be tested for truth value, for use in an "if" or\n' @@ -10390,8 +10348,7 @@ 'otherwise stated. (Important exception: the Boolean operations ' '"or"\n' 'and "and" always return one of their operands.)\n', - 'try': '\n' - 'The "try" statement\n' + 'try': 'The "try" statement\n' '*******************\n' '\n' 'The "try" statement specifies exception handlers and/or cleanup code\n' @@ -10538,8 +10495,7 @@ 'Exceptions, and information on using the "raise" statement to ' 'generate\n' 'exceptions may be found in section The raise statement.\n', - 'types': '\n' - 'The standard type hierarchy\n' + 'types': 'The standard type hierarchy\n' '***************************\n' '\n' 'Below is a list of the types that are built into Python. ' @@ -11097,6 +11053,27 @@ 'statements.\n' ' See also the Coroutine Objects section.\n' '\n' + ' Asynchronous generator functions\n' + ' A function or method which is defined using "async def" and\n' + ' which uses the "yield" statement is called a *asynchronous\n' + ' generator function*. Such a function, when called, returns ' + 'an\n' + ' asynchronous iterator object which can be used in an "async ' + 'for"\n' + ' statement to execute the body of the function.\n' + '\n' + ' Calling the asynchronous iterator\'s "aiterator.__anext__()"\n' + ' method will return an *awaitable* which when awaited will\n' + ' execute until it provides a value using the "yield" ' + 'expression.\n' + ' When the function executes an empty "return" statement or ' + 'falls\n' + ' off the end, a "StopAsyncIteration" exception is raised and ' + 'the\n' + ' asynchronous iterator will have reached the end of the set ' + 'of\n' + ' values to be yielded.\n' + '\n' ' Built-in functions\n' ' A built-in function object is a wrapper around a C function.\n' ' Examples of built-in functions are "len()" and "math.sin()"\n' @@ -11233,14 +11210,14 @@ 'the\n' ' dictionary containing the class\'s namespace; "__bases__" is a ' 'tuple\n' - ' (possibly empty or a singleton) containing the base classes, in ' - 'the\n' - ' order of their occurrence in the base class list; "__doc__" is ' - 'the\n' - ' class\'s documentation string, or "None" if undefined;\n' - ' "__annotations__" (optional) is a dictionary containing ' - '*variable\n' - ' annotations* collected during class body execution.\n' + ' containing the base classes, in the order of their occurrence ' + 'in\n' + ' the base class list; "__doc__" is the class\'s documentation ' + 'string,\n' + ' or "None" if undefined; "__annotations__" (optional) is a\n' + ' dictionary containing *variable annotations* collected during ' + 'class\n' + ' body execution.\n' '\n' 'Class instances\n' ' A class instance is created by calling a class object (see ' @@ -11520,8 +11497,7 @@ ' under "User-defined methods". Class method objects are ' 'created\n' ' by the built-in "classmethod()" constructor.\n', - 'typesfunctions': '\n' - 'Functions\n' + 'typesfunctions': 'Functions\n' '*********\n' '\n' 'Function objects are created by function definitions. The ' @@ -11538,8 +11514,7 @@ 'different object types.\n' '\n' 'See Function definitions for more information.\n', - 'typesmapping': '\n' - 'Mapping Types --- "dict"\n' + 'typesmapping': 'Mapping Types --- "dict"\n' '************************\n' '\n' 'A *mapping* object maps *hashable* values to arbitrary ' @@ -11896,8 +11871,7 @@ " {'bacon'}\n" " >>> keys ^ {'sausage', 'juice'}\n" " {'juice', 'sausage', 'bacon', 'spam'}\n", - 'typesmethods': '\n' - 'Methods\n' + 'typesmethods': 'Methods\n' '*******\n' '\n' 'Methods are functions that are called using the attribute ' @@ -11954,8 +11928,7 @@ " 'my name is method'\n" '\n' 'See The standard type hierarchy for more information.\n', - 'typesmodules': '\n' - 'Modules\n' + 'typesmodules': 'Modules\n' '*******\n' '\n' 'The only special operation on a module is attribute access: ' @@ -11992,8 +11965,7 @@ 'written as\n' '"".\n', - 'typesseq': '\n' - 'Sequence Types --- "list", "tuple", "range"\n' + 'typesseq': 'Sequence Types --- "list", "tuple", "range"\n' '*******************************************\n' '\n' 'There are three basic sequence types: lists, tuples, and range\n' @@ -12141,9 +12113,9 @@ '\n' '3. If *i* or *j* is negative, the index is relative to the end ' 'of\n' - ' the string: "len(s) + i" or "len(s) + j" is substituted. But ' - 'note\n' - ' that "-0" is still "0".\n' + ' sequence *s*: "len(s) + i" or "len(s) + j" is substituted. ' + 'But\n' + ' note that "-0" is still "0".\n' '\n' '4. The slice of *s* from *i* to *j* is defined as the sequence ' 'of\n' @@ -12162,12 +12134,17 @@ ' (j-i)/k". In other words, the indices are "i", "i+k", ' '"i+2*k",\n' ' "i+3*k" and so on, stopping when *j* is reached (but never\n' - ' including *j*). If *i* or *j* is greater than "len(s)", use\n' - ' "len(s)". If *i* or *j* are omitted or "None", they become ' - '"end"\n' - ' values (which end depends on the sign of *k*). Note, *k* ' - 'cannot be\n' - ' zero. If *k* is "None", it is treated like "1".\n' + ' including *j*). When *k* is positive, *i* and *j* are ' + 'reduced to\n' + ' "len(s)" if they are greater. When *k* is negative, *i* and ' + '*j* are\n' + ' reduced to "len(s) - 1" if they are greater. If *i* or *j* ' + 'are\n' + ' omitted or "None", they become "end" values (which end ' + 'depends on\n' + ' the sign of *k*). Note, *k* cannot be zero. If *k* is ' + '"None", it\n' + ' is treated like "1".\n' '\n' '6. Concatenating immutable sequences always results in a new\n' ' object. This means that building up a sequence by repeated\n' @@ -12685,8 +12662,7 @@ ' * The linspace recipe shows how to implement a lazy version ' 'of\n' ' range that suitable for floating point applications.\n', - 'typesseq-mutable': '\n' - 'Mutable Sequence Types\n' + 'typesseq-mutable': 'Mutable Sequence Types\n' '**********************\n' '\n' 'The operations in the following table are defined on ' @@ -12826,8 +12802,7 @@ 'referenced multiple\n' ' times, as explained for "s * n" under Common Sequence ' 'Operations.\n', - 'unary': '\n' - 'Unary arithmetic and bitwise operations\n' + 'unary': 'Unary arithmetic and bitwise operations\n' '***************************************\n' '\n' 'All unary arithmetic and bitwise operations have the same ' @@ -12849,8 +12824,7 @@ 'In all three cases, if the argument does not have the proper type, ' 'a\n' '"TypeError" exception is raised.\n', - 'while': '\n' - 'The "while" statement\n' + 'while': 'The "while" statement\n' '*********************\n' '\n' 'The "while" statement is used for repeated execution as long as an\n' @@ -12874,8 +12848,7 @@ 'executed in the first suite skips the rest of the suite and goes ' 'back\n' 'to testing the expression.\n', - 'with': '\n' - 'The "with" statement\n' + 'with': 'The "with" statement\n' '********************\n' '\n' 'The "with" statement is used to wrap the execution of a block with\n' @@ -12948,8 +12921,7 @@ ' The specification, background, and examples for the Python ' '"with"\n' ' statement.\n', - 'yield': '\n' - 'The "yield" statement\n' + 'yield': 'The "yield" statement\n' '*********************\n' '\n' ' yield_stmt ::= yield_expression\n' From e0fbe5feee4f9c00f09eb9659c2182183036261a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 4 Mar 2017 12:25:04 -0500 Subject: [PATCH 090/232] Update docs and patchlevel for 3.6.1rc1. --- Include/patchlevel.h | 8 ++++---- Misc/NEWS | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 49930e85565349..b42a0db2f82007 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 -#define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL -#define PY_RELEASE_SERIAL 0 +#define PY_MICRO_VERSION 1 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.0+" +#define PY_VERSION "3.6.1rc1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 9aee4974344e10..d715d306a34bc3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,10 +2,10 @@ Python News +++++++++++ -What's New in Python 3.6.1 release candidate 1? -=============================================== +What's New in Python 3.6.1 release candidate 1 +============================================== -*Release date: XXXX-XX-XX* +*Release date: 2017-03-04* Core and Builtins ----------------- From 43571dde7b81f8426e01875c611d29274135355a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 4 Mar 2017 15:40:48 -0800 Subject: [PATCH 091/232] Correct spelling "instanciate" (GH-465) (GH-467) (cherry picked from commit 6abaed0ddaa1dd9be727ede09f6cd801c467c2ec) --- Doc/library/asyncio-subprocess.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index dc93a74c6dee12..16ba9a3cd6c7f2 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -80,7 +80,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. however, where :class:`~subprocess.Popen` takes a single argument which is list of strings, :func:`subprocess_exec` takes multiple string arguments. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. Other parameters: @@ -123,7 +123,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. using the platform's "shell" syntax. This is similar to the standard library :class:`subprocess.Popen` class called with ``shell=True``. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. See :meth:`~AbstractEventLoop.subprocess_exec` for more details about From 9619ec99470100969a95a86ed8d7a1fab6c27546 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 4 Mar 2017 16:43:57 -0800 Subject: [PATCH 092/232] distutils docs: Fix a typo (GH-470) (GH-471) (cherry picked from commit 2a7bddaab7d6e1f7b243cdbb4fa6f6c8e266b18d) --- Doc/distutils/examples.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index 1f5be9cdb29a97..4e2761d8a7d046 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -321,7 +321,7 @@ You can read back this static file, by using the >>> metadata.description 'Easily download, build, install, upgrade, and uninstall Python packages' -Notice that the class can also be instanciated with a metadata file path to +Notice that the class can also be instantiated with a metadata file path to loads its values:: >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' From 38136e2a581ada0382d9bd1df5b32678cbd94b9a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 5 Mar 2017 05:33:38 -0500 Subject: [PATCH 093/232] Bump to v3.6.1rc1+. --- Include/patchlevel.h | 2 +- Misc/NEWS | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index b42a0db2f82007..b13794521f297c 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.1rc1" +#define PY_VERSION "3.6.1rc1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index d715d306a34bc3..b342738ac4b475 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,18 @@ Python News +++++++++++ +What's New in Python 3.6.1 final? +================================= + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +Library +------- + + What's New in Python 3.6.1 release candidate 1 ============================================== From 3405792b024e9c6b70c0d2355c55a23ac84e1e67 Mon Sep 17 00:00:00 2001 From: Petr Motejlek Date: Sun, 5 Mar 2017 17:14:06 +0100 Subject: [PATCH 094/232] bpo-29615: backport to 3.6 (#478) --- Lib/test/test_xmlrpc.py | 90 ++++++++++++++++++++++++++++++++++++++++- Lib/xmlrpc/server.py | 43 +++++++++++--------- Misc/NEWS | 4 ++ 3 files changed, 117 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index df9c79e3df0689..69de08e9c3b252 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -343,6 +343,94 @@ def run_server(): self.assertEqual(p.method(), 5) self.assertEqual(p.method(), 5) + +class SimpleXMLRPCDispatcherTestCase(unittest.TestCase): + class DispatchExc(Exception): + """Raised inside the dispatched functions when checking for + chained exceptions""" + + def test_call_registered_func(self): + """Calls explicitly registered function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + def dispatched_func(*params): + raise self.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(dispatched_func) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_instance_func(self): + """Calls a registered instance attribute as a function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + class DispatchedClass: + def dispatched_func(self, *params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(DispatchedClass()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_dispatch_func(self): + """Calls the registered instance's `_dispatch` function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_method = 'method' + exp_params = 1, 2, 3 + + class TestInstance: + def _dispatch(self, method, params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc( + method, params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(TestInstance()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch(exp_method, exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_method, exp_params)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_registered_func_is_none(self): + """Calls explicitly registered function which is None""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(None, name='method') + with self.assertRaises(Exception, expected_regex='method'): + dispatcher._dispatch('method', ('param',)) + + def test_instance_has_no_func(self): + """Attempts to call nonexistent function on a registered instance""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(object()) + with self.assertRaises(Exception, expected_regex='method'): + dispatcher._dispatch('method', ('param',)) + + def test_cannot_locate_func(self): + """Calls a function that the dispatcher cannot locate""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + with self.assertRaises(Exception, expected_regex='method'): + dispatcher._dispatch('method', ('param',)) + + class HelperTestCase(unittest.TestCase): def test_escape(self): self.assertEqual(xmlrpclib.escape("a&b"), "a&b") @@ -1312,7 +1400,7 @@ def test_main(): KeepaliveServerTestCase1, KeepaliveServerTestCase2, GzipServerTestCase, GzipUtilTestCase, MultiPathServerTestCase, ServerProxyTestCase, FailingServerTestCase, - CGIHandlerTestCase) + CGIHandlerTestCase, SimpleXMLRPCDispatcherTestCase) if __name__ == "__main__": diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 849bfddd84816f..6faa2d6f8fa319 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -386,31 +386,36 @@ def _dispatch(self, method, params): not be called. """ - func = None try: - # check to see if a matching function has been registered + # call the matching registered function func = self.funcs[method] except KeyError: - if self.instance is not None: - # check for a _dispatch method - if hasattr(self.instance, '_dispatch'): - return self.instance._dispatch(method, params) - else: - # call instance method directly - try: - func = resolve_dotted_attribute( - self.instance, - method, - self.allow_dotted_names - ) - except AttributeError: - pass - - if func is not None: - return func(*params) + pass else: + if func is not None: + return func(*params) raise Exception('method "%s" is not supported' % method) + if self.instance is not None: + if hasattr(self.instance, '_dispatch'): + # call the `_dispatch` method on the instance + return self.instance._dispatch(method, params) + + # call the instance's method directly + try: + func = resolve_dotted_attribute( + self.instance, + method, + self.allow_dotted_names + ) + except AttributeError: + pass + else: + if func is not None: + return func(*params) + + raise Exception('method "%s" is not supported' % method) + class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): """Simple XML-RPC request handler class. diff --git a/Misc/NEWS b/Misc/NEWS index b342738ac4b475..a12229857945c8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,10 @@ Core and Builtins Library ------- +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + What's New in Python 3.6.1 release candidate 1 ============================================== From b414e349eb6a6140a81ccec249bae3abe939e836 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 5 Mar 2017 20:18:43 +0100 Subject: [PATCH 095/232] bpo-29638: Fix spurious refleaks after typing is imported (#469) (#483) --- Lib/test/libregrtest/refleak.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 3b3d2458b1f2e0..e2e58bd872f094 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -143,9 +143,14 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + abs_classes = filter(isabstract, abs_classes) + if 'typing' in sys.modules: + t = sys.modules['typing'] + # These classes require special treatment because they do not appear + # in direct subclasses of collections.abc classes + abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] + for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() obj._abc_cache.clear() From e544b40faa5ab61b6aba691577d90b2b641f664d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sun, 5 Mar 2017 19:58:06 -0800 Subject: [PATCH 096/232] Fixes the upload script to purge the CDN correctly and display success output. (#466) (#497) --- Tools/msi/uploadrelease.proj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 305e84fc2d4b28..75840f2f851ecd 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -8,6 +8,7 @@ $(TARGET) /srv/www.python.org/ftp/python true + true false false @@ -91,6 +92,7 @@ echo." /> + From 4e1a065c20856a00d0fe88ce022b249170608058 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 6 Mar 2017 17:24:28 +0900 Subject: [PATCH 097/232] bpo-29719: Remove Date and Release field in whatsnew/3.6 (GH-500) (cherry picked from commit 2225ddaa9e64c086b2b6997b0c9ac50921f7aa85) --- Doc/whatsnew/3.6.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 96fd256b991b76..a696af4a999c84 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2,8 +2,6 @@ What's New In Python 3.6 **************************** -:Release: |release| -:Date: |today| :Editors: Elvis Pranskevichus , Yury Selivanov .. Rules for maintenance: From df6d7b406f3d1b2e4e2014751bfa25574c4df222 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 6 Mar 2017 18:17:10 +0800 Subject: [PATCH 098/232] [3.6] bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. (GH-504) --- Lib/test/test_bytes.py | 10 ++++++++++ Misc/NEWS | 3 +++ Objects/bytesobject.c | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a103a7d39cae24..cd82fa64570cb5 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -507,6 +507,11 @@ def test_mod(self): a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') self.assertIs(type(a), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b = b % b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_imod(self): b = self.type2test(b'hello, %b!') @@ -519,6 +524,11 @@ def test_imod(self): b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') self.assertIs(type(b), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b %= b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_rmod(self): with self.assertRaises(TypeError): diff --git a/Misc/NEWS b/Misc/NEWS index a12229857945c8..1fb78365bb89be 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 final? Core and Builtins ----------------- +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + Library ------- diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 5d484409600852..801711f7e6ccaa 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -619,11 +619,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, Py_ssize_t len; char *pos; - pos = strchr(fmt + 1, '%'); + pos = (char *)memchr(fmt + 1, '%', fmtcnt); if (pos != NULL) len = pos - fmt; else - len = format_len - (fmt - format); + len = fmtcnt + 1; assert(len != 0); memcpy(res, fmt, len); From e9effe6bb6b6dcbc9fb13e47811a251a84a807c5 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 6 Mar 2017 09:15:41 -0800 Subject: [PATCH 099/232] fix minor bug in pymalloc. (GH-335) (GH-476) (cherry picked from commit f669ffff6010a94f2d441200e0fd73e9dea2883e) --- Doc/c-api/memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 3ff545275fb3b8..873fb2ac1d3cad 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -391,7 +391,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: +:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: From 130c4ec5fbeb0878b88ec6aa06e47c9672566c17 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 6 Mar 2017 09:31:16 -0800 Subject: [PATCH 100/232] bpo-29557: Remove ambiguous line in binhex docs (GH-90) (GH-475) "appears to not work in all cases" does not inspire confidence in this module. I can find no context for what bug this was referencing so it should be removed. (cherry picked from commit 6de2b7817fa9403e81dc38f13f3690f0bbf3d064) --- Doc/library/binhex.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst index 359ab23b2f9787..2966e0dbfbcfe8 100644 --- a/Doc/library/binhex.rst +++ b/Doc/library/binhex.rst @@ -55,5 +55,3 @@ the source for details. If you code or decode textfiles on non-Macintosh platforms they will still use the old Macintosh newline convention (carriage-return as end of line). -As of this writing, :func:`hexbin` appears to not work in all cases. - From 284a2ba140cf11216e9804573d8cea6674ab90d9 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 7 Mar 2017 11:53:29 +0800 Subject: [PATCH 101/232] bpo-28728: clarify possible test failure due to ISP (GH-412) (GH-531) --- Lib/test/test_socket.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 97dc3cd76b24a5..5315d475df2218 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -803,11 +803,6 @@ def testHostnameRes(self): self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) def test_host_resolution(self): - for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', - '1:1:1:1:1:1:1:1:1']: - self.assertRaises(OSError, socket.gethostbyname, addr) - self.assertRaises(OSError, socket.gethostbyaddr, addr) - for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: self.assertEqual(socket.gethostbyname(addr), addr) @@ -816,6 +811,21 @@ def test_host_resolution(self): for host in [support.HOST]: self.assertIn(host, socket.gethostbyaddr(host)[2]) + def test_host_resolution_bad_address(self): + # These are all malformed IP addresses and expected not to resolve to + # any result. But some ISPs, e.g. AWS, may successfully resolve these + # IPs. + explanation = ( + "resolving an invalid IP address did not raise OSError; " + "can be caused by a broken DNS server" + ) + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + with self.assertRaises(OSError): + socket.gethostbyname(addr) + with self.assertRaises(OSError, msg=explanation): + socket.gethostbyaddr(addr) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") def test_sethostname(self): From 33d70d1721ed8a23d2744f33d5358657adccf4cd Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 7 Mar 2017 17:18:12 +0900 Subject: [PATCH 102/232] PCbuild: Add -q option to svn export (GH-537) Without this option, AppVeyor log is too unreadable. (cherry picked from commit 8886d5f39286dffa7d9337857b151e7fb4af23fd) --- PCbuild/get_externals.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 98a755d870ad90..3a2656e961ef7f 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -66,7 +66,7 @@ for %%e in (%libraries%) do ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - svn export %SVNROOT%%%e + svn export -q %SVNROOT%%%e ) ) From 23b26c4e206427a9e0a1d39240c17a049cf8c732 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 7 Mar 2017 02:44:11 -0800 Subject: [PATCH 103/232] Fixes bpo-29680: Older gdb does not have gdb.error. (GH-363) (GH-534) This change is required to make python-dbg.py compatible with GDB versions before 7.3. (cherry picked from commit 661ca8843fed1183e38db06e52d59ac300bf1c2a) --- Tools/gdb/libpython.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index cc1afbe16d01d1..d28851f9fd7f27 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -714,7 +714,7 @@ def _get_entries(self, keys): try: # <= Python 3.5 return keys['dk_entries'], dk_size - except gdb.error: + except RuntimeError: # >= Python 3.6 pass From 78ad039bcf1a8c494cbc8e18380cc30665869c3e Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 8 Mar 2017 11:43:35 +0800 Subject: [PATCH 104/232] bpo-26915: Test identity first in index() and count() of collections.abc.Sequence (GH-553) --- Lib/_collections_abc.py | 5 +++-- Lib/test/test_collections.py | 17 +++++++++++++---- Misc/NEWS | 3 +++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index b172f3f360e6ef..005d884572465f 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -908,7 +908,8 @@ def index(self, value, start=0, stop=None): i = start while stop is None or i < stop: try: - if self[i] == value: + v = self[i] + if v is value or v == value: return i except IndexError: break @@ -917,7 +918,7 @@ def index(self, value, start=0, stop=None): def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' - return sum(1 for v in self if v == value) + return sum(1 for v in self if v is value or v == value) Sequence.register(tuple) Sequence.register(str) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 87454cc6704cb0..47f756213ddf00 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1310,20 +1310,29 @@ def test_issue26915(self): class CustomEqualObject: def __eq__(self, other): return False - class CustomSequence(list): - def __contains__(self, value): - return Sequence.__contains__(self, value) + class CustomSequence(Sequence): + def __init__(self, seq): + self._seq = seq + def __getitem__(self, index): + return self._seq[index] + def __len__(self): + return len(self._seq) nan = float('nan') obj = CustomEqualObject() + seq = CustomSequence([nan, obj, nan]) containers = [ - CustomSequence([nan, obj]), + seq, ItemsView({1: nan, 2: obj}), ValuesView({1: nan, 2: obj}) ] for container in containers: for elem in container: self.assertIn(elem, container) + self.assertEqual(seq.index(nan), 0) + self.assertEqual(seq.index(obj), 1) + self.assertEqual(seq.count(nan), 2) + self.assertEqual(seq.count(obj), 1) def assertSameSet(self, s1, s2): # coerce both to a real set then check equality diff --git a/Misc/NEWS b/Misc/NEWS index 1fb78365bb89be..7a03a2d65e35bc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Core and Builtins Library ------- +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + - bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek. From 6a4b04cd337347d074ae0140fb13dca5bd4b11ef Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 7 Mar 2017 23:56:59 -0800 Subject: [PATCH 105/232] Revert "bpo-29571: Use correct locale encoding in test_re (#149)" (#554) (#555) This reverts commit ace5c0fdd9b962e6e886c29dbcea72c53f051dc4. --- Lib/test/test_re.py | 2 +- Misc/NEWS | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a506b98be4aeb2..b945cf094e99c1 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1402,7 +1402,7 @@ def test_ascii_and_unicode_flag(self): def test_locale_flag(self): import locale - enc = locale.getpreferredencoding(False) + _, enc = locale.getlocale(locale.LC_CTYPE) # Search non-ASCII letter for i in range(128, 256): try: diff --git a/Misc/NEWS b/Misc/NEWS index 7a03a2d65e35bc..bf3876948f0a26 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -134,6 +134,7 @@ Library - bpo-29110: Fix file object leak in aifc.open() when file is given as a filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. +- Issue #24932: Use proper command line parsing in _testembed - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, From 666165fddf499e80d10a5d9263846ec4534f6a2a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 8 Mar 2017 12:29:33 +0200 Subject: [PATCH 106/232] [3.6] bpo-28230: Document the pathlib support in tarfile and add tests. (#559) --- Doc/library/tarfile.rst | 28 ++++++++++++--- Lib/test/test_tarfile.py | 76 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index d8f809753dfdd7..337c061107299c 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -146,6 +146,10 @@ Some facts and figures: .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. class:: TarFile Class for reading and writing tar archives. Do not use this class directly: @@ -266,7 +270,8 @@ be finalized; only the internally used file object will be closed. See the All following arguments are optional and can be accessed as instance attributes as well. - *name* is the pathname of the archive. It can be omitted if *fileobj* is given. + *name* is the pathname of the archive. *name* may be a :term:`path-like object`. + It can be omitted if *fileobj* is given. In this case, the file object's :attr:`name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append @@ -319,6 +324,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. classmethod:: TarFile.open(...) Alternative constructor. The :func:`tarfile.open` function is actually a @@ -390,14 +399,17 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* may be a filename or a :class:`TarInfo` object. You can specify a different - directory using *path*. File attributes (owner, mtime, mode) are set unless - *set_attrs* is false. + directory using *path*. *path* may be a :term:`path-like object`. + File attributes (owner, mtime, mode) are set unless *set_attrs* is false. If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile are used to set the owner/group for the extracted files. Otherwise, the named @@ -418,6 +430,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + + .. method:: TarFile.extractfile(member) Extract a member from the archive as a file object. *member* may be a filename @@ -464,7 +480,8 @@ be finalized; only the internally used file object will be closed. See the Create a :class:`TarInfo` object from the result of :func:`os.stat` or equivalent on an existing file. The file is either named by *name*, or - specified as a :term:`file object` *fileobj* with a file descriptor. If + specified as a :term:`file object` *fileobj* with a file descriptor. + *name* may be a :term:`path-like object`. If given, *arcname* specifies an alternative name for the file in the archive, otherwise, the name is taken from *fileobj*’s :attr:`~io.FileIO.name` attribute, or the *name* argument. The name @@ -478,6 +495,9 @@ be finalized; only the internally used file object will be closed. See the The :attr:`~TarInfo.name` may also be modified, in which case *arcname* could be a dummy string. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + .. method:: TarFile.close() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 619cbc03b28abc..fc79055421161c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -4,6 +4,7 @@ from hashlib import md5 from contextlib import contextmanager from random import Random +import pathlib import unittest import unittest.mock @@ -440,6 +441,22 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) + def test_pathlike_name(self): + tarname = pathlib.Path(self.tarname) + with tarfile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with self.taropen(tarname) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with tarfile.TarFile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + if self.suffix == '': + with tarfile.TarFile(tarname, mode='r') as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + def test_illegal_mode_arg(self): with open(tmpname, 'wb'): pass @@ -582,6 +599,26 @@ def test_extract_directory(self): finally: support.rmtree(DIR) + def test_extractall_pathlike_name(self): + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + for tarinfo in directories: + path = DIR / tarinfo.name + self.assertEqual(os.path.getmtime(path), tarinfo.mtime) + + def test_extract_pathlike_name(self): + dirtype = "ustar/dirtype" + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + tarinfo = tar.getmember(dirtype) + tar.extract(tarinfo, path=DIR) + extracted = DIR / dirtype + self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) + def test_init_close_fobj(self): # Issue #7341: Close the internal file object in the TarFile # constructor in case of an error. For the test we rely on @@ -1092,6 +1129,17 @@ def test_directory_size(self): finally: support.rmdir(path) + def test_gettarinfo_pathlike_name(self): + with tarfile.open(tmpname, self.mode) as tar: + path = pathlib.Path(TEMPDIR) / "file" + with open(path, "wb") as fobj: + fobj.write(b"aaa") + tarinfo = tar.gettarinfo(path) + tarinfo2 = tar.gettarinfo(os.fspath(path)) + self.assertIsInstance(tarinfo.name, str) + self.assertEqual(tarinfo.name, tarinfo2.name) + self.assertEqual(tarinfo.size, 3) + @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1528,6 +1576,34 @@ def test_create_existing_taropen(self): self.assertEqual(len(names), 1) self.assertIn("spameggs42", names[0]) + def test_create_pathlike_name(self): + with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + def test_create_taropen_pathlike_name(self): + with self.taropen(pathlib.Path(tmpname), "x") as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + class GzipCreateTest(GzipTest, CreateTest): pass From eb65edd1029876a4a5bb70b009aeb914088ac749 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 8 Mar 2017 15:45:43 +0200 Subject: [PATCH 107/232] [3.6] bpo-28231: The zipfile module now accepts path-like objects for external paths. (#561) (cherry picked from commit 8606e9524a7a4065042f7f228dc57eb74f88e4d3) --- Doc/library/zipfile.rst | 24 +++++++- Lib/test/test_zipfile.py | 117 ++++++++++++++++++++++++++++++++++----- Lib/zipfile.py | 20 +++++-- Misc/NEWS | 3 + 4 files changed, 143 insertions(+), 21 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5eb6f103380661..4c91759c67f5d4 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -132,8 +132,9 @@ ZipFile Objects .. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) - Open a ZIP file, where *file* can be either a path to a file (a string) or a - file-like object. The *mode* parameter should be ``'r'`` to read an existing + Open a ZIP file, where *file* can be a path to a file (a string), a + file-like object or a :term:`path-like object`. + The *mode* parameter should be ``'r'`` to read an existing file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an existing file, or ``'x'`` to exclusively create and write a new file. If *mode* is ``'x'`` and *file* refers to an existing file, @@ -183,6 +184,9 @@ ZipFile Objects Previously, a plain :exc:`RuntimeError` was raised for unrecognized compression values. + .. versionchanged:: 3.6.2 + The *file* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.close() @@ -284,6 +288,9 @@ ZipFile Objects Calling :meth:`extract` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.extractall(path=None, members=None, pwd=None) @@ -304,6 +311,9 @@ ZipFile Objects Calling :meth:`extractall` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.printdir() @@ -403,6 +413,9 @@ ZipFile Objects The following data attributes are also available: +.. attribute:: ZipFile.filename + + Name of the ZIP file. .. attribute:: ZipFile.debug @@ -488,6 +501,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionadded:: 3.4 The *filterfunc* parameter. + .. versionchanged:: 3.6.2 + The *pathname* parameter accepts a :term:`path-like object`. + .. _zipinfo-objects: @@ -514,6 +530,10 @@ file: .. versionadded:: 3.6 + .. versionchanged:: 3.6.2 + The *filename* parameter accepts a :term:`path-like object`. + + Instances have the following methods and attributes: .. method:: ZipInfo.is_dir() diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 0a43b20e2bcf0d..d39f05fdb7788e 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2,6 +2,7 @@ import io import os import importlib.util +import pathlib import posixpath import time import struct @@ -13,7 +14,7 @@ from random import randint, random, getrandbits from test.support import script_helper -from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, +from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, temp_cwd, requires_zlib, requires_bz2, requires_lzma, captured_stdout, check_warnings) @@ -148,6 +149,12 @@ def test_open(self): for f in get_files(self): self.zip_open_test(f, self.compression) + def test_open_with_pathlike(self): + path = pathlib.Path(TESTFN2) + self.zip_open_test(path, self.compression) + with zipfile.ZipFile(path, "r", self.compression) as zipfp: + self.assertIsInstance(zipfp.filename, str) + def zip_random_open_test(self, f, compression): self.make_test_archive(f, compression) @@ -906,22 +913,56 @@ def test_write_pyfile_bad_syntax(self): finally: rmtree(TESTFN2) + def test_write_pathlike(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w") as fp: + fp.write("print(42)\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + finally: + rmtree(TESTFN2) + class ExtractTests(unittest.TestCase): - def test_extract(self): + + def make_test_file(self): with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: for fpath, fdata in SMALL_TEST_DATA: zipfp.writestr(fpath, fdata) + def test_extract(self): + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + writtenfile = zipfp.extract(fpath) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.normpath(correctfile) + + self.assertEqual(writtenfile, correctfile) + + # make sure correct data is in correct file + with open(writtenfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(writtenfile) + + def _test_extract_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: for fpath, fdata in SMALL_TEST_DATA: - writtenfile = zipfp.extract(fpath) + writtenfile = zipfp.extract(fpath, target) # make sure it was written to the right place - correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.join(target, fpath) correctfile = os.path.normpath(correctfile) - - self.assertEqual(writtenfile, correctfile) + self.assertTrue(os.path.samefile(writtenfile, correctfile), (writtenfile, target)) # make sure correct data is in correct file with open(writtenfile, "rb") as f: @@ -929,26 +970,50 @@ def test_extract(self): unlink(writtenfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_with_target(self): + with temp_dir() as extdir: + self._test_extract_with_target(extdir) + + def test_extract_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_with_target(pathlib.Path(extdir)) def test_extract_all(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + zipfp.extractall() + for fpath, fdata in SMALL_TEST_DATA: + outfile = os.path.join(os.getcwd(), fpath) + + with open(outfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(outfile) + def _test_extract_all_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: - zipfp.extractall() + zipfp.extractall(target) for fpath, fdata in SMALL_TEST_DATA: - outfile = os.path.join(os.getcwd(), fpath) + outfile = os.path.join(target, fpath) with open(outfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) unlink(outfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_all_with_target(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(extdir) + + def test_extract_all_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(pathlib.Path(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1188,6 +1253,8 @@ def test_is_zip_erroneous_file(self): with open(TESTFN, "w") as fp: fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a path-like object + self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2033,6 +2100,26 @@ def test_from_file(self): zi = zipfile.ZipInfo.from_file(__file__) self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_pathlike(self): + zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_bytes(self): + zi = zipfile.ZipInfo.from_file(os.fsencode(__file__), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_fileno(self): + with open(__file__, 'rb') as f: + zi = zipfile.ZipInfo.from_file(f.fileno(), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_dir(self): dirpath = os.path.dirname(os.path.abspath(__file__)) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 7f2b43ce1cc848..d7f5beba996e6e 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -479,6 +479,8 @@ def from_file(cls, filename, arcname=None): this will be the same as filename, but without a drive letter and with leading path separators removed). """ + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) @@ -1070,6 +1072,8 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): self._comment = b'' # Check if we were passed a file-like object + if isinstance(file, os.PathLike): + file = os.fspath(file) if isinstance(file, str): # No, it's a filename self._filePassed = 0 @@ -1470,11 +1474,10 @@ def extract(self, member, path=None, pwd=None): as possible. `member' may be a filename or a ZipInfo object. You can specify a different directory using `path'. """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - if path is None: path = os.getcwd() + else: + path = os.fspath(path) return self._extract_member(member, path, pwd) @@ -1487,8 +1490,13 @@ def extractall(self, path=None, members=None, pwd=None): if members is None: members = self.namelist() + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + for zipinfo in members: - self.extract(zipinfo, path, pwd) + self._extract_member(zipinfo, path, pwd) @classmethod def _sanitize_windows_name(cls, arcname, pathsep): @@ -1509,6 +1517,9 @@ def _extract_member(self, member, targetpath, pwd): """Extract the ZipInfo object 'member' to a physical file on the path targetpath. """ + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + # build the destination pathname, replacing # forward slashes to platform specific separators. arcname = member.filename.replace('/', os.path.sep) @@ -1801,6 +1812,7 @@ def writepy(self, pathname, basename="", filterfunc=None): If filterfunc(pathname) is given, it is called with every argument. When it is False, the file or directory is skipped. """ + pathname = os.fspath(pathname) if filterfunc and not filterfunc(pathname): if self.debug: label = 'path' if os.path.isdir(pathname) else 'file' diff --git a/Misc/NEWS b/Misc/NEWS index bf3876948f0a26..99ce8c73620714 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Core and Builtins Library ------- +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + - bpo-26915: index() and count() methods of collections.abc.Sequence now check identity before checking equality when do comparisons. From 7253aded71df8d4bf8684fc78d11c596079211b3 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 8 Mar 2017 08:19:57 -0800 Subject: [PATCH 108/232] bpo-29176: Fix name of the _curses.window class (#52) (#532) Set name to "_curses.window" instead of "_curses.curses window" (with a space!?). (cherry picked from commit 61e2bc74dfab1ceee332d3f480dcf86c478c87c5) --- Modules/_cursesmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 3bf2ca7ce15392..d88d06e78220ec 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -2082,7 +2082,7 @@ static PyGetSetDef PyCursesWindow_getsets[] = { PyTypeObject PyCursesWindow_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_curses.curses window", /*tp_name*/ + "_curses.window", /*tp_name*/ sizeof(PyCursesWindowObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ From 51b646a55a4a1c4d0df764c4404a062fbcc6b356 Mon Sep 17 00:00:00 2001 From: "n.d. parker" Date: Wed, 8 Mar 2017 23:27:46 +0100 Subject: [PATCH 109/232] Fix the only non-C90 comment to be C90 compatible. (#568) --- Include/pyport.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/pyport.h b/Include/pyport.h index 52a91a0d11d056..426822a81f075a 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -37,7 +37,7 @@ Used in: Py_SAFE_DOWNCAST * integral synonyms. Only define the ones we actually need. */ -// long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. +/* long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. */ #ifndef HAVE_LONG_LONG #define HAVE_LONG_LONG 1 #endif From 9cef253ae3251b22d2fbb475fcb7a28805b78c86 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 9 Mar 2017 10:51:44 +0200 Subject: [PATCH 110/232] [3.6] bpo-29768: Fixed compile-time check for expat version. (#576) (cherry picked from commit 22e707fa04476710ba5cc7e2206e4ac66743931b) --- Modules/pyexpat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b3259d57aa114c..47f70a27f15846 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1191,7 +1191,7 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec Py_DECREF(self); return NULL; } -#if ((XML_MAJOR_VERSION >= 2) && (XML_MINOR_VERSION >= 1)) || defined(XML_HAS_SET_HASH_SALT) +#if XML_COMBINED_VERSION >= 20100 || defined(XML_HAS_SET_HASH_SALT) /* This feature was added upstream in libexpat 2.1.0. Our expat copy * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT * to indicate that we can still use it. */ From 26d013e00f9d4adbcf3e084bbc890c799ff70407 Mon Sep 17 00:00:00 2001 From: orenmn Date: Thu, 9 Mar 2017 16:06:47 +0200 Subject: [PATCH 111/232] [3.6] bpo-28298: make array 'Q', 'L' and 'I' accept big intables as elements (#579) --- Lib/test/test_array.py | 39 +++++++++------ Misc/NEWS | 3 ++ Modules/arraymodule.c | 108 +++++++++++++++++++++++------------------ 3 files changed, 89 insertions(+), 61 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 1f8967ccff3e7c..d67f9195eba04c 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -14,14 +14,6 @@ import array from array import _array_reconstructor as array_reconstructor -try: - # Try to determine availability of long long independently - # of the array module under test - struct.calcsize('@q') - have_long_long = True -except struct.error: - have_long_long = False - sizeof_wchar = array.array('u').itemsize @@ -32,9 +24,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = "ubBhHiIlLfd" -if have_long_long: - typecodes += 'qQ' +typecodes = 'ubBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -1240,7 +1230,26 @@ def test_frombytearray(self): b = array.array(self.typecode, a) self.assertEqual(a, b) -class SignedNumberTest(NumberTest): +class IntegerNumberTest(NumberTest): + def test_type_error(self): + a = array.array(self.typecode) + a.append(42) + with self.assertRaises(TypeError): + a.append(42.0) + with self.assertRaises(TypeError): + a[0] = 42.0 + +class Intable: + def __init__(self, num): + self._num = num + def __int__(self): + return self._num + def __sub__(self, other): + return Intable(int(self) - int(other)) + def __add__(self, other): + return Intable(int(self) + int(other)) + +class SignedNumberTest(IntegerNumberTest): example = [-1, 0, 1, 42, 0x7f] smallerexample = [-1, 0, 1, 42, 0x7e] biggerexample = [-1, 0, 1, 43, 0x7f] @@ -1251,8 +1260,9 @@ def test_overflow(self): lower = -1 * int(pow(2, a.itemsize * 8 - 1)) upper = int(pow(2, a.itemsize * 8 - 1)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) -class UnsignedNumberTest(NumberTest): +class UnsignedNumberTest(IntegerNumberTest): example = [0, 1, 17, 23, 42, 0xff] smallerexample = [0, 1, 17, 23, 42, 0xfe] biggerexample = [0, 1, 17, 23, 43, 0xff] @@ -1263,6 +1273,7 @@ def test_overflow(self): lower = 0 upper = int(pow(2, a.itemsize * 8)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) def test_bytes_extend(self): s = bytes(self.example) @@ -1314,12 +1325,10 @@ class UnsignedLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'L' minitemsize = 4 -@unittest.skipIf(not have_long_long, 'need long long support') class LongLongTest(SignedNumberTest, unittest.TestCase): typecode = 'q' minitemsize = 8 -@unittest.skipIf(not have_long_long, 'need long long support') class UnsignedLongLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'Q' minitemsize = 8 diff --git a/Misc/NEWS b/Misc/NEWS index 99ce8c73620714..299a8388ff6e39 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Core and Builtins Library ------- +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + - bpo-28231: The zipfile module now accepts path-like objects for external paths. diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2caa8ee5a8e892..d4221fe4c6a163 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -331,35 +331,51 @@ II_getitem(arrayobject *ap, Py_ssize_t i) (unsigned long) ((unsigned int *)ap->ob_item)[i]); } +static PyObject * +get_int_unless_float(PyObject *v) +{ + if (PyFloat_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "array item must be integer"); + return NULL; + } + return (PyObject *)_PyLong_FromNbInt(v); +} + static int II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned int is less than minimum"); - return -1; + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long)y; - + return -1; } if (x > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, - "unsigned int is greater than maximum"); + "unsigned int is greater than maximum"); + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -390,31 +406,28 @@ static int LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) - return -1; - } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is less than minimum"); + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; } - x = (unsigned long)y; - + do_decref = 1; } - if (x > ULONG_MAX) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is greater than maximum"); + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -446,25 +459,28 @@ static int QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLongLong(v); - if (x == (unsigned long long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long long y; - if (!PyArg_Parse(v, "L;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long long is less than minimum"); - return -1; + x = PyLong_AsUnsignedLongLong(v); + if (x == (unsigned long long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long long)y; + return -1; } - if (i >= 0) ((unsigned long long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } From 68d29809405dc766966b2b973b8597212fbc3dbd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 9 Mar 2017 18:43:39 +0100 Subject: [PATCH 112/232] bpo-29619: Convert st_ino using unsigned integer (#557) (#584) bpo-29619: os.stat() and os.DirEntry.inodeo() now convert inode (st_ino) using unsigned integers. (cherry picked from commit 0f6d73343d342c106cda2219ebb8a6f0c4bd9b3c) (Misc/NEWS conflict handled manually.) --- Include/fileutils.h | 2 +- Modules/posixmodule.c | 13 ++++++++----- Python/fileutils.c | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Include/fileutils.h b/Include/fileutils.h index b933e985392e73..900c70faad719f 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -22,7 +22,7 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #ifdef MS_WINDOWS struct _Py_stat_struct { unsigned long st_dev; - __int64 st_ino; + uint64_t st_ino; unsigned short st_mode; int st_nlink; int st_uid; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8f8ba255ec48f6..2ea5e2def9acff 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1932,11 +1932,13 @@ _pystat_fromstructstat(STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); -#ifdef HAVE_LARGEFILE_SUPPORT +#if defined(HAVE_LARGEFILE_SUPPORT) || defined(MS_WINDOWS) + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(st->st_ino)); PyStructSequence_SET_ITEM(v, 1, - PyLong_FromLongLong((long long)st->st_ino)); + PyLong_FromUnsignedLongLong(st->st_ino)); #else - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long)st->st_ino)); + Py_BUILD_ASSERT(sizeof(unsigned long) >= sizeof(st->st_ino)); + PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLong(st->st_ino)); #endif #ifdef MS_WINDOWS PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); @@ -11156,7 +11158,7 @@ typedef struct { PyObject *lstat; #ifdef MS_WINDOWS struct _Py_stat_struct win32_lstat; - __int64 win32_file_index; + uint64_t win32_file_index; int got_file_index; #else /* POSIX */ #ifdef HAVE_DIRENT_D_TYPE @@ -11419,7 +11421,8 @@ DirEntry_inode(DirEntry *self) self->win32_file_index = stat.st_ino; self->got_file_index = 1; } - return PyLong_FromLongLong((long long)self->win32_file_index); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->win32_file_index)); + return PyLong_FromUnsignedLongLong(self->win32_file_index); #else /* POSIX */ #ifdef HAVE_LARGEFILE_SUPPORT return PyLong_FromLongLong((long long)self->d_ino); diff --git a/Python/fileutils.c b/Python/fileutils.c index e84d66e99a473d..f3764e4b3c5a2d 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -583,7 +583,7 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); result->st_nlink = info->nNumberOfLinks; - result->st_ino = (((__int64)info->nFileIndexHigh)<<32) + info->nFileIndexLow; + result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; if (reparse_tag == IO_REPARSE_TAG_SYMLINK) { /* first clear the S_IFMT bits */ result->st_mode ^= (result->st_mode & S_IFMT); @@ -653,7 +653,7 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) _Py_attribute_data_to_stat(&info, 0, status); /* specific to fstat() */ - status->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; + status->st_ino = (((uint64_t)info.nFileIndexHigh) << 32) + info.nFileIndexLow; return 0; #else return fstat(fd, status); From 80dfbe30b734f14f76a4e25c7fa8fdb7bdd3936a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 9 Mar 2017 21:01:49 +0200 Subject: [PATCH 113/232] [3.6] bpo-29773: Add more cases for testing string to float conversion errors. (#586) (cherry picked from commit 9e6ac83acae31de2b072e665e177db9fcdf7c049) --- Lib/test/test_float.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index ac8473db503b76..6491f458c3acc4 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -119,15 +119,27 @@ def test_float_memoryview(self): self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3) def test_error_message(self): - testlist = ('\xbd', '123\xbd', ' 123 456 ') - for s in testlist: - try: + def check(s): + with self.assertRaises(ValueError, msg='float(%r)' % (s,)) as cm: float(s) - except ValueError as e: - self.assertIn(s.strip(), e.args[0]) - else: - self.fail("Expected int(%r) to raise a ValueError", s) - + self.assertEqual(str(cm.exception), + 'could not convert string to float: %r' % (s,)) + + check('\xbd') + check('123\xbd') + check(' 123 456 ') + check(b' 123 456 ') + + # non-ascii digits (error came from non-digit '!') + check('\u0663\u0661\u0664!') + # embedded NUL + check('123\x00') + check('123\x00 245') + check('123\x00245') + # byte string with embedded NUL + check(b'123\x00') + # non-UTF-8 byte string + check(b'123\xa0') @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') def test_float_with_comma(self): From d9b0b8f63c3db38dc547e0c115e9060f13f2dbcf Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Fri, 10 Mar 2017 01:06:43 -0600 Subject: [PATCH 114/232] Add Appveyor (GH-324 backport) (GH-491) --- .github/appveyor.yml | 6 ++++++ README.rst | 4 ++++ 2 files changed, 10 insertions(+) create mode 100644 .github/appveyor.yml diff --git a/.github/appveyor.yml b/.github/appveyor.yml new file mode 100644 index 00000000000000..5a40005660b781 --- /dev/null +++ b/.github/appveyor.yml @@ -0,0 +1,6 @@ +version: 3.6.1rc1+.{build} +clone_depth: 5 +build_script: +- cmd: PCbuild\build.bat -e +test_script: +- cmd: PCbuild\rt.bat -q -uall -rwW --slowest --timeout=1200 -j0 diff --git a/README.rst b/README.rst index 242572c5e361db..6b4c2dba306ca5 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,10 @@ This is Python version 3.6.1 release candidate 1 :alt: CPython build status on Travis CI :target: https://travis-ci.org/python/cpython +.. image:: https://ci.appveyor.com/api/projects/status/4mew1a93xdkbf5ua/branch/3.6?svg=true + :alt: CPython build status on Appveyor + :target: https://ci.appveyor.com/project/python/cpython/branch/3.6 + .. image:: https://codecov.io/gh/python/cpython/branch/3.6/graph/badge.svg :alt: CPython code coverage on Codecov :target: https://codecov.io/gh/python/cpython From faaa80fb9887950e7068a9506e3517bcd6562ac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 10 Mar 2017 05:30:05 -0800 Subject: [PATCH 115/232] Ignore Include/pydtrace_probes.h (#596) This is a generated file when --with-dtrace is passed. This entry is present in .hgignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ed4ebfbbd9de74..db27597aac57da 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ .gdb_history Doc/build/ Doc/venv/ +Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* From ff6f3716279e75b2519133a82b9de0c3601963d9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 10 Mar 2017 09:52:03 -0800 Subject: [PATCH 116/232] bpo-28739: Document that f-strings cannot be used as docstring (GH-592) (GH-600) (cherry picked from commit d4e89287b397c7382c12d3f3d9fd901fd8243b3c) --- Doc/reference/lexical_analysis.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index da7017afff0a3d..7f9c66481785c1 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -696,6 +696,17 @@ a temporary variable. >>> f"newline: {newline}" 'newline: 10' +Formatted string literals cannot be used as docstrings, even if they do not +include expressions. + +:: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> foo.__doc__ is None + True + See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. From 68c5e4c592495f1e0f4f60acb3483beb6aa87be1 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 10 Mar 2017 14:44:48 -0800 Subject: [PATCH 117/232] bpo-28810: Document BUILD_TUPLE_UNPACK_WITH_CALL bytecode (GH-605) (cherry picked from commit 7e52c3e7aefb4cdaa0662fc01ff68a5e976b77ca) --- Doc/library/dis.rst | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 1c1e1a2aa5d4b3..686118f5201857 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -807,6 +807,15 @@ All of the following opcodes use their arguments. .. versionadded:: 3.5 +.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, + but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position + ``count + 1`` should be the corresponding callable ``f``. + + .. versionadded:: 3.6 + + .. opcode:: BUILD_LIST_UNPACK (count) This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list @@ -834,14 +843,16 @@ All of the following opcodes use their arguments. .. versionadded:: 3.5 -.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (oparg) +.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (count) This is similar to :opcode:`BUILD_MAP_UNPACK`, - but is used for ``f(**x, **y, **z)`` call syntax. The lowest byte of - *oparg* is the count of mappings, the relative position of the - corresponding callable ``f`` is encoded in the second byte of *oparg*. + but is used for ``f(**x, **y, **z)`` call syntax. The stack item at + position ``count + 2`` should be the corresponding callable ``f``. .. versionadded:: 3.5 + .. versionchanged:: 3.6 + The position of the callable is determined by adding 2 to the opcode + argument instead of encoding it in the second byte of the argument. .. opcode:: LOAD_ATTR (namei) From 4807fa8386f825d0c71192c59f7e33b99a94bebe Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 10 Mar 2017 15:08:52 -0800 Subject: [PATCH 118/232] bpo-28810: Document changes to CALL_FUNCTION opcodes (GH-607) (cherry picked from commit 4b2a2a425a906c8e4eb8daee14ab1793e225f726) --- Doc/library/dis.rst | 69 +++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 686118f5201857..11d056901b7ad5 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1009,14 +1009,45 @@ All of the following opcodes use their arguments. .. opcode:: CALL_FUNCTION (argc) - Calls a function. The low byte of *argc* indicates the number of positional - parameters, the high byte the number of keyword parameters. On the stack, the - opcode finds the keyword parameters first. For each keyword argument, the - value is on top of the key. Below the keyword parameters, the positional - parameters are on the stack, with the right-most parameter on top. Below the - parameters, the function object to call is on the stack. Pops all function - arguments, and the function itself off the stack, and pushes the return - value. + Calls a function. *argc* indicates the number of positional arguments. + The positional arguments are on the stack, with the right-most argument + on top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + This opcode is used only for calls with positional arguments. + + +.. opcode:: CALL_FUNCTION_KW (argc) + + Calls a function. *argc* indicates the number of arguments (positional + and keyword). The top element on the stack contains a tuple of keyword + argument names. Below the tuple, keyword arguments are on the stack, in + the order corresponding to the tuple. Below the keyword arguments, the + positional arguments are on the stack, with the right-most parameter on + top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + Keyword arguments are packed in a tuple instead of a dictionary, + *argc* indicates the total number of arguments + + +.. opcode:: CALL_FUNCTION_EX (flags) + + Calls a function. The lowest bit of *flags* indicates whether the + var-keyword argument is placed at the top of the stack. Below the + var-keyword argument, the var-positional argument is on the stack. + Below the arguments, the function object to call is placed. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. Note that this opcode pops at most three items + from the stack. Var-positional and var-keyword arguments are packed + by :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. + + .. versionadded:: 3.6 .. opcode:: MAKE_FUNCTION (argc) @@ -1049,28 +1080,6 @@ All of the following opcodes use their arguments. two most-significant bytes. -.. opcode:: CALL_FUNCTION_VAR (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the variable argument list, followed by - keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by explicit keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_VAR_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by the variable-arguments tuple, followed by explicit keyword and positional - arguments. - - .. opcode:: FORMAT_VALUE (flags) Used for implementing formatted literal strings (f-strings). Pops From 701f13ab930b62a634579951dc610f737f461135 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 10 Mar 2017 19:36:26 -0800 Subject: [PATCH 119/232] bpo-29784: Fix the reference to shutil.copy in the docs (GH-602) (GH-608) (cherry picked from commit 70ee0cd5c2a3dba82cb8e0c0742c012f9134c040) --- Doc/library/shutil.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index a85bf339ae1bc8..41e5bafa53d468 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -153,7 +153,7 @@ Directory and files operations is true and *src* is a symbolic link, *dst* will be a copy of the file *src* refers to. - :func:`copy` copies the file data and the file's permission + :func:`~shutil.copy` copies the file data and the file's permission mode (see :func:`os.chmod`). Other metadata, like the file's creation and modification times, is not preserved. To preserve all file metadata from the original, use @@ -302,7 +302,7 @@ Directory and files operations *src* and *dst*, and will be used to copy *src* to *dest* if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the :func:`copy_function`. The - default *copy_function* is :func:`copy2`. Using :func:`copy` as the + default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. From 16416c22f9b8d9e067506d3a98f661756c80389c Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Sat, 11 Mar 2017 14:07:30 +0800 Subject: [PATCH 120/232] bpo-29770: remove outdated PYO related info (GH-590) (GH-612) --- Doc/glossary.rst | 2 +- Doc/library/zipfile.rst | 6 +++--- Doc/library/zipimport.rst | 2 +- Doc/using/cmdline.rst | 4 ++-- Modules/main.c | 2 +- Modules/zipimport.c | 6 +++--- PC/getpathp.c | 4 ++-- PCbuild/rmpyc.py | 20 +++++++------------- PCbuild/rt.bat | 8 ++++---- Python/pylifecycle.c | 2 +- 10 files changed, 25 insertions(+), 31 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 41ee3d83b311fa..e07ab0df2fd68c 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -155,7 +155,7 @@ Glossary bytecode Python source code is compiled into bytecode, the internal representation of a Python program in the CPython interpreter. The bytecode is also - cached in ``.pyc`` and ``.pyo`` files so that executing the same file is + cached in ``.pyc`` files so that executing the same file is faster the second time (recompilation from source to bytecode can be avoided). This "intermediate language" is said to run on a :term:`virtual machine` that executes the machine code corresponding to diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 4c91759c67f5d4..a5d42118ba5176 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -464,12 +464,12 @@ The :class:`PyZipFile` constructor takes the same parameters as the added to the archive, compiling if necessary. If *pathname* is a file, the filename must end with :file:`.py`, and - just the (corresponding :file:`\*.py[co]`) file is added at the top level + just the (corresponding :file:`\*.pyc`) file is added at the top level (no path information). If *pathname* is a file that does not end with :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, and the directory is not a package directory, then all the files - :file:`\*.py[co]` are added at the top level. If the directory is a - package directory, then all :file:`\*.py[co]` are added under the package + :file:`\*.pyc` are added at the top level. If the directory is a + package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, all of these are added recursively. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 46b8c245f7b3fe..eaae2bb04b7482 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -9,7 +9,7 @@ -------------- This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.py[co]`) and packages from ZIP-format archives. It is usually not +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not needed to use the :mod:`zipimport` module explicitly; it is automatically used by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths to ZIP archives. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 195f63f0a319da..08dc311d837be9 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -532,8 +532,8 @@ conflict. .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set to a non-empty string, Python won't try to write ``.pyc`` or - ``.pyo`` files on the import of source modules. This is equivalent to + If this is set to a non-empty string, Python won't try to write ``.pyc`` + files on the import of source modules. This is equivalent to specifying the :option:`-B` option. diff --git a/Modules/main.c b/Modules/main.c index 2e6a60b1673f88..2a0b1b37da69fb 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,7 +53,7 @@ static const char usage_1[] = "\ Options and arguments (and corresponding environment variables):\n\ -b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ and comparing bytes/bytearray with str. (-bb: issue errors)\n\ --B : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x\n\ +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : debug output from parser; also PYTHONDEBUG=x\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 59046aaae41f51..cccc033d2d9fab 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -1102,7 +1102,7 @@ get_decompress_func(void) _Py_IDENTIFIER(decompress); if (importing_zlib != 0) - /* Someone has a zlib.py[co] in their Zip file; + /* Someone has a zlib.pyc in their Zip file; let's avoid a stack overflow. */ return NULL; importing_zlib = 1; @@ -1261,7 +1261,7 @@ eq_mtime(time_t t1, time_t t2) return d <= 1; } -/* Given the contents of a .py[co] file in a buffer, unmarshal the data +/* Given the contents of a .pyc file in a buffer, unmarshal the data and return the code object. Return None if it the magic word doesn't match (we do this instead of raising an exception as we fall back to .py if available and we don't want to mask other errors). @@ -1403,7 +1403,7 @@ get_mtime_of_source(ZipImporter *self, PyObject *path) PyObject *toc_entry, *stripped; time_t mtime; - /* strip 'c' or 'o' from *.py[co] */ + /* strip 'c' from *.pyc */ if (PyUnicode_READY(path) == -1) return (time_t)-1; stripped = PyUnicode_FromKindAndData(PyUnicode_KIND(path), diff --git a/PC/getpathp.c b/PC/getpathp.c index 1eeebfe9c19451..e7be704a9a78dd 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -185,7 +185,7 @@ exists(wchar_t *filename) may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/.pyo too */ +ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ { size_t n; @@ -196,7 +196,7 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/ n = wcsnlen_s(filename, MAXPATHLEN+1); if (n < MAXPATHLEN) { int exist = 0; - filename[n] = Py_OptimizeFlag ? L'o' : L'c'; + filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); if (!update_filename) diff --git a/PCbuild/rmpyc.py b/PCbuild/rmpyc.py index a1e75bb7ae5b4b..0b58f687729f7e 100644 --- a/PCbuild/rmpyc.py +++ b/PCbuild/rmpyc.py @@ -1,25 +1,19 @@ -# Remove all the .pyc and .pyo files under ../Lib. +# Remove all the .pyc files under ../Lib. def deltree(root): import os from os.path import join - npyc = npyo = 0 + npyc = 0 for root, dirs, files in os.walk(root): for name in files: - delete = False - if name.endswith('.pyc'): - delete = True + # to be thorough + if name.endswith(('.pyc', '.pyo')): npyc += 1 - elif name.endswith('.pyo'): - delete = True - npyo += 1 - - if delete: os.remove(join(root, name)) - return npyc, npyo + return npyc -npyc, npyo = deltree("../Lib") -print(npyc, ".pyc deleted,", npyo, ".pyo deleted") +npyc = deltree("../Lib") +print(npyc, ".pyc deleted") diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 35826727f3fa27..e73ac0457646e9 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -4,8 +4,8 @@ rem Usage: rt [-d] [-O] [-q] [-x64] regrtest_args rem -d Run Debug build (python_d.exe). Else release build. rem -O Run python.exe or python_d.exe (see -d) with -O. rem -q "quick" -- normally the tests are run twice, the first time -rem after deleting all the .py[co] files reachable from Lib/. -rem -q runs the tests just once, and without deleting .py[co] files. +rem after deleting all the .pyc files reachable from Lib/. +rem -q runs the tests just once, and without deleting .pyc files. rem -x64 Run the 64-bit build of python (or python_d if -d was specified) rem from the 'amd64' dir instead of the 32-bit build in this dir. rem All leading instances of these switches are shifted off, and @@ -45,7 +45,7 @@ set exe=%prefix%python%suffix%.exe set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% if defined qmode goto Qmode -echo Deleting .pyc/.pyo files ... +echo Deleting .pyc files ... "%exe%" "%pcbuild%rmpyc.py" echo Cleaning _pth files ... @@ -55,7 +55,7 @@ echo on %cmd% @echo off -echo About to run again without deleting .pyc/.pyo first: +echo About to run again without deleting .pyc first: pause :Qmode diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a4f7f823bc695d..af96d6d5fcb63c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -88,7 +88,7 @@ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ -int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ +int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.pyc) */ int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ From d7a8d32ce2d2b17ea9584759a281235ebfe81a89 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 11 Mar 2017 10:19:51 -0800 Subject: [PATCH 121/232] tempfile.rst: Fix some typos (GH-610) (GH-618) (cherry picked from commit d3b8f98696b905367da3975f73b482bac380b51c) --- Doc/library/tempfile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 665261ffb21f94..c59aca1e189086 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -245,12 +245,12 @@ The module uses a global variable to store the name of the directory used for temporary files returned by :func:`gettempdir`. It can be set directly to override the selection process, but this is discouraged. All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommend approach. +to specify the directory and this is the recommended approach. .. data:: tempdir When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to all the functions defined in this + default value for the *dir* argument to the functions defined in this module. If ``tempdir`` is unset or ``None`` at any call to any of the above From b3536df9f8b041142f5e3bf6037c11be06eb7f59 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 11 Mar 2017 15:42:25 -0600 Subject: [PATCH 122/232] Only trigger AppVeyor on code or config changes (GH-611 backport) (GH-620) --- .github/appveyor.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/appveyor.yml b/.github/appveyor.yml index 5a40005660b781..a369e52415bb17 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -4,3 +4,20 @@ build_script: - cmd: PCbuild\build.bat -e test_script: - cmd: PCbuild\rt.bat -q -uall -rwW --slowest --timeout=1200 -j0 + +# Only trigger AppVeyor if actual code or its configuration changes +only_commits: + files: + - .github/appveyor.yml + - .gitattributes + - Grammar/ + - Include/ + - Lib/ + - Modules/ + - Objects/ + - PC/ + - PCBuild/ + - Parser/ + - Programs/ + - Python/ + - Tools/ From df6f3f6f5743a061d6c3b939136a11ac170ae69e Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 11 Mar 2017 17:16:34 -0600 Subject: [PATCH 123/232] fix subprocess on Windows (GH-391 backport) (GH-623) --- Lib/subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 822ddb459e2615..0db6571f291dae 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -986,7 +986,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, int(not close_fds), creationflags, env, - cwd, + os.fspath(cwd) if cwd is not None else None, startupinfo) finally: # Child is launched. Close the parent's copy of those pipe From 7c6d6e056460753c916348feee0e4916a97d7c86 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sat, 11 Mar 2017 22:39:54 -0600 Subject: [PATCH 124/232] bpo-29763: Use support.unlink instead of os.unlink (GH-624) (GH-625) support.unlink waits for the files to be removed before returning --- Lib/test/test_site.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0620b242766057..342ec9e43d5de3 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -501,15 +501,15 @@ def _create_underpth_exe(self, lines): print(line, file=f) return exe_file except: - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) raise @classmethod def _cleanup_underpth_exe(self, exe_file): _pth_file = os.path.splitext(exe_file)[0] + '._pth' - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) @classmethod def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): From 2f386254e2cf054cc7ee5103b54c1cf16d9fa979 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Mar 2017 16:17:46 +1000 Subject: [PATCH 125/232] bpo-29656: Handle PR branches in 'make patchcheck' (#302) (#626) (cherry picked from commit 482f7a274fa52b7ba34ff308cd9acdcac9f41ba5) --- Tools/scripts/patchcheck.py | 58 +++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 58b081a9c53ff5..7a04aafa82a8a1 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -12,7 +12,6 @@ SRCDIR = sysconfig.get_config_var('srcdir') - def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -46,27 +45,73 @@ def mq_patches_applied(): return st.returncode == 0 and bstdout +def get_git_branch(): + """Get the symbolic name for the current git branch""" + cmd = "git rev-parse --abbrev-ref HEAD".split() + try: + return subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return None + + +def get_git_upstream_remote(): + """Get the remote name to use for upstream branches + + Uses "upstream" if it exists, "origin" otherwise + """ + cmd = "git remote get-url upstream".split() + try: + subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return "origin" + return "upstream" + + +@status("Getting base branch for PR", + info=lambda x: x if x is not None else "not a PR branch") +def get_base_branch(): + if not os.path.isdir(os.path.join(SRCDIR, '.git')): + # Not a git checkout, so there's no base branch + return None + version = sys.version_info + if version.releaselevel == 'alpha': + base_branch = "master" + else: + base_branch = "{0.major}.{0.minor}".format(version) + this_branch = get_git_branch() + if this_branch is None or this_branch == base_branch: + # Not on a git PR branch, so there's no base branch + return None + upstream_remote = get_git_upstream_remote() + return upstream_remote + "/" + base_branch + + @status("Getting the list of files that have been added/changed", info=lambda x: n_files_str(len(x))) -def changed_files(): +def changed_files(base_branch=None): """Get the list of changed or added files from Mercurial or git.""" if os.path.isdir(os.path.join(SRCDIR, '.hg')): + if base_branch is not None: + sys.exit('need a git checkout to check PR status') cmd = 'hg status --added --modified --no-status' if mq_patches_applied(): cmd += ' --rev qparent' with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] elif os.path.isdir(os.path.join(SRCDIR, '.git')): - cmd = 'git status --porcelain' + if base_branch: + cmd = 'git diff --name-status ' + base_branch + else: + cmd = 'git status --porcelain' filenames = [] with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: for line in st.stdout: line = line.decode().rstrip() - status = set(line[:2]) + status_text, filename = line.split(maxsplit=1) + status = set(status_text) # modified, added or unmerged files if not status.intersection('MAU'): continue - filename = line[3:] if ' -> ' in filename: # file is renamed filename = filename.split(' -> ', 2)[1].strip() @@ -165,7 +210,8 @@ def regenerated_pyconfig_h_in(file_paths): return "not needed" def main(): - file_paths = changed_files() + base_branch = get_base_branch() + file_paths = changed_files(base_branch) python_files = [fn for fn in file_paths if fn.endswith('.py')] c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] doc_files = [fn for fn in file_paths if fn.startswith('Doc') and From 4652d82a51e6530a4f820588a6209d3295963bdb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 12 Mar 2017 10:05:05 +0200 Subject: [PATCH 126/232] bpo-29746: Update marshal docs to Python 3. (#547) (#631) (cherry picked from commit c611a5b1d4fab0123bf622f06c3bfa510221dc32) --- Doc/c-api/marshal.rst | 8 ++++---- Doc/glossary.rst | 7 +++++++ Doc/library/marshal.rst | 19 ++++++++----------- Python/marshal.c | 26 ++++++++++++-------------- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index a6d0f4688d1b78..c6d1d02a2fa510 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -34,7 +34,7 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) - Return a string object containing the marshalled representation of *value*. + Return a bytes object containing the marshalled representation of *value*. *version* indicates the file format. @@ -88,10 +88,10 @@ written using these routines? :exc:`TypeError`) and returns *NULL*. -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *string, Py_ssize_t len) +.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) - Return a Python object from the data stream in a character buffer - containing *len* bytes pointed to by *string*. + Return a Python object from the data stream in a byte buffer + containing *len* bytes pointed to by *data*. On error, sets the appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index e07ab0df2fd68c..495934afe74c8b 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -131,6 +131,10 @@ Glossary binary file A :term:`file object` able to read and write :term:`bytes-like objects `. + Examples of binary files are files opened in binary mode (``'rb'``, + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, + :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and + :class:`gzip.GzipFile`. .. seealso:: A :term:`text file` reads and writes :class:`str` objects. @@ -966,6 +970,9 @@ Glossary A :term:`file object` able to read and write :class:`str` objects. Often, a text file actually accesses a byte-oriented datastream and handles the :term:`text encoding` automatically. + Examples of text files are files opened in text mode (``'r'`` or ``'w'``), + :data:`sys.stdin`, :data:`sys.stdout`, and instances of + :class:`io.StringIO`. .. seealso:: A :term:`binary file` reads and write :class:`bytes` objects. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 1ffc6effc7142d..d65afc20041133 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -49,7 +49,7 @@ For format *version* lower than 3, recursive lists, sets and dictionaries cannot be written (see below). There are functions that read/write files as well as functions operating on -strings. +bytes-like objects. The module defines these functions: @@ -57,9 +57,7 @@ The module defines these functions: .. function:: dump(value, file[, version]) Write the value on the open file. The value must be a supported type. The - file must be an open file object such as ``sys.stdout`` or returned by - :func:`open` or :func:`os.popen`. It must be opened in binary mode (``'wb'`` - or ``'w+b'``). + file must be a writeable :term:`binary file`. If the value has (or contains an object that has) an unsupported type, a :exc:`ValueError` exception is raised --- but garbage data will also be written @@ -74,8 +72,7 @@ The module defines these functions: Read one value from the open file and return it. If no valid value is read (e.g. because the data has a different Python version's incompatible marshal format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be an open file object opened in binary mode (``'rb'`` or - ``'r+b'``). + file must be a readable :term:`binary file`. .. note:: @@ -85,7 +82,7 @@ The module defines these functions: .. function:: dumps(value[, version]) - Return the string that would be written to a file by ``dump(value, file)``. The + Return the bytes object that would be written to a file by ``dump(value, file)``. The value must be a supported type. Raise a :exc:`ValueError` exception if value has (or contains an object that has) an unsupported type. @@ -93,11 +90,11 @@ The module defines these functions: (see below). -.. function:: loads(string) +.. function:: loads(bytes) - Convert the string to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra characters in the - string are ignored. + Convert the :term:`bytes-like object` to a value. If no valid value is found, raise + :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the + input are ignored. In addition, the following constants are defined: diff --git a/Python/marshal.c b/Python/marshal.c index 87a4b240a41c87..7b12ab7510dcfb 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -549,7 +549,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_lnotab, p); } else if (PyObject_CheckBuffer(v)) { - /* Write unknown bytes-like objects as a byte string */ + /* Write unknown bytes-like objects as a bytes object */ Py_buffer view; if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) { w_byte(TYPE_UNKNOWN, p); @@ -1086,7 +1086,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (bytes object size out of range)"); break; } v = PyBytes_FromStringAndSize((char *)NULL, n); @@ -1110,7 +1110,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } goto _read_ascii; @@ -1150,7 +1150,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } if (n != 0) { @@ -1612,7 +1612,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) if (wf.ptr - base > PY_SSIZE_T_MAX) { Py_DECREF(wf.str); PyErr_SetString(PyExc_OverflowError, - "too much marshal data for a string"); + "too much marshal data for a bytes object"); return NULL; } if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) @@ -1658,8 +1658,7 @@ PyDoc_STRVAR(dump_doc, "dump(value, file[, version])\n\ \n\ Write the value on the open file. The value must be a supported type.\n\ -The file must be an open file object such as sys.stdout or returned by\n\ -open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\ +The file must be a writeable binary file.\n\ \n\ If the value has (or contains an object that has) an unsupported type, a\n\ ValueError exception is raised - but garbage data will also be written\n\ @@ -1715,8 +1714,7 @@ PyDoc_STRVAR(load_doc, Read one value from the open file and return it. If no valid value is\n\ read (e.g. because the data has a different Python version's\n\ incompatible marshal format), raise EOFError, ValueError or TypeError.\n\ -The file must be an open file object opened in binary mode ('rb' or\n\ -'r+b').\n\ +The file must be a readable binary file.\n\ \n\ Note: If an object containing an unsupported type was marshalled with\n\ dump(), load() will substitute None for the unmarshallable type."); @@ -1735,7 +1733,7 @@ marshal_dumps(PyObject *self, PyObject *args) PyDoc_STRVAR(dumps_doc, "dumps(value[, version])\n\ \n\ -Return the string that would be written to a file by dump(value, file).\n\ +Return the bytes object that would be written to a file by dump(value, file).\n\ The value must be a supported type. Raise a ValueError exception if\n\ value has (or contains an object that has) an unsupported type.\n\ \n\ @@ -1771,8 +1769,8 @@ marshal_loads(PyObject *self, PyObject *args) PyDoc_STRVAR(loads_doc, "loads(bytes)\n\ \n\ -Convert the bytes object to a value. If no valid value is found, raise\n\ -EOFError, ValueError or TypeError. Extra characters in the input are\n\ +Convert the bytes-like object to a value. If no valid value is found,\n\ +raise EOFError, ValueError or TypeError. Extra bytes in the input are\n\ ignored."); static PyMethodDef marshal_methods[] = { @@ -1810,8 +1808,8 @@ Functions:\n\ \n\ dump() -- write value to a file\n\ load() -- read value from a file\n\ -dumps() -- write value to a string\n\ -loads() -- read value from a string"); +dumps() -- marshal value as a bytes object\n\ +loads() -- read value from a bytes-like object"); From 61a82a5fbf1a674b91764652ab70f5dbb9ba50f2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Mar 2017 20:00:20 +1000 Subject: [PATCH 127/232] bpo-29798: Handle git worktree in `make patchcheck` (#629) (#633) In git worktree directories, `.git` is a configuration file rather than a subdirectory (cherry picked from commit 6a6d090612dd7deaac2bc0399fad743e5e2db606) --- Tools/scripts/patchcheck.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 7a04aafa82a8a1..f4ec7d8a30ea23 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -98,7 +98,10 @@ def changed_files(base_branch=None): cmd += ' --rev qparent' with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] - elif os.path.isdir(os.path.join(SRCDIR, '.git')): + elif os.path.exists(os.path.join(SRCDIR, '.git')): + # We just use an existence check here as: + # directory = normal git checkout/clone + # file = git worktree directory if base_branch: cmd = 'git diff --name-status ' + base_branch else: From c60948464fb0ec116ea227f6bce8a4bb8fb75257 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Mar 2017 21:34:22 +1000 Subject: [PATCH 128/232] [3.6] bpo-29723: Consistently configure sys.path[0] (#636) Directory and zipfile execution previously added the parent directory of the directory or zipfile as sys.path[0] and then subsequently overwrote it with the directory or zipfile itself. This caused problems in isolated mode, as it overwrote the "stdlib as a zip archive" entry in sys.path, as the parent directory was never added. The attempted fix to that issue in bpo-29319 created the opposite problem in *non*-isolated mode, by potentially leaving the parent directory on sys.path instead of overwriting it. This change fixes the root cause of the problem by removing the whole "add-and-overwrite" dance for sys.path[0], and instead simply never adds the parent directory to sys.path in the first place. (cherry picked from commit d2977a3ae2cc6802921b1e3b6e9d13fcfbda872d) --- Lib/test/test_cmd_line_script.py | 67 +++++++++++++++++++++++++++++ Misc/NEWS | 8 ++++ Modules/main.c | 72 +++++++++++++++++++------------- 3 files changed, 119 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e058ecd086df3c..1587daf8f582dd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -572,6 +572,73 @@ def test_syntaxerror_indented_caret_position(self): self.assertNotIn("\f", text) self.assertIn("\n 1 + 1 = 2\n ^", text) + def test_consistent_sys_path_for_direct_execution(self): + # This test case ensures that the following all give the same + # sys.path configuration: + # + # ./python -s script_dir/__main__.py + # ./python -s script_dir + # ./python -I script_dir + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir, support.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from directly executing __main__.py + # We omit PYTHONPATH and user site to align with isolated mode + p = spawn_python("-Es", script_name, cwd=work_dir) + out_by_name = kill_python(p).decode().splitlines() + self.assertEqual(out_by_name[0], script_dir) + self.assertNotIn(work_dir, out_by_name) + # Directory execution should give the same output + p = spawn_python("-Es", script_dir, cwd=work_dir) + out_by_dir = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir, out_by_name) + # As should directory execution in isolated mode + p = spawn_python("-I", script_dir, cwd=work_dir) + out_by_dir_isolated = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir_isolated, out_by_dir, out_by_name) + + def test_consistent_sys_path_for_module_execution(self): + # This test case ensures that the following both give the same + # sys.path configuration: + # ./python -sm script_pkg.__main__ + # ./python -sm script_pkg + # + # And that this fails as unable to find the package: + # ./python -Im script_pkg + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from `-m script_pkg.__main__` + # We omit PYTHONPATH and user site to better align with the + # direct execution test cases + p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) + out_by_module = kill_python(p).decode().splitlines() + self.assertEqual(out_by_module[0], '') + self.assertNotIn(script_dir, out_by_module) + # Package execution should give the same output + p = spawn_python("-sm", "script_pkg", cwd=work_dir) + out_by_package = kill_python(p).decode().splitlines() + self.assertEqual(out_by_package, out_by_module) + # Isolated mode should fail with an import error + exitcode, stdout, stderr = assert_python_failure( + "-Im", "script_pkg", cwd=work_dir + ) + traceback_lines = stderr.decode().splitlines() + self.assertIn("No module named script_pkg", traceback_lines[-1]) def test_main(): support.run_unittest(CmdLineTest) diff --git a/Misc/NEWS b/Misc/NEWS index 299a8388ff6e39..640c783ed7f4be 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,14 @@ What's New in Python 3.6.1 final? Core and Builtins ----------------- +- bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a + regression by revealing an inconsistency in how sys.path is initialized when + executing ``__main__`` from a zipfile, directory, or other import location. + The interpreter now consistently avoids ever adding the import location's + parent directory to ``sys.path``, and ensures no other ``sys.path`` entries + are inadvertently modified when inserting the import location named on the + command line. + - bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. diff --git a/Modules/main.c b/Modules/main.c index 2a0b1b37da69fb..475a2fdc36db9b 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -225,55 +225,60 @@ static int RunModule(wchar_t *modname, int set_argv0) return 0; } -static int -RunMainFromImporter(wchar_t *filename) +static PyObject * +AsImportPathEntry(wchar_t *filename) { - PyObject *argv0 = NULL, *importer, *sys_path, *sys_path0; - int sts; + PyObject *sys_path0 = NULL, *importer; - argv0 = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (argv0 == NULL) + sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); + if (sys_path0 == NULL) goto error; - importer = PyImport_GetImporter(argv0); + importer = PyImport_GetImporter(sys_path0); if (importer == NULL) goto error; if (importer == Py_None) { - Py_DECREF(argv0); + Py_DECREF(sys_path0); Py_DECREF(importer); - return -1; + return NULL; } Py_DECREF(importer); + return sys_path0; + +error: + Py_XDECREF(sys_path0); + PySys_WriteStderr("Failed checking if argv[0] is an import path entry\n"); + PyErr_Print(); + PyErr_Clear(); + return NULL; +} + + +static int +RunMainFromImporter(PyObject *sys_path0) +{ + PyObject *sys_path; + int sts; - /* argv0 is usable as an import source, so put it in sys.path[0] - and import __main__ */ + /* Assume sys_path0 has already been checked by AsImportPathEntry, + * so put it in sys.path[0] and import __main__ */ sys_path = PySys_GetObject("path"); if (sys_path == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); goto error; } - sys_path0 = PyList_GetItem(sys_path, 0); - sts = 0; - if (!sys_path0) { - PyErr_Clear(); - sts = PyList_Append(sys_path, argv0); - } else if (PyObject_IsTrue(sys_path0)) { - sts = PyList_Insert(sys_path, 0, argv0); - } else { - sts = PyList_SetItem(sys_path, 0, argv0); - } + sts = PyList_Insert(sys_path, 0, sys_path0); if (sts) { - argv0 = NULL; + sys_path0 = NULL; goto error; } - Py_INCREF(argv0); sts = RunModule(L"__main__", 0); return sts != 0; error: - Py_XDECREF(argv0); + Py_XDECREF(sys_path0); PyErr_Print(); return 1; } @@ -358,6 +363,7 @@ Py_Main(int argc, wchar_t **argv) int saw_unbuffered_flag = 0; char *opt; PyCompilerFlags cf; + PyObject *main_importer_path = NULL; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -714,7 +720,17 @@ Py_Main(int argc, wchar_t **argv) argv[_PyOS_optind] = L"-m"; } - PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + if (filename != NULL) { + main_importer_path = AsImportPathEntry(filename); + } + + if (main_importer_path != NULL) { + /* Let RunMainFromImporter adjust sys.path[0] later */ + PySys_SetArgvEx(argc-_PyOS_optind, argv+_PyOS_optind, 0); + } else { + /* Use config settings to decide whether or not to update sys.path[0] */ + PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + } if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) && isatty(fileno(stdin)) && @@ -744,11 +760,11 @@ Py_Main(int argc, wchar_t **argv) sts = -1; /* keep track of whether we've already run __main__ */ - if (filename != NULL) { - sts = RunMainFromImporter(filename); + if (main_importer_path != NULL) { + sts = RunMainFromImporter(main_importer_path); } - if (sts==-1 && filename!=NULL) { + if (sts==-1 && filename != NULL) { fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; From aac875fa2f03cab61ceeaa2621c4c5534c7bcfc2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 12 Mar 2017 21:52:17 +0200 Subject: [PATCH 129/232] [3.6] bpo-8256: Fixed possible failing or crashing input() (#641) if attributes "encoding" or "errors" of sys.stdin or sys.stdout are not set or are not strings. --- Misc/NEWS | 3 +++ Python/bltinmodule.c | 21 ++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 640c783ed7f4be..eef4dfa3352286 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,9 @@ Core and Builtins Library ------- +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + - bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big intables (objects that have __int__) as elements. Patch by Oren Milman. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ef5a34cae900b6..597e26ec69a95a 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1927,12 +1927,15 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *result; size_t len; + /* stdin is a text stream, so it must have an encoding. */ stdin_encoding = _PyObject_GetAttrId(fin, &PyId_encoding); stdin_errors = _PyObject_GetAttrId(fin, &PyId_errors); - if (!stdin_encoding || !stdin_errors) - /* stdin is a text stream, so it must have an - encoding. */ + if (!stdin_encoding || !stdin_errors || + !PyUnicode_Check(stdin_encoding) || + !PyUnicode_Check(stdin_errors)) { + tty = 0; goto _readline_errors; + } stdin_encoding_str = PyUnicode_AsUTF8(stdin_encoding); stdin_errors_str = PyUnicode_AsUTF8(stdin_errors); if (!stdin_encoding_str || !stdin_errors_str) @@ -1948,8 +1951,12 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *stringpo; stdout_encoding = _PyObject_GetAttrId(fout, &PyId_encoding); stdout_errors = _PyObject_GetAttrId(fout, &PyId_errors); - if (!stdout_encoding || !stdout_errors) + if (!stdout_encoding || !stdout_errors || + !PyUnicode_Check(stdout_encoding) || + !PyUnicode_Check(stdout_errors)) { + tty = 0; goto _readline_errors; + } stdout_encoding_str = PyUnicode_AsUTF8(stdout_encoding); stdout_errors_str = PyUnicode_AsUTF8(stdout_errors); if (!stdout_encoding_str || !stdout_errors_str) @@ -2003,13 +2010,17 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_XDECREF(po); PyMem_FREE(s); return result; + _readline_errors: Py_XDECREF(stdin_encoding); Py_XDECREF(stdout_encoding); Py_XDECREF(stdin_errors); Py_XDECREF(stdout_errors); Py_XDECREF(po); - return NULL; + if (tty) + return NULL; + + PyErr_Clear(); } /* Fallback if we're not interactive */ From fa448de97de85d242491d7ad259ade0732f05db8 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 12 Mar 2017 17:04:06 -0400 Subject: [PATCH 130/232] Fix wrapping into StopIteration of return values in generators and coroutines (#644) (#647) --- Lib/test/test_coroutines.py | 15 +++++++++++++++ Misc/NEWS | 2 ++ Objects/genobject.c | 3 +-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index b4c7b5be6e208c..a69583b5f95ae6 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1103,6 +1103,21 @@ async def waiter(coro): "coroutine is being awaited already"): waiter(coro).send(None) + def test_await_16(self): + # See https://bugs.python.org/issue29600 for details. + + async def f(): + return ValueError() + + async def g(): + try: + raise KeyError + except: + return await f() + + _, result = run_async(g()) + self.assertIsNone(result.__context__) + def test_with_1(self): class Manager: def __init__(self, name): diff --git a/Misc/NEWS b/Misc/NEWS index eef4dfa3352286..504b977bc87b4b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.6.1 final? Core and Builtins ----------------- +- bpo-29600: Fix wrapping coroutine return values in StopIteration. + - bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a regression by revealing an inconsistency in how sys.path is initialized when executing ``__main__`` from a zipfile, directory, or other import location. diff --git a/Objects/genobject.c b/Objects/genobject.c index 2680ab0e129da4..1c29e296afe2a8 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -575,8 +575,7 @@ _PyGen_SetStopIterationValue(PyObject *value) PyObject *e; if (value == NULL || - (!PyTuple_Check(value) && - !PyObject_TypeCheck(value, (PyTypeObject *) PyExc_StopIteration))) + (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) { /* Delay exception instantiation if we can */ PyErr_SetObject(PyExc_StopIteration, value); From 99f8d33a94c8951fd6b2d5f079d20be698150993 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 12 Mar 2017 17:06:16 -0400 Subject: [PATCH 131/232] bpo-29742: asyncio get_extra_info() throws exception (#525) (#645) --- Lib/asyncio/sslproto.py | 4 +++- Lib/test/test_asyncio/test_sslproto.py | 12 ++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 7ad28d6aa0089a..ab7ff0bf93d076 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -543,8 +543,10 @@ def eof_received(self): def _get_extra_info(self, name, default=None): if name in self._extra: return self._extra[name] - else: + elif self._transport is not None: return self._transport.get_extra_info(name, default) + else: + return default def _start_shutdown(self): if self._in_shutdown: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 59ff0f6967e5b5..f1771c5561afea 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -95,5 +95,17 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_get_extra_info_on_closed_connection(self): + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + default = object() + self.assertIs(ssl_proto._get_extra_info('socket', default), default) + self.connection_made(ssl_proto) + self.assertIsNotNone(ssl_proto._get_extra_info('socket')) + ssl_proto.connection_lost(None) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 504b977bc87b4b..25a24f52008a6f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,9 @@ Core and Builtins Library ------- +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + - bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" or "errors" of sys.stdin or sys.stdout are not set or are not strings. From 7927abba3449e12d0ae3518b722fdfeb383b4039 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 13 Mar 2017 10:36:34 +0800 Subject: [PATCH 132/232] fix the name of argument to ftplib.FTP.set_pasv and fix wording (GH-653) (GH-654) --- Doc/library/ftplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index b8c1dcfef2d98e..1153ce77ddabcb 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -255,9 +255,9 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. prints the line to ``sys.stdout``. -.. method:: FTP.set_pasv(boolean) +.. method:: FTP.set_pasv(val) - Enable "passive" mode if *boolean* is true, other disable passive mode. + Enable "passive" mode if *val* is true, otherwise disable passive mode. Passive mode is on by default. From 388e2568fc919a177df2d2949b575c553aea84ac Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Mon, 13 Mar 2017 11:03:36 +0800 Subject: [PATCH 133/232] ftplib.FTP.retrbinary callback gets a bytes, not a str (GH-652) (GH-657) --- Doc/library/ftplib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 1153ce77ddabcb..7291dfe84811c6 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -235,7 +235,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Retrieve a file in binary transfer mode. *cmd* should be an appropriate ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single string argument giving the data + each block of data received, with a single bytes argument giving the data block. The optional *blocksize* argument specifies the maximum chunk size to read on the low-level socket object created to do the actual transfer (which will also be the largest size of the data blocks passed to *callback*). A From faa2cc63e45bc7d7ffab84bebe5a9f4fe065bd96 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 14 Mar 2017 15:27:01 +0800 Subject: [PATCH 134/232] bpo-28856: Let %b format for bytes support objects that follow the buffer protocol (GH-664) --- Lib/test/test_format.py | 8 ++++++-- Misc/NEWS | 3 +++ Objects/bytesobject.c | 15 ++++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 83eb29faf88d6c..b6ba2e566b295f 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -315,10 +315,12 @@ def __bytes__(self): testcommon(b"%b", b"abc", b"abc") testcommon(b"%b", bytearray(b"def"), b"def") testcommon(b"%b", fb, b"123") + testcommon(b"%b", memoryview(b"abc"), b"abc") # # %s is an alias for %b -- should only be used for Py2/3 code testcommon(b"%s", b"abc", b"abc") testcommon(b"%s", bytearray(b"def"), b"def") testcommon(b"%s", fb, b"123") + testcommon(b"%s", memoryview(b"abc"), b"abc") # %a will give the equivalent of # repr(some_obj).encode('ascii', 'backslashreplace') testcommon(b"%a", 3.14, b"3.14") @@ -377,9 +379,11 @@ def test_exc(formatstr, args, exception, excmsg): test_exc(b"%c", 3.14, TypeError, "%c requires an integer in range(256) or a single byte") test_exc(b"%b", "Xc", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") test_exc(b"%s", "Wd", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: diff --git a/Misc/NEWS b/Misc/NEWS index 25a24f52008a6f..284a2031eeafc8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ Core and Builtins - bpo-29600: Fix wrapping coroutine return values in StopIteration. +- bpo-28856: Fix an oversight that %b format for bytes should support objects + follow the buffer protocol. + - bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a regression by revealing an inconsistency in how sys.path is initialized when executing ``__main__`` from a zipfile, directory, or other import location. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 801711f7e6ccaa..30c1a7e24532da 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -528,6 +528,8 @@ byte_converter(PyObject *arg, char *p) return 0; } +static PyObject *_PyBytes_FromBuffer(PyObject *x); + static PyObject * format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) { @@ -564,8 +566,19 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) *plen = PyBytes_GET_SIZE(result); return result; } + /* does it support buffer protocol? */ + if (PyObject_CheckBuffer(v)) { + /* maybe we can avoid making a copy of the buffer object here? */ + result = _PyBytes_FromBuffer(v); + if (result == NULL) + return NULL; + *pbuf = PyBytes_AS_STRING(result); + *plen = PyBytes_GET_SIZE(result); + return result; + } PyErr_Format(PyExc_TypeError, - "%%b requires bytes, or an object that implements __bytes__, not '%.100s'", + "%%b requires a bytes-like object, " + "or an object that implements __bytes__, not '%.100s'", Py_TYPE(v)->tp_name); return NULL; } From 53b2667dcf2a7d13af466a5fb91844f5125a920d Mon Sep 17 00:00:00 2001 From: Michael Seifert Date: Wed, 15 Mar 2017 08:42:02 +0100 Subject: [PATCH 135/232] bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords are not strings (#649) (#671) --- Lib/test/test_functools.py | 26 ++++++++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/_functoolsmodule.c | 5 ++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index b7d648d0b15a55..cd4664cec08745 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -402,6 +402,32 @@ def test_attributes_unwritable(self): else: self.fail('partial object allowed __dict__ to be deleted') + def test_manually_adding_non_string_keyword(self): + p = self.partial(capture) + # Adding a non-string/unicode keyword to partial kwargs + p.keywords[1234] = 'value' + r = repr(p) + self.assertIn('1234', r) + self.assertIn("'value'", r) + with self.assertRaises(TypeError): + p() + + def test_keystr_replaces_value(self): + p = self.partial(capture) + + class MutatesYourDict(object): + def __str__(self): + p.keywords[self] = ['sth2'] + return 'astr' + + # Raplacing the value during key formatting should keep the original + # value alive (at least long enough). + p.keywords[MutatesYourDict()] = ['sth'] + r = repr(p) + self.assertIn('astr', r) + self.assertIn("['sth']", r) + + class TestPartialPy(TestPartial, unittest.TestCase): partial = py_functools.partial diff --git a/Misc/ACKS b/Misc/ACKS index c3b29a42a34f75..03afeb8f3876ee 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1371,6 +1371,7 @@ Federico Schwindt Barry Scott Steven Scott Nick Seidenman +Michael Seifert Ćœiga Seilnacht Yury Selivanov Fred Sells diff --git a/Misc/NEWS b/Misc/NEWS index 284a2031eeafc8..4c5efefb659378 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,9 @@ Core and Builtins Library ------- +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + - bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. Patch by Nikolay Kim. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 7abc9f464027f7..1bcf16a7e00f53 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -250,8 +250,11 @@ partial_repr(partialobject *pto) /* Pack keyword arguments */ assert (PyDict_Check(pto->kw)); for (i = 0; PyDict_Next(pto->kw, &i, &key, &value);) { - Py_SETREF(arglist, PyUnicode_FromFormat("%U, %U=%R", arglist, + /* Prevent key.__str__ from deleting the value. */ + Py_INCREF(value); + Py_SETREF(arglist, PyUnicode_FromFormat("%U, %S=%R", arglist, key, value)); + Py_DECREF(value); if (arglist == NULL) goto done; } From 02c6fa573a0a7f417e21ad9e81bcff3ac61a9fa8 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 15 Mar 2017 14:12:22 +0300 Subject: [PATCH 136/232] Delete duplicate entry in Misc/NEWS (#673) It has already been added in the build section. --- Misc/NEWS | 2 -- 1 file changed, 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 4c5efefb659378..03f70dc885d818 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -73,8 +73,6 @@ Core and Builtins - Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. -- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. - - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. From 78add3309b53d68c2c5c118ec63f7485bbf93c9a Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 15 Mar 2017 14:12:53 +0300 Subject: [PATCH 137/232] Change assertRaises to assertRaisesRegex in test_xmlrpc (#481) (#674) (cherry picked from commit c6b448b36d22769c684bb3276f85c1b47d15ab63) --- Lib/test/test_xmlrpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 69de08e9c3b252..30025e388ddf2e 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -412,7 +412,7 @@ def test_registered_func_is_none(self): dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() dispatcher.register_function(None, name='method') - with self.assertRaises(Exception, expected_regex='method'): + with self.assertRaisesRegex(Exception, 'method'): dispatcher._dispatch('method', ('param',)) def test_instance_has_no_func(self): @@ -420,14 +420,14 @@ def test_instance_has_no_func(self): dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() dispatcher.register_instance(object()) - with self.assertRaises(Exception, expected_regex='method'): + with self.assertRaisesRegex(Exception, 'method'): dispatcher._dispatch('method', ('param',)) def test_cannot_locate_func(self): """Calls a function that the dispatcher cannot locate""" dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() - with self.assertRaises(Exception, expected_regex='method'): + with self.assertRaisesRegex(Exception, 'method'): dispatcher._dispatch('method', ('param',)) From 0dfd18a1e14e08b0d6459c2dd1898e5bb38020c5 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 16 Mar 2017 16:22:35 +0300 Subject: [PATCH 138/232] Fix stderr bug in json.tool test (#346) (#676) See https://github.com/python/cpython/pull/201#discussion_r103229425.(cherry picked from commit b4e9087e7b77e8f76feac76f9c1ab21b49c0c766) --- Lib/test/test_json/test_tool.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 15f373664e1278..9d93f931ca3958 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -2,7 +2,7 @@ import sys import textwrap import unittest -import subprocess +from subprocess import Popen, PIPE from test import support from test.support.script_helper import assert_python_ok @@ -61,12 +61,11 @@ class TestTool(unittest.TestCase): """) def test_stdin_stdout(self): - with subprocess.Popen( - (sys.executable, '-m', 'json.tool'), - stdin=subprocess.PIPE, stdout=subprocess.PIPE) as proc: + args = sys.executable, '-m', 'json.tool' + with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: out, err = proc.communicate(self.data.encode()) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) - self.assertEqual(err, None) + self.assertEqual(err, b'') def _create_infile(self): infile = support.TESTFN From 9cd5e87bac51d7b901e3c36bf22728bb1693da59 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 16 Mar 2017 11:03:03 -0700 Subject: [PATCH 139/232] Takes vcruntime140.dll from the correct source. (#684) --- Tools/msi/make_zip.proj | 9 +++------ Tools/nuget/make_pkg.proj | 8 +++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj index f78e6ffa28fb72..b3588b7a0bae90 100644 --- a/Tools/msi/make_zip.proj +++ b/Tools/msi/make_zip.proj @@ -17,15 +17,12 @@ rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) - set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) - + diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj index d7e932cee54d3c..464ef0456af47a 100644 --- a/Tools/nuget/make_pkg.proj +++ b/Tools/nuget/make_pkg.proj @@ -34,9 +34,8 @@ $(NugetArguments) -Version "$(NuspecVersion)" $(NugetArguments) -NoPackageAnalysis -NonInteractive - setlocal -set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) @@ -45,8 +44,7 @@ set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140. - + From 3fce38c540681a736b7e5ca7611758f7bc159c9e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 16 Mar 2017 19:56:56 -0700 Subject: [PATCH 140/232] bpo-29820: othergui.rst: Remove outdated information (GH-685) (GH-688) (cherry picked from commit 1bb0f3762ec5104014aeed0ae6e9d64598d8fcac) --- Doc/library/othergui.rst | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index ee1ce50b6bd5e1..d40abe167653b7 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -9,14 +9,15 @@ available for Python: .. seealso:: `PyGObject `_ - provides introspection bindings for C libraries using + PyGObject provides introspection bindings for C libraries using `GObject `_. One of these libraries is the `GTK+ 3 `_ widget set. GTK+ comes with many more widgets than Tkinter provides. An online `Python GTK+ 3 Tutorial `_ is available. - `PyGTK `_ provides bindings for an older version + `PyGTK `_ + PyGTK provides bindings for an older version of the library, GTK+ 2. It provides an object oriented interface that is slightly higher level than the C one. There are also bindings to `GNOME `_. An online `tutorial @@ -27,15 +28,10 @@ available for Python: extensive C++ GUI application development framework that is available for Unix, Windows and Mac OS X. :program:`sip` is a tool for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. The *PyQt3* bindings have a - book, `GUI Programming with Python: QT Edition - `_ by Boudewijn - Rempt. The *PyQt4* bindings also have a book, `Rapid GUI Programming - with Python and Qt `_, by Mark - Summerfield. + is specifically designed for Python. `PySide `_ - is a newer binding to the Qt toolkit, provided by Nokia. + PySide is a newer binding to the Qt toolkit, provided by Nokia. Compared to PyQt, its licensing scheme is friendlier to non-open source applications. @@ -49,9 +45,7 @@ available for Python: documentation and context sensitive help, printing, HTML viewing, low-level device context drawing, drag and drop, system clipboard access, an XML-based resource format and more, including an ever growing library - of user-contributed modules. wxPython has a book, `wxPython in Action - `_, by Noel Rappin and - Robin Dunn. + of user-contributed modules. PyGTK, PyQt, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for From 4ac01f0ff3e3c9c02f5cc8c55f2fbe4639808f5e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 16 Mar 2017 20:58:42 -0700 Subject: [PATCH 141/232] update test_socket AEAD test for kernel 4.9 and up (GH-133) (GH-548) (cherry picked from commit 9764c151c51480a7ca6042b1ccd69be2620ff360) --- Lib/test/test_socket.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 5315d475df2218..1508f20c9f0598 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5479,7 +5479,7 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 3) # see test_aes_cbc + @support.requires_linux_version(4, 9) # see issue29324 def test_aead_aes_gcm(self): key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') @@ -5502,8 +5502,7 @@ def test_aead_aes_gcm(self): op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen, flags=socket.MSG_MORE) op.sendall(assoc, socket.MSG_MORE) - op.sendall(plain, socket.MSG_MORE) - op.sendall(b'\x00' * taglen) + op.sendall(plain) res = op.recv(assoclen + len(plain) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5511,7 +5510,7 @@ def test_aead_aes_gcm(self): # now with msg op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen) res = op.recv(assoclen + len(plain) + taglen) @@ -5522,7 +5521,7 @@ def test_aead_aes_gcm(self): pack_uint32 = struct.Struct('I').pack op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg( [msg], ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], @@ -5530,7 +5529,7 @@ def test_aead_aes_gcm(self): [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], ) ) - res = op.recv(len(msg)) + res = op.recv(len(msg) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5540,8 +5539,8 @@ def test_aead_aes_gcm(self): msg = assoc + expected_ct + expected_tag op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, assoclen=assoclen) - res = op.recv(len(msg)) - self.assertEqual(plain, res[assoclen:-taglen]) + res = op.recv(len(msg) - taglen) + self.assertEqual(plain, res[assoclen:]) @support.requires_linux_version(4, 3) # see test_aes_cbc def test_drbg_pr_sha256(self): From 7c2081122c2a6b8ea27c3111da7f19f7824fe18a Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 16 Mar 2017 20:59:36 -0700 Subject: [PATCH 142/232] Add sockaddr_alg to sock_addr_t (GH-234) (GH-533) (cherry picked from commit d37c068e695f8ec72b5c1b5a5a5ece2337fda768) --- Modules/socketmodule.c | 30 ------------------------------ Modules/socketmodule.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f4edc062fd6539..f3654c97e76ce9 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -288,36 +288,6 @@ if_indextoname(index) -- return the corresponding interface name\n\ #include #endif -#ifdef HAVE_SOCKADDR_ALG -#include -#ifndef AF_ALG -#define AF_ALG 38 -#endif -#ifndef SOL_ALG -#define SOL_ALG 279 -#endif - -/* Linux 3.19 */ -#ifndef ALG_SET_AEAD_ASSOCLEN -#define ALG_SET_AEAD_ASSOCLEN 4 -#endif -#ifndef ALG_SET_AEAD_AUTHSIZE -#define ALG_SET_AEAD_AUTHSIZE 5 -#endif -/* Linux 4.8 */ -#ifndef ALG_SET_PUBKEY -#define ALG_SET_PUBKEY 6 -#endif - -#ifndef ALG_OP_SIGN -#define ALG_OP_SIGN 2 -#endif -#ifndef ALG_OP_VERIFY -#define ALG_OP_VERIFY 3 -#endif - -#endif /* HAVE_SOCKADDR_ALG */ - /* Generic socket object definitions and includes */ #define PySocket_BUILDING_SOCKET #include "socketmodule.h" diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 3cce927e0b35f0..03f982b91083f3 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -98,6 +98,37 @@ typedef int socklen_t; #include #endif +#ifdef HAVE_SOCKADDR_ALG +#include +#ifndef AF_ALG +#define AF_ALG 38 +#endif +#ifndef SOL_ALG +#define SOL_ALG 279 +#endif + +/* Linux 3.19 */ +#ifndef ALG_SET_AEAD_ASSOCLEN +#define ALG_SET_AEAD_ASSOCLEN 4 +#endif +#ifndef ALG_SET_AEAD_AUTHSIZE +#define ALG_SET_AEAD_AUTHSIZE 5 +#endif +/* Linux 4.8 */ +#ifndef ALG_SET_PUBKEY +#define ALG_SET_PUBKEY 6 +#endif + +#ifndef ALG_OP_SIGN +#define ALG_OP_SIGN 2 +#endif +#ifndef ALG_OP_VERIFY +#define ALG_OP_VERIFY 3 +#endif + +#endif /* HAVE_SOCKADDR_ALG */ + + #ifndef Py__SOCKET_H #define Py__SOCKET_H #ifdef __cplusplus @@ -159,6 +190,9 @@ typedef union sock_addr { #ifdef HAVE_SYS_KERN_CONTROL_H struct sockaddr_ctl ctl; #endif +#ifdef HAVE_SOCKADDR_ALG + struct sockaddr_alg alg; +#endif } sock_addr_t; /* The object holding a socket. It holds some extra information, From 948171bf999cf8b3e12048851041d2e04ae3a78c Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 17 Mar 2017 14:59:16 +0300 Subject: [PATCH 143/232] bpo-16355: Clarify when inspect.getcomments() returns None (#428) (#690) Initial patch by Vajrasky Kok. (cherry picked from commit 3f2155ffe683080f2a1b28408fa48d43ba92f943) --- Doc/library/inspect.rst | 4 +++- Lib/test/test_inspect.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 3fa44a0c99891c..4ff2187b45f8eb 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -442,7 +442,9 @@ Retrieving source code Return in a single string any lines of comments immediately preceding the object's source code (for a class, function, or method), or at the top of the - Python source file (if the object is a module). + Python source file (if the object is a module). If the object's source code + is unavailable, return ``None``. This could happen if the object has been + defined in C or the interactive shell. .. function:: getfile(object) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 88eaabe6765ab2..cfea281c70db11 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -387,6 +387,11 @@ def test_cleandoc(self): def test_getcomments(self): self.assertEqual(inspect.getcomments(mod), '# line 1\n') self.assertEqual(inspect.getcomments(mod.StupidGit), '# line 20\n') + # If the object source file is not available, return None. + co = compile('x=1', '_non_existing_filename.py', 'exec') + self.assertIsNone(inspect.getcomments(co)) + # If the object has been defined in C, return None. + self.assertIsNone(inspect.getcomments(list)) def test_getmodule(self): # Check actual module From 46e81d3345b979f835740a228857906dca0f59c0 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Fri, 17 Mar 2017 19:52:56 +0000 Subject: [PATCH 144/232] bpo-29808: Do not fail in SysLogHandler constructor if syslog isn't available. (#696) bpo-29808: SysLogHandler: Do not fail if initial connect to syslog failed. (cherry picked from commit 1b038e073807ecb6fd176edaf3386a8e3205416e) --- Lib/logging/handlers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 7d779734f37b0d..2356f8d3d2d8ce 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -815,7 +815,14 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), if isinstance(address, str): self.unixsocket = True - self._connect_unixsocket(address) + # Syslog server may be unavailable during handler initialisation. + # C's openlog() function also ignores connection errors. + # Moreover, we ignore these errors while logging, so it not worse + # to ignore it also here. + try: + self._connect_unixsocket(address) + except OSError: + pass else: self.unixsocket = False if socktype is None: From 49fc153342f98e0d4ae5a96b88f3e91e4c5861d7 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 17 Mar 2017 15:41:35 -0700 Subject: [PATCH 145/232] Combine the C++ header CI build into the main C build (GH-697) (GH-704) This will eliminate one of the builds in Travis, allowing for CI overall to complete faster. (cherry picked from commit 993d4b3440f2282976901ce66879037c4443868a) --- .travis.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27b63c6c08b376..36961e38790a4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,15 +64,6 @@ matrix: # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. - source ./venv/bin/activate - bash <(curl -s https://codecov.io/bash) - - os: linux - language: cpp - compiler: clang - env: - - TESTING="C++ header compatibility" - before_script: - - ./configure - script: - - echo '#include "Python.h"' > test.cc && $CXX -c test.cc -o /dev/null -I ./Include -I . # Travis provides only 2 cores, so don't overdue the parallelism and waste memory. before_script: @@ -88,6 +79,8 @@ before_script: script: # `-r -w` implicitly provided through `make buildbottest`. - make buildbottest TESTOPTS="-j4" + # Test for C++ header compatibility. + - echo '#include "Python.h"' > test.cc && $CXX -c test.cc -o /dev/null -I ./Include -I . notifications: email: false From b044120f048571030552eefab8ba2329a9e76ceb Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 19 Mar 2017 10:03:39 -0700 Subject: [PATCH 146/232] Drop C++ header compatibility test (#718) (#719) The $CXX environment variable is not exported under the 'c' language on Travis. (cherry picked from commit 77ed11552da3e01dd235b7d68988076866b1f604) --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36961e38790a4d..66f03dc7161a7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,8 +79,6 @@ before_script: script: # `-r -w` implicitly provided through `make buildbottest`. - make buildbottest TESTOPTS="-j4" - # Test for C++ header compatibility. - - echo '#include "Python.h"' > test.cc && $CXX -c test.cc -o /dev/null -I ./Include -I . notifications: email: false From 69eab3123ed1de4bed4b7dedecabe415f6139bb6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 19 Mar 2017 20:26:26 +0200 Subject: [PATCH 147/232] bpo-28749: Fixed the documentation of the mapping codec APIs. (#487) (#714) Added the documentation for PyUnicode_Translate(). (cherry picked from commit c85a26628ceb9624c96c3064e8b99033c026d8a3) --- Doc/c-api/unicode.rst | 95 +++++++++++++++++++++-------------------- Include/unicodeobject.h | 45 ++++++++----------- 2 files changed, 66 insertions(+), 74 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 02f7ada7be7ee4..6e91576ee8f004 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1393,77 +1393,78 @@ Character Map Codecs This codec is special in that it can be used to implement many different codecs (and this is in fact what was done to obtain most of the standard codecs included in the :mod:`encodings` package). The codec uses mapping to encode and -decode characters. - -Decoding mappings must map single string characters to single Unicode -characters, integers (which are then interpreted as Unicode ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -Encoding mappings must map single Unicode characters to single string -characters, integers (which are then interpreted as Latin-1 ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -The mapping objects provided must only support the __getitem__ mapping -interface. - -If a character lookup fails with a LookupError, the character is copied as-is -meaning that its ordinal value will be interpreted as Unicode or Latin-1 ordinal -resp. Because of this, mappings only need to contain those mappings which map -characters to different code points. +decode characters. The mapping objects provided must support the +:meth:`__getitem__` mapping interface; dictionaries and sequences work well. These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* using - the given *mapping* object. Return *NULL* if an exception was raised by the - codec. If *mapping* is *NULL* latin-1 decoding will be done. Else it can be a - dictionary mapping byte or a unicode string, which is treated as a lookup table. - Byte values greater that the length of the string and U+FFFE "characters" are - treated as "undefined mapping". + Create a Unicode object by decoding *size* bytes of the encoded string *s* + using the given *mapping* object. Return *NULL* if an exception was raised + by the codec. + + If *mapping* is *NULL*, Latin-1 decoding will be applied. Else + *mapping* must map bytes ordinals (integers in the range from 0 to 255) + to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or ``None``. Unmapped data bytes -- ones which cause a + :exc:`LookupError`, as well as ones which get mapped to ``None``, + ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause + an error. .. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) - Encode a Unicode object using the given *mapping* object and return the result - as Python string object. Error handling is "strict". Return *NULL* if an + Encode a Unicode object using the given *mapping* object and return the + result as a bytes object. Error handling is "strict". Return *NULL* if an exception was raised by the codec. -The following codec API is special in that maps Unicode to Unicode. - + The *mapping* object must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or ``None``. Unmapped character + ordinals (ones which cause a :exc:`LookupError`) as well as mapped to + ``None`` are treated as "undefined mapping" and cause an error. -.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ - PyObject *table, const char *errors) - - Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a - character mapping *table* to it and return the resulting Unicode object. Return - *NULL* when an exception was raised by the codec. - The *mapping* table must map Unicode ordinal integers to Unicode ordinal - integers or ``None`` (causing deletion of the character). +.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. + Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given + *mapping* object and return the result as a bytes object. Return *NULL* if + an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API - ` + :c:func:`PyUnicode_AsCharmapString` or + :c:func:`PyUnicode_AsEncodedString`. -.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ +The following codec API is special in that maps Unicode to Unicode. + +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ PyObject *mapping, const char *errors) - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return a Python string object. Return *NULL* if an - exception was raised by the codec. + Translate a Unicode object using the given *mapping* object and return the + resulting Unicode object. Return *NULL* if an exception was raised by the + codec. + + The *mapping* object must map Unicode ordinal integers to Unicode strings, + integers (which are then interpreted as Unicode ordinals) or ``None`` + (causing deletion of the character). Unmapped character ordinals (ones + which cause a :exc:`LookupError`) are left untouched and are copied as-is. + + +.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) + + Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a + character *mapping* table to it and return the resulting Unicode object. + Return *NULL* when an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsCharmapString` or - :c:func:`PyUnicode_AsEncodedString`. + :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API + ` MBCS codecs for Windows diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 587cf03e3697eb..2d0d77e804d9d8 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1609,50 +1609,41 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII( This codec uses mappings to encode and decode characters. - Decoding mappings must map single string characters to single - Unicode characters, integers (which are then interpreted as Unicode - ordinals) or None (meaning "undefined mapping" and causing an - error). - - Encoding mappings must map single Unicode characters to single - string characters, integers (which are then interpreted as Latin-1 - ordinals) or None (meaning "undefined mapping" and causing an - error). - - If a character lookup fails with a LookupError, the character is - copied as-is meaning that its ordinal value will be interpreted as - Unicode or Latin-1 ordinal resp. Because of this mappings only need - to contain those mappings which map characters to different code - points. + Decoding mappings must map byte ordinals (integers in the range from 0 to + 255) to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or None. Unmapped data bytes (ones which cause a LookupError) + as well as mapped to None, 0xFFFE or '\ufffe' are treated as "undefined + mapping" and cause an error. + + Encoding mappings must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or None. Unmapped character + ordinals (ones which cause a LookupError) as well as mapped to + None are treated as "undefined mapping" and cause an error. */ PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap( const char *string, /* Encoded string */ Py_ssize_t length, /* size of string */ - PyObject *mapping, /* character mapping - (char ordinal -> unicode ordinal) */ + PyObject *mapping, /* decoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString( PyObject *unicode, /* Unicode object */ - PyObject *mapping /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping /* encoding mapping */ ); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( PyObject *unicode, /* Unicode object */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); #endif @@ -1661,8 +1652,8 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and @@ -1960,8 +1951,8 @@ PyAPI_FUNC(PyObject*) PyUnicode_RSplit( /* Translate a string by applying a character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and From fca705d533970011e50b3f278aab81cead39b00d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 19 Mar 2017 20:27:16 +0200 Subject: [PATCH 148/232] bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. (#514) (#722) (cherry picked from commit a5af6e1af77ee0f9294c5776478a9c24d9fbab94) --- Lib/test/test_fileio.py | 8 +- Lib/test/test_io.py | 20 + Misc/NEWS | 800 +++++++++++++++++++++++++++++++++++++++ Modules/_io/bufferedio.c | 14 +- Modules/_io/fileio.c | 16 +- Modules/_io/textio.c | 23 +- 6 files changed, 872 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 3da210ae1997ad..57a02656206ff4 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,7 @@ from functools import wraps from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest, - make_bad_fd, cpython_only) + make_bad_fd, cpython_only, swap_attr) from collections import UserList import _io # C implementation of io @@ -176,6 +176,12 @@ def testReprNoCloseFD(self): finally: os.close(fd) + def testRecursiveRepr(self): + # Issue #25455 + with swap_attr(self.f, 'name', self.f): + with self.assertRaises(RuntimeError): + repr(self.f) # Should not crash + def testErrors(self): f = self.f self.assertFalse(f.isatty()) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index aaa64eadffe1ea..8f895fe0bfb885 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1014,6 +1014,16 @@ def test_repr(self): raw.name = b"dummy" self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname) + def test_recursive_repr(self): + # Issue #25455 + raw = self.MockRawIO() + b = self.tp(raw) + with support.swap_attr(raw, 'name', b): + try: + repr(b) # Should not crash + except RuntimeError: + pass + def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush # and that flush() is called before file closed. @@ -2424,6 +2434,16 @@ def test_repr(self): t.buffer.detach() repr(t) # Should not raise an exception + def test_recursive_repr(self): + # Issue #25455 + raw = self.BytesIO() + t = self.TextIOWrapper(raw) + with support.swap_attr(raw, 'name', t): + try: + repr(t) # Should not crash + except RuntimeError: + pass + def test_line_buffering(self): r = self.BytesIO() b = self.BufferedWriter(r, 1000) diff --git a/Misc/NEWS b/Misc/NEWS index 03f70dc885d818..310e982d78f03a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,806 @@ Core and Builtins - bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. +- bpo-29568: Escaped percent "%%" in the format string for classic string + formatting no longer allows any characters between two percents. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +- bpo-29695: bool(), float(), list() and tuple() no longer take keyword arguments. + The first argument of int() can now be passes only as positional argument. + +- bpo-20087: Prefer glibc's list of locale aliases to the X11 ones. + +- bpo-28893: Set correct __cause__ for errors about invalid awaitables + returned from __aiter__ and __anext__. + +- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by + Brian Coleman. + +- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. + It should raise TypeError when kwargs is not a dict. But it might + cause segv when args=NULL and kwargs is not a dict. + +- bpo-28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. + Patch by Matthieu Dartiailh. + +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + +- bpo-29347: Fixed possibly dereferencing undefined pointers + when creating weakref objects. + +- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef, + and AsyncFunctionDef ast nodes. docstring is not first stmt in their body + anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object + for module and class. + +- bpo-29438: Fixed use-after-free problem in key sharing dict. + +- bpo-29546: Set the 'path' and 'name' attribute on ImportError for ``from ... import ...``. + +- bpo-29546: Improve from-import error message with location + +- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. + +- Issue #29337: Fixed possible BytesWarning when compare the code objects. + Warnings could be emitted at compile time. + +- Issue #29327: Fixed a crash when pass the iterable keyword argument to + sorted(). + +- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). + +- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. + +- Issue #29049: Call _PyObject_GC_TRACK() lazily when calling Python function. + Calling function is up to 5% faster. + +- Issue #28927: bytes.fromhex() and bytearray.fromhex() now ignore all ASCII + whitespace, not only spaces. Patch by Robert Xiao. + +- Issue #28932: Do not include if it does not exist. + +- Issue #25677: Correct the positioning of the syntax error caret for + indented blocks. Based on patch by Michael Layzell. + +- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate + form. + +- Issue #18896: Python function can now have more than 255 parameters. + collections.namedtuple() now supports tuples with more than 255 elements. + +- Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by + Chi Hsuan Yen. + +- Issue #26919: On Android, operating system data is now always encoded/decoded + to/from UTF-8, instead of the locale encoding to avoid inconsistencies with + os.fsencode() and os.fsdecode() which are already using UTF-8. + +- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy + bug triggerable by a monkey-patched len() function. + +- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() + must not convert combined table into split table. Patch written by INADA + Naoki. + +- Issue #28739: f-string expressions are no longer accepted as docstrings and + by ast.literal_eval() even if they do not include expressions. + +- Issue #28512: Fixed setting the offset attribute of SyntaxError by + PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). + +- Issue #28918: Fix the cross compilation of xxlimited when Python has been + built with Py_DEBUG defined. + +- Issue #23722: Rather than silently producing a class that doesn't support + zero-argument ``super()`` in methods, failing to pass the new + ``__classcell__`` namespace entry up to ``type.__new__`` now results in a + ``DeprecationWarning`` and a class that supports zero-argument ``super()``. + +- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of + a descriptor that is used inside that class no longer prevents calling the + __set_name__ method of other descriptors. + +- Issue #28799: Remove the ``PyEval_GetCallStats()`` function and deprecate + the untested and undocumented ``sys.callstats()`` function. Remove the + ``CALL_PROFILE`` special build: use the :func:`sys.setprofile` function, + :mod:`cProfile` or :mod:`profile` to profile function calls. + +- Issue #12844: More than 255 arguments can now be passed to a function. + +- Issue #28782: Fix a bug in the implementation ``yield from`` when checking + if the next instruction is YIELD_FROM. Regression introduced by WORDCODE + (issue #26647). + +- Issue #28774: Fix error position of the unicode error in ASCII and Latin1 + encoders when a string returned by the error handler contains multiple + non-encodable characters (non-ASCII for the ASCII codec, characters out + of the U+0000-U+00FF range for Latin1). + +- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. + Improve speed of dict literal with constant keys up to 30%. + +- Issue #28532: Show sys.version when -V option is supplied twice. + +- Issue #27100: The with-statement now checks for __enter__ before it + checks for __exit__. This gives less confusing error messages when + both methods are missing. Patch by Jonathan Ellington. + +- Issue #28746: Fix the set_inheritable() file descriptor method on platforms + that do not have the ioctl FIOCLEX and FIONCLEX commands. + +- Issue #26920: Fix not getting the locale's charset upon initializing the + interpreter, on platforms that do not have langinfo. + +- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X + when decode astral characters. Patch by Xiang Zhang. + +- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. + +- Issue #19398: Extra slash no longer added to sys.path components in case of + empty compile-time PYTHONPATH components. + +- Issue #28621: Sped up converting int to float by reusing faster bits counting + implementation. Patch by Adrian Wielgosik. + +- Issue #28580: Optimize iterating split table values. + Patch by Xiang Zhang. + +- Issue #28583: PyDict_SetDefault didn't combine split table when needed. + Patch by Xiang Zhang. + +- Issue #28128: Deprecation warning for invalid str and byte escape + sequences now prints better information about where the error + occurs. Patch by Serhiy Storchaka and Eric Smith. + +- Issue #28509: dict.update() no longer allocate unnecessary large memory. + +- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug + build. + +- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused + keeping unreachable code. + +- Issue #28214: Improved exception reporting for problematic __set_name__ + attributes. + +- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception + loss in PyTraceBack_Here(). + +- Issue #28183: Optimize and cleanup dict iteration. + +- Issue #26081: Added C implementation of asyncio.Future. + Original patch by Yury Selivanov. + +- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). + Patch by Xiang Zhang. + +- Issue #28376: The type of long range iterator is now registered as Iterator. + Patch by Oren Milman. + +- Issue #28376: Creating instances of range_iterator by calling range_iterator + type now is disallowed. Calling iter() on range instance is the only way. + Patch by Oren Milman. + +- Issue #26906: Resolving special methods of uninitialized type now causes + implicit initialization of the type instead of a fail. + +- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. + Original patch by Niklas Koep. + +- Issue #24098: Fixed possible crash when AST is changed in process of + compiling it. + +- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when + hashes have same lower bits. + +- Issue #28350: String constants with null character no longer interned. + +- Issue #26617: Fix crash when GC runs during weakref callbacks. + +- Issue #27942: String constants now interned recursively in tuples and frozensets. + +- Issue #28289: ImportError.__init__ now resets not specified attributes. + +- Issue #21578: Fixed misleading error message when ImportError called with + invalid keyword args. + +- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. + Patch by Soumya Sharma. + +- Issue #28086: Single var-positional argument of tuple subtype was passed + unscathed to the C-defined function. Now it is converted to exact tuple. + +- Issue #28214: Now __set_name__ is looked up on the class instead of the + instance. + +- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() + syscall fails with EPERM, for example when blocked by SECCOMP. + +- Issue #28192: Don't import readline in isolated mode. + +- Issue #27441: Remove some redundant assignments to ob_size in longobject.c. + Thanks Oren Milman. + +- Issue #27222: Clean up redundant code in long_rshift function. Thanks + Oren Milman. + +- Upgrade internal unicode databases to Unicode version 9.0.0. + +- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport + should use the same optimization level as the interpreter. + +- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly + optimize memcpy(). + +- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a + "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. + +- Issue #26182: Raise DeprecationWarning when async and await keywords are + used as variable/attribute/class/function name. + +- Issue #26182: Fix a refleak in code that raises DeprecationWarning. + +- Issue #28721: Fix asynchronous generators aclose() and athrow() to + handle StopAsyncIteration propagation properly. + +- Issue #26110: Speed-up method calls: add LOAD_METHOD and CALL_METHOD + opcodes. + +Extension Modules +----------------- + +- Issue #29169: Update zlib to 1.2.11. + +Library +------- + +- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. + +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + +- bpo-28692: Using non-integer value for selecting a plural form in gettext is + now deprecated. + +- bpo-26121: Use C library implementation for math functions erf() and erfc(). + +- bpo-29619: os.stat() and os.DirEntry.inode() now convert inode (st_ino) using + unsigned integers. + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. + +- bpo-29645: Speed up importing the webbrowser module. webbrowser.register() + is now thread-safe. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-28682: Added support for bytes paths in os.fwalk(). + +- bpo-29623: Allow use of path-like object as a single argument in + ConfigParser.read(). Patch by David Ellis. + +- bpo-9303: Migrate sqlite3 module to _v2 API. Patch by Aviv Palivoda. + +- bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback + implemented in C. + +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before + all pipes are closed. + +- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C + to accept None argument as their pure Python implementation. + +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + +- bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher + and its subclasses can now be used as a decorator. + +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + +- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts + PathLike objects. Patch by Sayan Chowdhury. + +- bpo-28518: Start a transaction implicitly before a DML statement. + Patch by Aviv Palivoda. + +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + +- Issue #16285: urrlib.parse.quote is now based on RFC 3986 and hence includes + '~' in the set of characters that is not quoted by default. Patch by + Christian Theune and Ratnadeep Debnath. + +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + +- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from + the platform about whether generated UUIDs are generated with a + multiprocessing safe method. + +- bpo-29576: Improve some deprecations in importlib. Some deprecated methods + now emit DeprecationWarnings and have better descriptive messages. + +- bpo-29534: Fixed different behaviour of Decimal.from_float() + for _decimal and _pydecimal. Thanks Andrew Nester. + +- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, + improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, + Manuel Krebber, and Ɓukasz Langa. + +- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python + 3.6.0: check minimum and maximum years. + +- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir + +- Issue #29444: Fixed out-of-bounds buffer access in the group() method of + the match object. Based on patch by WGH. + +- Issue #29377: Add SlotWrapperType, MethodWrapperType, and + MethodDescriptorType built-in types to types module. + Original patch by Manuel Krebber. + +- Issue #29218: Unused install_misc command is now removed. It has been + documented as unused since 2000. Patch by Eric N. Vander Weele. + +- Issue #29368: The extend() method is now called instead of the append() + method when unpickle collections.deque and other list-like objects. + This can speed up unpickling to 2 times. + +- Issue #29338: The help of a builtin or extension class now includes the + constructor signature if __text_signature__ is provided for the class. + +- Issue #29335: Fix subprocess.Popen.wait() when the child process has + exited to a stopped instead of terminated state (ex: when under ptrace). + +- Issue #29290: Fix a regression in argparse that help messages would wrap at + non-breaking spaces. + +- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. + +- Issue #29197: Removed deprecated function ntpath.splitunc(). + +- Issue #29210: Removed support of deprecated argument "exclude" in + tarfile.TarFile.add(). + +- Issue #29219: Fixed infinite recursion in the repr of uninitialized + ctypes.CDLL instances. + +- Issue #29192: Removed deprecated features in the http.cookies module. + +- Issue #29193: A format string argument for string.Formatter.format() + is now positional-only. + +- Issue #29195: Removed support of deprecated undocumented keyword arguments + in methods of regular expression objects. + +- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. + KeyError could be raised when cached function with full cache was + simultaneously called from differen threads with the same uncached arguments. + +- Issue #20804: The unittest.mock.sentinel attributes now preserve their + identity when they are copied or pickled. + +- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with + leading dots could match related hostnames again (e.g. .b.c matches a.b.c). + Patch by Milan Oberkirch. + +- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter + anymore. Patch written by Jiajun Huang. + +- Issue #15812: inspect.getframeinfo() now correctly shows the first line of + a context. Patch by Sam Breese. + +- Issue #28985: Update authorizer constants in sqlite3 module. + Patch by Dingyuan Wang. + +- Issue #29094: Offsets in a ZIP file created with extern file object and modes + "w" and "x" now are relative to the start of the file. + +- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows + +- Issue #13051: Fixed recursion errors in large or resized + curses.textpad.Textbox. Based on patch by Tycho Andersen. + +- Issue #9770: curses.ascii predicates now work correctly with negative + integers. + +- Issue #28427: old keys should not remove new values from + WeakValueDictionary when collecting from another thread. + +- Issue 28923: Remove editor artifacts from Tix.py. + +- Issue #28871: Fixed a crash when deallocate deep ElementTree. + +- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and + WeakValueDictionary.pop() when a GC collection happens in another + thread. + +- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that + doesn't own its elements as limits. + +- Issue #16255: subprocess.Popen uses /system/bin/sh on Android as the shell, + instead of /bin/sh. + +- Issue #28779: multiprocessing.set_forkserver_preload() would crash the + forkserver process if a preloaded module instantiated some + multiprocessing objects such as locks. + +- Issue #26937: The chown() method of the tarfile.TarFile class does not fail + now when the grp module cannot be imported, as for example on Android + platforms. + +- Issue #28847: dbm.dumb now supports reading read-only files and no longer + writes the index file when it is not changed. A deprecation warning is now + emitted if the index file is missed and recreated in the 'r' and 'w' modes + (will be an error in future Python releases). + +- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in + re.sub() replacement templates regular expressions now are errors. + +- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): + call warnings.showwarning() if it was overriden inside the context manager. + +- Issue #27172: To assist with upgrades from 2.7, the previously documented + deprecation of ``inspect.getfullargspec()`` has been reversed. This decision + may be revisited again after the Python 2.7 branch is no longer officially + supported. + +- Issue #28740: Add sys.getandroidapilevel(): return the build time API version + of Android as an integer. Function only available on Android. + +- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and + :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by + Omar Sandoval. + +- Issue #28752: Restored the __reduce__() methods of datetime objects. + +- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created + by re.compile(), become comparable (only x==y and x!=y operators). This + change should fix the issue #18383: don't duplicate warning filters when the + warnings module is reloaded (thing usually only done in unit tests). + +- Issue #20572: Remove the subprocess.Popen.wait endtime parameter. It was + deprecated in 3.4 and undocumented prior to that. + +- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and + from_buffer_copy() methods on abstract classes like Array. + +- Issue #28548: In the "http.server" module, parse the protocol version if + possible, to avoid using HTTP 0.9 in some error responses. + +- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. + Patch by Vajrasky Kok + +- Issue #28563: Fixed possible DoS and arbitrary code execution when handle + plural form selections in the gettext module. The expression parser now + supports exact syntax supported by GNU gettext. + +- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when + the garbage collector is invoked in other thread. Based on patch by + Sebastian Cufre. + +- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if + given empty data twice. Patch by Benjamin Fogle. + +- Issue #28549: Fixed segfault in curses's addch() with ncurses6. + +- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar + file with compression before trying to open it without compression. Otherwise + it had 50% chance failed with ignore_zeros=True. + +- Issue #23262: The webbrowser module now supports Firefox 36+ and derived + browsers. Based on patch by Oleg Broytman. + +- Issue #24241: The webbrowser in an X environment now prefers using the + default browser directly. Also, the webbrowser register() function now has + a documented 'preferred' argument, to specify browsers to be returned by + get() with no arguments. Patch by David Steele + +- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused + by representing the scale as float value internally in Tk. tkinter.IntVar + now works if float value is set to underlying Tk variable. + +- Issue #28255: calendar.TextCalendar.prweek() no longer prints a space after + a weeks's calendar. calendar.TextCalendar.pryear() no longer prints redundant + newline after a year's calendar. Based on patch by Xiang Zhang. + +- Issue #28255: calendar.TextCalendar.prmonth() no longer prints a space + at the start of new line after printing a month's calendar. Patch by + Xiang Zhang. + +- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. + Based on patch by Kaarle Ritvanen. + +- Issue #28353: os.fwalk() no longer fails on broken links. + +- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept + non-None value is passed to it.send(val). + +- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix + for readability (was "`"). + +- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin + a workaround to Tix library bug. + +- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. + +- Issue #25953: re.sub() now raises an error for invalid numerical group + reference in replacement template even if the pattern is not found in + the string. Error message for invalid group reference now includes the + group index and the position of the reference. + Based on patch by SilentGhost. + +- Issue #28469: timeit now uses the sequence 1, 2, 5, 10, 20, 50,... instead + of 1, 10, 100,... for autoranging. + +- Issue #28115: Command-line interface of the zipfile module now uses argparse. + Added support of long options. + +- Issue #18219: Optimize csv.DictWriter for large number of columns. + Patch by Mariatta Wijaya. + +- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. + +- Issue #23214: In the "io" module, the argument to BufferedReader and + BytesIO's read1() methods is now optional and can be -1, matching the + BufferedIOBase specification. + +- Issue #28480: Fix error building socket module when multithreading is + disabled. + +- Issue #28240: timeit: remove ``-c/--clock`` and ``-t/--time`` command line + options which were deprecated since Python 3.3. + +- Issue #28240: timeit now repeats the benchmarks 5 times instead of only 3 + to make benchmarks more reliable. + +- Issue #28240: timeit autorange now uses a single loop iteration if the + benchmark takes less than 10 seconds, instead of 10 iterations. + "python3 -m timeit -s 'import time' 'time.sleep(1)'" now takes 4 seconds + instead of 40 seconds. + +- Distutils.sdist now looks for README and setup.py files with case + sensitivity. This behavior matches that found in Setuptools 6.0 and + later. See `setuptools 100 + `_ for rationale. + +- Issue #24452: Make webbrowser support Chrome on Mac OS X. Patch by + Ned Batchelder. + +- Issue #20766: Fix references leaked by pdb in the handling of SIGINT + handlers. + +- Issue #27998: Fixed bytes path support in os.scandir() on Windows. + Patch by Eryk Sun. + +- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. + +- Issue #26293: Fixed writing ZIP files that starts not from the start of the + file. Offsets in ZIP file now are relative to the start of the archive in + conforming to the specification. + +- Issue #28380: unittest.mock Mock autospec functions now properly support + assert_called, assert_not_called, and assert_called_once. + +- Issue #28229: lzma module now supports pathlib. + +- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. + +- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. + +- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. + +- Issue #28332: Deprecated silent truncations in socket.htons and socket.ntohs. + Original patch by Oren Milman. + +- Issue #27358: Optimized merging var-keyword arguments and improved error + message when passing a non-mapping as a var-keyword argument. + +- Issue #28257: Improved error message when passing a non-iterable as + a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. + +- Issue #28322: Fixed possible crashes when unpickle itertools objects from + incorrect pickle data. Based on patch by John Leitch. + +- Issue #28228: imghdr now supports pathlib. + +- Issue #28226: compileall now supports pathlib. + +- Issue #28314: Fix function declaration (C flags) for the getiterator() method + of xml.etree.ElementTree.Element. + +- Issue #28148: Stop using localtime() and gmtime() in the time + module. + + Introduced platform independent _PyTime_localtime API that is + similar to POSIX localtime_r, but available on all platforms. Patch + by Ed Schouten. + +- Issue #28253: Fixed calendar functions for extreme months: 0001-01 + and 9999-12. + + Methods itermonthdays() and itermonthdays2() are reimplemented so + that they don't call itermonthdates() which can cause datetime.date + under/overflow. + +- Issue #28275: Fixed possible use after free in the decompress() + methods of the LZMADecompressor and BZ2Decompressor classes. + Original patch by John Leitch. + +- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() + if pass invalid string-like object as a name. Patch by Xiang Zhang. + +- Issue #18844: random.choices() now has k as a keyword-only argument + to improve the readability of common cases and come into line + with the signature used in other languages. + +- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. + Patch by Madison May. + +- Issue #27611: Fixed support of default root window in the tkinter.tix module. + Added the master parameter in the DisplayStyle constructor. + +- Issue #27348: In the traceback module, restore the formatting of exception + messages like "Exception: None". This fixes a regression introduced in + 3.5a2. + +- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). + +- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is + interrupted by a signal and a signal handler raises a Python exception. + +- Issue #28200: Fix memory leak on Windows in the os module (fix + path_converter() function). + +- Issue #25400: RobotFileParser now correctly returns default values for + crawl_delay and request_rate. Initial patch by Peter Wirtz. + +- Issue #27932: Prevent memory leak in win32_ver(). + +- Fix UnboundLocalError in socket._sendfile_use_sendfile. + +- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of + os.stat(). Patch by Eryk Sun. + +- Issue #22493: Warning message emitted by using inline flags in the middle of + regular expression now contains a (truncated) regex pattern. + Patch by Tim Graham. + +- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when + an empty bytestring is passed. + +- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. + +- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. + Patch by Gergely Imreh and Markus Holtermann. + +- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. + Patch by Eryk Sun. + +- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). + +- Issue #27906: Fix socket accept exhaustion during high TCP traffic. + Patch by Kevin Conway. + +- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. + Patch by Seth Michael Larson. + +- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. + Patch by iceboy. + +- Issue #26909: Fix slow pipes IO in asyncio. + Patch by INADA Naoki. + +- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. + +- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. + Patch by Mark Williams. + +- Issue #28325: Remove vestigial MacOS 9 macurl2path module and its tests. + +- Issue #28368: Refuse monitoring processes if the child watcher has + no loop attached. + Patch by Vincent Michel. + +- Issue #28369: Raise RuntimeError when transport's FD is used with + add_reader, add_writer, etc. + +- Issue #28370: Speedup asyncio.StreamReader.readexactly. + Patch by ĐšĐŸŃ€Đ”ĐœĐ±Đ”Ń€Đł МарĐș. + +- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. + +- Issue #28372: Fix asyncio to support formatting of non-python coroutines. + +- Issue #28399: Remove UNIX socket from FS before binding. + Patch by ĐšĐŸŃ€Đ”ĐœĐ±Đ”Ń€Đł МарĐș. + +- Issue #27972: Prohibit Tasks to await on themselves. + +- Issue #24142: Reading a corrupt config file left configparser in an + invalid state. Original patch by Florian Höch. + +- Issue #29581: ABCMeta.__new__ now accepts **kwargs, allowing abstract base + classes to use keyword parameters in __init_subclass__. Patch by Nate Soares. + +Windows +------- + +- bpo-29579: Removes readme.txt from the installer. + +- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) + +- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default + +- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c + +- Issue #28402: Adds signed catalog files for stdlib on Windows. + +- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by + Eryk Sun) + +- Issue #28251: Improvements to help manuals on Windows. + +- Issue #28110: launcher.msi has different product codes between 32-bit and + 64-bit + +- Issue #28161: Opening CON for write access fails + +- Issue #28162: WindowsConsoleIO readall() fails if first line starts with + Ctrl+Z + +- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to + _open_osfhandle + +- Issue #28164: _PyIO_get_console_type fails for various paths + +- Issue #28137: Renames Windows path file to ._pth + +- Issue #28138: Windows ._pth file should allow import site + +C API +----- + +- Issue #27867: Function PySlice_GetIndicesEx() is deprecated and replaced with + a macro if Py_LIMITED_API is not set or set to the value between 0x03050400 + and 0x03060000 (not including) or 0x03060100 or higher. Added functions + PySlice_Unpack() and PySlice_AdjustIndices(). + +- Issue #29083: Fixed the declaration of some public API functions. + PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in + limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and + Py_BuildValue() were not available in limited API of version < 3.3 when + PY_SSIZE_T_CLEAN is defined. + +- Issue #28769: The result of PyUnicode_AsUTF8AndSize() and PyUnicode_AsUTF8() + is now of type ``const char *`` rather of ``char *``. + +- Issue #29058: All stable API extensions added after Python 3.2 are now + available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of + the minimum Python version supporting this API. Library ------- diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index cbe7425eaefa65..efc7d05b7d01b9 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1416,8 +1416,18 @@ buffered_repr(buffered *self) res = PyUnicode_FromFormat("<%s>", Py_TYPE(self)->tp_name); } else { - res = PyUnicode_FromFormat("<%s name=%R>", - Py_TYPE(self)->tp_name, nameobj); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat("<%s name=%R>", + Py_TYPE(self)->tp_name, nameobj); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 7f3bcab9625ef5..833ea8e7b50894 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -1082,9 +1082,19 @@ fileio_repr(fileio *self) self->fd, mode_string(self), self->closefd ? "True" : "False"); } else { - res = PyUnicode_FromFormat( - "<_io.FileIO name=%R mode='%s' closefd=%s>", - nameobj, mode_string(self), self->closefd ? "True" : "False"); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat( + "<_io.FileIO name=%R mode='%s' closefd=%s>", + nameobj, mode_string(self), self->closefd ? "True" : "False"); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 4df55626d4bc23..bc8d11efa5258c 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2483,6 +2483,7 @@ static PyObject * textiowrapper_repr(textio *self) { PyObject *nameobj, *modeobj, *res, *s; + int status; CHECK_INITIALIZED(self); @@ -2490,6 +2491,15 @@ textiowrapper_repr(textio *self) if (res == NULL) return NULL; + status = Py_ReprEnter((PyObject *)self); + if (status != 0) { + if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } + goto error; + } nameobj = _PyObject_GetAttrId((PyObject *) self, &PyId_name); if (nameobj == NULL) { if (PyErr_ExceptionMatches(PyExc_Exception)) @@ -2504,7 +2514,7 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } modeobj = _PyObject_GetAttrId((PyObject *) self, &PyId_mode); if (modeobj == NULL) { @@ -2520,14 +2530,21 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } s = PyUnicode_FromFormat("%U encoding=%R>", res, self->encoding); Py_DECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return s; -error: + + error: Py_XDECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return NULL; } From da88596a19d2cad0e85a2b3c3290b5b1d97a793d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 19 Mar 2017 20:53:42 +0200 Subject: [PATCH 149/232] bpo-29845: Mark tests that use _testcapi as CPython-only (#711) (#725) (cherry picked from commit 24c738a9e91b8f46da6166663d8ce7ec18cec784) --- Lib/ctypes/test/test_structures.py | 5 ++++- Lib/test/test_atexit.py | 1 + Lib/test/test_coroutines.py | 1 + Lib/test/test_socket.py | 1 + Lib/test/test_tracemalloc.py | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 3eded7749ed95e..2e778fb1b43740 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -2,8 +2,8 @@ from ctypes import * from ctypes.test import need_symbol from struct import calcsize -import _testcapi import _ctypes_test +import test.support class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -202,7 +202,10 @@ class X(Structure): "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + @test.support.cpython_only + def test_packed_c_limits(self): # Issue 15989 + import _testcapi d = {"_fields_": [("a", c_byte)], "_pack_": _testcapi.INT_MAX + 1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 172bd25419cea7..c761076c4a0225 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -143,6 +143,7 @@ def test_bound_methods(self): self.assertEqual(l, [5]) +@support.cpython_only class SubinterpreterTest(unittest.TestCase): def test_callbacks_leak(self): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index a69583b5f95ae6..2b79a17ea703f5 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2117,6 +2117,7 @@ def wrap(gen): sys.set_coroutine_wrapper(None) +@support.cpython_only class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 1508f20c9f0598..2497e47c668bc3 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -906,6 +906,7 @@ def testNtoH(self): self.assertEqual(swapped & mask, mask) self.assertRaises(OverflowError, func, 1<<34) + @support.cpython_only def testNtoHErrors(self): good_values = [ 1, 2, 3, 1, 2, 3 ] bad_values = [ -1, -2, -3, -1, -2, -3 ] diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 790ab7ee606d60..742259b4396b98 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -865,6 +865,7 @@ def test_sys_xoptions_invalid(self): b'number of frames', stderr) + @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_pymem_alloc0(self): # Issue #21639: Check that PyMem_Malloc(0) with tracemalloc enabled # does not crash. From 3d258b1eb453bcbc412d6b252f5bdceae0303f07 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 19 Mar 2017 21:06:29 +0200 Subject: [PATCH 150/232] bpo-29116: Fix error messages for concatenating bytes and bytearray with unsupported type. (#709) (#723) (cherry picked from commit 6b5a9ec4788770c652bac3bf5d5a0a3b710b82ae) --- Objects/bytearrayobject.c | 2 +- Objects/bytesobject.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a8d69802508736..d456678d462f2f 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -254,7 +254,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 30c1a7e24532da..c95a46d1846649 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1438,7 +1438,7 @@ bytes_concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } From d1f566f6bddc67e9870eaa75ae477208b9a199be Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 19 Mar 2017 20:55:16 -0700 Subject: [PATCH 151/232] bpo-29856: Fix typo in curses documentation (GH-730) (GH-731) From Shifted Dxit -> Shifted Exit in Doc/library/curses.rst (cherry picked from commit 64508780d72769e4c7afc67a511c057261c578f6) --- Doc/library/curses.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index d746eafaea9b5c..d51085506c2a96 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1443,7 +1443,7 @@ The exact keycaps available are system dependent. +-------------------+--------------------------------------------+ | ``KEY_SEOL`` | Shifted Clear line | +-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Dxit | +| ``KEY_SEXIT`` | Shifted Exit | +-------------------+--------------------------------------------+ | ``KEY_SFIND`` | Shifted Find | +-------------------+--------------------------------------------+ From 6fad4090ec9a27f8572bb00661b9890f01fb62f7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 20 Mar 2017 09:13:47 +0200 Subject: [PATCH 152/232] bpo-28876: bool of large range raises OverflowError (#699) (#734) (cherry picked from commit e46fb8611867fa3b407a813f53137929b7cb4a10) --- Lib/test/test_range.py | 17 +++++++++++++---- Misc/NEWS | 3 +++ Objects/rangeobject.c | 12 +++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 9e11e518f61e1f..679759ec6d676a 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -99,20 +99,24 @@ def test_large_operands(self): x = range(10**20+10, 10**20, 3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20, 10**20+10, -3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20+10, 10**20, -3) self.assertEqual(len(x), 4) self.assertEqual(len(list(x)), 4) + self.assertTrue(x) # Now test range() with longs - self.assertEqual(list(range(-2**100)), []) - self.assertEqual(list(range(0, -2**100)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) + for x in [range(-2**100), + range(0, -2**100), + range(0, 2**100, -1)]: + self.assertEqual(list(x), []) + self.assertFalse(x) a = int(10 * sys.maxsize) b = int(100 * sys.maxsize) @@ -153,6 +157,7 @@ def _range_len(x): step = x[1] - x[0] length = 1 + ((x[-1] - x[0]) // step) return length + a = -sys.maxsize b = sys.maxsize expected_len = b - a @@ -160,6 +165,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -177,6 +183,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -195,6 +202,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -213,6 +221,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 diff --git a/Misc/NEWS b/Misc/NEWS index 310e982d78f03a..7e87a28145780b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -39,6 +39,9 @@ Core and Builtins - bpo-28893: Set correct __cause__ for errors about invalid awaitables returned from __aiter__ and __anext__. +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + - bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by Brian Coleman. diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 8449fc7a24d394..8f5fc434bd4955 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -668,6 +668,16 @@ static PyMappingMethods range_as_mapping = { (objobjargproc)0, /* mp_ass_subscript */ }; +static int +range_bool(rangeobject* self) +{ + return PyObject_IsTrue(self->length); +} + +static PyNumberMethods range_as_number = { + .nb_bool = (inquiry)range_bool, +}; + static PyObject * range_iter(PyObject *seq); static PyObject * range_reverse(PyObject *seq); @@ -707,7 +717,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_setattr */ 0, /* tp_reserved */ (reprfunc)range_repr, /* tp_repr */ - 0, /* tp_as_number */ + &range_as_number, /* tp_as_number */ &range_as_sequence, /* tp_as_sequence */ &range_as_mapping, /* tp_as_mapping */ (hashfunc)range_hash, /* tp_hash */ From fcd79ada4e4cf335372538b34c8eb89cf09f84b2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 20 Mar 2017 14:54:52 +0200 Subject: [PATCH 153/232] Fix "NotImplentedError" typo in constants documentation (#692) (#739) `NotImplentedError` --> `NotImplementedError` (cherry picked from commit 05f53735c8912f8df1077e897f052571e13c3496) --- Doc/library/constants.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index f0742cee55bd55..469a3eed606ff0 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -46,7 +46,7 @@ A small number of constants live in the built-in namespace. They are: .. note:: - ``NotImplentedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. From e9213d929d7b0075539e87416f6a6fb86c27454b Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 00:35:08 -0400 Subject: [PATCH 154/232] bpo-27593: Revise git SCM build info. (#744) (#745) Use --short form of git hash. Use output from "git describe" for tag. Expected outputs: 1. previous hg 2. previous git 3. updated git Release (tagged) build: 1. Python 3.7.0a0 (v3.7.0a0:4def2a2901a5, ... 2. Python 3.7.0a0 (v3.7.0a0^0:05f53735c8912f8df1077e897f052571e13c3496, ... 3. Python 3.7.0a0 (v3.7.0a0:05f53735c8, ... Development build: 1. Python 3.7.0a0 (default:41df79263a11, ... 2. Python 3.7.0a0 (master:05f53735c8912f8df1077e897f052571e13c3496, ... 3. Python 3.7.0a0 (heads/master-dirty:05f53735c8, ... "dirty" means the working tree has uncommitted changes. See "git help describe" for more info. (cherry picked from commit 554626ada769abf82a5dabe6966afa4265acb6a6) --- PCbuild/pythoncore.vcxproj | 4 ++-- configure | 4 ++-- configure.ac | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d7e9473638cbb2..6ea184877f99cd 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -414,8 +414,8 @@ - - + + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) diff --git a/configure b/configure index d4eccb1d68d856..abe1dc5abd4e1c 100755 --- a/configure +++ b/configure @@ -2743,8 +2743,8 @@ HAS_GIT=no-repository fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse HEAD" - GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else GITVERSION="" diff --git a/configure.ac b/configure.ac index f00a2a6fa498f7..9eacf52559e8c2 100644 --- a/configure.ac +++ b/configure.ac @@ -37,8 +37,8 @@ HAS_GIT=no-repository fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse HEAD" - GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else GITVERSION="" From 6efc812280b9303d37f51fbd061f7c31379fb72e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sun, 5 Mar 2017 19:58:06 -0800 Subject: [PATCH 155/232] Fixes the upload script to purge the CDN correctly and display success output. (#466) (#497) (cherry picked from commit e544b40faa5ab61b6aba691577d90b2b641f664d) --- Tools/msi/uploadrelease.proj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 305e84fc2d4b28..75840f2f851ecd 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -8,6 +8,7 @@ $(TARGET) /srv/www.python.org/ftp/python true + true false false @@ -91,6 +92,7 @@ echo." /> + From 0acdea79cba3883c5e7035c7336fb0ce54435c03 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 6 Mar 2017 17:24:28 +0900 Subject: [PATCH 156/232] bpo-29719: Remove Date and Release field in whatsnew/3.6 (GH-500) (cherry picked from commit 2225ddaa9e64c086b2b6997b0c9ac50921f7aa85) (cherry picked from commit 4e1a065c20856a00d0fe88ce022b249170608058) --- Doc/whatsnew/3.6.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 96fd256b991b76..a696af4a999c84 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2,8 +2,6 @@ What's New In Python 3.6 **************************** -:Release: |release| -:Date: |today| :Editors: Elvis Pranskevichus , Yury Selivanov .. Rules for maintenance: From 07e6cbd7b9e5b6047a608cedd5cc9b20891a3972 Mon Sep 17 00:00:00 2001 From: "n.d. parker" Date: Wed, 8 Mar 2017 23:27:46 +0100 Subject: [PATCH 157/232] Fix the only non-C90 comment to be C90 compatible. (#568) (cherry picked from commit 51b646a55a4a1c4d0df764c4404a062fbcc6b356) --- Include/pyport.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/pyport.h b/Include/pyport.h index 52a91a0d11d056..426822a81f075a 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -37,7 +37,7 @@ Used in: Py_SAFE_DOWNCAST * integral synonyms. Only define the ones we actually need. */ -// long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. +/* long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. */ #ifndef HAVE_LONG_LONG #define HAVE_LONG_LONG 1 #endif From 75345c552d0889f4f63039d6063f371846c8f41f Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Mar 2017 21:34:22 +1000 Subject: [PATCH 158/232] [3.6] bpo-29723: Consistently configure sys.path[0] (#636) Directory and zipfile execution previously added the parent directory of the directory or zipfile as sys.path[0] and then subsequently overwrote it with the directory or zipfile itself. This caused problems in isolated mode, as it overwrote the "stdlib as a zip archive" entry in sys.path, as the parent directory was never added. The attempted fix to that issue in bpo-29319 created the opposite problem in *non*-isolated mode, by potentially leaving the parent directory on sys.path instead of overwriting it. This change fixes the root cause of the problem by removing the whole "add-and-overwrite" dance for sys.path[0], and instead simply never adds the parent directory to sys.path in the first place. (cherry picked from commit d2977a3ae2cc6802921b1e3b6e9d13fcfbda872d) (cherry picked from commit c60948464fb0ec116ea227f6bce8a4bb8fb75257) --- Lib/test/test_cmd_line_script.py | 67 +++++++++++++++++++++++++++++ Misc/NEWS | 36 ++++++++++++++++ Modules/main.c | 72 +++++++++++++++++++------------- 3 files changed, 147 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e058ecd086df3c..1587daf8f582dd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -572,6 +572,73 @@ def test_syntaxerror_indented_caret_position(self): self.assertNotIn("\f", text) self.assertIn("\n 1 + 1 = 2\n ^", text) + def test_consistent_sys_path_for_direct_execution(self): + # This test case ensures that the following all give the same + # sys.path configuration: + # + # ./python -s script_dir/__main__.py + # ./python -s script_dir + # ./python -I script_dir + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir, support.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from directly executing __main__.py + # We omit PYTHONPATH and user site to align with isolated mode + p = spawn_python("-Es", script_name, cwd=work_dir) + out_by_name = kill_python(p).decode().splitlines() + self.assertEqual(out_by_name[0], script_dir) + self.assertNotIn(work_dir, out_by_name) + # Directory execution should give the same output + p = spawn_python("-Es", script_dir, cwd=work_dir) + out_by_dir = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir, out_by_name) + # As should directory execution in isolated mode + p = spawn_python("-I", script_dir, cwd=work_dir) + out_by_dir_isolated = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir_isolated, out_by_dir, out_by_name) + + def test_consistent_sys_path_for_module_execution(self): + # This test case ensures that the following both give the same + # sys.path configuration: + # ./python -sm script_pkg.__main__ + # ./python -sm script_pkg + # + # And that this fails as unable to find the package: + # ./python -Im script_pkg + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from `-m script_pkg.__main__` + # We omit PYTHONPATH and user site to better align with the + # direct execution test cases + p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) + out_by_module = kill_python(p).decode().splitlines() + self.assertEqual(out_by_module[0], '') + self.assertNotIn(script_dir, out_by_module) + # Package execution should give the same output + p = spawn_python("-sm", "script_pkg", cwd=work_dir) + out_by_package = kill_python(p).decode().splitlines() + self.assertEqual(out_by_package, out_by_module) + # Isolated mode should fail with an import error + exitcode, stdout, stderr = assert_python_failure( + "-Im", "script_pkg", cwd=work_dir + ) + traceback_lines = stderr.decode().splitlines() + self.assertIn("No module named script_pkg", traceback_lines[-1]) def test_main(): support.run_unittest(CmdLineTest) diff --git a/Misc/NEWS b/Misc/NEWS index d715d306a34bc3..653510bb411791 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,42 @@ Python News +++++++++++ +What's New in Python 3.6.1 final? +================================= + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +- bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a + regression by revealing an inconsistency in how sys.path is initialized when + executing ``__main__`` from a zipfile, directory, or other import location. + The interpreter now consistently avoids ever adding the import location's + parent directory to ``sys.path``, and ensures no other ``sys.path`` entries + are inadvertently modified when inserting the import location named on the + command line. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +Library +------- + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + + What's New in Python 3.6.1 release candidate 1 ============================================== diff --git a/Modules/main.c b/Modules/main.c index 2e6a60b1673f88..dd502119d985e9 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -225,55 +225,60 @@ static int RunModule(wchar_t *modname, int set_argv0) return 0; } -static int -RunMainFromImporter(wchar_t *filename) +static PyObject * +AsImportPathEntry(wchar_t *filename) { - PyObject *argv0 = NULL, *importer, *sys_path, *sys_path0; - int sts; + PyObject *sys_path0 = NULL, *importer; - argv0 = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (argv0 == NULL) + sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); + if (sys_path0 == NULL) goto error; - importer = PyImport_GetImporter(argv0); + importer = PyImport_GetImporter(sys_path0); if (importer == NULL) goto error; if (importer == Py_None) { - Py_DECREF(argv0); + Py_DECREF(sys_path0); Py_DECREF(importer); - return -1; + return NULL; } Py_DECREF(importer); + return sys_path0; + +error: + Py_XDECREF(sys_path0); + PySys_WriteStderr("Failed checking if argv[0] is an import path entry\n"); + PyErr_Print(); + PyErr_Clear(); + return NULL; +} + + +static int +RunMainFromImporter(PyObject *sys_path0) +{ + PyObject *sys_path; + int sts; - /* argv0 is usable as an import source, so put it in sys.path[0] - and import __main__ */ + /* Assume sys_path0 has already been checked by AsImportPathEntry, + * so put it in sys.path[0] and import __main__ */ sys_path = PySys_GetObject("path"); if (sys_path == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); goto error; } - sys_path0 = PyList_GetItem(sys_path, 0); - sts = 0; - if (!sys_path0) { - PyErr_Clear(); - sts = PyList_Append(sys_path, argv0); - } else if (PyObject_IsTrue(sys_path0)) { - sts = PyList_Insert(sys_path, 0, argv0); - } else { - sts = PyList_SetItem(sys_path, 0, argv0); - } + sts = PyList_Insert(sys_path, 0, sys_path0); if (sts) { - argv0 = NULL; + sys_path0 = NULL; goto error; } - Py_INCREF(argv0); sts = RunModule(L"__main__", 0); return sts != 0; error: - Py_XDECREF(argv0); + Py_XDECREF(sys_path0); PyErr_Print(); return 1; } @@ -358,6 +363,7 @@ Py_Main(int argc, wchar_t **argv) int saw_unbuffered_flag = 0; char *opt; PyCompilerFlags cf; + PyObject *main_importer_path = NULL; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -714,7 +720,17 @@ Py_Main(int argc, wchar_t **argv) argv[_PyOS_optind] = L"-m"; } - PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + if (filename != NULL) { + main_importer_path = AsImportPathEntry(filename); + } + + if (main_importer_path != NULL) { + /* Let RunMainFromImporter adjust sys.path[0] later */ + PySys_SetArgvEx(argc-_PyOS_optind, argv+_PyOS_optind, 0); + } else { + /* Use config settings to decide whether or not to update sys.path[0] */ + PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + } if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) && isatty(fileno(stdin)) && @@ -744,11 +760,11 @@ Py_Main(int argc, wchar_t **argv) sts = -1; /* keep track of whether we've already run __main__ */ - if (filename != NULL) { - sts = RunMainFromImporter(filename); + if (main_importer_path != NULL) { + sts = RunMainFromImporter(main_importer_path); } - if (sts==-1 && filename!=NULL) { + if (sts==-1 && filename != NULL) { fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; From 360c49b9cf3c164fa1ee3228e1a68282d77c5e4e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 16 Mar 2017 11:03:03 -0700 Subject: [PATCH 159/232] Takes vcruntime140.dll from the correct source. (#684) (cherry picked from commit 9cd5e87bac51d7b901e3c36bf22728bb1693da59) --- Tools/msi/make_zip.proj | 9 +++------ Tools/nuget/make_pkg.proj | 8 +++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj index f78e6ffa28fb72..b3588b7a0bae90 100644 --- a/Tools/msi/make_zip.proj +++ b/Tools/msi/make_zip.proj @@ -17,15 +17,12 @@ rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) - set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) - + diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj index d7e932cee54d3c..464ef0456af47a 100644 --- a/Tools/nuget/make_pkg.proj +++ b/Tools/nuget/make_pkg.proj @@ -34,9 +34,8 @@ $(NugetArguments) -Version "$(NuspecVersion)" $(NugetArguments) -NoPackageAnalysis -NonInteractive - setlocal -set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) @@ -45,8 +44,7 @@ set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140. - + From 8c18fbeed1c7721b67f1726a6e9c41acef823135 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 00:35:08 -0400 Subject: [PATCH 160/232] bpo-27593: Revise git SCM build info. (#744) (#745) Use --short form of git hash. Use output from "git describe" for tag. Expected outputs: 1. previous hg 2. previous git 3. updated git Release (tagged) build: 1. Python 3.7.0a0 (v3.7.0a0:4def2a2901a5, ... 2. Python 3.7.0a0 (v3.7.0a0^0:05f53735c8912f8df1077e897f052571e13c3496, ... 3. Python 3.7.0a0 (v3.7.0a0:05f53735c8, ... Development build: 1. Python 3.7.0a0 (default:41df79263a11, ... 2. Python 3.7.0a0 (master:05f53735c8912f8df1077e897f052571e13c3496, ... 3. Python 3.7.0a0 (heads/master-dirty:05f53735c8, ... "dirty" means the working tree has uncommitted changes. See "git help describe" for more info. (cherry picked from commit 554626ada769abf82a5dabe6966afa4265acb6a6) (cherry picked from commit e9213d929d7b0075539e87416f6a6fb86c27454b) --- PCbuild/pythoncore.vcxproj | 4 ++-- configure | 4 ++-- configure.ac | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index d7e9473638cbb2..6ea184877f99cd 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -414,8 +414,8 @@ - - + + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) diff --git a/configure b/configure index d4eccb1d68d856..abe1dc5abd4e1c 100755 --- a/configure +++ b/configure @@ -2743,8 +2743,8 @@ HAS_GIT=no-repository fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse HEAD" - GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else GITVERSION="" diff --git a/configure.ac b/configure.ac index f00a2a6fa498f7..9eacf52559e8c2 100644 --- a/configure.ac +++ b/configure.ac @@ -37,8 +37,8 @@ HAS_GIT=no-repository fi if test $HAS_GIT = found then - GITVERSION="git -C \$(srcdir) rev-parse HEAD" - GITTAG="git -C \$(srcdir) name-rev --tags --name-only HEAD" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else GITVERSION="" From 69c0db5050f623e8895b72dfe970392b1f9a0e2e Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 02:32:38 -0400 Subject: [PATCH 161/232] Update docs and patchlevel for 3.6.1 final --- Include/patchlevel.h | 6 +++--- Misc/NEWS | 26 +++++++------------------- README.rst | 4 ++-- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index b42a0db2f82007..54a9ab6ff78896 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -19,11 +19,11 @@ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 1 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.1rc1" +#define PY_VERSION "3.6.1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 653510bb411791..4807bd0fcc3b3f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,10 +2,10 @@ Python News +++++++++++ -What's New in Python 3.6.1 final? -================================= +What's New in Python 3.6.1? +=========================== -*Release date: XXXX-XX-XX* +*Release date: 2017-03-21* Core and Builtins ----------------- @@ -18,24 +18,12 @@ Core and Builtins are inadvertently modified when inserting the import location named on the command line. -- bpo-29714: Fix a regression that bytes format may fail when containing zero - bytes inside. - -Library -------- - -- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big - intables (objects that have __int__) as elements. Patch by Oren Milman. - -- bpo-28231: The zipfile module now accepts path-like objects for external - paths. +Build +----- -- bpo-26915: index() and count() methods of collections.abc.Sequence now - check identity before checking equality when do comparisons. +- bpo-27593: fix format of git information used in sys.version -- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other - exception) to exception(s) raised in the dispatched methods. - Patch by Petr Motejlek. +- Fix incompatible comment in python.h What's New in Python 3.6.1 release candidate 1 diff --git a/README.rst b/README.rst index 242572c5e361db..9ea036066877c9 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.6.1 release candidate 1 -================================================ +This is Python version 3.6.1 +============================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 :alt: CPython build status on Travis CI From fe30339534c602af1123e1402e44a1463f91f2e5 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 22 Mar 2017 03:46:26 +0900 Subject: [PATCH 162/232] bpo-29859: Fix error messages from return codes for pthread_* calls (GH-753) (cherry picked from commit d7fa6b259e00fca04dbf816bfcf4115fdda14bb7) --- Misc/NEWS | 3 +++ Python/thread_pthread.h | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7e87a28145780b..aad92480693e6a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.1 final? Core and Builtins ----------------- +- bpo-29859: Show correct error messages when any of the pthread_* calls in + thread_pthread.h fails. + - bpo-29600: Fix wrapping coroutine return values in StopIteration. - bpo-28856: Fix an oversight that %b format for bytes should support objects diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 27e0dc84bcb52e..ba7393f03de6ae 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -143,6 +143,8 @@ typedef struct { } pthread_lock; #define CHECK_STATUS(name) if (status != 0) { perror(name); error = 1; } +#define CHECK_STATUS_PTHREAD(name) if (status != 0) { fprintf(stderr, \ + "%s: %s\n", name, strerror(status)); error = 1; } /* * Initialization. @@ -417,7 +419,7 @@ PyThread_allocate_lock(void) status = pthread_mutex_init(&lock->mut, pthread_mutexattr_default); - CHECK_STATUS("pthread_mutex_init"); + CHECK_STATUS_PTHREAD("pthread_mutex_init"); /* Mark the pthread mutex underlying a Python mutex as pure happens-before. We can't simply mark the Python-level mutex as a mutex because it can be @@ -427,7 +429,7 @@ PyThread_allocate_lock(void) status = pthread_cond_init(&lock->lock_released, pthread_condattr_default); - CHECK_STATUS("pthread_cond_init"); + CHECK_STATUS_PTHREAD("pthread_cond_init"); if (error) { PyMem_RawFree((void *)lock); @@ -452,10 +454,10 @@ PyThread_free_lock(PyThread_type_lock lock) * and must have the cond destroyed first. */ status = pthread_cond_destroy( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_destroy"); + CHECK_STATUS_PTHREAD("pthread_cond_destroy"); status = pthread_mutex_destroy( &thelock->mut ); - CHECK_STATUS("pthread_mutex_destroy"); + CHECK_STATUS_PTHREAD("pthread_mutex_destroy"); PyMem_RawFree((void *)thelock); } @@ -472,7 +474,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, lock, microseconds, intr_flag)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); if (thelock->locked == 0) { success = PY_LOCK_ACQUIRED; @@ -494,13 +496,13 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, &thelock->mut, &ts); if (status == ETIMEDOUT) break; - CHECK_STATUS("pthread_cond_timed_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); } else { status = pthread_cond_wait( &thelock->lock_released, &thelock->mut); - CHECK_STATUS("pthread_cond_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_wait"); } if (intr_flag && status == 0 && thelock->locked) { @@ -518,7 +520,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, } if (success == PY_LOCK_ACQUIRED) thelock->locked = 1; status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); if (error) success = PY_LOCK_FAILURE; dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", @@ -536,16 +538,16 @@ PyThread_release_lock(PyThread_type_lock lock) dprintf(("PyThread_release_lock(%p) called\n", lock)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[3]"); thelock->locked = 0; /* wake up someone (anyone, if any) waiting on the lock */ status = pthread_cond_signal( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_signal"); + CHECK_STATUS_PTHREAD("pthread_cond_signal"); status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[3]"); } #endif /* USE_SEMAPHORES */ From 677df6ecade6ff503d3c3c93cf157ed12fec9386 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 20:21:49 -0400 Subject: [PATCH 163/232] Revert and fix Misc/NEWS after merge error in fca705d53397. (#755) --- Misc/NEWS | 805 +----------------------------------------------------- 1 file changed, 3 insertions(+), 802 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index aad92480693e6a..2e0f00be111427 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins - bpo-29859: Show correct error messages when any of the pthread_* calls in thread_pthread.h fails. +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + - bpo-29600: Fix wrapping coroutine return values in StopIteration. - bpo-28856: Fix an oversight that %b format for bytes should support objects @@ -28,813 +31,12 @@ Core and Builtins - bpo-29714: Fix a regression that bytes format may fail when containing zero bytes inside. -- bpo-29568: Escaped percent "%%" in the format string for classic string - formatting no longer allows any characters between two percents. - -- bpo-29714: Fix a regression that bytes format may fail when containing zero - bytes inside. - -- bpo-29695: bool(), float(), list() and tuple() no longer take keyword arguments. - The first argument of int() can now be passes only as positional argument. - -- bpo-20087: Prefer glibc's list of locale aliases to the X11 ones. - -- bpo-28893: Set correct __cause__ for errors about invalid awaitables - returned from __aiter__ and __anext__. - -- bpo-28876: ``bool(range)`` works even if ``len(range)`` - raises :exc:`OverflowError`. - -- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by - Brian Coleman. - -- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. - It should raise TypeError when kwargs is not a dict. But it might - cause segv when args=NULL and kwargs is not a dict. - -- bpo-28598: Support __rmod__ for subclasses of str being called before - str.__mod__. Patch by Martijn Pieters. - -- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. - Patch by Matthieu Dartiailh. - -- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for - complex subclasses and for inputs having a __complex__ method. Patch - by Serhiy Storchaka. - -- bpo-29347: Fixed possibly dereferencing undefined pointers - when creating weakref objects. - -- bpo-29463: Add ``docstring`` field to Module, ClassDef, FunctionDef, - and AsyncFunctionDef ast nodes. docstring is not first stmt in their body - anymore. It affects ``co_firstlineno`` and ``co_lnotab`` of code object - for module and class. - -- bpo-29438: Fixed use-after-free problem in key sharing dict. - -- bpo-29546: Set the 'path' and 'name' attribute on ImportError for ``from ... import ...``. - -- bpo-29546: Improve from-import error message with location - -- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - -- Issue #29337: Fixed possible BytesWarning when compare the code objects. - Warnings could be emitted at compile time. - -- Issue #29327: Fixed a crash when pass the iterable keyword argument to - sorted(). - -- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). - -- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. - -- Issue #29049: Call _PyObject_GC_TRACK() lazily when calling Python function. - Calling function is up to 5% faster. - -- Issue #28927: bytes.fromhex() and bytearray.fromhex() now ignore all ASCII - whitespace, not only spaces. Patch by Robert Xiao. - -- Issue #28932: Do not include if it does not exist. - -- Issue #25677: Correct the positioning of the syntax error caret for - indented blocks. Based on patch by Michael Layzell. - -- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate - form. - -- Issue #18896: Python function can now have more than 255 parameters. - collections.namedtuple() now supports tuples with more than 255 elements. - -- Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by - Chi Hsuan Yen. - -- Issue #26919: On Android, operating system data is now always encoded/decoded - to/from UTF-8, instead of the locale encoding to avoid inconsistencies with - os.fsencode() and os.fsdecode() which are already using UTF-8. - -- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy - bug triggerable by a monkey-patched len() function. - -- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() - must not convert combined table into split table. Patch written by INADA - Naoki. - -- Issue #28739: f-string expressions are no longer accepted as docstrings and - by ast.literal_eval() even if they do not include expressions. - -- Issue #28512: Fixed setting the offset attribute of SyntaxError by - PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). - -- Issue #28918: Fix the cross compilation of xxlimited when Python has been - built with Py_DEBUG defined. - -- Issue #23722: Rather than silently producing a class that doesn't support - zero-argument ``super()`` in methods, failing to pass the new - ``__classcell__`` namespace entry up to ``type.__new__`` now results in a - ``DeprecationWarning`` and a class that supports zero-argument ``super()``. - -- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of - a descriptor that is used inside that class no longer prevents calling the - __set_name__ method of other descriptors. - -- Issue #28799: Remove the ``PyEval_GetCallStats()`` function and deprecate - the untested and undocumented ``sys.callstats()`` function. Remove the - ``CALL_PROFILE`` special build: use the :func:`sys.setprofile` function, - :mod:`cProfile` or :mod:`profile` to profile function calls. - -- Issue #12844: More than 255 arguments can now be passed to a function. - -- Issue #28782: Fix a bug in the implementation ``yield from`` when checking - if the next instruction is YIELD_FROM. Regression introduced by WORDCODE - (issue #26647). - -- Issue #28774: Fix error position of the unicode error in ASCII and Latin1 - encoders when a string returned by the error handler contains multiple - non-encodable characters (non-ASCII for the ASCII codec, characters out - of the U+0000-U+00FF range for Latin1). - -- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. - Improve speed of dict literal with constant keys up to 30%. - -- Issue #28532: Show sys.version when -V option is supplied twice. - -- Issue #27100: The with-statement now checks for __enter__ before it - checks for __exit__. This gives less confusing error messages when - both methods are missing. Patch by Jonathan Ellington. - -- Issue #28746: Fix the set_inheritable() file descriptor method on platforms - that do not have the ioctl FIOCLEX and FIONCLEX commands. - -- Issue #26920: Fix not getting the locale's charset upon initializing the - interpreter, on platforms that do not have langinfo. - -- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X - when decode astral characters. Patch by Xiang Zhang. - -- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. - -- Issue #19398: Extra slash no longer added to sys.path components in case of - empty compile-time PYTHONPATH components. - -- Issue #28621: Sped up converting int to float by reusing faster bits counting - implementation. Patch by Adrian Wielgosik. - -- Issue #28580: Optimize iterating split table values. - Patch by Xiang Zhang. - -- Issue #28583: PyDict_SetDefault didn't combine split table when needed. - Patch by Xiang Zhang. - -- Issue #28128: Deprecation warning for invalid str and byte escape - sequences now prints better information about where the error - occurs. Patch by Serhiy Storchaka and Eric Smith. - -- Issue #28509: dict.update() no longer allocate unnecessary large memory. - -- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug - build. - -- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused - keeping unreachable code. - -- Issue #28214: Improved exception reporting for problematic __set_name__ - attributes. - -- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception - loss in PyTraceBack_Here(). - -- Issue #28183: Optimize and cleanup dict iteration. - -- Issue #26081: Added C implementation of asyncio.Future. - Original patch by Yury Selivanov. - -- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). - Patch by Xiang Zhang. - -- Issue #28376: The type of long range iterator is now registered as Iterator. - Patch by Oren Milman. - -- Issue #28376: Creating instances of range_iterator by calling range_iterator - type now is disallowed. Calling iter() on range instance is the only way. - Patch by Oren Milman. - -- Issue #26906: Resolving special methods of uninitialized type now causes - implicit initialization of the type instead of a fail. - -- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. - Original patch by Niklas Koep. - -- Issue #24098: Fixed possible crash when AST is changed in process of - compiling it. - -- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when - hashes have same lower bits. - -- Issue #28350: String constants with null character no longer interned. - -- Issue #26617: Fix crash when GC runs during weakref callbacks. - -- Issue #27942: String constants now interned recursively in tuples and frozensets. - -- Issue #28289: ImportError.__init__ now resets not specified attributes. - -- Issue #21578: Fixed misleading error message when ImportError called with - invalid keyword args. - -- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. - Patch by Soumya Sharma. - -- Issue #28086: Single var-positional argument of tuple subtype was passed - unscathed to the C-defined function. Now it is converted to exact tuple. - -- Issue #28214: Now __set_name__ is looked up on the class instead of the - instance. - -- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() - syscall fails with EPERM, for example when blocked by SECCOMP. - -- Issue #28192: Don't import readline in isolated mode. - -- Issue #27441: Remove some redundant assignments to ob_size in longobject.c. - Thanks Oren Milman. - -- Issue #27222: Clean up redundant code in long_rshift function. Thanks - Oren Milman. - -- Upgrade internal unicode databases to Unicode version 9.0.0. - -- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport - should use the same optimization level as the interpreter. - -- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly - optimize memcpy(). - -- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a - "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. - -- Issue #26182: Raise DeprecationWarning when async and await keywords are - used as variable/attribute/class/function name. - -- Issue #26182: Fix a refleak in code that raises DeprecationWarning. - -- Issue #28721: Fix asynchronous generators aclose() and athrow() to - handle StopAsyncIteration propagation properly. - -- Issue #26110: Speed-up method calls: add LOAD_METHOD and CALL_METHOD - opcodes. - -Extension Modules ------------------ - -- Issue #29169: Update zlib to 1.2.11. Library ------- - bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. -- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords - are not strings. Patch by Michael Seifert. - -- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" - or "errors" of sys.stdin or sys.stdout are not set or are not strings. - -- bpo-28692: Using non-integer value for selecting a plural form in gettext is - now deprecated. - -- bpo-26121: Use C library implementation for math functions erf() and erfc(). - -- bpo-29619: os.stat() and os.DirEntry.inode() now convert inode (st_ino) using - unsigned integers. - -- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big - intables (objects that have __int__) as elements. - -- bpo-29645: Speed up importing the webbrowser module. webbrowser.register() - is now thread-safe. - -- bpo-28231: The zipfile module now accepts path-like objects for external - paths. - -- bpo-26915: index() and count() methods of collections.abc.Sequence now - check identity before checking equality when do comparisons. - -- bpo-28682: Added support for bytes paths in os.fwalk(). - -- bpo-29623: Allow use of path-like object as a single argument in - ConfigParser.read(). Patch by David Ellis. - -- bpo-9303: Migrate sqlite3 module to _v2 API. Patch by Aviv Palivoda. - -- bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback - implemented in C. - -- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before - all pipes are closed. - -- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C - to accept None argument as their pure Python implementation. - -- bpo-29703: Fix asyncio to support instantiation of new event loops - in child processes. - -- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other - exception) to exception(s) raised in the dispatched methods. - Patch by Petr Motejlek. - -- bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher - and its subclasses can now be used as a decorator. - -- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). - -- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts - PathLike objects. Patch by Sayan Chowdhury. - -- bpo-28518: Start a transaction implicitly before a DML statement. - Patch by Aviv Palivoda. - -- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. - Patch by Nikolay Kim. - -- Issue #16285: urrlib.parse.quote is now based on RFC 3986 and hence includes - '~' in the set of characters that is not quoted by default. Patch by - Christian Theune and Ratnadeep Debnath. - -- bpo-29532: Altering a kwarg dictionary passed to functools.partial() - no longer affects a partial object after creation. - -- bpo-29110: Fix file object leak in aifc.open() when file is given as a - filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. - -- bpo-22807: Add uuid.SafeUUID and uuid.UUID.is_safe to relay information from - the platform about whether generated UUIDs are generated with a - multiprocessing safe method. - -- bpo-29576: Improve some deprecations in importlib. Some deprecated methods - now emit DeprecationWarnings and have better descriptive messages. - -- bpo-29534: Fixed different behaviour of Decimal.from_float() - for _decimal and _pydecimal. Thanks Andrew Nester. - -- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, - improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, - Manuel Krebber, and Ɓukasz Langa. - -- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python - 3.6.0: check minimum and maximum years. - -- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir - -- Issue #29444: Fixed out-of-bounds buffer access in the group() method of - the match object. Based on patch by WGH. - -- Issue #29377: Add SlotWrapperType, MethodWrapperType, and - MethodDescriptorType built-in types to types module. - Original patch by Manuel Krebber. - -- Issue #29218: Unused install_misc command is now removed. It has been - documented as unused since 2000. Patch by Eric N. Vander Weele. - -- Issue #29368: The extend() method is now called instead of the append() - method when unpickle collections.deque and other list-like objects. - This can speed up unpickling to 2 times. - -- Issue #29338: The help of a builtin or extension class now includes the - constructor signature if __text_signature__ is provided for the class. - -- Issue #29335: Fix subprocess.Popen.wait() when the child process has - exited to a stopped instead of terminated state (ex: when under ptrace). - -- Issue #29290: Fix a regression in argparse that help messages would wrap at - non-breaking spaces. - -- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. - -- Issue #29197: Removed deprecated function ntpath.splitunc(). - -- Issue #29210: Removed support of deprecated argument "exclude" in - tarfile.TarFile.add(). - -- Issue #29219: Fixed infinite recursion in the repr of uninitialized - ctypes.CDLL instances. - -- Issue #29192: Removed deprecated features in the http.cookies module. - -- Issue #29193: A format string argument for string.Formatter.format() - is now positional-only. - -- Issue #29195: Removed support of deprecated undocumented keyword arguments - in methods of regular expression objects. - -- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. - KeyError could be raised when cached function with full cache was - simultaneously called from differen threads with the same uncached arguments. - -- Issue #20804: The unittest.mock.sentinel attributes now preserve their - identity when they are copied or pickled. - -- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with - leading dots could match related hostnames again (e.g. .b.c matches a.b.c). - Patch by Milan Oberkirch. - -- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter - anymore. Patch written by Jiajun Huang. - -- Issue #15812: inspect.getframeinfo() now correctly shows the first line of - a context. Patch by Sam Breese. - -- Issue #28985: Update authorizer constants in sqlite3 module. - Patch by Dingyuan Wang. - -- Issue #29094: Offsets in a ZIP file created with extern file object and modes - "w" and "x" now are relative to the start of the file. - -- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows - -- Issue #13051: Fixed recursion errors in large or resized - curses.textpad.Textbox. Based on patch by Tycho Andersen. - -- Issue #9770: curses.ascii predicates now work correctly with negative - integers. - -- Issue #28427: old keys should not remove new values from - WeakValueDictionary when collecting from another thread. - -- Issue 28923: Remove editor artifacts from Tix.py. - -- Issue #28871: Fixed a crash when deallocate deep ElementTree. - -- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and - WeakValueDictionary.pop() when a GC collection happens in another - thread. - -- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that - doesn't own its elements as limits. - -- Issue #16255: subprocess.Popen uses /system/bin/sh on Android as the shell, - instead of /bin/sh. - -- Issue #28779: multiprocessing.set_forkserver_preload() would crash the - forkserver process if a preloaded module instantiated some - multiprocessing objects such as locks. - -- Issue #26937: The chown() method of the tarfile.TarFile class does not fail - now when the grp module cannot be imported, as for example on Android - platforms. - -- Issue #28847: dbm.dumb now supports reading read-only files and no longer - writes the index file when it is not changed. A deprecation warning is now - emitted if the index file is missed and recreated in the 'r' and 'w' modes - (will be an error in future Python releases). - -- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in - re.sub() replacement templates regular expressions now are errors. - -- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): - call warnings.showwarning() if it was overriden inside the context manager. - -- Issue #27172: To assist with upgrades from 2.7, the previously documented - deprecation of ``inspect.getfullargspec()`` has been reversed. This decision - may be revisited again after the Python 2.7 branch is no longer officially - supported. - -- Issue #28740: Add sys.getandroidapilevel(): return the build time API version - of Android as an integer. Function only available on Android. - -- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and - :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by - Omar Sandoval. - -- Issue #28752: Restored the __reduce__() methods of datetime objects. - -- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created - by re.compile(), become comparable (only x==y and x!=y operators). This - change should fix the issue #18383: don't duplicate warning filters when the - warnings module is reloaded (thing usually only done in unit tests). - -- Issue #20572: Remove the subprocess.Popen.wait endtime parameter. It was - deprecated in 3.4 and undocumented prior to that. - -- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and - from_buffer_copy() methods on abstract classes like Array. - -- Issue #28548: In the "http.server" module, parse the protocol version if - possible, to avoid using HTTP 0.9 in some error responses. - -- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. - Patch by Vajrasky Kok - -- Issue #28563: Fixed possible DoS and arbitrary code execution when handle - plural form selections in the gettext module. The expression parser now - supports exact syntax supported by GNU gettext. - -- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when - the garbage collector is invoked in other thread. Based on patch by - Sebastian Cufre. - -- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if - given empty data twice. Patch by Benjamin Fogle. - -- Issue #28549: Fixed segfault in curses's addch() with ncurses6. - -- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar - file with compression before trying to open it without compression. Otherwise - it had 50% chance failed with ignore_zeros=True. - -- Issue #23262: The webbrowser module now supports Firefox 36+ and derived - browsers. Based on patch by Oleg Broytman. - -- Issue #24241: The webbrowser in an X environment now prefers using the - default browser directly. Also, the webbrowser register() function now has - a documented 'preferred' argument, to specify browsers to be returned by - get() with no arguments. Patch by David Steele - -- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused - by representing the scale as float value internally in Tk. tkinter.IntVar - now works if float value is set to underlying Tk variable. - -- Issue #28255: calendar.TextCalendar.prweek() no longer prints a space after - a weeks's calendar. calendar.TextCalendar.pryear() no longer prints redundant - newline after a year's calendar. Based on patch by Xiang Zhang. - -- Issue #28255: calendar.TextCalendar.prmonth() no longer prints a space - at the start of new line after printing a month's calendar. Patch by - Xiang Zhang. - -- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. - Based on patch by Kaarle Ritvanen. - -- Issue #28353: os.fwalk() no longer fails on broken links. - -- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept - non-None value is passed to it.send(val). - -- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix - for readability (was "`"). - -- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin - a workaround to Tix library bug. - -- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. - -- Issue #25953: re.sub() now raises an error for invalid numerical group - reference in replacement template even if the pattern is not found in - the string. Error message for invalid group reference now includes the - group index and the position of the reference. - Based on patch by SilentGhost. - -- Issue #28469: timeit now uses the sequence 1, 2, 5, 10, 20, 50,... instead - of 1, 10, 100,... for autoranging. - -- Issue #28115: Command-line interface of the zipfile module now uses argparse. - Added support of long options. - -- Issue #18219: Optimize csv.DictWriter for large number of columns. - Patch by Mariatta Wijaya. - -- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. - -- Issue #23214: In the "io" module, the argument to BufferedReader and - BytesIO's read1() methods is now optional and can be -1, matching the - BufferedIOBase specification. - -- Issue #28480: Fix error building socket module when multithreading is - disabled. - -- Issue #28240: timeit: remove ``-c/--clock`` and ``-t/--time`` command line - options which were deprecated since Python 3.3. - -- Issue #28240: timeit now repeats the benchmarks 5 times instead of only 3 - to make benchmarks more reliable. - -- Issue #28240: timeit autorange now uses a single loop iteration if the - benchmark takes less than 10 seconds, instead of 10 iterations. - "python3 -m timeit -s 'import time' 'time.sleep(1)'" now takes 4 seconds - instead of 40 seconds. - -- Distutils.sdist now looks for README and setup.py files with case - sensitivity. This behavior matches that found in Setuptools 6.0 and - later. See `setuptools 100 - `_ for rationale. - -- Issue #24452: Make webbrowser support Chrome on Mac OS X. Patch by - Ned Batchelder. - -- Issue #20766: Fix references leaked by pdb in the handling of SIGINT - handlers. - -- Issue #27998: Fixed bytes path support in os.scandir() on Windows. - Patch by Eryk Sun. - -- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. - -- Issue #26293: Fixed writing ZIP files that starts not from the start of the - file. Offsets in ZIP file now are relative to the start of the archive in - conforming to the specification. - -- Issue #28380: unittest.mock Mock autospec functions now properly support - assert_called, assert_not_called, and assert_called_once. - -- Issue #28229: lzma module now supports pathlib. - -- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. - -- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. - -- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. - -- Issue #28332: Deprecated silent truncations in socket.htons and socket.ntohs. - Original patch by Oren Milman. - -- Issue #27358: Optimized merging var-keyword arguments and improved error - message when passing a non-mapping as a var-keyword argument. - -- Issue #28257: Improved error message when passing a non-iterable as - a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. - -- Issue #28322: Fixed possible crashes when unpickle itertools objects from - incorrect pickle data. Based on patch by John Leitch. - -- Issue #28228: imghdr now supports pathlib. - -- Issue #28226: compileall now supports pathlib. - -- Issue #28314: Fix function declaration (C flags) for the getiterator() method - of xml.etree.ElementTree.Element. - -- Issue #28148: Stop using localtime() and gmtime() in the time - module. - - Introduced platform independent _PyTime_localtime API that is - similar to POSIX localtime_r, but available on all platforms. Patch - by Ed Schouten. - -- Issue #28253: Fixed calendar functions for extreme months: 0001-01 - and 9999-12. - - Methods itermonthdays() and itermonthdays2() are reimplemented so - that they don't call itermonthdates() which can cause datetime.date - under/overflow. - -- Issue #28275: Fixed possible use after free in the decompress() - methods of the LZMADecompressor and BZ2Decompressor classes. - Original patch by John Leitch. - -- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() - if pass invalid string-like object as a name. Patch by Xiang Zhang. - -- Issue #18844: random.choices() now has k as a keyword-only argument - to improve the readability of common cases and come into line - with the signature used in other languages. - -- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. - Patch by Madison May. - -- Issue #27611: Fixed support of default root window in the tkinter.tix module. - Added the master parameter in the DisplayStyle constructor. - -- Issue #27348: In the traceback module, restore the formatting of exception - messages like "Exception: None". This fixes a regression introduced in - 3.5a2. - -- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). - -- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is - interrupted by a signal and a signal handler raises a Python exception. - -- Issue #28200: Fix memory leak on Windows in the os module (fix - path_converter() function). - -- Issue #25400: RobotFileParser now correctly returns default values for - crawl_delay and request_rate. Initial patch by Peter Wirtz. - -- Issue #27932: Prevent memory leak in win32_ver(). - -- Fix UnboundLocalError in socket._sendfile_use_sendfile. - -- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of - os.stat(). Patch by Eryk Sun. - -- Issue #22493: Warning message emitted by using inline flags in the middle of - regular expression now contains a (truncated) regex pattern. - Patch by Tim Graham. - -- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when - an empty bytestring is passed. - -- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. - -- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. - Patch by Gergely Imreh and Markus Holtermann. - -- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. - Patch by Eryk Sun. - -- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). - -- Issue #27906: Fix socket accept exhaustion during high TCP traffic. - Patch by Kevin Conway. - -- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. - Patch by Seth Michael Larson. - -- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. - Patch by iceboy. - -- Issue #26909: Fix slow pipes IO in asyncio. - Patch by INADA Naoki. - -- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. - -- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. - Patch by Mark Williams. - -- Issue #28325: Remove vestigial MacOS 9 macurl2path module and its tests. - -- Issue #28368: Refuse monitoring processes if the child watcher has - no loop attached. - Patch by Vincent Michel. - -- Issue #28369: Raise RuntimeError when transport's FD is used with - add_reader, add_writer, etc. - -- Issue #28370: Speedup asyncio.StreamReader.readexactly. - Patch by ĐšĐŸŃ€Đ”ĐœĐ±Đ”Ń€Đł МарĐș. - -- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. - -- Issue #28372: Fix asyncio to support formatting of non-python coroutines. - -- Issue #28399: Remove UNIX socket from FS before binding. - Patch by ĐšĐŸŃ€Đ”ĐœĐ±Đ”Ń€Đł МарĐș. - -- Issue #27972: Prohibit Tasks to await on themselves. - -- Issue #24142: Reading a corrupt config file left configparser in an - invalid state. Original patch by Florian Höch. - -- Issue #29581: ABCMeta.__new__ now accepts **kwargs, allowing abstract base - classes to use keyword parameters in __init_subclass__. Patch by Nate Soares. - -Windows -------- - -- bpo-29579: Removes readme.txt from the installer. - -- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) - -- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default - -- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c - -- Issue #28402: Adds signed catalog files for stdlib on Windows. - -- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by - Eryk Sun) - -- Issue #28251: Improvements to help manuals on Windows. - -- Issue #28110: launcher.msi has different product codes between 32-bit and - 64-bit - -- Issue #28161: Opening CON for write access fails - -- Issue #28162: WindowsConsoleIO readall() fails if first line starts with - Ctrl+Z - -- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to - _open_osfhandle - -- Issue #28164: _PyIO_get_console_type fails for various paths - -- Issue #28137: Renames Windows path file to ._pth - -- Issue #28138: Windows ._pth file should allow import site - -C API ------ - -- Issue #27867: Function PySlice_GetIndicesEx() is deprecated and replaced with - a macro if Py_LIMITED_API is not set or set to the value between 0x03050400 - and 0x03060000 (not including) or 0x03060100 or higher. Added functions - PySlice_Unpack() and PySlice_AdjustIndices(). - -- Issue #29083: Fixed the declaration of some public API functions. - PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in - limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and - Py_BuildValue() were not available in limited API of version < 3.3 when - PY_SSIZE_T_CLEAN is defined. - -- Issue #28769: The result of PyUnicode_AsUTF8AndSize() and PyUnicode_AsUTF8() - is now of type ``const char *`` rather of ``char *``. - -- Issue #29058: All stable API extensions added after Python 3.2 are now - available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of - the minimum Python version supporting this API. - -Library -------- - - bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords are not strings. Patch by Michael Seifert. @@ -966,7 +168,6 @@ Library - bpo-29110: Fix file object leak in aifc.open() when file is given as a filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. -- Issue #24932: Use proper command line parsing in _testembed - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, From 1688e64925967ba41df682211efaa8d066c4b3e2 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 21 Mar 2017 20:39:58 -0400 Subject: [PATCH 164/232] Bump to 3.6.2rc1 development. --- Include/patchlevel.h | 2 +- Misc/NEWS | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 54a9ab6ff78896..a658e9f5840e1c 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.1" +#define PY_VERSION "3.6.1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 4807bd0fcc3b3f..bb37f0479ebc34 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,56 @@ Python News +++++++++++ +What's New in Python 3.6.2 release candidate 1? +=============================================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +- bpo-29859: Show correct error messages when any of the pthread_* calls in + thread_pthread.h fails. + +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + +- bpo-29600: Fix wrapping coroutine return values in StopIteration. + +- bpo-28856: Fix an oversight that %b format for bytes should support objects + follow the buffer protocol. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +Library +------- + +- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. + +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + + What's New in Python 3.6.1? =========================== From af839fe2fef12dbbc8083fc7f01daee48a85a916 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 22 Mar 2017 07:45:23 +0200 Subject: [PATCH 165/232] bpo-29864: Don't use Py_SIZE for dict object. (#747) (#750) --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a0c1977037b9e6..8bcc931148beaa 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1929,7 +1929,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, ESTIMATE_SIZE(Py_SIZE(iterable)))) { + if (dictresize(mp, ESTIMATE_SIZE(((PyDictObject *)iterable)->ma_used))) { Py_DECREF(d); return NULL; } From 72c51136c6269112f4c37ef14f972c89b92c272c Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 22 Mar 2017 15:26:53 +0800 Subject: [PATCH 166/232] fix function name in tabnanny documentation (GH-762) --- Doc/library/tabnanny.rst | 8 ++++---- Lib/tabnanny.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 1edb0fbabb2023..dfe688a2f93e0c 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -48,14 +48,14 @@ described below. .. exception:: NannyNag - Raised by :func:`tokeneater` if detecting an ambiguous indent. Captured and + Raised by :func:`process_tokens` if detecting an ambiguous indent. Captured and handled in :func:`check`. -.. function:: tokeneater(type, token, start, end, line) +.. function:: process_tokens(tokens) - This function is used by :func:`check` as a callback parameter to the function - :func:`tokenize.tokenize`. + This function is used by :func:`check` to process tokens generated by the + :mod:`tokenize` module. .. XXX document errprint, format_witnesses, Whitespace, check_equal, indents, reset_globals diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 46e0f56a392ae6..bfb670c9027d76 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -59,7 +59,7 @@ def main(): class NannyNag(Exception): """ - Raised by tokeneater() if detecting an ambiguous indent. + Raised by process_tokens() if detecting an ambiguous indent. Captured and handled in check(). """ def __init__(self, lineno, msg, line): From c0f3e21a23356133c30926b6cdf3e0e70cbc8486 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 22 Mar 2017 17:14:03 +0900 Subject: [PATCH 167/232] doc: minor fix for library/profile (GH-766) (cherry picked from commit bd3d8ba3b22da0bad018b53a3e6610ae03c5aa49) --- Doc/library/profile.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index bd67fe486abf77..b4b1479e2ceba4 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -85,11 +85,11 @@ next line: ``Ordered by: standard name``, indicates that the text string in the far right column was used to sort the output. The column headings include: ncalls - for the number of calls, + for the number of calls. tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) + for the total time spent in the given function (and excluding time made in + calls to sub-functions) percall is the quotient of ``tottime`` divided by ``ncalls`` From 906118d8c68160ed4a3d15cec803d1ee57836517 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 22 Mar 2017 21:09:41 +0900 Subject: [PATCH 168/232] bpo-28331: fix impl-detail label is removed when content is translated. (GH-769) (cherry picked from commit c351ce6a2c923c5016e48ecbf7b1e4833031d154) --- Doc/tools/extensions/pyspecific.py | 14 ++++++++++++-- Doc/tools/templates/dummy.html | 6 ++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 Doc/tools/templates/dummy.html diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 1141d6c5ac1891..865f2edee02205 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -21,6 +21,7 @@ from sphinx import addnodes from sphinx.builders import Builder +from sphinx.locale import translators from sphinx.util.nodes import split_explicit_title from sphinx.util.compat import Directive from sphinx.writers.html import HTMLTranslator @@ -103,16 +104,25 @@ class ImplementationDetail(Directive): optional_arguments = 1 final_argument_whitespace = True + # This text is copied to templates/dummy.html + label_text = 'CPython implementation detail:' + def run(self): pnode = nodes.compound(classes=['impl-detail']) + label = translators['sphinx'].gettext(self.label_text) content = self.content - add_text = nodes.strong('CPython implementation detail:', - 'CPython implementation detail:') + add_text = nodes.strong(label, label) if self.arguments: n, m = self.state.inline_text(self.arguments[0], self.lineno) pnode.append(nodes.paragraph('', '', *(n + m))) self.state.nested_parse(content, self.content_offset, pnode) if pnode.children and isinstance(pnode[0], nodes.paragraph): + content = nodes.inline(pnode[0].rawsource, translatable=True) + content.source = pnode[0].source + content.line = pnode[0].line + content += pnode[0].children + pnode[0].replace_self(nodes.paragraph('', '', content, + translatable=False)) pnode[0].insert(0, add_text) pnode[0].insert(1, nodes.Text(' ')) else: diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html new file mode 100644 index 00000000000000..6e43be23230b54 --- /dev/null +++ b/Doc/tools/templates/dummy.html @@ -0,0 +1,6 @@ +This file is not an actual template, but used to add some +texts in extensions to sphinx.pot file. + +In extensions/pyspecific.py: + +{% trans %}CPython implementation detail:{% endtrans %} From ee51327a2329eb9c2bfbb6ea673cfab4299450b0 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 22 Mar 2017 18:56:56 -0700 Subject: [PATCH 169/232] Remove an outdated statement in execution model docs (GH-754) (GH-775) (cherry picked from commit fad7f1560669af1766c583c7ef242c55d8c8de41) --- Doc/reference/executionmodel.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 5f1ea92ed460f3..f0dbbd1edf6321 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -194,12 +194,6 @@ This means that the following code will print 42:: i = 42 f() -There are several cases where Python statements are illegal when used in -conjunction with nested scopes that contain free variables. - -If a variable is referenced in an enclosing scope, it is illegal to delete the -name. An error will be reported at compile time. - .. XXX from * also invalid with relative imports (at least currently) The :func:`eval` and :func:`exec` functions do not have access to the full From 90eafdb1546604c0e9bc207723484b6816570cfe Mon Sep 17 00:00:00 2001 From: Christophe Zeitouny Date: Fri, 24 Mar 2017 04:20:40 -0700 Subject: [PATCH 170/232] faulthandler: Restore the old sigaltstack during teardown (GH-777) (GH-797) (cherry picked from commit 20fbf8accd494fd15b0fc4c84928178c71ead4d1) --- Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/faulthandler.c | 17 ++++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Misc/ACKS b/Misc/ACKS index 03afeb8f3876ee..2b23702f22910c 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1709,6 +1709,7 @@ Artur Zaprzala Mike Zarnstorff Yury V. Zaytsev Siebren van der Zee +Christophe Zeitouny Nickolai Zeldovich Yuxiao Zeng Uwe Zessin diff --git a/Misc/NEWS b/Misc/NEWS index 1835d1e389d725..687c5c06ef9bdb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- bpo-29884: faulthandler: Restore the old sigaltstack during teardown. + Patch by Christophe Zeitouny. + - bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. - bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1c1e4fb7d17760..2f8b624fd160d9 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -124,6 +124,7 @@ static const size_t faulthandler_nsignals = \ #ifdef HAVE_SIGALTSTACK static stack_t stack; +static stack_t old_stack; #endif @@ -1310,7 +1311,7 @@ int _PyFaulthandler_Init(void) stack.ss_size = SIGSTKSZ; stack.ss_sp = PyMem_Malloc(stack.ss_size); if (stack.ss_sp != NULL) { - err = sigaltstack(&stack, NULL); + err = sigaltstack(&stack, &old_stack); if (err) { PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; @@ -1366,6 +1367,20 @@ void _PyFaulthandler_Fini(void) faulthandler_disable(); #ifdef HAVE_SIGALTSTACK if (stack.ss_sp != NULL) { + /* Fetch the current alt stack */ + stack_t current_stack; + if (sigaltstack(NULL, ¤t_stack) == 0) { + if (current_stack.ss_sp == stack.ss_sp) { + /* The current alt stack is the one that we installed. + It is safe to restore the old stack that we found when + we installed ours */ + sigaltstack(&old_stack, NULL); + } else { + /* Someone switched to a different alt stack and didn't + restore ours when they were done (if they're done). + There's not much we can do in this unlikely case */ + } + } PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; } From cc3331fec8b7a61c3f06c097eac85bfa38490758 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 24 Mar 2017 14:45:34 +0100 Subject: [PATCH 171/232] bpo-29861: release references to multiprocessing Pool tasks (#743) (#800) * bpo-29861: release references to multiprocessing Pool tasks (#743) * bpo-29861: release references to multiprocessing Pool tasks Release references to tasks, their arguments and their results as soon as they are finished, instead of keeping them alive until another task arrives. * Comments in test (cherry picked from commit 8988945cdc27ffa86ba8c624e095b51c459f5154) * Fix Misc/NEWS (hopefully) --- Lib/multiprocessing/pool.py | 7 ++++++- Lib/test/_test_multiprocessing.py | 28 ++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ffdf42614d59eb..ae8cec44796b21 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -128,6 +128,8 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) @@ -402,10 +404,11 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): if set_length: util.debug('doing set_length()') set_length(i+1) + finally: + task = taskseq = job = None else: util.debug('task handler got sentinel') - try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') @@ -445,6 +448,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None while cache and thread._state != TERMINATE: try: @@ -461,6 +465,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b5f47825466de6..1d3bb0f8bae781 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -18,6 +18,7 @@ import logging import struct import operator +import weakref import test.support import test.support.script_helper @@ -1738,6 +1739,19 @@ def raise_large_valuerror(wait): time.sleep(wait) raise ValueError("x" * 1024**2) +def identity(x): + return x + +class CountedObject(object): + n_instances = 0 + + def __new__(cls): + cls.n_instances += 1 + return object.__new__(cls) + + def __del__(self): + type(self).n_instances -= 1 + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): @@ -1746,6 +1760,7 @@ def exception_throwing_generator(total, when): raise SayWhenError("Somebody said when") yield i + class _TestPool(BaseTestCase): @classmethod @@ -2000,6 +2015,19 @@ def test_map_no_failfast(self): # check that we indeed waited for all jobs self.assertGreater(time.time() - t_start, 0.9) + def test_release_task_refs(self): + # Issue #29861: task arguments and results should not be kept + # alive after we are done with them. + objs = [CountedObject() for i in range(10)] + refs = [weakref.ref(o) for o in objs] + self.pool.map(identity, objs) + + del objs + self.assertEqual(set(wr() for wr in refs), {None}) + # With a process pool, copies of the objects are returned, check + # they were released too. + self.assertEqual(CountedObject.n_instances, 0) + def raising(): raise KeyError("key") diff --git a/Misc/NEWS b/Misc/NEWS index 687c5c06ef9bdb..0c7eaea5a8b65c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- bpo-29861: Release references to tasks, their arguments and their results + as soon as they are finished in multiprocessing.Pool. + - bpo-29884: faulthandler: Restore the old sigaltstack during teardown. Patch by Christophe Zeitouny. From 8c8785b2f8e4048cef350f89c686266f4519b67c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 24 Mar 2017 21:46:25 +0200 Subject: [PATCH 172/232] =?UTF-8?q?bpo-25803:=20Avoid=20incorrect=20errors?= =?UTF-8?q?=20raised=20by=20Path.mkdir(exist=5Fok=3DTrue)=E2=80=A6=20(#806?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit when the OS gives priority to errors such as EACCES over EEXIST. (cherry picked from commit af7b9ec5c855366feef4c67dc492d64b3baf84ca) --- Lib/pathlib.py | 32 +++++++++++++++----------------- Lib/test/test_pathlib.py | 5 +++++ Misc/NEWS | 3 +++ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 9f347216b183a9..8c1cb96bad3075 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1220,25 +1220,23 @@ def touch(self, mode=0o666, exist_ok=True): os.close(fd) def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ if self._closed: self._raise_closed() - if not parents: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - else: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - except OSError as e: - if e.errno != ENOENT or self.parent == self: - raise - self.parent.mkdir(parents=True) - self._accessor.mkdir(self, mode) + try: + self._accessor.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True) + self._accessor.mkdir(self, mode) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise def chmod(self, mode): """ diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 88a93e5802c1cb..3ff9726d4359ec 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1776,6 +1776,11 @@ def test_mkdir_exist_ok_with_parent(self): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + @only_nt # XXX: not sure how to test this on POSIX def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': diff --git a/Misc/NEWS b/Misc/NEWS index 0c7eaea5a8b65c..9798a3a4490cbe 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- bpo-25803: Avoid incorrect errors raised by Path.mkdir(exist_ok=True) + when the OS gives priority to errors such as EACCES over EEXIST. + - bpo-29861: Release references to tasks, their arguments and their results as soon as they are finished in multiprocessing.Pool. From 74bfcc314b188e1e8c90e442270e36d6510755ee Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 24 Mar 2017 14:18:53 -0700 Subject: [PATCH 173/232] bpo-28810: Document remaining bytecode changes in 3.6 (GH-651) (GH-808) (cherry picked from commit 8f9e1bbf2dbdf46a0bf920279568a31460043376) --- Doc/library/dis.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 11d056901b7ad5..c795782034a464 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -20,6 +20,10 @@ interpreter. between versions of Python. Use of this module should not be considered to work across Python VMs or Python releases. + .. versionchanged:: 3.6 + Use 2 bytes for each instruction. Previously the number of bytes varied + by instruction. + Example: Given the function :func:`myfunc`:: @@ -210,6 +214,11 @@ operation is being performed, so the intermediate analysis object isn't useful: This generator function uses the ``co_firstlineno`` and ``co_lnotab`` attributes of the code object *code* to find the offsets which are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. + See :source:`Objects/lnotab_notes.txt` for the ``co_lnotab`` format and + how to decode it. + + .. versionchanged:: 3.6 + Line numbers can be decreasing. Before, they were always increasing. .. function:: findlabels(code) @@ -1105,8 +1114,13 @@ All of the following opcodes use their arguments. .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between - opcodes which don't take arguments ``< HAVE_ARGUMENT`` and those which do - ``>= HAVE_ARGUMENT``. + opcodes which don't use their argument and those that do + (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). + + .. versionchanged:: 3.6 + Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` + ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. + .. _opcode_collections: From 5dafaece67cb576ef96579fb5ddcbf62e0827b0d Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 24 Mar 2017 20:04:40 -0700 Subject: [PATCH 174/232] bpo-29892: Fix wrong markup on doc-lib-functions (GH-802) (GH-810) (cherry picked from commit 29540cdf6c66df9f806375a95078c0c63192ef78) --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c26037bd9eef19..64c9569766cc7d 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1072,7 +1072,7 @@ are always available. They are listed here in alphabetical order. * The ``'x'`` mode was added. * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. * :exc:`FileExistsError` is now raised if the file opened in exclusive - * creation mode (``'x'``) already exists. + creation mode (``'x'``) already exists. .. versionchanged:: 3.4 From 8b82236952619c3838865ff535e5ce77b59b4a78 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sat, 25 Mar 2017 03:41:59 -0700 Subject: [PATCH 175/232] bpo-29862: Fix grammar in importlib.reload() exception (GH-809) (GH-811) (cherry picked from commit 9f0aa4843f8c26937d5817f27cac4aae9c0a034f) --- Lib/importlib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index b6a9f82e05f268..8b11d22b024ffa 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -136,7 +136,7 @@ def reload(module): """ if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be module") + raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: From ceff32fb833e7be42508ffa73344e2e56865e54b Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 26 Mar 2017 13:58:13 -0400 Subject: [PATCH 176/232] bpo-29888: Fix the link referring to the "Python download page" (GH-824) (GH-826) (cherry picked from commit f8beb9831acd5cf80b9c56aea5864e16118c5400) --- Doc/tools/templates/download.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index de84ae3abc81c7..3a05cb697938fd 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -42,7 +42,7 @@

Download Python {{ release }} Documentation

These archives contain all the content in the documentation.

HTML Help (.chm) files are made available in the "Windows" section -on the Python +on the Python download page.

From d2fc7824104172b47cdcea09d7ce699d5bfc2ba6 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 26 Mar 2017 19:53:37 -0700 Subject: [PATCH 177/232] import sys before we use it on line 9 (GH-828) (GH-833) (cherry picked from commit 0579e81f30d00da562f021760d5b6a9c35186520) --- Lib/idlelib/pyshell.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index f3ee391ca006fc..dd6c997abc587b 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -1,5 +1,7 @@ #! /usr/bin/env python3 +import sys + try: from tkinter import * except ImportError: @@ -25,7 +27,6 @@ import re import socket import subprocess -import sys import threading import time import tokenize From abc68484b2c523b500802cb8e8913531e0ff9a8d Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Sun, 26 Mar 2017 22:53:57 -0500 Subject: [PATCH 178/232] Treat Sphinx warnings as errors (GH-832) (GH-834) * Treat Sphinx warnings as errors (GH-832) (cherry picked from commit 334e9ec938ea9876baadef15edb135d6d2aff30c) * Remove unused suspicious rules --- .travis.yml | 2 +- Doc/tools/susp-ignored.csv | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66f03dc7161a7f..3496a1abba2c97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: - cd Doc - make venv script: - - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q" + - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q -W" - os: linux language: c compiler: clang diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index c6e03119ae9730..166374cb90cec1 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -324,6 +324,4 @@ whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') whatsnew/3.5,,:root,ERROR:root:exception whatsnew/3.5,,:exception,ERROR:root:exception whatsnew/changelog,,:version,import sys; I = version[:version.index(' ')] -whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't" -whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as" whatsnew/changelog,,`,"for readability (was ""`"")." From bc2031470eac4b199692921b4e97730c85ceb678 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 27 Mar 2017 00:35:56 -0500 Subject: [PATCH 179/232] Minor test cleanup (GH-837) (GH-838) * Remove unused test file * Remove duplicated text in sndhdrdata README (cherry picked from commit b8a7daf077dab18e9e3701c5380b542ae0aa9a94) --- Lib/test/185test.db | Bin 16384 -> 0 bytes Lib/test/sndhdrdata/README | 7 ------- 2 files changed, 7 deletions(-) delete mode 100644 Lib/test/185test.db diff --git a/Lib/test/185test.db b/Lib/test/185test.db deleted file mode 100644 index 14cb5e258bc0961fa22527dcb5d947a6a0fd5480..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI!F$%&k6vpwdEl39?E*_+VxVZETo - -Sound file samples used by Lib/test/test_sndhdr.py and generated using the -following commands: - - dd if=/dev/zero of=sndhdr.raw bs=20 count=1 - sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - From 08f9f71162dedeec51ad99bac1b26eb0ab014400 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 27 Mar 2017 16:46:31 -0700 Subject: [PATCH 180/232] Drop the standard gcc test build on Travis (GH-853) (GH-859) Instead have gcc be used for the coverage build so gcc is exercised in at least one place. (cherry picked from commit ad2f9e2c8a0b44b3e6aec9d28ba59e13239236f7) --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3496a1abba2c97..5fee6cd7673324 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,11 @@ os: - linux # macOS builds are disabled as the machines are under-provisioned on Travis, # adding up to an extra hour completing a full CI run. - #- osx compiler: - clang - - gcc + # gcc also works, but to keep the # of concurrent builds down, we use one C + # compiler here and the other to run the coverage build. env: - TESTING=cpython @@ -32,7 +32,7 @@ matrix: include: - os: linux language: python - python: 3.5 + python: 3.6 env: - TESTING=docs before_script: @@ -42,7 +42,7 @@ matrix: - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q -W" - os: linux language: c - compiler: clang + compiler: gcc env: - TESTING=coverage before_script: From 596506216104613591218c2896cdb49fa0b7e5bb Mon Sep 17 00:00:00 2001 From: Mariatta Date: Mon, 27 Mar 2017 17:02:29 -0700 Subject: [PATCH 181/232] bpo-29677: DOC: clarify documentation for `round` (GH-357) (GH-862) (cherry picked from commit 6003db7db5fec545c01923c198a5fdfca5a91538) --- Doc/library/functions.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 64c9569766cc7d..2c0ea71c47f413 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1262,7 +1262,8 @@ are always available. They are listed here in alphabetical order. closest multiple of 10 to the power minus *ndigits*; if two multiples are equally close, rounding is done toward the even choice (so, for example, both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). The return value is an integer if called with one argument, + ``2``). Any integer value is valid for *ndigits* (positive, zero, or + negative). The return value is an integer if called with one argument, otherwise of the same type as *number*. .. note:: From f01de61a8efea8319c65365898982f929d59a895 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 29 Mar 2017 00:25:26 +0900 Subject: [PATCH 182/232] bpo-29643: Fix check for --enable-optimizations (GH-869) The presence of the ``--enable-optimizations`` flag is indicated by the value of ``$enableval``, but the configure script was checking ``$withval``, resulting in the ``--enable-optimizations`` flag being effectively ignored. (cherry picked from commit 8cea5929f52801b0ce5928b46ef836e99a24321a) --- Misc/NEWS | 5 +++++ configure | 2 +- configure.ac | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 9798a3a4490cbe..45c5c7828701f8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -60,6 +60,11 @@ Library exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek. +Build +----- + +- bpo-29643: Fix ``--enable-optimization`` didn't work. + What's New in Python 3.6.1? =========================== diff --git a/configure b/configure index abe1dc5abd4e1c..8bdb82ba4a7bce 100755 --- a/configure +++ b/configure @@ -6521,7 +6521,7 @@ $as_echo_n "checking for --enable-optimizations... " >&6; } # Check whether --enable-optimizations was given. if test "${enable_optimizations+set}" = set; then : enableval=$enable_optimizations; -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 diff --git a/configure.ac b/configure.ac index 9eacf52559e8c2..6b331d5742745f 100644 --- a/configure.ac +++ b/configure.ac @@ -1274,7 +1274,7 @@ Py_OPT='false' AC_MSG_CHECKING(for --enable-optimizations) AC_ARG_ENABLE(optimizations, AS_HELP_STRING([--enable-optimizations], [Enable expensive optimizations (PGO, etc). Disabled by default.]), [ -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' AC_MSG_RESULT(yes); From c4021af50526f488c0c280e7c7eaa83ef80ae1df Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 28 Mar 2017 09:33:54 -0700 Subject: [PATCH 183/232] bpo-16011: clarify that 'in' always returns a boolean value (GH-874) (cherry picked from commit 0ae7c8bd614d3aa1fcaf2d71a10ff1148c80d9b5) --- Doc/reference/expressions.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index f4a82699b0be12..d80768ac07dc0e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1431,28 +1431,29 @@ Membership test operations -------------------------- The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in -s`` evaluates to true if *x* is a member of *s*, and false otherwise. ``x not -in s`` returns the negation of ``x in s``. All built-in sequences and set types -support this as well as dictionary, for which :keyword:`in` tests whether the -dictionary has a given key. For container types such as list, tuple, set, -frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent +s`` evaluates to ``True`` if *x* is a member of *s*, and ``False`` otherwise. +``x not in s`` returns the negation of ``x in s``. All built-in sequences and +set types support this as well as dictionary, for which :keyword:`in` tests +whether the dictionary has a given key. For container types such as list, tuple, +set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent to ``any(x is e or x == e for e in y)``. -For the string and bytes types, ``x in y`` is true if and only if *x* is a +For the string and bytes types, ``x in y`` is ``True`` if and only if *x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are always considered to be a substring of any other string, so ``"" in "abc"`` will return ``True``. For user-defined classes which define the :meth:`__contains__` method, ``x in -y`` is true if and only if ``y.__contains__(x)`` is true. +y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and +``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is true if some value ``z`` with ``x == z`` is +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is produced while iterating over ``y``. If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is true if and only if there is a non-negative +:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x == y[i]``, and all lower integer indices do not raise :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). From 346dcd65e6b832a35b4cfc15b7309b51a38e9ca2 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Wed, 29 Mar 2017 12:50:28 +0800 Subject: [PATCH 184/232] bpo-28699: fix abnormal behaviour of pools in multiprocessing.pool (GH-882) an exception raised at the very first of an iterable would cause pools behave abnormally (swallow the exception or hang) --- Lib/multiprocessing/pool.py | 79 +++++++++++++++++++++---------- Lib/test/_test_multiprocessing.py | 59 ++++++++++++++++++++++- Misc/NEWS | 4 ++ 3 files changed, 117 insertions(+), 25 deletions(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ae8cec44796b21..a545f3c1a18961 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -118,7 +118,7 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, try: result = (True, func(*args, **kwds)) except Exception as e: - if wrap_exception: + if wrap_exception and func is not _helper_reraises_exception: e = ExceptionWithTraceback(e, e.__traceback__) result = (False, e) try: @@ -133,6 +133,10 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, completed += 1 util.debug('worker exiting after %d tasks' % completed) +def _helper_reraises_exception(ex): + 'Pickle-able helper function for use by _guarded_task_generation.' + raise ex + # # Class representing a process pool # @@ -277,6 +281,17 @@ def starmap_async(self, func, iterable, chunksize=None, callback=None, return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) + def _guarded_task_generation(self, result_job, func, iterable): + '''Provides a generator of tasks for imap and imap_unordered with + appropriate handling for iterables which throw exceptions during + iteration.''' + try: + i = -1 + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + except Exception as e: + yield (result_job, i+1, _helper_reraises_exception, (e,), {}) + def imap(self, func, iterable, chunksize=1): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. @@ -285,15 +300,23 @@ def imap(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def imap_unordered(self, func, iterable, chunksize=1): @@ -304,15 +327,23 @@ def imap_unordered(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def apply_async(self, func, args=(), kwds={}, callback=None, @@ -323,7 +354,7 @@ def apply_async(self, func, args=(), kwds={}, callback=None, if self._state != RUN: raise ValueError("Pool not running") result = ApplyResult(self._cache, callback, error_callback) - self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) + self._taskqueue.put(([(result._job, 0, func, args, kwds)], None)) return result def map_async(self, func, iterable, chunksize=None, callback=None, @@ -354,8 +385,14 @@ def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, task_batches = Pool._get_tasks(func, iterable, chunksize) result = MapResult(self._cache, chunksize, len(iterable), callback, error_callback=error_callback) - self._taskqueue.put((((result._job, i, mapper, (x,), {}) - for i, x in enumerate(task_batches)), None)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapper, + task_batches), + None + ) + ) return result @staticmethod @@ -377,33 +414,27 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): for taskseq, set_length in iter(taskqueue.get, None): task = None - i = -1 try: - for i, task in enumerate(taskseq): + # iterating taskseq cannot fail + for task in taskseq: if thread._state: util.debug('task handler found thread._state != RUN') break try: put(task) except Exception as e: - job, ind = task[:2] + job, idx = task[:2] try: - cache[job]._set(ind, (False, e)) + cache[job]._set(idx, (False, e)) except KeyError: pass else: if set_length: util.debug('doing set_length()') - set_length(i+1) + idx = task[1] if task else -1 + set_length(idx + 1) continue break - except Exception as ex: - job, ind = task[:2] if task else (0, 0) - if job in cache: - cache[job]._set(ind + 1, (False, ex)) - if set_length: - util.debug('doing set_length()') - set_length(i+1) finally: task = taskseq = job = None else: diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1d3bb0f8bae781..771bbf24265b91 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1755,6 +1755,8 @@ def __del__(self): class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): + if when == -1: + raise SayWhenError("Somebody said when") for i in range(total): if i == when: raise SayWhenError("Somebody said when") @@ -1833,6 +1835,32 @@ def test_map_chunksize(self): except multiprocessing.TimeoutError: self.fail("pool.map_async with chunksize stalled on null list") + def test_map_handle_iterable_exception(self): + if self.TYPE == 'manager': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + # SayWhenError seen at the very first of the iterable + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + # again, make sure it's reentrant + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(10, 3), 1) + + class SpecialIterable: + def __iter__(self): + return self + def __next__(self): + raise SayWhenError + def __len__(self): + return 1 + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + def test_async(self): res = self.pool.apply_async(sqr, (7, TIMEOUT1,)) get = TimingWrapper(res.get) @@ -1863,6 +1891,13 @@ def test_imap_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap(sqr, exception_throwing_generator(10, 3), 1) for i in range(3): self.assertEqual(next(it), i*i) @@ -1889,6 +1924,17 @@ def test_imap_unordered_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap_unordered(sqr, exception_throwing_generator(10, 3), 1) @@ -1970,7 +2016,7 @@ def test_traceback(self): except Exception as e: exc = e else: - raise AssertionError('expected RuntimeError') + self.fail('expected RuntimeError') self.assertIs(type(exc), RuntimeError) self.assertEqual(exc.args, (123,)) cause = exc.__cause__ @@ -1984,6 +2030,17 @@ def test_traceback(self): sys.excepthook(*sys.exc_info()) self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + # _helper_reraises_exception should not make the error + # a remote exception + with self.Pool(1) as p: + try: + p.map(sqr, exception_throwing_generator(1, -1), 1) + except Exception as e: + exc = e + else: + self.fail('expected SayWhenError') + self.assertIs(type(exc), SayWhenError) + self.assertIs(exc.__cause__, None) @classmethod def _test_wrapped_exception(cls): diff --git a/Misc/NEWS b/Misc/NEWS index 45c5c7828701f8..fbaa840638ff97 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,10 @@ Core and Builtins Library ------- +- bpo-28699: Fixed a bug in pools in multiprocessing.pool that raising an + exception at the very first of an iterable may swallow the exception or + make the program hang. Patch by Davin Potts and Xiang Zhang. + - bpo-25803: Avoid incorrect errors raised by Path.mkdir(exist_ok=True) when the OS gives priority to errors such as EACCES over EEXIST. From 2609c9ee78c53d501914a5a90dbe094d9a8c3c97 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 29 Mar 2017 19:10:07 -0700 Subject: [PATCH 185/232] bpo-29677: DOC: clarify documentation for `round` (GH-877) (GH-892) (cherry picked from commit 85deefcf61d3cc192846f41a4ccc6df17da60c98) --- Doc/library/functions.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2c0ea71c47f413..90fb07af227d98 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1254,9 +1254,9 @@ are always available. They are listed here in alphabetical order. .. function:: round(number[, ndigits]) - Return the floating point value *number* rounded to *ndigits* digits after - the decimal point. If *ndigits* is omitted or is ``None``, it returns the - nearest integer to its input. Delegates to ``number.__round__(ndigits)``. + Return *number* rounded to *ndigits* precision after the decimal + point. If *ndigits* is omitted or is ``None``, it returns the + nearest integer to its input. For the built-in types supporting :func:`round`, values are rounded to the closest multiple of 10 to the power minus *ndigits*; if two multiples are @@ -1266,6 +1266,9 @@ are always available. They are listed here in alphabetical order. negative). The return value is an integer if called with one argument, otherwise of the same type as *number*. + For a general Python object ``number``, ``round(number, ndigits)`` delegates to + ``number.__round__(ndigits)``. + .. note:: The behavior of :func:`round` for floats can be surprising: for example, From da6ad2f780d187fbfdea330d1037766ae7bdb778 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 29 Mar 2017 22:29:00 -0700 Subject: [PATCH 186/232] bpo-29917: DOC: Remove link from PyMethodDef (#890) (#894) (cherry picked from commit c3c7ef088583cc12bd218138036d1edb6de9c63f) --- Doc/c-api/structures.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f48119391f2bbd..c080f317bee9d9 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -241,7 +241,7 @@ definition with the same method name. +==================+=============+===============================+ | :attr:`name` | char \* | name of the member | +------------------+-------------+-------------------------------+ - | :attr:`type` | int | the type of the member in the | + | :attr:`!type` | int | the type of the member in the | | | | C struct | +------------------+-------------+-------------------------------+ | :attr:`offset` | Py_ssize_t | the offset in bytes that the | @@ -256,7 +256,7 @@ definition with the same method name. | | | docstring | +------------------+-------------+-------------------------------+ - :attr:`type` can be one of many ``T_`` macros corresponding to various C + :attr:`!type` can be one of many ``T_`` macros corresponding to various C types. When the member is accessed in Python, it will be converted to the equivalent Python type. From 1b43a959fb1dd2e08dbf51bbc644df5ff2eb357e Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 29 Mar 2017 23:56:52 -0700 Subject: [PATCH 187/232] Remove an unrequired TODO in test_urllib2. (#897) (#900) (cherry picked from commit e6911a44f69c0d302db60f49952a9cf69da69a2b) --- Lib/test/test_urllib2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 34329f87162a62..308acb3724cad1 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1414,7 +1414,6 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") - # TODO: This should be only for OSX @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { From a6b4e1902250d6f28ca6d083ce1c8d7e9b91974b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 30 Mar 2017 18:08:21 +0300 Subject: [PATCH 188/232] bpo-27863: Fixed multiple crashes in ElementTree. (#765) (#903) (cherry picked from commit 576def096ec7b64814e038f03290031f172886c3) --- Lib/test/test_xml_etree.py | 112 +++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 + Modules/_elementtree.c | 100 +++++++++++++++++---------------- 3 files changed, 167 insertions(+), 48 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c0144d1cb8febf..dbdad23a7423f2 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1879,6 +1879,118 @@ def test_recursive_repr(self): with self.assertRaises(RuntimeError): repr(e) # Should not crash + def test_element_get_text(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem.text + except NameError: + pass + + b = ET.TreeBuilder() + b.start('tag', {}) + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('tag') + + elem = b.close() + self.assertEqual(elem.text, 'ABCDEFGHIJKL') + + def test_element_get_tail(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem[0].tail + except NameError: + pass + + b = ET.TreeBuilder() + b.start('root', {}) + b.start('tag', {}) + b.end('tag') + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('root') + + elem = b.close() + self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') + + def test_element_iter(self): + # Issue #27863 + state = { + 'tag': 'tag', + '_children': [None], # non-Element + 'attrib': 'attr', + 'tail': 'tail', + 'text': 'text', + } + + e = ET.Element('tag') + try: + e.__setstate__(state) + except AttributeError: + e.__dict__ = state + + it = e.iter() + self.assertIs(next(it), e) + self.assertRaises(AttributeError, next, it) + + def test_subscr(self): + # Issue #27863 + class X: + def __index__(self): + del e[:] + return 1 + + e = ET.Element('elem') + e.append(ET.Element('child')) + e[:X()] # shouldn't crash + + e.append(ET.Element('child')) + e[0:10:X()] # shouldn't crash + + def test_ass_subscr(self): + # Issue #27863 + class X: + def __index__(self): + e[:] = [] + return 1 + + e = ET.Element('elem') + for _ in range(10): + e.insert(0, ET.Element('child')) + + e[0:10:X()] = [] # shouldn't crash + + def test_treebuilder_start(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.start, 'tag2', {}) + del b + gc_collect() + + def test_treebuilder_end(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.end, 'tag') + del b + gc_collect() + + class MutatingElementPath(str): def __new__(cls, elem, *args): self = str.__new__(cls, *args) diff --git a/Misc/NEWS b/Misc/NEWS index fbaa840638ff97..2caa8f360d122d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- bpo-27863: Fixed multiple crashes in ElementTree caused by race conditions + and wrong types. + - bpo-28699: Fixed a bug in pools in multiprocessing.pool that raising an exception at the very first of an iterable may swallow the exception or make the program hang. Patch by Davin Potts and Xiang Zhang. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 2cda98e61127d6..e3350d194da18a 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -131,7 +131,7 @@ elementtree_free(void *m) LOCAL(PyObject*) list_join(PyObject* list) { - /* join list elements (destroying the list in the process) */ + /* join list elements */ PyObject* joiner; PyObject* result; @@ -140,8 +140,6 @@ list_join(PyObject* list) return NULL; result = PyUnicode_Join(joiner, list); Py_DECREF(joiner); - if (result) - Py_DECREF(list); return result; } @@ -508,15 +506,17 @@ element_get_text(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->text; + PyObject *res = self->text; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->text = res; + self->text = tmp; + Py_DECREF(res); + res = tmp; } } @@ -528,15 +528,17 @@ element_get_tail(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->tail; + PyObject *res = self->tail; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->tail = res; + self->tail = tmp; + Py_DECREF(res); + res = tmp; } } @@ -2147,6 +2149,12 @@ elementiter_next(ElementIterObject *it) continue; } + if (!PyObject_TypeCheck(extra->children[child_index], &Element_Type)) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute 'iter'", + Py_TYPE(extra->children[child_index])->tp_name); + return NULL; + } elem = (ElementObject *)extra->children[child_index]; item->child_index++; Py_INCREF(elem); @@ -2396,40 +2404,51 @@ treebuilder_dealloc(TreeBuilderObject *self) /* helpers for handling of arbitrary element-like objects */ static int -treebuilder_set_element_text_or_tail(PyObject *element, PyObject *data, +treebuilder_set_element_text_or_tail(PyObject *element, PyObject **data, PyObject **dest, _Py_Identifier *name) { if (Element_CheckExact(element)) { - Py_DECREF(JOIN_OBJ(*dest)); - *dest = JOIN_SET(data, PyList_CheckExact(data)); + PyObject *tmp = JOIN_OBJ(*dest); + *dest = JOIN_SET(*data, PyList_CheckExact(*data)); + *data = NULL; + Py_DECREF(tmp); return 0; } else { - PyObject *joined = list_join(data); + PyObject *joined = list_join(*data); int r; if (joined == NULL) return -1; r = _PyObject_SetAttrId(element, name, joined); Py_DECREF(joined); - return r; + if (r < 0) + return -1; + Py_CLEAR(*data); + return 0; } } -/* These two functions steal a reference to data */ -static int -treebuilder_set_element_text(PyObject *element, PyObject *data) +LOCAL(int) +treebuilder_flush_data(TreeBuilderObject* self) { - _Py_IDENTIFIER(text); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->text, &PyId_text); -} + PyObject *element = self->last; -static int -treebuilder_set_element_tail(PyObject *element, PyObject *data) -{ - _Py_IDENTIFIER(tail); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->tail, &PyId_tail); + if (!self->data) { + return 0; + } + + if (self->this == element) { + _Py_IDENTIFIER(text); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->text, &PyId_text); + } + else { + _Py_IDENTIFIER(tail); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->tail, &PyId_tail); + } } static int @@ -2479,16 +2498,8 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, PyObject* this; elementtreestate *st = ET_STATE_GLOBAL; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } - else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (!self->element_factory || self->element_factory == Py_None) { @@ -2591,15 +2602,8 @@ treebuilder_handle_end(TreeBuilderObject* self, PyObject* tag) { PyObject* item; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (self->index == 0) { From bf4bb2e43030661e568d5d4b046e8b9351cc164c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 30 Mar 2017 19:46:59 +0300 Subject: [PATCH 189/232] bpo-29935: Fixed error messages in the index() method of tuple, list and deque (#887) (#907) when pass indices of wrong type. (cherry picked from commit d4edfc9abffca965e76ebc5957a92031a4d6c4d4) --- Include/ceval.h | 1 + Misc/NEWS | 3 +++ Modules/_collectionsmodule.c | 4 ++-- Objects/listobject.c | 4 ++-- Objects/tupleobject.c | 4 ++-- Python/ceval.c | 26 +++++++++++++++++++++----- 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index 89c6062f11e17c..1e482729a1cc98 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -216,6 +216,7 @@ PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); +PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #endif diff --git a/Misc/NEWS b/Misc/NEWS index 2caa8f360d122d..e8cd8f0221102e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-29935: Fixed error messages in the index() method of tuple, list and deque + when pass indices of wrong type. + - bpo-29859: Show correct error messages when any of the pthread_* calls in thread_pthread.h fails. diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index e6111c64e77de2..30157701d70b47 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1051,8 +1051,8 @@ deque_index(dequeobject *deque, PyObject *args) int cmp; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(deque); diff --git a/Objects/listobject.c b/Objects/listobject.c index dcd7b5efe5b0fe..cde281a0f6f9dc 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2153,8 +2153,8 @@ listindex(PyListObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c0ff499e72c7dd..52f20f4fe8a2b1 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -522,8 +522,8 @@ tupleindex(PyTupleObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); diff --git a/Python/ceval.c b/Python/ceval.c index 9cac771abdf65b..bce86ab12cf2fb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5074,14 +5074,10 @@ do_call_core(PyObject *func, PyObject *callargs, PyObject *kwdict) and silently boost values less than -PY_SSIZE_T_MAX-1 to -PY_SSIZE_T_MAX-1. Return 0 on error, 1 on success. */ -/* Note: If v is NULL, return success without storing into *pi. This - is because_PyEval_SliceIndex() is called by apply_slice(), which can be - called by the SLICE opcode with v and/or w equal to NULL. -*/ int _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) { - if (v != NULL) { + if (v != Py_None) { Py_ssize_t x; if (PyIndex_Check(v)) { x = PyNumber_AsSsize_t(v, NULL); @@ -5099,6 +5095,26 @@ _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) return 1; } +int +_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) +{ + Py_ssize_t x; + if (PyIndex_Check(v)) { + x = PyNumber_AsSsize_t(v, NULL); + if (x == -1 && PyErr_Occurred()) + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + return 0; + } + *pi = x; + return 1; +} + + #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" From 7b5b1379ac2c1e89ebf90b88b5d32457910e975e Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 30 Mar 2017 12:27:19 -0700 Subject: [PATCH 190/232] bpo-29928: Add f-string to the Glossary (GH-864) (GH-914) (cherry picked from commit 33db068dac7686e37736f7ecf8abb2aee0345cf2) --- Doc/glossary.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 495934afe74c8b..81238414efe49d 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -320,6 +320,11 @@ Glossary A module written in C or C++, using Python's C API to interact with the core and with user code. + f-string + String literals prefixed with ``'f'`` or ``'F'`` are commonly called + "f-strings" which is short for + :ref:`formatted string literals `. See also :pep:`498`. + file object An object exposing a file-oriented API (with methods such as :meth:`read()` or :meth:`write()`) to an underlying resource. Depending From 599bb181036f724629a515317f0f39520950d51c Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Thu, 30 Mar 2017 12:48:23 -0700 Subject: [PATCH 191/232] bpo-29942: Fix the use of recursion in itertools.chain.from_iterable. (#911) * bpo-29942: Fix the use of recursion in itertools.chain.from_iterable. Fix the use of recursion in itertools.chain.from_iterable. Using recursion is unnecessary, and can easily cause stack overflows, especially when building in low optimization modes or with Py_DEBUG enabled. (cherry picked from commit 5466d4af5fe76ec0a5fbc8a05675287d9e8e9d14) --- Lib/test/test_itertools.py | 8 ++++++ Misc/NEWS | 3 +++ Modules/itertoolsmodule.c | 52 ++++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index ea1f57caade12b..c431f0dc6e1d7f 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1976,6 +1976,14 @@ def gen2(x): self.assertRaises(AssertionError, list, cycle(gen1())) self.assertEqual(hist, [0,1]) + def test_long_chain_of_empty_iterables(self): + # Make sure itertools.chain doesn't run into recursion limits when + # dealing with long chains of empty iterables. Even with a high + # number this would probably only fail in Py_DEBUG mode. + it = chain.from_iterable(() for unused in range(10000000)) + with self.assertRaises(StopIteration): + next(it) + class SubclassWithKwargsTest(unittest.TestCase): def test_keywords_in_subclass(self): # count is not subclassable... diff --git a/Misc/NEWS b/Misc/NEWS index e8cd8f0221102e..5ea4f66e76fb39 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -30,6 +30,9 @@ Core and Builtins Library ------- +- bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering + long runs of empty iterables. + - bpo-27863: Fixed multiple crashes in ElementTree caused by race conditions and wrong types. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6bf04cbee31c35..8e22ec9c4b7cc1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1864,33 +1864,37 @@ chain_next(chainobject *lz) { PyObject *item; - if (lz->source == NULL) - return NULL; /* already stopped */ - - if (lz->active == NULL) { - PyObject *iterable = PyIter_Next(lz->source); - if (iterable == NULL) { - Py_CLEAR(lz->source); - return NULL; /* no more input sources */ - } - lz->active = PyObject_GetIter(iterable); - Py_DECREF(iterable); + /* lz->source is the iterator of iterables. If it's NULL, we've already + * consumed them all. lz->active is the current iterator. If it's NULL, + * we should grab a new one from lz->source. */ + while (lz->source != NULL) { if (lz->active == NULL) { - Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + PyObject *iterable = PyIter_Next(lz->source); + if (iterable == NULL) { + Py_CLEAR(lz->source); + return NULL; /* no more input sources */ + } + lz->active = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (lz->active == NULL) { + Py_CLEAR(lz->source); + return NULL; /* input not iterable */ + } } + item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); + if (item != NULL) + return item; + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else + return NULL; /* input raised an exception */ + } + /* lz->active is consumed, try with the next iterable. */ + Py_CLEAR(lz->active); } - item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); - if (item != NULL) - return item; - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - return NULL; /* input raised an exception */ - } - Py_CLEAR(lz->active); - return chain_next(lz); /* recurse and use next active */ + /* Everything had been consumed already. */ + return NULL; } static PyObject * From 8d1f935945ca8cf3665315e0f82358fffe689353 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 30 Mar 2017 23:15:51 -0700 Subject: [PATCH 192/232] Remove catching OSError in ftphandler test. Only URLError is raised in urllib.request module. (#918) (#920) (cherry picked from commit ed3dd1c02af6872bd0748f7b9a5dadb89f7b830f) --- Lib/test/test_urllib2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 308acb3724cad1..b537fca3c31adf 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -830,7 +830,6 @@ def test_file(self): for url, ftp in [ ("file://ftp.example.com//foo.txt", False), ("file://ftp.example.com///foo.txt", False), -# XXXX bug: fails with OSError, should be URLError ("file://ftp.example.com/foo.txt", False), ("file://somehost//foo/something.txt", False), ("file://localhost//foo/something.txt", False), @@ -839,7 +838,7 @@ def test_file(self): try: h.file_open(req) # XXXX remove OSError when bug fixed - except (urllib.error.URLError, OSError): + except urllib.error.URLError: self.assertFalse(ftp) else: self.assertIs(o.req, req) @@ -1689,7 +1688,6 @@ def test_invalid_closed(self): self.assertTrue(conn.fakesock.closed, "Connection not closed") - class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): From 4c75fbb485c0e42181aab95c2ae92c597915827c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 31 Mar 2017 15:43:10 +0900 Subject: [PATCH 193/232] bpo-29952: Use usual terminology of dict (GH-922) s/keys and elements/keys and values/ (cherry picked from commit cdcac039fb447f2ab04efcacbe663751bb2cb4ec) --- Doc/reference/expressions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index d80768ac07dc0e..c4f6c55c7ca3aa 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1355,7 +1355,7 @@ built-in types. true). * Mappings (instances of :class:`dict`) compare equal if and only if they have - equal `(key, value)` pairs. Equality comparison of the keys and elements + equal `(key, value)` pairs. Equality comparison of the keys and values enforces reflexivity. Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. From 0a17e584461b14ff65ec287048f53911dbb22222 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 31 Mar 2017 02:12:18 -0700 Subject: [PATCH 194/232] bpo-28810: Update lnotab_notes.txt (GH-665) (GH-919) (cherry picked from commit 9135275cba680902e6caf29461f0423dc570190d) --- Objects/lnotab_notes.txt | 50 ++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 515375772e6317..3dab2b98661695 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,17 +1,18 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes -disguised as a Python string. It is used to map bytecode offsets to source code -line #s for tracebacks and to identify line number boundaries for line tracing. +disguised as a Python bytes object. It is used to map bytecode offsets to +source code line #s for tracebacks and to identify line number boundaries for +line tracing. The array is conceptually a compressed list of (bytecode offset increment, line number increment) pairs. The details are important and delicate, best illustrated by example: byte code offset source code line number - 0 1 - 6 2 - 50 7 + 0 1 + 6 2 + 50 7 350 207 361 208 @@ -24,7 +25,8 @@ look like: The above doesn't really work, but it's a start. An unsigned byte (byte code offset) can't hold negative values, or values larger than 255, a signed byte (line number) can't hold values larger than 127 or less than -128, and the -above example contains two such values. So we make two tweaks: +above example contains two such values. (Note that before 3.6, line number +was also encoded by an unsigned byte.) So we make two tweaks: (a) there's a deep assumption that byte code offsets increase monotonically, and @@ -52,7 +54,7 @@ the example above, assemble_lnotab in compile.c should not (as was actually done until 2.2) expand 300, 200 to 255, 255, 45, 45, but to - 255, 0, 45, 128, 0, 72. + 255, 0, 45, 127, 0, 73. The above is sufficient to reconstruct line numbers for tracebacks, but not for line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c @@ -83,30 +85,34 @@ Consider this code: 1: def f(a): 2: while a: -3: print 1, +3: print(1) 4: break 5: else: -6: print 2, +6: print(2) which compiles to this: - 2 0 SETUP_LOOP 19 (to 22) - >> 3 LOAD_FAST 0 (a) - 6 POP_JUMP_IF_FALSE 17 + 2 0 SETUP_LOOP 26 (to 28) + >> 2 LOAD_FAST 0 (a) + 4 POP_JUMP_IF_FALSE 18 - 3 9 LOAD_CONST 1 (1) - 12 PRINT_ITEM + 3 6 LOAD_GLOBAL 0 (print) + 8 LOAD_CONST 1 (1) + 10 CALL_FUNCTION 1 + 12 POP_TOP - 4 13 BREAK_LOOP - 14 JUMP_ABSOLUTE 3 - >> 17 POP_BLOCK + 4 14 BREAK_LOOP + 16 JUMP_ABSOLUTE 2 + >> 18 POP_BLOCK - 6 18 LOAD_CONST 2 (2) - 21 PRINT_ITEM - >> 22 LOAD_CONST 0 (None) - 25 RETURN_VALUE + 6 20 LOAD_GLOBAL 0 (print) + 22 LOAD_CONST 2 (2) + 24 CALL_FUNCTION 1 + 26 POP_TOP + >> 28 LOAD_CONST 0 (None) + 30 RETURN_VALUE -If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 17 +If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 18 and the co_lnotab will claim that execution has moved to line 4, which is wrong. In this case, we could instead associate the POP_BLOCK with line 5, but that would break jumps around loops without else clauses. From 7d5d13d8d003ae5b62bb8c9ef1d1f310eaabc506 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 31 Mar 2017 23:23:49 +0300 Subject: [PATCH 195/232] =?UTF-8?q?bpo-29953:=20Fix=20memory=20leaks=20in?= =?UTF-8?q?=20the=20replace()=20method=20of=20datetime=20and=20t=E2=80=A6?= =?UTF-8?q?=20(#933)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit objects when pass out of bound fold argument. (cherry picked from commit 314d6fca36a4eaa0541218431d14804fadec6488) --- Lib/test/datetimetester.py | 5 +++++ Misc/NEWS | 3 +++ Modules/_datetimemodule.c | 21 ++++++++++----------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2350125f6d6dbe..bccd97aa3c7f60 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -4313,6 +4313,11 @@ def test_replace(self): dt = dt.replace(fold=1, tzinfo=Eastern) self.assertEqual(t.replace(tzinfo=None).fold, 1) self.assertEqual(dt.replace(tzinfo=None).fold, 1) + # Out of bounds. + with self.assertRaises(ValueError): + t.replace(fold=2) + with self.assertRaises(ValueError): + dt.replace(fold=2) # Check that fold is a keyword-only argument with self.assertRaises(TypeError): t.replace(1, 1, 1, None, 1) diff --git a/Misc/NEWS b/Misc/NEWS index 5ea4f66e76fb39..c1bcdc1e93fc5d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -30,6 +30,9 @@ Core and Builtins Library ------- +- bpo-29953: Fixed memory leaks in the replace() method of datetime and time + objects when pass out of bound fold argument. + - bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering long runs of empty iterables. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c784d0f4a90999..c2ad9a203e9652 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3926,16 +3926,16 @@ time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) time_kws, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = time_new(Py_TYPE(self), tuple, NULL); if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } TIME_SET_FOLD(clone, fold); } Py_DECREF(tuple); @@ -5019,17 +5019,16 @@ datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) &y, &m, &d, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } DATE_SET_FOLD(clone, fold); } Py_DECREF(tuple); From efde51ad54c58353f25ff80c8d30dbee82ef33a3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 1 Apr 2017 23:29:31 +0900 Subject: [PATCH 196/232] bpo-29949: Fix set memory usage regression (GH-945) Revert "Minor factoring: move redundant resize scaling logic into the resize function." This reverts commit 4897300276d870f99459c82b937f0ac22450f0b6. (cherry picked from commit e82cf8675bacd7a03de508ed11865fc2701dcef5) --- Misc/NEWS | 2 ++ Objects/setobject.c | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index c1bcdc1e93fc5d..e61a1652381fa0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.6.2 release candidate 1? Core and Builtins ----------------- +- bpo-29949: Fix memory usage regression of set and frozenset object. + - bpo-29935: Fixed error messages in the index() method of tuple, list and deque when pass indices of wrong type. diff --git a/Objects/setobject.c b/Objects/setobject.c index fdb9d3600d9fad..9fe28138c0012b 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -236,7 +236,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); found_active: Py_DECREF(key); @@ -304,7 +304,6 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry small_copy[PySet_MINSIZE]; assert(minused >= 0); - minused = (minused > 50000) ? minused * 2 : minused * 4; /* Find the smallest table size > minused. */ /* XXX speed-up with intrinsics */ @@ -646,8 +645,8 @@ set_merge(PySetObject *so, PyObject *otherset) * that there will be no (or few) overlapping keys. */ if ((so->fill + other->used)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + other->used) != 0) - return -1; + if (set_table_resize(so, (so->used + other->used)*2) != 0) + return -1; } so_entry = so->table; other_entry = other->table; @@ -990,7 +989,7 @@ set_update_internal(PySetObject *so, PyObject *other) if (dictsize < 0) return -1; if ((so->fill + dictsize)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + dictsize) != 0) + if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { @@ -1511,7 +1510,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /* If more than 1/4th are dummies, then resize them away. */ if ((size_t)(so->fill - so->used) <= (size_t)so->mask / 4) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } static PyObject * From 4415b823a08163a8728d13ff06565efb12353fc0 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 1 Apr 2017 19:57:04 -0700 Subject: [PATCH 197/232] Add helpful explaination to test_password_manager tests. (#936) (#941) Also uncomment and fix a path test. (cherry picked from commit 1f5425ff69ea0531d869b4f9fa28bd3f66ca3de7) --- Lib/test/test_urllib2.py | 41 +++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index b537fca3c31adf..d50211c59857e1 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -141,44 +141,55 @@ def test_password_manager(self): mgr = urllib.request.HTTPPasswordMgr() add = mgr.add_password find_user_pass = mgr.find_user_password + add("Some Realm", "http://example.com/", "joe", "password") add("Some Realm", "http://example.com/ni", "ni", "ni") - add("c", "http://example.com/foo", "foo", "ni") - add("c", "http://example.com/bar", "bar", "nini") - add("b", "http://example.com/", "first", "blah") - add("b", "http://example.com/", "second", "spam") - add("a", "http://example.com", "1", "a") add("Some Realm", "http://c.example.com:3128", "3", "c") add("Some Realm", "d.example.com", "4", "d") add("Some Realm", "e.example.com:3128", "5", "e") + # For the same realm, password set the highest path is the winner. self.assertEqual(find_user_pass("Some Realm", "example.com"), ('joe', 'password')) - - #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), - # ('ni', 'ni')) - + self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), + ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com"), ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam"), - ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam/spam"), - ('joe', 'password')) + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam"), + ('joe', 'password')) + + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam/spam"), + ('joe', 'password')) + + # You can have different passwords for different paths. + + add("c", "http://example.com/foo", "foo", "ni") + add("c", "http://example.com/bar", "bar", "nini") + self.assertEqual(find_user_pass("c", "http://example.com/foo"), ('foo', 'ni')) + self.assertEqual(find_user_pass("c", "http://example.com/bar"), ('bar', 'nini')) + + # For the same path, newer password should be considered. + + add("b", "http://example.com/", "first", "blah") + add("b", "http://example.com/", "second", "spam") + self.assertEqual(find_user_pass("b", "http://example.com/"), ('second', 'spam')) # No special relationship between a.example.com and example.com: + add("a", "http://example.com", "1", "a") self.assertEqual(find_user_pass("a", "http://example.com/"), ('1', 'a')) + self.assertEqual(find_user_pass("a", "http://a.example.com/"), (None, None)) From a71a3ad54d14483cfaebd8e0fb96c97c798d9e32 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 1 Apr 2017 20:00:36 -0700 Subject: [PATCH 198/232] bpo-26947: DOC: clarify wording on hashable in glossary (#948) (#957) (cherry picked from commit 64c887ab3a400cf91bde4f0c5ef69eacc88bc5e1) --- Doc/glossary.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 81238414efe49d..dba9186d935a6a 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -467,9 +467,9 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable, while no mutable - containers (such as lists or dictionaries) are. Objects which are - instances of user-defined classes are hashable by default; they all + All of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not. Objects which are + instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. From 90e3518225bafaff01469ed48c472ae7db5686f0 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Sat, 1 Apr 2017 20:20:05 -0700 Subject: [PATCH 199/232] bpo-29941: Assert fixes (#886) (#955) Make a non-Py_DEBUG, asserts-enabled build of CPython possible. This means making sure helper functions are defined when NDEBUG is not defined, not just when Py_DEBUG is defined. Also fix a division-by-zero in obmalloc.c that went unnoticed because in Py_DEBUG mode, elsize is never zero. (cherry picked from commit a00c3fd12d421e41b769debd7df717d17b0deed5 and 06bb4873d6a9ac303701d08a851d6cd9a51e02a3) --- Include/unicodeobject.h | 4 ++++ Objects/dictobject.c | 2 +- Objects/obmalloc.c | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 2d0d77e804d9d8..5b877185f258d3 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -2309,6 +2309,10 @@ PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy( PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); +#elif !defined(NDEBUG) +/* For asserts that call _PyUnicode_CheckConsistency(), which would + * otherwise be a problem when building with asserts but without Py_DEBUG. */ +#define _PyUnicode_CheckConsistency(op, check_content) PyUnicode_Check(op) #endif #ifndef Py_LIMITED_API diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 8bcc931148beaa..566d1a5ac8d893 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -436,7 +436,7 @@ static PyObject *empty_values[1] = { NULL }; /* #define DEBUG_PYDICT */ -#ifdef Py_DEBUG +#ifndef NDEBUG static int _PyDict_CheckConsistency(PyDictObject *mp) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index a1142f3b09ad9e..32e7ecbe1e0436 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1227,7 +1227,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) _Py_AllocatedBlocks++; - assert(nelem <= PY_SSIZE_T_MAX / elsize); + assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; #ifdef WITH_VALGRIND From c4e557ed9933499e1c97644ea12ad77ba064e829 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 2 Apr 2017 01:36:58 -0700 Subject: [PATCH 200/232] Minor spell fix and formatting fixes in urllib tests. (#959) (#961) (cherry picked from commit efbd4ea65dbb9f87b1afeec6a760802756badee5) --- Lib/test/test_urllib.py | 6 +++++- Lib/test/test_urllibnet.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 5084486e5ab479..fa3757cc94be52 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -206,6 +206,7 @@ def test_iter(self): def test_relativelocalfile(self): self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + class ProxyTests(unittest.TestCase): def setUp(self): @@ -259,6 +260,7 @@ def test_proxy_bypass_environment_host_match(self): self.assertFalse(bypass('newdomain.com')) # no port self.assertFalse(bypass('newdomain.com:1235')) # wrong port + class ProxyTests_withOrderedEnv(unittest.TestCase): def setUp(self): @@ -294,6 +296,7 @@ def test_getproxies_environment_prefer_lowercase(self): proxies = urllib.request.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http']) + class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): """Test urlopen() opening a fake http connection.""" @@ -432,7 +435,6 @@ def test_ftp_cache_pruning(self): finally: self.unfakeftp() - def test_userpass_inurl(self): self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") try: @@ -476,6 +478,7 @@ def test_cafile_and_context(self): "https://localhost", cafile="/nonexistent/path", context=context ) + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -549,6 +552,7 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 949716c2b567e0..865a7f4f5fb172 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -29,7 +29,7 @@ def testURLread(self): class urlopenNetworkTests(unittest.TestCase): - """Tests urllib.reqest.urlopen using the network. + """Tests urllib.request.urlopen using the network. These tests are not exhaustive. Assuming that testing using files does a good job overall of some of the basic interface features. There are no From d184c20e359907e326696724a3395ce1d542f42e Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Tue, 4 Apr 2017 01:26:32 +0900 Subject: [PATCH 201/232] Keep the c-api exception doc up-to-date (#966) cherry-pick'ed from ec1f5df..e3d6db3 --- Doc/c-api/exceptions.rst | 205 ++++++++++++++++++++++++++++----------- Misc/ACKS | 1 + 2 files changed, 150 insertions(+), 56 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 037b85cfd11754..3516a1690384f7 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -750,6 +750,61 @@ All standard Python exceptions are available as global variables whose names are :c:type:`PyObject\*`; they are all class objects. For completeness, here are all the variables: +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | +=========================================+=================================+==========+ @@ -759,8 +814,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | @@ -769,27 +822,31 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportError` | :exc:`ImportError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_IndexError` | :exc:`IndexError` | | +-----------------------------------------+---------------------------------+----------+ @@ -801,8 +858,12 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NameError` | :exc:`NameError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | @@ -823,16 +884,32 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemError` | :exc:`SystemError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ValueError` | :exc:`ValueError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | @@ -849,11 +926,18 @@ the variables: and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. .. versionadded:: 3.5 - :c:data:`PyExc_RecursionError`. + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. These are compatibility aliases to :c:data:`PyExc_OSError`: +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ @@ -867,52 +951,6 @@ These are compatibility aliases to :c:data:`PyExc_OSError`: .. versionchanged:: 3.3 These aliases used to be separate exception types. - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_LookupError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_ConnectionError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_ImportError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_MemoryError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - Notes: (1) @@ -924,3 +962,58 @@ Notes: (3) Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. + +Standard Warnings +================= + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DepricationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDepricationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Misc/ACKS b/Misc/ACKS index 2b23702f22910c..b5da02a63ad0aa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -775,6 +775,7 @@ Jason Killen Jan Kim Taek Joo Kim Sam Kimbrel +Tomohiko Kinebuchi James King W. Trevor King Paul Kippes From cb1e002c07622e027e80a3843d27a623d1617430 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 3 Apr 2017 22:27:15 -0700 Subject: [PATCH 202/232] bpo-29725: DOC: add text for arraysize in sqlite3.Cursor (#947) (#985) (cherry picked from commit 02e12138000da834f23719521a011fa93763384d) --- Doc/library/sqlite3.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index d1f7a6f120620b..9fef7d7f03f1f0 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -640,6 +640,11 @@ Cursor Objects .. versionchanged:: 3.6 Added support for the ``REPLACE`` statement. + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + .. attribute:: description This read-only attribute provides the column names of the last query. To From ce92be1825c5ca1d3e6bff24598f3aa54c000b22 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Tue, 4 Apr 2017 11:49:14 +0200 Subject: [PATCH 203/232] remove merge=union attribute for Misc/NEWS (GH-460) (GH-988) (cherry picked from commit 060d2d776a29341c079cce37220324f9775140ba) --- .gitattributes | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index 55a3f49922bf19..82694d81f276b2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,3 @@ -Misc/NEWS merge=union - *.pck binary Lib/test/cjkencodings/* binary Lib/test/decimaltestdata/*.decTest binary From ba980e8ef01dfaf484defd6e7a19ea5375de1338 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Apr 2017 13:18:18 +0300 Subject: [PATCH 204/232] Miscellaneous minor fixes of Misc/NEWS formatting. (#1002) (#1003) (cherry picked from commit a0157b5f11e621f2196af4e918b9f07688a6cd1c) --- Misc/NEWS | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index e61a1652381fa0..fa20ea16cd0144 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -290,7 +290,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #29055: Neaten-up empty population error on random.choice() by suppressing the upstream exception. @@ -1058,7 +1058,7 @@ Core and Builtins alpha releases, where backslashes are allowed anywhere in an f-string. Also, require that expressions inside f-strings be enclosed within literal braces, and not escapes like - f'\x7b"hi"\x7d'. + ``f'\x7b"hi"\x7d'``. - Issue #28046: Remove platform-specific directories from sys.path. @@ -1277,7 +1277,7 @@ Library - Issue #24277: The new email API is no longer provisional, and the docs have been reorganized and rewritten to emphasize the new API. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -1296,9 +1296,9 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27331: The email.mime classes now all accept an optional policy keyword. +- Issue #27331: The email.mime classes now all accept an optional policy keyword. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #16113: Add SHA-3 and SHAKE support to hashlib module. @@ -2406,7 +2406,7 @@ Core and Builtins in ``def f(): 1.0``. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen FĂŒrstenau and + (``*``) unpacking in function calls. Based on patch by Hagen FĂŒrstenau and Daniel Urban. - Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used @@ -2964,7 +2964,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -3100,7 +3100,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -3735,7 +3735,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #28871: Fixed a crash when deallocate deep ElementTree. @@ -3858,7 +3858,7 @@ Library - Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). -- Issue #19003:m email.generator now replaces only \r and/or \n line +- Issue #19003:m email.generator now replaces only ``\r`` and/or ``\n`` line endings, per the RFC, instead of all unicode line endings. - Issue #28019: itertools.count() no longer rounds non-integer step in range @@ -3881,7 +3881,7 @@ Library - Issue #27445: Don't pass str(_charset) to MIMEText.set_payload(). Patch by Claude Paroz. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -3896,7 +3896,7 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #27691: Fix ssl module's parsing of GEN_RID subject alternative name fields in X.509 certs. @@ -4347,7 +4347,7 @@ Core and Builtins cookie names. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen FĂŒrstenau and + (``*``) unpacking in function calls. Based on patch by Hagen FĂŒrstenau and Daniel Urban. - Issue #27138: Fix the doc comment for FileFinder.find_spec(). @@ -4727,7 +4727,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -5137,7 +5137,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -6680,7 +6680,7 @@ Library - Issue #23521: Corrected pure python implementation of timedelta division. - * Eliminated OverflowError from timedelta * float for some floats; + * Eliminated OverflowError from ``timedelta * float`` for some floats; * Corrected rounding in timedlta true division. - Issue #21619: Popen objects no longer leave a zombie after exit in the with @@ -7479,7 +7479,7 @@ Library character instead of truncating it. Based on patch by Victor Stinner. - Issue #13968: The glob module now supports recursive search in - subdirectories using the "**" pattern. + subdirectories using the ``**`` pattern. - Issue #21951: Fixed a crash in Tkinter on AIX when called Tcl command with empty string or tuple argument. From ed278448daa4afb78d1af57b0c179097477f9832 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 6 Apr 2017 08:38:38 -0700 Subject: [PATCH 205/232] correct parse_qs and parse_qsl test case descriptions. (#968) (#997) (cherry picked from commit 257b980b316a5206ecf6c23b958e2b7c4df4f3de) --- Lib/test/test_urlparse.py | 12 ++++++------ Lib/urllib/parse.py | 30 +++++++++++++++++------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 99c5c033e30d20..3c89ed928ff51f 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -6,8 +6,8 @@ RFC3986_BASE = 'http://a/b/c/d;p?q' SIMPLE_BASE = 'http://a/b/c/d' -# A list of test cases. Each test case is a two-tuple that contains -# a string with the query and a dictionary with the expected result. +# Each parse_qsl testcase is a two-tuple that contains +# a string with the query and a list with the expected result. parse_qsl_test_cases = [ ("", []), @@ -42,6 +42,9 @@ (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), ] +# Each parse_qs testcase is a two-tuple that contains +# a string with the query and a dictionary with the expected result. + parse_qs_test_cases = [ ("", {}), ("&", {}), @@ -290,7 +293,6 @@ def test_RFC2368(self): def test_RFC2396(self): # cases from RFC 2396 - self.checkJoin(RFC2396_BASE, 'g:h', 'g:h') self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g') self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g') @@ -333,9 +335,7 @@ def test_RFC2396(self): # self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g') # self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g') - def test_RFC3986(self): - # Test cases from RFC3986 self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y') self.checkJoin(RFC3986_BASE, ';x', 'http://a/b/c/;x') self.checkJoin(RFC3986_BASE, 'g:h','g:h') @@ -363,7 +363,7 @@ def test_RFC3986(self): self.checkJoin(RFC3986_BASE, '../../g','http://a/g') self.checkJoin(RFC3986_BASE, '../../../g', 'http://a/g') - #Abnormal Examples + # Abnormal Examples # The 'abnormal scenarios' are incompatible with RFC2986 parsing # Tests are here for reference. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 958767a08d7e77..32de25b374da93 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -612,6 +612,7 @@ def unquote(string, encoding='utf-8', errors='replace'): append(bits[i + 1]) return ''.join(res) + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. @@ -633,6 +634,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. + + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, @@ -644,28 +647,29 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result[name] = [value] return parsed_result + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. - Arguments: + Arguments: - qs: percent-encoded query string to be parsed + qs: percent-encoded query string to be parsed - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. + keep_blank_values: flag indicating whether blank values in + percent-encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. + encoding and errors: specify how to decode percent-encoded sequences + into Unicode characters, as accepted by the bytes.decode() method. - Returns a list, as G-d intended. + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] From 19d61b265fabed01ebfb8eae9855f2ee59ff591b Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Fri, 7 Apr 2017 00:56:59 -0700 Subject: [PATCH 206/232] Remove Invalid comment in test_urllib2.py. (#1022) (cherry picked from commit fd0cd07a5a3c964c084f4efc5bbcb89dd2193ee6) --- Lib/test/test_urllib2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index d50211c59857e1..876fcd4199fb92 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -848,7 +848,6 @@ def test_file(self): req = Request(url) try: h.file_open(req) - # XXXX remove OSError when bug fixed except urllib.error.URLError: self.assertFalse(ftp) else: From 616d3eb7ef96f1a27bbff68fba6e5d229257d936 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 7 Apr 2017 16:50:13 -0400 Subject: [PATCH 207/232] Fix a minor typo. (#1032) (#1037) (cherry picked from commit dd9a0a14c89d57e43898d4b866b8c161e4ff8506) --- Doc/library/asyncio-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 482ffbbed35b1e..3f55506c669a29 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -384,7 +384,7 @@ The following callbacks are called on :class:`Protocol` instances: .. method:: Protocol.eof_received() - Calls when the other end signals it won't send any more data + Called when the other end signals it won't send any more data (for example by calling :meth:`write_eof`, if the other end also uses asyncio). From d0d575a6db8cb3b2a720be9f404af3d754da9a5d Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Fri, 7 Apr 2017 23:23:08 +0100 Subject: [PATCH 208/232] bpo-29939: suppress compiler warnings in _ctypes_test (#1038) bpo-29939: Changed test code to suppress a compiler warning, while taking care to avoid the code being optimized out by the compiler. (cherry picked from commit 164d30eb1e66575dafee6af4fca4cbf52c7fbe6a) --- Modules/_ctypes/_ctypes_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 59d56d0d218732..6119ecdaf90dc6 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -52,9 +52,9 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) EXPORT(void) _testfunc_large_struct_update_value(Test in) { - in.first = 0x0badf00d; - in.second = 0x0badf00d; - in.third = 0x0badf00d; + ((volatile Test *)&in)->first = 0x0badf00d; + ((volatile Test *)&in)->second = 0x0badf00d; + ((volatile Test *)&in)->third = 0x0badf00d; } EXPORT(void)testfunc_array(int values[4]) From c26b19d5c7aba51b50a4d7fb5f8291036cb9da24 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 8 Apr 2017 11:18:14 +0300 Subject: [PATCH 209/232] Expand the PySlice_GetIndicesEx macro. (#1023) (#1044) (cherry picked from commit b879fe82e7e5c3f7673c9a7fa4aad42bd05445d8) --- Modules/_ctypes/_ctypes.c | 10 ++++------ Modules/_elementtree.c | 12 ++++++------ Modules/_testbuffer.c | 7 ++++--- Modules/arraymodule.c | 11 ++++++----- Modules/mmapmodule.c | 9 ++++----- Objects/bytearrayobject.c | 12 ++++++------ Objects/bytesobject.c | 6 +++--- Objects/listobject.c | 10 ++++++---- Objects/memoryobject.c | 4 ++-- Objects/tupleobject.c | 6 +++--- Objects/unicodeobject.c | 5 +++-- 11 files changed, 47 insertions(+), 45 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index df3aedec6eeab5..12234e254e1a7a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4273,11 +4273,10 @@ Array_subscript(PyObject *myself, PyObject *item) PyObject *np; Py_ssize_t start, stop, step, slicelen, cur, i; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for array object instances */ @@ -4414,11 +4413,10 @@ Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, otherlen, i, cur; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); if ((step < 0 && start < stop) || (step > 0 && start > stop)) stop = start; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index e3350d194da18a..bef702ebe69c93 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1712,11 +1712,11 @@ element_subscr(PyObject* self_, PyObject* item) if (!self->extra) return PyList_New(0); - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (slicelen <= 0) return PyList_New(0); @@ -1768,11 +1768,11 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) return -1; } - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (value == NULL) { /* Delete slice */ diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 4e1ce6851a5eb6..6b8ab34d931cd3 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1715,10 +1715,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { @@ -1935,9 +1935,10 @@ slice_indices(PyObject *self, PyObject *args) "first argument must be a slice object"); return NULL; } - if (PySlice_GetIndicesEx(key, len, &s[0], &s[1], &s[2], &s[3]) < 0) { + if (PySlice_Unpack(key, &s[0], &s[1], &s[2]) < 0) { return NULL; } + s[3] = PySlice_AdjustIndices(len, &s[0], &s[1], s[2]); ret = PyTuple_New(4); if (ret == NULL) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index d4221fe4c6a163..64e0f172fd7d82 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2297,10 +2297,11 @@ array_subscr(arrayobject* self, PyObject* item) arrayobject* ar; int itemsize = self->ob_descr->itemsize; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return newarrayobject(&Arraytype, 0, self->ob_descr); @@ -2368,11 +2369,11 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) return (*self->ob_descr->setitem)(self, i, value); } else if (PySlice_Check(item)) { - if (PySlice_GetIndicesEx(item, - Py_SIZE(self), &start, &stop, - &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); } else { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5f1615ff82992c..426b7cacebb573 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -806,10 +806,10 @@ mmap_subscript(mmap_object *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->size, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); @@ -932,11 +932,10 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) Py_ssize_t start, stop, step, slicelen; Py_buffer vbuf; - if (PySlice_GetIndicesEx(item, - self->size, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "mmap object doesn't support slice deletion"); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index d456678d462f2f..a9c8ca6f1f35a8 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -400,11 +400,11 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) } else if (PySlice_Check(index)) { Py_ssize_t start, stop, step, slicelength, cur, i; - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), + &start, &stop, step); if (slicelength <= 0) return PyByteArray_FromStringAndSize("", 0); @@ -630,11 +630,11 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu } } else if (PySlice_Check(index)) { - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), &start, + &stop, step); } else { PyErr_Format(PyExc_TypeError, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index c95a46d1846649..4c55294d9d8c8a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1683,11 +1683,11 @@ bytes_subscript(PyBytesObject* self, PyObject* item) char* result_buf; PyObject* result; - if (PySlice_GetIndicesEx(item, - PyBytes_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyBytes_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyBytes_FromStringAndSize("", 0); diff --git a/Objects/listobject.c b/Objects/listobject.c index cde281a0f6f9dc..547bdf0b95f1ee 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2420,10 +2420,11 @@ list_subscript(PyListObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return PyList_New(0); @@ -2469,10 +2470,11 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (step == 1) return list_ass_slice(self, start, stop, value); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index b1798a2073d108..e1ac7281783f38 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2285,10 +2285,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 52f20f4fe8a2b1..0dada74dc7c986 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -720,11 +720,11 @@ tuplesubscript(PyTupleObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, - PyTuple_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyTuple_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyTuple_New(0); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a5ae454b495dee..949205adf9347e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13915,10 +13915,11 @@ unicode_subscript(PyObject* self, PyObject* item) int src_kind, dest_kind; Py_UCS4 ch, max_char, kind_limit; - if (PySlice_GetIndicesEx(item, PyUnicode_GET_LENGTH(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyUnicode_GET_LENGTH(self), + &start, &stop, step); if (slicelength <= 0) { _Py_RETURN_UNICODE_EMPTY(); From af685f9050416da8050e0ec11a8dff9afd4130e7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 8 Apr 2017 11:25:47 +0300 Subject: [PATCH 210/232] bpo-29998: Pickling and copying ImportError now preserves name and path (#1010) (#1042) attributes. (cherry picked from commit b785396ab451b0c9d6ae9ee5a9e56c810209a6cb) --- Lib/test/test_exceptions.py | 20 ++++++++++++++++ Misc/NEWS | 3 +++ Objects/exceptions.c | 48 +++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 48379222c37659..960fc0f1bf9486 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1,5 +1,6 @@ # Python test set -- part 5, built-in exceptions +import copy import os import sys import unittest @@ -1119,6 +1120,25 @@ def test_non_str_argument(self): exc = ImportError(arg) self.assertEqual(str(arg), str(exc)) + def test_copy_pickle(self): + for kwargs in (dict(), + dict(name='somename'), + dict(path='somepath'), + dict(name='somename', path='somepath')): + orig = ImportError('test', **kwargs) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + exc = pickle.loads(pickle.dumps(orig, proto)) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + for c in copy.copy, copy.deepcopy: + exc = c(orig) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index fa20ea16cd0144..83f9e7a6852560 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- bpo-29998: Pickling and copying ImportError now preserves name and path + attributes. + - bpo-29953: Fixed memory leaks in the replace() method of datetime and time objects when pass out of bound fold argument. diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f63f06a145bcb1..d158b9768d45f5 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -686,6 +686,53 @@ ImportError_str(PyImportErrorObject *self) } } +static PyObject * +ImportError_getstate(PyImportErrorObject *self) +{ + PyObject *dict = ((PyBaseExceptionObject *)self)->dict; + if (self->name || self->path) { + _Py_IDENTIFIER(name); + _Py_IDENTIFIER(path); + dict = dict ? PyDict_Copy(dict) : PyDict_New(); + if (dict == NULL) + return NULL; + if (self->name && _PyDict_SetItemId(dict, &PyId_name, self->name) < 0) { + Py_DECREF(dict); + return NULL; + } + if (self->path && _PyDict_SetItemId(dict, &PyId_path, self->path) < 0) { + Py_DECREF(dict); + return NULL; + } + return dict; + } + else if (dict) { + Py_INCREF(dict); + return dict; + } + else { + Py_RETURN_NONE; + } +} + +/* Pickling support */ +static PyObject * +ImportError_reduce(PyImportErrorObject *self) +{ + PyObject *res; + PyObject *args; + PyObject *state = ImportError_getstate(self); + if (state == NULL) + return NULL; + args = ((PyBaseExceptionObject *)self)->args; + if (state == Py_None) + res = PyTuple_Pack(2, Py_TYPE(self), args); + else + res = PyTuple_Pack(3, Py_TYPE(self), args, state); + Py_DECREF(state); + return res; +} + static PyMemberDef ImportError_members[] = { {"msg", T_OBJECT, offsetof(PyImportErrorObject, msg), 0, PyDoc_STR("exception message")}, @@ -697,6 +744,7 @@ static PyMemberDef ImportError_members[] = { }; static PyMethodDef ImportError_methods[] = { + {"__reduce__", (PyCFunction)ImportError_reduce, METH_NOARGS}, {NULL} }; From 4c116cb3b3ee38cae229f7e0df0cd2045fe73c27 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 9 Apr 2017 19:22:36 +1000 Subject: [PATCH 211/232] bpo-29798: Handle git worktree in patchcheck (#1058) (#1060) The original attempted fix missed an `isdir()` call in `get_base_branch()`. (cherry picked from commit 2abfdf5a81383d3b1ed6b7321903a9a168c373c5) --- Tools/scripts/patchcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index f4ec7d8a30ea23..33a9fead879325 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -70,7 +70,7 @@ def get_git_upstream_remote(): @status("Getting base branch for PR", info=lambda x: x if x is not None else "not a PR branch") def get_base_branch(): - if not os.path.isdir(os.path.join(SRCDIR, '.git')): + if not os.path.exists(os.path.join(SRCDIR, '.git')): # Not a git checkout, so there's no base branch return None version = sys.version_info From a524d6307273e3920d410452ace89610c843f051 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 9 Apr 2017 20:57:04 +1000 Subject: [PATCH 212/232] bpo-29506: Clarify deep copy note in copy module The reference to administrative data was confusing to readers, so this simplifies the note to explain that deep copying may copy more then you intended, such as data that you expected to be shared between copies. (cherry picked from commit 19e04942562a980ad2519f6ff79c455a7472783b) --- Doc/library/copy.rst | 4 ++-- Misc/ACKS | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index d0b861d469bc05..2041d9175ea587 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -47,8 +47,8 @@ copy operations: * Recursive objects (compound objects that, directly or indirectly, contain a reference to themselves) may cause a recursive loop. -* Because deep copy copies *everything* it may copy too much, e.g., - even administrative data structures that should be shared even between copies. +* Because deep copy copies everything it may copy too much, such as data + which is intended to be shared between copies. The :func:`deepcopy` function avoids these problems by: diff --git a/Misc/ACKS b/Misc/ACKS index b5da02a63ad0aa..72070e19f65701 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -770,6 +770,7 @@ Lawrence Kesteloot Vivek Khera Dhiru Kholia Akshit Khurana +Sanyam Khurana Mads Kiilerich Jason Killen Jan Kim From 854ca174a92970d11d9b063af46e00d132832944 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 9 Apr 2017 10:15:06 -0700 Subject: [PATCH 213/232] Remove invalid comment in urllib.request. (#1055) (cherry picked from commit a2a9ddd923a849124bdd1c484f70f02df6fde0e9) --- Lib/urllib/request.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index b6690c3d0e4bf5..3e82000466432f 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1658,14 +1658,10 @@ def pathname2url(pathname): of the 'file' scheme; not recommended for general use.""" return quote(pathname) -# This really consists of two pieces: -# (1) a class which handles opening of all sorts of URLs -# (plus assorted utilities etc.) -# (2) a set of functions for parsing URLs -# XXX Should these be separated out into different modules? - ftpcache = {} + + class URLopener: """Class to open URLs. This is a class rather than just a subroutine because we may need From 90eb7a9ff4ce38b5c4e3662bdd42a0ed513893a2 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 9 Apr 2017 15:15:57 -0700 Subject: [PATCH 214/232] [3.6] Correct typo in configparser.rst (GH-1012) (GH-1025) (cherry picked from commit 01fa9ae5460b00bf1ced500c797176ebd3fb060d) --- Doc/library/configparser.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index af57cba2b5661d..04c2a820921759 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -34,8 +34,8 @@ can be customized by end users easily. .. seealso:: Module :mod:`shlex` - Support for a creating Unix shell-like mini-languages which can be used - as an alternate format for application configuration files. + Support for creating Unix shell-like mini-languages which can be used as + an alternate format for application configuration files. Module :mod:`json` The json module implements a subset of JavaScript syntax which can also From 577fc04a7157f6e904cffd6a0e1ad83d3460acd6 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 9 Apr 2017 15:17:06 -0700 Subject: [PATCH 215/232] [3.6] bpo-29549: Fixes docstring for str.index (GH-256) (GH-1028) (cherry picked from commit 43ba8861e0ad044efafa46a7cc04e12ac5df640e) --- Objects/bytes_methods.c | 13 ++++++++++--- Objects/unicodeobject.c | 12 ++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index d5c4fe6346fc53..625e242d563d89 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -542,7 +542,11 @@ _Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_index__doc__, "B.index(sub[, start[, end]]) -> int\n\ \n\ -Like B.find() but raise ValueError when the subsection is not found."); +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the subsection is not found."); PyObject * _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) @@ -579,7 +583,11 @@ _Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_rindex__doc__, "B.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like B.rfind() but raise ValueError when the subsection is not found."); +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raise ValueError when the subsection is not found."); PyObject * _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) @@ -811,4 +819,3 @@ PyDoc_STRVAR_shared(_Py_zfill__doc__, "\n" "Pad a numeric string B with zeros on the left, to fill a field\n" "of the specified width. B is never truncated."); - diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 949205adf9347e..7871043681533d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -11701,7 +11701,11 @@ unicode_hash(PyObject *self) PyDoc_STRVAR(index__doc__, "S.index(sub[, start[, end]]) -> int\n\ \n\ -Like S.find() but raise ValueError when the substring is not found."); +Return the lowest index in S where substring sub is found, \n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_index(PyObject *self, PyObject *args) @@ -12758,7 +12762,11 @@ unicode_rfind(PyObject *self, PyObject *args) PyDoc_STRVAR(rindex__doc__, "S.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like S.rfind() but raise ValueError when the substring is not found."); +Return the highest index in S where substring sub is found,\n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_rindex(PyObject *self, PyObject *args) From 28ddd3c905836907bf0c4df909067d3a054358c1 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Sun, 9 Apr 2017 15:17:28 -0700 Subject: [PATCH 216/232] [3.6] Improvements to typing documentation (GH-967) (GH-1053) Documents a few omitted classes and adds NamedTuple methods. (cherry picked from commit 45d22c256bce3afcf57f49032a6b20fdec4f26ad) --- Doc/library/typing.rst | 50 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index d130e1759d8f59..2cfc37f695ef2d 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -508,6 +508,14 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__float__``. +.. class:: SupportsComplex + + An ABC with one abstract method ``__complex__``. + +.. class:: SupportsBytes + + An ABC with one abstract method ``__bytes__``. + .. class:: SupportsAbs An ABC with one abstract method ``__abs__`` that is covariant @@ -658,7 +666,19 @@ The module defines the following classes, functions and decorators: .. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - A generic version of :class:`collections.defaultdict` + A generic version of :class:`collections.defaultdict`. + +.. class:: Counter(collections.Counter, Dict[T, int]) + + A generic version of :class:`collections.Counter`. + + .. versionadded:: 3.6.1 + +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + + A generic version of :class:`collections.ChainMap`. + + .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -742,9 +762,12 @@ The module defines the following classes, functions and decorators: This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These representing the types of I/O streams such as returned by + These represent the types of I/O streams such as returned by :func:`open`. + These types are also accessible directly as ``typing.IO``, + ``typing.TextIO``, and ``typing.BinaryIO``. + .. class:: re Wrapper namespace for regular expression matching types. @@ -756,6 +779,9 @@ The module defines the following classes, functions and decorators: ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. + These types are also accessible directly as ``typing.Pattern`` + and ``typing.Match``. + .. class:: NamedTuple Typed version of namedtuple. @@ -782,10 +808,20 @@ The module defines the following classes, functions and decorators: Fields with a default value must come after any fields without a default. The resulting class has two extra attributes: ``_field_types``, - giving a dict mapping field names to types, and ``field_defaults``, a dict + giving a dict mapping field names to types, and ``_field_defaults``, a dict mapping field names to default values. (The field names are in the ``_fields`` attribute, which is part of the namedtuple API.) + ``NamedTuple`` subclasses can also have docstrings and methods:: + + class Employee(NamedTuple): + """Represents an employee.""" + name: str + id: int = 3 + + def __repr__(self) -> str: + return f'' + Backward-compatible usage:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) @@ -794,7 +830,7 @@ The module defines the following classes, functions and decorators: Added support for :pep:`526` variable annotation syntax. .. versionchanged:: 3.6.1 - Added support for default values. + Added support for default values, methods, and docstrings. .. function:: NewType(typ) @@ -972,9 +1008,9 @@ The module defines the following classes, functions and decorators: :data:`ClassVar` is not a class itself, and should not be used with :func:`isinstance` or :func:`issubclass`. - Note that :data:`ClassVar` does not change Python runtime behavior; - it can be used by 3rd party type checkers, so that the following - code might flagged as an error by those:: + :data:`ClassVar` does not change Python runtime behavior, but + it can be used by third-party type checkers. For example, a type checker + might flag the following code as an error:: enterprise_d = Starship(3000) enterprise_d.stats = {} # Error, setting class variable on instance From 03066a05f5cdf96bc3bf15718f586877860f2d47 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 9 Apr 2017 20:37:44 -0700 Subject: [PATCH 217/232] Remove OSError related comment in urllib.request. (#1072) (cherry picked from commit 6dfcc81f6b1c82a71a1c876e14424fb8b3573447) --- Lib/urllib/request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 3e82000466432f..96921440edea11 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1488,7 +1488,6 @@ def open_local_file(self, req): origurl = 'file://' + filename return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - # users shouldn't expect OSErrors coming from urlopen() raise URLError(exp) raise URLError('file not on local host') From f0c416799b33742af36d7c6608b9a422615fb3d2 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 12 Apr 2017 20:02:21 -0700 Subject: [PATCH 218/232] [3.6] bpo-30047: Fix a typo in Doc/library/select.rst (GH-1086) (GH-1098) (cherry picked from commit 3e0f1fc4e0ffcfcc706015fa3d67c262948ef171) --- Doc/library/select.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f97118ebe05788..bd5442c6a27aa0 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -290,7 +290,7 @@ Edge and Level Trigger Polling (epoll) Objects | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | | | associated fd has an event. The default (if | | | this flag is not set) is to wake all epoll | - | | objects polling on on a fd. | + | | objects polling on a fd. | +-------------------------+-----------------------------------------------+ | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | | | down writing half of connection. | From 14944c62300f741488c2f28cb91ad1e3fef7343b Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Thu, 13 Apr 2017 11:14:17 +0800 Subject: [PATCH 219/232] bpo-26985: Add missing info of code object in inspect documentation (GH-1090) (GH-1099) --- Doc/library/inspect.rst | 379 +++++++++++++++++++++------------------- Lib/inspect.py | 30 ++-- Misc/NEWS | 5 + 3 files changed, 223 insertions(+), 191 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 4ff2187b45f8eb..6be28a2b31cb66 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -34,185 +34,198 @@ provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special attributes: -+-----------+-----------------+---------------------------+ -| Type | Attribute | Description | -+===========+=================+===========================+ -| module | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __file__ | filename (missing for | -| | | built-in modules) | -+-----------+-----------------+---------------------------+ -| class | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-----------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-----------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-----------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-----------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-----------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-----------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-----------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-----------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-----------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-----------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-----------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_restricted | 0 or 1 if frame is in | -| | | restricted execution mode | -+-----------+-----------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-----------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including \* or \*\* | -| | | args) | -+-----------+-----------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-----------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-----------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-----------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | `| -+-----------+-----------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-----------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-----------------+---------------------------+ -| | co_names | tuple of names of local | -| | | variables | -+-----------+-----------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-----------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-----------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-----------------+---------------------------+ -| generator | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | gi_frame | frame | -+-----------+-----------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-----------------+---------------------------+ -| | gi_code | code | -+-----------+-----------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-----------------+---------------------------+ -| | cr_frame | frame | -+-----------+-----------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-----------------+---------------------------+ -| | cr_code | code | -+-----------+-----------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ ++-----------+-------------------+---------------------------+ +| Type | Attribute | Description | ++===========+===================+===========================+ +| module | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __file__ | filename (missing for | +| | | built-in modules) | ++-----------+-------------------+---------------------------+ +| class | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_restricted | 0 or 1 if frame is in | +| | | restricted execution mode | ++-----------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | `| ++-----------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------+-------------------+---------------------------+ +| | co_names | tuple of names of local | +| | | variables | ++-----------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------+-------------------+---------------------------+ +| | gi_code | code | ++-----------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------+-------------------+---------------------------+ +| | cr_code | code | ++-----------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -1268,6 +1281,10 @@ Code Objects Bit Flags Python code objects have a ``co_flags`` attribute, which is a bitmap of the following flags: +.. data:: CO_OPTIMIZED + + The code object is optimized, using fast locals. + .. data:: CO_NEWLOCALS If set, a new dict will be created for the frame's ``f_locals`` when @@ -1281,6 +1298,10 @@ the following flags: The code object has a variable keyword parameter (``**kwargs``-like). +.. data:: CO_NESTED + + The flag is set when the code object is a nested function. + .. data:: CO_GENERATOR The flag is set when the code object is a generator function, i.e. diff --git a/Lib/inspect.py b/Lib/inspect.py index 4d56ef5d41b47f..a2dcb888a0c609 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -253,18 +253,24 @@ def iscode(object): """Return true if the object is a code object. Code objects provide these attributes: - co_argcount number of arguments (not including * or ** args) - co_code string of raw compiled bytecode - co_consts tuple of constants used in the bytecode - co_filename name of file in which this code object was created - co_firstlineno number of first line in Python source code - co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg - co_lnotab encoded mapping of line numbers to bytecode indices - co_name name with which this code object was defined - co_names tuple of names of local variables - co_nlocals number of local variables - co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_argcount number of arguments (not including *, ** args + or keyword only arguments) + co_code string of raw compiled bytecode + co_cellvars tuple of names of cell variables + co_consts tuple of constants used in the bytecode + co_filename name of file in which this code object was created + co_firstlineno number of first line in Python source code + co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg + | 16=nested | 32=generator | 64=nofree | 128=coroutine + | 256=iterable_coroutine | 512=async_generator + co_freevars tuple of names of free variables + co_kwonlyargcount number of keyword only arguments (not including ** arg) + co_lnotab encoded mapping of line numbers to bytecode indices + co_name name with which this code object was defined + co_names tuple of names of local variables + co_nlocals number of local variables + co_stacksize virtual machine stack space required + co_varnames tuple of names of arguments and local variables""" return isinstance(object, types.CodeType) def isbuiltin(object): diff --git a/Misc/NEWS b/Misc/NEWS index 83f9e7a6852560..dd1fa12d7c0be8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -86,6 +86,11 @@ Build - bpo-29643: Fix ``--enable-optimization`` didn't work. +Documentation +------------- + +- bpo-26985: Add missing info of code object in inspect documentation. + What's New in Python 3.6.1? =========================== From bc9025012fc5ae007283e9ef345dba0ba08cf2b9 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 12 Apr 2017 20:45:54 -0700 Subject: [PATCH 220/232] [3.6] Remove superfluous comment in urllib.error. (GH-1076) (GH-1101) (cherry picked from commit 6fab78e9027f9ebd6414995580781b480433e595) --- Lib/urllib/error.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index c5b675d16188b8..851515bc307d62 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -16,10 +16,6 @@ __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] -# do these error classes make sense? -# make sure all of the OSError stuff is overridden. we just want to be -# subtypes. - class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of # the implementation. need to override __init__ and __str__. From bd1173f202f5a3990063d980368e7ad1edc9b5b5 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Wed, 12 Apr 2017 20:52:39 -0700 Subject: [PATCH 221/232] [3.6] Clarify exception handler scope in contextlib (GH-1103) Moved explicit raise from inside try to try...else. (cherry picked from commit 680e20beee8bbce9f857b8e7795009191f98b0ba) --- Lib/contextlib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 8421968525947e..e91cf460e53bfd 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -98,7 +98,6 @@ def __exit__(self, type, value, traceback): value = type() try: self.gen.throw(type, value, traceback) - raise RuntimeError("generator didn't stop after throw()") except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration @@ -124,6 +123,8 @@ def __exit__(self, type, value, traceback): # if sys.exc_info()[1] is not value: raise + else: + raise RuntimeError("generator didn't stop after throw()") def contextmanager(func): From 9b409ff41ceb2d7ea7e8d25a7bbf5eb7d46625f3 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 02:50:21 -0700 Subject: [PATCH 222/232] [3.6] bpo-29692: contextlib.contextmanager may incorrectly unchain RuntimeError (GH-949) (#1105) contextlib._GeneratorContextManager.__exit__ includes a special case to deal with PEP 479 RuntimeErrors created when `StopIteration` is thrown into the context manager body. Previously this check was too permissive, and undid one level of chaining on *all* RuntimeError instances, not just those that wrapped a StopIteration instance. (cherry picked from commit 00c75e9a45ff0366c185e9e8a2e23af5a35481b0) --- Lib/contextlib.py | 12 ++++++------ Lib/test/test_contextlib.py | 23 +++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index e91cf460e53bfd..5e47054954ba5a 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -88,7 +88,7 @@ def __exit__(self, type, value, traceback): try: next(self.gen) except StopIteration: - return + return False else: raise RuntimeError("generator didn't stop") else: @@ -110,7 +110,7 @@ def __exit__(self, type, value, traceback): # Likewise, avoid suppressing if a StopIteration exception # was passed to throw() and later wrapped into a RuntimeError # (see PEP 479). - if exc.__cause__ is value: + if type is StopIteration and exc.__cause__ is value: return False raise except: @@ -121,10 +121,10 @@ def __exit__(self, type, value, traceback): # fixes the impedance mismatch between the throw() protocol # and the __exit__() protocol. # - if sys.exc_info()[1] is not value: - raise - else: - raise RuntimeError("generator didn't stop after throw()") + if sys.exc_info()[1] is value: + return False + raise + raise RuntimeError("generator didn't stop after throw()") def contextmanager(func): diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index c04c804af57047..b1a467d952da0f 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -152,6 +152,29 @@ def woohoo(): else: self.fail('StopIteration was suppressed') + def test_contextmanager_do_not_unchain_non_stopiteration_exceptions(self): + @contextmanager + def test_issue29692(): + try: + yield + except Exception as exc: + raise RuntimeError('issue29692:Chained') from exc + try: + with test_issue29692(): + raise ZeroDivisionError + except Exception as ex: + self.assertIs(type(ex), RuntimeError) + self.assertEqual(ex.args[0], 'issue29692:Chained') + self.assertIsInstance(ex.__cause__, ZeroDivisionError) + + try: + with test_issue29692(): + raise StopIteration('issue29692:Unchained') + except Exception as ex: + self.assertIs(type(ex), StopIteration) + self.assertEqual(ex.args[0], 'issue29692:Unchained') + self.assertIsNone(ex.__cause__) + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): diff --git a/Misc/NEWS b/Misc/NEWS index dd1fa12d7c0be8..6e2db58a92f451 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -31,6 +31,9 @@ Core and Builtins Library ------- +- bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in + contextlib.contextmanager. + Patch by Siddharth Velankar. - bpo-29998: Pickling and copying ImportError now preserves name and path attributes. From 97a40b7a5b2979fb17e1751c139fd4ba1ebd5276 Mon Sep 17 00:00:00 2001 From: Nevada Sanchez Date: Thu, 13 Apr 2017 06:21:05 -0400 Subject: [PATCH 223/232] bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-752) --- Lib/lib2to3/pgen2/tokenize.py | 16 ++++++++-------- Lib/lib2to3/tests/data/py3_test_grammar.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60f7da89e..fba0fa2c00dccd 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index cf31a5411a855a..0b9bee0ab42ea6 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,6 +72,28 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 + def testUnderscoresInNumbers(self): + # Integers + x = 1_0 + x = 123_456_7_89 + x = 0xabc_123_4_5 + x = 0X_abc_123 + x = 0B11_01 + x = 0b_11_01 + x = 0o45_67 + x = 0O_45_67 + + # Floats + x = 3_1.4 + x = 03_1.4 + x = 3_1. + x = .3_1 + x = 3.1_4 + x = 0_3.1_4 + x = 3e1_4 + x = 3_1e+4_1 + x = 3_1E-4_1 + def testFloats(self): x = 3.14 x = 314. From 84c2d75489a84174d8993aea292828662e35a50f Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 04:03:16 -0700 Subject: [PATCH 224/232] Revert "bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-752)" (GH-1109) This reverts commit 97a40b7a5b2979fb17e1751c139fd4ba1ebd5276. The commit is supposed to go to the master branch first. --- Lib/lib2to3/pgen2/tokenize.py | 16 ++++++++-------- Lib/lib2to3/tests/data/py3_test_grammar.py | 22 ---------------------- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index fba0fa2c00dccd..d14db60f7da89e 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' -Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' -Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' -Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') +Binnumber = r'0[bB][01]*' +Hexnumber = r'0[xX][\da-fA-F]*[lL]?' +Octnumber = r'0[oO]?[0-7]*[lL]?' +Decnumber = r'[1-9]\d*[lL]?' Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+(?:_\d+)*' -Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) -Expfloat = r'\d+(?:_\d+)*' + Exponent +Exponent = r'[eE][-+]?\d+' +Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) +Expfloat = r'\d+' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index 0b9bee0ab42ea6..cf31a5411a855a 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,28 +72,6 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 - def testUnderscoresInNumbers(self): - # Integers - x = 1_0 - x = 123_456_7_89 - x = 0xabc_123_4_5 - x = 0X_abc_123 - x = 0B11_01 - x = 0b_11_01 - x = 0o45_67 - x = 0O_45_67 - - # Floats - x = 3_1.4 - x = 03_1.4 - x = 3_1. - x = .3_1 - x = 3.1_4 - x = 0_3.1_4 - x = 3e1_4 - x = 3_1e+4_1 - x = 3_1E-4_1 - def testFloats(self): x = 3.14 x = 314. From 119d94ad37a99ecb0b8329467d04cd9d909e310e Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 13 Apr 2017 16:17:01 +0300 Subject: [PATCH 225/232] bpo-29791: Clarify that flush is keyword-only argument (GH-1093) Reported by Lucio Ricardo Montero Valenzuela. (cherry picked from commit 61b9ac93712df8092a25223cd56fa6528359792b) --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 90fb07af227d98..9126535eecb85c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1125,7 +1125,7 @@ are always available. They are listed here in alphabetical order. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end* and *file*, if present, must be given as keyword + by *end*. *sep*, *end*, *file* and *flush*, if present, must be given as keyword arguments. All non-keyword arguments are converted to strings like :func:`str` does and From 936633282220768a6fec9bd6bb53ee9e3ec354df Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Apr 2017 19:41:19 +0300 Subject: [PATCH 226/232] bpo-30021: Add examples for re.escape(). (#1048) (#1115) And fix the parameter name. (cherry picked from commit 8fc7bc2b7631ee819ee614e47b6f44bacebe1574) --- Doc/library/re.rst | 17 ++++++++++++++--- Doc/tools/susp-ignored.csv | 2 ++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 9cced512706554..944c6fb10d6c25 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -784,11 +784,22 @@ form. Unmatched groups are replaced with an empty string. -.. function:: escape(string) +.. function:: escape(pattern) - Escape all the characters in pattern except ASCII letters, numbers and ``'_'``. + Escape all the characters in *pattern* except ASCII letters, numbers and ``'_'``. This is useful if you want to match an arbitrary literal string that may - have regular expression metacharacters in it. + have regular expression metacharacters in it. For example:: + + >>> print(re.escape('python.exe')) + python\.exe + + >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" + >>> print('[%s]+' % re.escape(legal_chars)) + [abcdefghijklmnopqrstuvwxyz0123456789\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\:]+ + + >>> operators = ['+', '-', '*', '/', '**'] + >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) + \/|\-|\+|\*\*|\* .. versionchanged:: 3.3 The ``'_'`` character is no longer escaped. diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 166374cb90cec1..57b0cf12f53712 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -302,6 +302,8 @@ whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe: whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" whatsnew/3.2,,:location,zope9-location = ${zope9:location} whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf +library/re,,`,!#$%&'*+-.^_`|~: +library/re,,`,\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\: library/tarfile,,:xz,'x:xz' library/xml.etree.elementtree,,:sometag,prefix:sometag library/xml.etree.elementtree,,:fictional," Date: Thu, 13 Apr 2017 14:11:48 -0700 Subject: [PATCH 227/232] convert from long long to PyLong loselessly (#1106) (#1121) --- Modules/_testcapimodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f09205f63cc5ab..b25f4434ffa35c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3812,7 +3812,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (_PyTime_AsTimeval(t, &tv, round) < 0) return NULL; - seconds = PyLong_FromLong((long long)tv.tv_sec); + seconds = PyLong_FromLongLong(tv.tv_sec); if (seconds == NULL) return NULL; return Py_BuildValue("Nl", seconds, tv.tv_usec); From c5c24cfaf9fe9a38f0901756cd797bd12524a8b7 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 16:37:51 -0700 Subject: [PATCH 228/232] [3.6] Fix a typo in Doc/library/functions.rst (GH-1117) (GH-1123) Replace `For object's ... ` with `For objects ...` (cherry picked from commit 873ef20d0007b4b120933473e6252d2309a70102) --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 9126535eecb85c..138f7d027cb86f 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -610,7 +610,7 @@ are always available. They are listed here in alphabetical order. .. note:: - For object's with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`__hash__` methods, note that :func:`hash` truncates the return value based on the bit width of the host machine. See :meth:`__hash__` for details. From 2cdf087d1fd48f7d0f95b5a0b31b9a624fa84751 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 16:54:49 -0700 Subject: [PATCH 229/232] [3.6] bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-1119) (GH-1122) * Allow underscores in numeric literals in lib2to3. * Stricter literal parsing for Python 3.6 in lib2to3.pgen2.tokenize. * Add test case for underscores in literals in Python 3. (cherry picked from commit a6e395dffadf8c5124903c01ad69fefa36b1a935) --- Lib/lib2to3/pgen2/tokenize.py | 16 ++++++++-------- Lib/lib2to3/tests/data/py3_test_grammar.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60f7da89e..fba0fa2c00dccd 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index cf31a5411a855a..0b9bee0ab42ea6 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,6 +72,28 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 + def testUnderscoresInNumbers(self): + # Integers + x = 1_0 + x = 123_456_7_89 + x = 0xabc_123_4_5 + x = 0X_abc_123 + x = 0B11_01 + x = 0b_11_01 + x = 0o45_67 + x = 0O_45_67 + + # Floats + x = 3_1.4 + x = 03_1.4 + x = 3_1. + x = .3_1 + x = 3.1_4 + x = 0_3.1_4 + x = 3e1_4 + x = 3_1e+4_1 + x = 3_1E-4_1 + def testFloats(self): x = 3.14 x = 314. From cbc46afa59dcc43c2c8c90ae7a0a0dc404325a89 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Thu, 13 Apr 2017 19:26:16 -0700 Subject: [PATCH 230/232] [3.6] bpo-29694: race condition in pathlib mkdir with flags parents=True (GH-1089). (GH-1126) (cherry picked from commit 22a594a0047d7706537ff2ac676cdc0f1dcb329c) --- Lib/pathlib.py | 4 ++-- Lib/test/test_pathlib.py | 30 ++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 8c1cb96bad3075..70f5cba76ffa8a 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1230,8 +1230,8 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): except FileNotFoundError: if not parents or self.parent == self: raise - self.parent.mkdir(parents=True) - self._accessor.mkdir(self, mode) + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) except OSError: # Cannot rely on checking for EEXIST, since the operating system # could give priority to other errors like EACCES or EROFS diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 3ff9726d4359ec..846f721e8d9abb 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -8,6 +8,7 @@ import stat import tempfile import unittest +from unittest import mock from test import support android_not_root = support.android_not_root @@ -1816,6 +1817,35 @@ def test_mkdir_no_parents_file(self): p.mkdir(exist_ok=True) self.assertEqual(cm.exception.errno, errno.EEXIST) + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(BASE, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + os.mkdir(path, mode) # from another process + concurrently_created.add(path) + os.mkdir(path, mode) # our real call + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("pathlib._normal_accessor.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + @with_symlinks def test_symlink_to(self): P = self.cls(BASE) diff --git a/Misc/NEWS b/Misc/NEWS index 6e2db58a92f451..e623457cdd18ce 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -31,6 +31,10 @@ Core and Builtins Library ------- + +- bpo-29694: Fixed race condition in pathlib mkdir with flags + parents=True. Patch by Armin Rigo. + - bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in contextlib.contextmanager. Patch by Siddharth Velankar. From 413a8913aee255d0635e4ddbb9c343d9e5e76fea Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 14 Apr 2017 13:37:46 +0200 Subject: [PATCH 231/232] Relax test timing (bpo-29861) to avoid sporadic failures (#1120) (#1132) (cherry picked from commit 685cdb9acc3fca04a9897d88b89771ddfd50e772) --- Lib/test/_test_multiprocessing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 771bbf24265b91..57ada127465d75 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2080,6 +2080,7 @@ def test_release_task_refs(self): self.pool.map(identity, objs) del objs + time.sleep(DELTA) # let threaded cleanup code run self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check # they were released too. From d390bebd6afb57b75bdd3a015a0670d6c972a107 Mon Sep 17 00:00:00 2001 From: Jack McCracken Date: Fri, 14 Apr 2017 07:35:30 -0400 Subject: [PATCH 232/232] Fix misplaced positional argument in OS X support library --- Lib/_osx_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index eadf06f20e2507..e37852e2536c33 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -210,7 +210,7 @@ def _remove_universal_flags(_config_vars): # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub(r'-arch\s+\w+\s', ' ', flags, re.ASCII) + flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) flags = re.sub('-isysroot [^ \t]*', ' ', flags) _save_modified_value(_config_vars, cv, flags)