diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000000000..fcf9df6a7a698e4 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,35 @@ +codecov: + notify: + require_ci_to_pass: true +comment: off +ignore: + - "Doc/**/*" + - "Misc/*" + - "Mac/**/*" + - "PC/**/*" + - "PCbuild/**/*" + - "Tools/**/*" + - "Grammar/*" +coverage: + precision: 2 + range: + - 70.0 + - 100.0 + round: down + status: + changes: off + project: off + patch: + default: + target: 100% + only_pulls: true + threshold: 0.05 +parsers: + gcov: + branch_detection: + conditional: true + loop: true + macro: false + method: false + javascript: + enable_partials: false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000000000..82694d81f276b2c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +*.pck binary +Lib/test/cjkencodings/* binary +Lib/test/decimaltestdata/*.decTest binary +Lib/test/sndhdrdata/sndhdr.* binary +Lib/test/test_email/data/msg_26.txt binary +Lib/test/xmltestdata/* binary +Lib/venv/scripts/nt/* binary +Lib/test/coding20731.py binary diff --git a/.github/appveyor.yml b/.github/appveyor.yml new file mode 100644 index 000000000000000..a369e52415bb178 --- /dev/null +++ b/.github/appveyor.yml @@ -0,0 +1,23 @@ +version: 3.6.1rc1+.{build} +clone_depth: 5 +build_script: +- cmd: PCbuild\build.bat -e +test_script: +- cmd: PCbuild\rt.bat -q -uall -rwW --slowest --timeout=1200 -j0 + +# Only trigger AppVeyor if actual code or its configuration changes +only_commits: + files: + - .github/appveyor.yml + - .gitattributes + - Grammar/ + - Include/ + - Lib/ + - Modules/ + - Objects/ + - PC/ + - PCBuild/ + - Parser/ + - Programs/ + - Python/ + - Tools/ diff --git a/.gitignore b/.gitignore index ed4ebfbbd9de74e..db27597aac57da1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ .gdb_history Doc/build/ Doc/venv/ +Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff --git a/.mention-bot b/.mention-bot new file mode 100644 index 000000000000000..cb53b993fb1e8a4 --- /dev/null +++ b/.mention-bot @@ -0,0 +1,3 @@ +{ + "findPotentialReviewers": false +} diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000000..5fee6cd76733242 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,94 @@ +language: c +dist: trusty +sudo: false +group: beta + +# To cache doc-building dependencies. +cache: pip + +branches: + only: + - master + - /^\d\.\d$/ + +os: + - linux + # macOS builds are disabled as the machines are under-provisioned on Travis, + # adding up to an extra hour completing a full CI run. + +compiler: + - clang + # gcc also works, but to keep the # of concurrent builds down, we use one C + # compiler here and the other to run the coverage build. + +env: + - TESTING=cpython + +matrix: + fast_finish: true + allow_failures: + - env: + - TESTING=coverage + include: + - os: linux + language: python + python: 3.6 + env: + - TESTING=docs + before_script: + - cd Doc + - make venv + script: + - make check suspicious html PYTHON="./venv/bin/python" SPHINXBUILD="./venv/bin/python -m sphinx" SPHINXOPTS="-q -W" + - os: linux + language: c + compiler: gcc + env: + - TESTING=coverage + before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure + make -s -j4 + # Need a venv that can parse covered code. + ./python -m venv venv + ./venv/bin/python -m pip install -U coverage + script: + # Skip tests that re-run the entire test suite. + - ./venv/bin/python -m coverage run --pylib -m test -uall -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn + after_script: # Probably should be after_success once test suite updated to run under coverage.py. + # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. + - source ./venv/bin/activate + - bash <(curl -s https://codecov.io/bash) + +# Travis provides only 2 cores, so don't overdue the parallelism and waste memory. +before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.(rst|yml)$)|(^Doc)/' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure --with-pydebug + make -j4 + +script: + # `-r -w` implicitly provided through `make buildbottest`. + - make buildbottest TESTOPTS="-j4" + +notifications: + email: false + irc: + channels: + # This is set to a secure variable to prevent forks from notifying the + # IRC channel whenever they fail a build. This can be removed when travis + # implements https://github.com/travis-ci/travis-ci/issues/1094. + # The actual value here is: irc.freenode.net#python-dev + - secure: "s7kAkpcom2yUJ8XqyjFI0obJmhAGrn1xmoivdaPdgBIA++X47TBp1x4pgDsbEsoalef7bEwa4l07KdT4qa+DOd/c4QxaWom7fbN3BuLVsZuVfODnl79+gYq/TAbGfyH+yDs18DXrUfPgwD7C5aW32ugsqAOd4iWzfGJQ5OrOZzqzGjYdYQUEkJFXgxDEIb4aHvxNDWGO3Po9uKISrhb5saQ0l776yLo1Ur7M4oxl8RTbCdgX0vf5TzPg52BgvZpOgt3DHOUYPeiJLKNjAE6ibg0U95sEvMfHX77nz4aFY4/3UI6FFaRla34rZ+mYKrn0TdxOhera1QOgPmM6HzdO4K44FpfK1DS0Xxk9U9/uApq+cG0bU3W+cVUHDBe5+90lpRBAXHeHCgT7TI8gec614aiT8lEr3+yH8OBRYGzkjNK8E2LJZ/SxnVxDe7aLF6AWcoWLfS6/ziAIBFQ5Nc4U72CT8fGVSkl8ywPiRlvixKdvTODMSZo0jMqlfZSNaAPTsNRx4wu5Uis4qekwe32Fz4aB6KGpsuuVjBi+H6v0RKxNJNGY3JKDiEH2TK0UE2auJ5GvLW48aUVFcQMB7euCWYXlSWVRHh3WLU8QXF29Dw4JduRZqUpOdRgMHU79UHRq+mkE0jAS/nBcS6CvsmxCpTSrfVYuMOu32yt18QQoTyU=" + on_success: change + on_failure: always + skip_join: true diff --git a/Doc/Makefile b/Doc/Makefile index 91f937f985831af..94697f956718943 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -4,13 +4,13 @@ # # You can set these variables from the command line. -PYTHON = python +PYTHON = python3 SPHINXBUILD = sphinx-build PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) .PHONY: help build html htmlhelp latex text changes linkcheck \ @@ -153,7 +153,7 @@ dist: cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub check: - $(PYTHON) tools/rstlint.py -i tools -i venv + $(PYTHON) tools/rstlint.py -i tools -i venv -i README.rst serve: ../Tools/scripts/serve.py build/html diff --git a/Doc/README.txt b/Doc/README.rst similarity index 83% rename from Doc/README.txt rename to Doc/README.rst index 4f8e9f8f1417fbc..dcd3d6e80ff3c48 100644 --- a/Doc/README.txt +++ b/Doc/README.rst @@ -2,20 +2,21 @@ Python Documentation README ~~~~~~~~~~~~~~~~~~~~~~~~~~~ This directory contains the reStructuredText (reST) sources to the Python -documentation. You don't need to build them yourself, prebuilt versions are -available at . +documentation. You don't need to build them yourself, `prebuilt versions are +available `_. Documentation on authoring Python documentation, including information about -both style and markup, is available in the "Documenting Python" chapter of the -developers guide . +both style and markup, is available in the "`Documenting Python +`_" chapter of the +developers guide. Building the docs ================= -You need to have Sphinx installed; it is the toolset +You need to have `Sphinx `_ installed; it is the toolset used to build the docs. It is not included in this tree, but maintained -separately and available from PyPI . +separately and `available from PyPI `_. Using make @@ -108,11 +109,11 @@ see the make targets above). Contributing ============ -Bugs in the content should be reported to the Python bug tracker at -https://bugs.python.org. +Bugs in the content should be reported to the +`Python bug tracker `_. -Bugs in the toolset should be reported in the Sphinx bug tracker at -https://www.bitbucket.org/birkenfeld/sphinx/issues/. +Bugs in the toolset should be reported in the +`Sphinx bug tracker `_. You can also send a mail to the Python Documentation Team at docs@python.org, and we will process your request as soon as possible. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 037b85cfd117542..3516a1690384f73 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -750,6 +750,61 @@ All standard Python exceptions are available as global variables whose names are :c:type:`PyObject\*`; they are all class objects. For completeness, here are all the variables: +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | +=========================================+=================================+==========+ @@ -759,8 +814,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | @@ -769,27 +822,31 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportError` | :exc:`ImportError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_IndexError` | :exc:`IndexError` | | +-----------------------------------------+---------------------------------+----------+ @@ -801,8 +858,12 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NameError` | :exc:`NameError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | @@ -823,16 +884,32 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemError` | :exc:`SystemError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ValueError` | :exc:`ValueError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | @@ -849,11 +926,18 @@ the variables: and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. .. versionadded:: 3.5 - :c:data:`PyExc_RecursionError`. + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. These are compatibility aliases to :c:data:`PyExc_OSError`: +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ @@ -867,52 +951,6 @@ These are compatibility aliases to :c:data:`PyExc_OSError`: .. versionchanged:: 3.3 These aliases used to be separate exception types. - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_LookupError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_ConnectionError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_ImportError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_MemoryError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - Notes: (1) @@ -924,3 +962,58 @@ Notes: (3) Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. + +Standard Warnings +================= + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DepricationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDepricationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index a6d0f4688d1b78b..c6d1d02a2fa510e 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -34,7 +34,7 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) - Return a string object containing the marshalled representation of *value*. + Return a bytes object containing the marshalled representation of *value*. *version* indicates the file format. @@ -88,10 +88,10 @@ written using these routines? :exc:`TypeError`) and returns *NULL*. -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *string, Py_ssize_t len) +.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) - Return a Python object from the data stream in a character buffer - containing *len* bytes pointed to by *string*. + Return a Python object from the data stream in a byte buffer + containing *len* bytes pointed to by *data*. On error, sets the appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 3ff545275fb3b8a..873fb2ac1d3cad8 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -391,7 +391,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: +:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f48119391f2bbd9..c080f317bee9d9d 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -241,7 +241,7 @@ definition with the same method name. +==================+=============+===============================+ | :attr:`name` | char \* | name of the member | +------------------+-------------+-------------------------------+ - | :attr:`type` | int | the type of the member in the | + | :attr:`!type` | int | the type of the member in the | | | | C struct | +------------------+-------------+-------------------------------+ | :attr:`offset` | Py_ssize_t | the offset in bytes that the | @@ -256,7 +256,7 @@ definition with the same method name. | | | docstring | +------------------+-------------+-------------------------------+ - :attr:`type` can be one of many ``T_`` macros corresponding to various C + :attr:`!type` can be one of many ``T_`` macros corresponding to various C types. When the member is accessed in Python, it will be converted to the equivalent Python type. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 02f7ada7be7ee4e..6e91576ee8f0045 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1393,77 +1393,78 @@ Character Map Codecs This codec is special in that it can be used to implement many different codecs (and this is in fact what was done to obtain most of the standard codecs included in the :mod:`encodings` package). The codec uses mapping to encode and -decode characters. - -Decoding mappings must map single string characters to single Unicode -characters, integers (which are then interpreted as Unicode ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -Encoding mappings must map single Unicode characters to single string -characters, integers (which are then interpreted as Latin-1 ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -The mapping objects provided must only support the __getitem__ mapping -interface. - -If a character lookup fails with a LookupError, the character is copied as-is -meaning that its ordinal value will be interpreted as Unicode or Latin-1 ordinal -resp. Because of this, mappings only need to contain those mappings which map -characters to different code points. +decode characters. The mapping objects provided must support the +:meth:`__getitem__` mapping interface; dictionaries and sequences work well. These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* using - the given *mapping* object. Return *NULL* if an exception was raised by the - codec. If *mapping* is *NULL* latin-1 decoding will be done. Else it can be a - dictionary mapping byte or a unicode string, which is treated as a lookup table. - Byte values greater that the length of the string and U+FFFE "characters" are - treated as "undefined mapping". + Create a Unicode object by decoding *size* bytes of the encoded string *s* + using the given *mapping* object. Return *NULL* if an exception was raised + by the codec. + + If *mapping* is *NULL*, Latin-1 decoding will be applied. Else + *mapping* must map bytes ordinals (integers in the range from 0 to 255) + to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or ``None``. Unmapped data bytes -- ones which cause a + :exc:`LookupError`, as well as ones which get mapped to ``None``, + ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause + an error. .. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) - Encode a Unicode object using the given *mapping* object and return the result - as Python string object. Error handling is "strict". Return *NULL* if an + Encode a Unicode object using the given *mapping* object and return the + result as a bytes object. Error handling is "strict". Return *NULL* if an exception was raised by the codec. -The following codec API is special in that maps Unicode to Unicode. - + The *mapping* object must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or ``None``. Unmapped character + ordinals (ones which cause a :exc:`LookupError`) as well as mapped to + ``None`` are treated as "undefined mapping" and cause an error. -.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ - PyObject *table, const char *errors) - - Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a - character mapping *table* to it and return the resulting Unicode object. Return - *NULL* when an exception was raised by the codec. - The *mapping* table must map Unicode ordinal integers to Unicode ordinal - integers or ``None`` (causing deletion of the character). +.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. + Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given + *mapping* object and return the result as a bytes object. Return *NULL* if + an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API - ` + :c:func:`PyUnicode_AsCharmapString` or + :c:func:`PyUnicode_AsEncodedString`. -.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ +The following codec API is special in that maps Unicode to Unicode. + +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ PyObject *mapping, const char *errors) - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return a Python string object. Return *NULL* if an - exception was raised by the codec. + Translate a Unicode object using the given *mapping* object and return the + resulting Unicode object. Return *NULL* if an exception was raised by the + codec. + + The *mapping* object must map Unicode ordinal integers to Unicode strings, + integers (which are then interpreted as Unicode ordinals) or ``None`` + (causing deletion of the character). Unmapped character ordinals (ones + which cause a :exc:`LookupError`) are left untouched and are copied as-is. + + +.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) + + Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a + character *mapping* table to it and return the resulting Unicode object. + Return *NULL* when an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsCharmapString` or - :c:func:`PyUnicode_AsEncodedString`. + :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API + ` MBCS codecs for Windows diff --git a/Doc/conf.py b/Doc/conf.py index b1bb6208bb4b8e6..18aebb68a8d8dfc 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -37,7 +37,7 @@ needs_sphinx = '1.2' # Ignore any .rst files in the venv/ directory. -exclude_patterns = ['venv/*'] +exclude_patterns = ['venv/*', 'README.rst'] # Options for HTML output @@ -88,11 +88,24 @@ # Options for LaTeX output # ------------------------ +# Get LaTeX to handle Unicode correctly +latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} + +# Additional stuff for the LaTeX preamble. +latex_elements['preamble'] = r''' +\authoraddress{ + \strong{Python Software Foundation}\\ + Email: \email{docs@python.org} +} +\let\Verbatim=\OriginalVerbatim +\let\endVerbatim=\endOriginalVerbatim +''' + # The paper size ('letter' or 'a4'). -latex_paper_size = 'a4' +latex_elements['papersize'] = 'a4' # The font size ('10pt', '11pt' or '12pt'). -latex_font_size = '10pt' +latex_elements['font_size'] = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -125,22 +138,9 @@ for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst') -# Additional stuff for the LaTeX preamble. -latex_preamble = r''' -\authoraddress{ - \strong{Python Software Foundation}\\ - Email: \email{docs@python.org} -} -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim -''' - # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] -# Get LaTeX to handle Unicode correctly -latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} - # Options for Epub output # ----------------------- diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index 1f5be9cdb29a974..4e2761d8a7d0460 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -321,7 +321,7 @@ You can read back this static file, by using the >>> metadata.description 'Easily download, build, install, upgrade, and uninstall Python packages' -Notice that the class can also be instanciated with a metadata file path to +Notice that the class can also be instantiated with a metadata file path to loads its values:: >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index f1e33afdabf8a58..8f6a907a8a2fdac 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -159,7 +159,7 @@ How do I obtain a copy of the Python source? The latest Python source distribution is always available from python.org, at https://www.python.org/downloads/. The latest development sources can be obtained -via anonymous Mercurial access at https://hg.python.org/cpython. +at https://github.com/python/cpython/. The source distribution is a gzipped tar file containing the complete C source, Sphinx-formatted documentation, Python library modules, example programs, and @@ -222,8 +222,8 @@ releases are announced on the comp.lang.python and comp.lang.python.announce newsgroups and on the Python home page at https://www.python.org/; an RSS feed of news is available. -You can also access the development version of Python through Mercurial. See -https://docs.python.org/devguide/faq.html for details. +You can also access the development version of Python through Git. See +`The Python Developer's Guide `_ for details. How do I submit bug reports and patches for Python? diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index d7253436beaf61d..6ac83e45d2e8152 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -300,9 +300,10 @@ this respect, and is easily configured to use spaces: Take :menuselection:`Tools --> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent size" to 4, and select the "Insert spaces" radio button. -If you suspect mixed tabs and spaces are causing problems in leading whitespace, -run Python with the :option:`-t` switch or run ``Tools/Scripts/tabnanny.py`` to -check a directory tree in batch mode. +Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs +and spaces are causing problems in leading whitespace. +You may also run the :mod:`tabnanny` module to check a directory tree +in batch mode. How do I check for a keypress without blocking? diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 41ee3d83b311fa5..dba9186d935a6a8 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -131,6 +131,10 @@ Glossary binary file A :term:`file object` able to read and write :term:`bytes-like objects `. + Examples of binary files are files opened in binary mode (``'rb'``, + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, + :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and + :class:`gzip.GzipFile`. .. seealso:: A :term:`text file` reads and writes :class:`str` objects. @@ -155,7 +159,7 @@ Glossary bytecode Python source code is compiled into bytecode, the internal representation of a Python program in the CPython interpreter. The bytecode is also - cached in ``.pyc`` and ``.pyo`` files so that executing the same file is + cached in ``.pyc`` files so that executing the same file is faster the second time (recompilation from source to bytecode can be avoided). This "intermediate language" is said to run on a :term:`virtual machine` that executes the machine code corresponding to @@ -316,6 +320,11 @@ Glossary A module written in C or C++, using Python's C API to interact with the core and with user code. + f-string + String literals prefixed with ``'f'`` or ``'F'`` are commonly called + "f-strings" which is short for + :ref:`formatted string literals `. See also :pep:`498`. + file object An object exposing a file-oriented API (with methods such as :meth:`read()` or :meth:`write()`) to an underlying resource. Depending @@ -458,9 +467,9 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable, while no mutable - containers (such as lists or dictionaries) are. Objects which are - instances of user-defined classes are hashable by default; they all + All of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not. Objects which are + instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. @@ -966,6 +975,9 @@ Glossary A :term:`file object` able to read and write :class:`str` objects. Often, a text file actually accesses a byte-oriented datastream and handles the :term:`text encoding` automatically. + Examples of text files are files opened in text mode (``'r'`` or ``'w'``), + :data:`sys.stdin`, :data:`sys.stdout`, and instances of + :class:`io.StringIO`. .. seealso:: A :term:`binary file` reads and write :class:`bytes` objects. diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py index b853d23b170985a..a38a39de3e7c86c 100644 --- a/Doc/includes/setup.py +++ b/Doc/includes/setup.py @@ -5,4 +5,5 @@ Extension("noddy2", ["noddy2.c"]), Extension("noddy3", ["noddy3.c"]), Extension("noddy4", ["noddy4.c"]), + Extension("shoddy", ["shoddy.c"]), ]) diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c index 07a272124ceabaa..0c6d412b3c4cda4 100644 --- a/Doc/includes/shoddy.c +++ b/Doc/includes/shoddy.c @@ -31,7 +31,7 @@ Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds) static PyTypeObject ShoddyType = { - PyObject_HEAD_INIT(NULL) + PyVarObject_HEAD_INIT(NULL, 0) "shoddy.Shoddy", /* tp_name */ sizeof(Shoddy), /* tp_basicsize */ 0, /* tp_itemsize */ diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3fbf51058673d92..3f55506c669a29f 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -36,7 +36,7 @@ BaseTransport Base class for transports. - .. method:: close(self) + .. method:: close() Close the transport. If the transport has a buffer for outgoing data, buffered data will be flushed asynchronously. No more data @@ -44,7 +44,7 @@ BaseTransport protocol's :meth:`connection_lost` method will be called with :const:`None` as its argument. - .. method:: is_closing(self) + .. method:: is_closing() Return ``True`` if the transport is closing or is closed. @@ -251,7 +251,7 @@ BaseSubprocessTransport if it hasn't returned, similarly to the :attr:`subprocess.Popen.returncode` attribute. - .. method:: kill(self) + .. method:: kill() Kill the subprocess, as in :meth:`subprocess.Popen.kill`. @@ -384,7 +384,7 @@ The following callbacks are called on :class:`Protocol` instances: .. method:: Protocol.eof_received() - Calls when the other end signals it won't send any more data + Called when the other end signals it won't send any more data (for example by calling :meth:`write_eof`, if the other end also uses asyncio). diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index dc93a74c6dee126..16ba9a3cd6c7f24 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -80,7 +80,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. however, where :class:`~subprocess.Popen` takes a single argument which is list of strings, :func:`subprocess_exec` takes multiple string arguments. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. Other parameters: @@ -123,7 +123,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. using the platform's "shell" syntax. This is similar to the standard library :class:`subprocess.Popen` class called with ``shell=True``. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. See :meth:`~AbstractEventLoop.subprocess_exec` for more details about diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 080d9d77ec984ea..ceecf17cba23e3c 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -237,14 +237,18 @@ The legacy interface: .. function:: decodebytes(s) - decodestring(s) Decode the :term:`bytes-like object` *s*, which must contain one or more lines of base64 encoded data, and return the decoded :class:`bytes`. - ``decodestring`` is a deprecated alias. .. versionadded:: 3.1 +.. function:: decodestring(s) + + Deprecated alias of :func:`decodebytes`. + + .. deprecated:: 3.1 + .. function:: encode(input, output) @@ -257,14 +261,19 @@ The legacy interface: .. function:: encodebytes(s) - encodestring(s) Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary data, and return :class:`bytes` containing the base64-encoded data, with newlines (``b'\n'``) inserted after every 76 bytes of output, and ensuring that there is a trailing newline, as per :rfc:`2045` (MIME). - ``encodestring`` is a deprecated alias. + .. versionadded:: 3.1 + +.. function:: encodestring(s) + + Deprecated alias of :func:`encodebytes`. + + .. deprecated:: 3.1 An example usage of the module: diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst index 359ab23b2f9787d..2966e0dbfbcfe87 100644 --- a/Doc/library/binhex.rst +++ b/Doc/library/binhex.rst @@ -55,5 +55,3 @@ the source for details. If you code or decode textfiles on non-Macintosh platforms they will still use the old Macintosh newline convention (carriage-return as end of line). -As of this writing, :func:`hexbin` appears to not work in all cases. - diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index e09562dc9046c78..04c2a8209217597 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -34,8 +34,8 @@ can be customized by end users easily. .. seealso:: Module :mod:`shlex` - Support for a creating Unix shell-like mini-languages which can be used - as an alternate format for application configuration files. + Support for creating Unix shell-like mini-languages which can be used as + an alternate format for application configuration files. Module :mod:`json` The json module implements a subset of JavaScript syntax which can also @@ -983,13 +983,16 @@ ConfigParser Objects .. method:: read(filenames, encoding=None) Attempt to read and parse a list of filenames, returning a list of - filenames which were successfully parsed. If *filenames* is a string, it - is treated as a single filename. If a file named in *filenames* cannot - be opened, that file will be ignored. This is designed so that you can - specify a list of potential configuration file locations (for example, - the current directory, the user's home directory, and some system-wide - directory), and all existing configuration files in the list will be - read. If none of the named files exist, the :class:`ConfigParser` + filenames which were successfully parsed. + + If *filenames* is a string or :term:`path-like object`, it is treated as + a single filename. If a file named in *filenames* cannot be opened, that + file will be ignored. This is designed so that you can specify a list of + potential configuration file locations (for example, the current + directory, the user's home directory, and some system-wide directory), + and all existing configuration files in the list will be read. + + If none of the named files exist, the :class:`ConfigParser` instance will contain an empty dataset. An application which requires initial values to be loaded from a file should load the required file or files using :meth:`read_file` before calling :meth:`read` for any @@ -1006,6 +1009,9 @@ ConfigParser Objects The *encoding* parameter. Previously, all files were read using the default encoding for :func:`open`. + .. versionadded:: 3.6.1 + The *filenames* parameter accepts a :term:`path-like object`. + .. method:: read_file(f, source=None) diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index f0742cee55bd552..469a3eed606ff07 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -46,7 +46,7 @@ A small number of constants live in the built-in namespace. They are: .. note:: - ``NotImplentedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index d0b861d469bc05f..2041d9175ea5878 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -47,8 +47,8 @@ copy operations: * Recursive objects (compound objects that, directly or indirectly, contain a reference to themselves) may cause a recursive loop. -* Because deep copy copies *everything* it may copy too much, e.g., - even administrative data structures that should be shared even between copies. +* Because deep copy copies everything it may copy too much, such as data + which is intended to be shared between copies. The :func:`deepcopy` function avoids these problems by: diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index d746eafaea9b5ca..d51085506c2a96e 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1443,7 +1443,7 @@ The exact keycaps available are system dependent. +-------------------+--------------------------------------------+ | ``KEY_SEOL`` | Shifted Clear line | +-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Dxit | +| ``KEY_SEXIT`` | Shifted Exit | +-------------------+--------------------------------------------+ | ``KEY_SFIND`` | Shifted Find | +-------------------+--------------------------------------------+ diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a15690ba489a66b..c795782034a4648 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -20,6 +20,10 @@ interpreter. between versions of Python. Use of this module should not be considered to work across Python VMs or Python releases. + .. versionchanged:: 3.6 + Use 2 bytes for each instruction. Previously the number of bytes varied + by instruction. + Example: Given the function :func:`myfunc`:: @@ -210,6 +214,11 @@ operation is being performed, so the intermediate analysis object isn't useful: This generator function uses the ``co_firstlineno`` and ``co_lnotab`` attributes of the code object *code* to find the offsets which are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. + See :source:`Objects/lnotab_notes.txt` for the ``co_lnotab`` format and + how to decode it. + + .. versionchanged:: 3.6 + Line numbers can be decreasing. Before, they were always increasing. .. function:: findlabels(code) @@ -772,8 +781,13 @@ All of the following opcodes use their arguments. .. opcode:: BUILD_MAP (count) - Pushes a new dictionary object onto the stack. The dictionary is pre-sized - to hold *count* entries. + Pushes a new dictionary object onto the stack. Pops ``2 * count`` items + so that the dictionary holds *count* entries: + ``{..., TOS3: TOS2, TOS1: TOS}``. + + .. versionchanged:: 3.5 + The dictionary is created from stack items instead of creating an + empty dictionary pre-sized to hold *count* items. .. opcode:: BUILD_CONST_KEY_MAP (count) @@ -793,6 +807,63 @@ All of the following opcodes use their arguments. .. versionadded:: 3.6 +.. opcode:: BUILD_TUPLE_UNPACK (count) + + Pops *count* iterables from the stack, joins them in a single tuple, + and pushes the result. Implements iterable unpacking in tuple + displays ``(*x, *y, *z)``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, + but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position + ``count + 1`` should be the corresponding callable ``f``. + + .. versionadded:: 3.6 + + +.. opcode:: BUILD_LIST_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list + instead of tuple. Implements iterable unpacking in list + displays ``[*x, *y, *z]``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_SET_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a set + instead of tuple. Implements iterable unpacking in set + displays ``{*x, *y, *z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK (count) + + Pops *count* mappings from the stack, merges them into a single dictionary, + and pushes the result. Implements dictionary unpacking in dictionary + displays ``{**x, **y, **z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_MAP_UNPACK`, + but is used for ``f(**x, **y, **z)`` call syntax. The stack item at + position ``count + 2`` should be the corresponding callable ``f``. + + .. versionadded:: 3.5 + .. versionchanged:: 3.6 + The position of the callable is determined by adding 2 to the opcode + argument instead of encoding it in the second byte of the argument. + + .. opcode:: LOAD_ATTR (namei) Replaces TOS with ``getattr(TOS, co_names[namei])``. @@ -947,14 +1018,45 @@ All of the following opcodes use their arguments. .. opcode:: CALL_FUNCTION (argc) - Calls a function. The low byte of *argc* indicates the number of positional - parameters, the high byte the number of keyword parameters. On the stack, the - opcode finds the keyword parameters first. For each keyword argument, the - value is on top of the key. Below the keyword parameters, the positional - parameters are on the stack, with the right-most parameter on top. Below the - parameters, the function object to call is on the stack. Pops all function - arguments, and the function itself off the stack, and pushes the return - value. + Calls a function. *argc* indicates the number of positional arguments. + The positional arguments are on the stack, with the right-most argument + on top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + This opcode is used only for calls with positional arguments. + + +.. opcode:: CALL_FUNCTION_KW (argc) + + Calls a function. *argc* indicates the number of arguments (positional + and keyword). The top element on the stack contains a tuple of keyword + argument names. Below the tuple, keyword arguments are on the stack, in + the order corresponding to the tuple. Below the keyword arguments, the + positional arguments are on the stack, with the right-most parameter on + top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + Keyword arguments are packed in a tuple instead of a dictionary, + *argc* indicates the total number of arguments + + +.. opcode:: CALL_FUNCTION_EX (flags) + + Calls a function. The lowest bit of *flags* indicates whether the + var-keyword argument is placed at the top of the stack. Below the + var-keyword argument, the var-positional argument is on the stack. + Below the arguments, the function object to call is placed. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. Note that this opcode pops at most three items + from the stack. Var-positional and var-keyword arguments are packed + by :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. + + .. versionadded:: 3.6 .. opcode:: MAKE_FUNCTION (argc) @@ -987,28 +1089,6 @@ All of the following opcodes use their arguments. two most-significant bytes. -.. opcode:: CALL_FUNCTION_VAR (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the variable argument list, followed by - keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by explicit keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_VAR_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by the variable-arguments tuple, followed by explicit keyword and positional - arguments. - - .. opcode:: FORMAT_VALUE (flags) Used for implementing formatted literal strings (f-strings). Pops @@ -1034,8 +1114,13 @@ All of the following opcodes use their arguments. .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between - opcodes which don't take arguments ``< HAVE_ARGUMENT`` and those which do - ``>= HAVE_ARGUMENT``. + opcodes which don't use their argument and those that do + (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). + + .. versionchanged:: 3.6 + Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` + ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. + .. _opcode_collections: diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 2c65079ebdc3f1d..b0707646e64ac65 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -33,11 +33,11 @@ having a MIME type such as :mimetype:`multipart/\*` or The conceptual model provided by a :class:`Message` object is that of an ordered dictionary of headers with additional methods for accessing both specialized information from the headers, for accessing the payload, for -generating a serialized version of the mssage, and for recursively walking over -the object tree. Note that duplicate headers are supported but special methods -must be used to access them. +generating a serialized version of the message, and for recursively walking +over the object tree. Note that duplicate headers are supported but special +methods must be used to access them. -The :class:`Message` psuedo-dictionary is indexed by the header names, which +The :class:`Message` pseudo-dictionary is indexed by the header names, which must be ASCII values. The values of the dictionary are strings that are supposed to contain only ASCII characters; there is some special handling for non-ASCII input, but it doesn't always produce the correct results. Headers @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: This is a legacy method. On the :class:`~email.emailmessage.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the - realted ``make`` and ``add`` methods. + related ``make`` and ``add`` methods. .. method:: get_payload(i=None, decode=False) diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 32852e706986b48..d36e769f763cd89 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -364,7 +364,7 @@ message objects. *header* specifies an alternative header to :mailheader:`Content-Type`. If the value contains non-ASCII characters, the charset and language may - be explicity specified using the optional *charset* and *language* + be explicitly specified using the optional *charset* and *language* parameters. Optional *language* specifies the :rfc:`2231` language, defaulting to the empty string. Both *charset* and *language* should be strings. The default is to use the ``utf8`` *charset* and ``None`` for diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 5cd6472f3e2f45b..6548adf789da17e 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -24,8 +24,8 @@ Module Contents --------------- This module defines four enumeration classes that can be used to define unique -sets of names and values: :class:`Enum`, :class:`IntEnum`, and -:class:`IntFlags`. It also defines one decorator, :func:`unique`, and one +sets of names and values: :class:`Enum`, :class:`IntEnum`, :class:`Flag`, and +:class:`IntFlag`. It also defines one decorator, :func:`unique`, and one helper, :class:`auto`. .. class:: Enum diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index b8c1dcfef2d98e5..7291dfe84811c63 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -235,7 +235,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Retrieve a file in binary transfer mode. *cmd* should be an appropriate ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single string argument giving the data + each block of data received, with a single bytes argument giving the data block. The optional *blocksize* argument specifies the maximum chunk size to read on the low-level socket object created to do the actual transfer (which will also be the largest size of the data blocks passed to *callback*). A @@ -255,9 +255,9 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. prints the line to ``sys.stdout``. -.. method:: FTP.set_pasv(boolean) +.. method:: FTP.set_pasv(val) - Enable "passive" mode if *boolean* is true, other disable passive mode. + Enable "passive" mode if *val* is true, otherwise disable passive mode. Passive mode is on by default. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c26037bd9eef192..138f7d027cb86fe 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -610,7 +610,7 @@ are always available. They are listed here in alphabetical order. .. note:: - For object's with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`__hash__` methods, note that :func:`hash` truncates the return value based on the bit width of the host machine. See :meth:`__hash__` for details. @@ -1072,7 +1072,7 @@ are always available. They are listed here in alphabetical order. * The ``'x'`` mode was added. * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. * :exc:`FileExistsError` is now raised if the file opened in exclusive - * creation mode (``'x'``) already exists. + creation mode (``'x'``) already exists. .. versionchanged:: 3.4 @@ -1125,7 +1125,7 @@ are always available. They are listed here in alphabetical order. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end* and *file*, if present, must be given as keyword + by *end*. *sep*, *end*, *file* and *flush*, if present, must be given as keyword arguments. All non-keyword arguments are converted to strings like :func:`str` does and @@ -1254,17 +1254,21 @@ are always available. They are listed here in alphabetical order. .. function:: round(number[, ndigits]) - Return the floating point value *number* rounded to *ndigits* digits after - the decimal point. If *ndigits* is omitted or is ``None``, it returns the - nearest integer to its input. Delegates to ``number.__round__(ndigits)``. + Return *number* rounded to *ndigits* precision after the decimal + point. If *ndigits* is omitted or is ``None``, it returns the + nearest integer to its input. For the built-in types supporting :func:`round`, values are rounded to the closest multiple of 10 to the power minus *ndigits*; if two multiples are equally close, rounding is done toward the even choice (so, for example, both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). The return value is an integer if called with one argument, + ``2``). Any integer value is valid for *ndigits* (positive, zero, or + negative). The return value is an integer if called with one argument, otherwise of the same type as *number*. + For a general Python object ``number``, ``round(number, ndigits)`` delegates to + ``number.__round__(ndigits)``. + .. note:: The behavior of :func:`round` for floats can be surprising: for example, diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index a629bc50dbc7495..07c2a25d6a012c2 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -446,7 +446,7 @@ longer or disable the extension. Calltips ^^^^^^^^ -A calltip is shown when one types :kbd:`(` after the name of an *acccessible* +A calltip is shown when one types :kbd:`(` after the name of an *accessible* function. A name expression may include dots and subscripts. A calltip remains until it is clicked, the cursor is moved out of the argument area, or :kbd:`)` is typed. When the cursor is in the argument part of a definition, diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 41a784d982d6c26..6be28a2b31cb66a 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -34,185 +34,198 @@ provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special attributes: -+-----------+-----------------+---------------------------+ -| Type | Attribute | Description | -+===========+=================+===========================+ -| module | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __file__ | filename (missing for | -| | | built-in modules) | -+-----------+-----------------+---------------------------+ -| class | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-----------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-----------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-----------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-----------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-----------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-----------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-----------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-----------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-----------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-----------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-----------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_restricted | 0 or 1 if frame is in | -| | | restricted execution mode | -+-----------+-----------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-----------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including \* or \*\* | -| | | args) | -+-----------+-----------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-----------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-----------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-----------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | `| -+-----------+-----------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-----------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-----------------+---------------------------+ -| | co_names | tuple of names of local | -| | | variables | -+-----------+-----------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-----------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-----------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-----------------+---------------------------+ -| generator | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | gi_frame | frame | -+-----------+-----------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-----------------+---------------------------+ -| | gi_code | code | -+-----------+-----------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-----------------+---------------------------+ -| | cr_frame | frame | -+-----------+-----------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-----------------+---------------------------+ -| | cr_code | code | -+-----------+-----------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ ++-----------+-------------------+---------------------------+ +| Type | Attribute | Description | ++===========+===================+===========================+ +| module | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __file__ | filename (missing for | +| | | built-in modules) | ++-----------+-------------------+---------------------------+ +| class | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_restricted | 0 or 1 if frame is in | +| | | restricted execution mode | ++-----------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | `| ++-----------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------+-------------------+---------------------------+ +| | co_names | tuple of names of local | +| | | variables | ++-----------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------+-------------------+---------------------------+ +| | gi_code | code | ++-----------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------+-------------------+---------------------------+ +| | cr_code | code | ++-----------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -442,7 +455,9 @@ Retrieving source code Return in a single string any lines of comments immediately preceding the object's source code (for a class, function, or method), or at the top of the - Python source file (if the object is a module). + Python source file (if the object is a module). If the object's source code + is unavailable, return ``None``. This could happen if the object has been + defined in C or the interactive shell. .. function:: getfile(object) @@ -905,10 +920,8 @@ Classes and functions are the names of the ``*`` and ``**`` arguments or ``None``. *locals* is the locals dictionary of the given frame. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) @@ -944,10 +957,8 @@ Classes and functions :func:`getargvalues`. The format\* arguments are the corresponding optional formatting functions that are called to turn names and values into strings. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: getmro(cls) @@ -1270,6 +1281,10 @@ Code Objects Bit Flags Python code objects have a ``co_flags`` attribute, which is a bitmap of the following flags: +.. data:: CO_OPTIMIZED + + The code object is optimized, using fast locals. + .. data:: CO_NEWLOCALS If set, a new dict will be created for the frame's ``f_locals`` when @@ -1283,6 +1298,10 @@ the following flags: The code object has a variable keyword parameter (``**kwargs``-like). +.. data:: CO_NESTED + + The flag is set when the code object is a nested function. + .. data:: CO_GENERATOR The flag is set when the code object is a generator function, i.e. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 5edb23de83bcff1..cce6c23e611e31d 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -39,7 +39,7 @@ Reading and writing compressed files object`. The *filename* argument can be either an actual file name (given as a - :class:`str`, :class:`bytes` or :term:`path-like object` object), in + :class:`str`, :class:`bytes` or :term:`path-like ` object), in which case the named file is opened, or it can be an existing file object to read from or write to. @@ -76,7 +76,7 @@ Reading and writing compressed files An :class:`LZMAFile` can wrap an already-open :term:`file object`, or operate directly on a named file. The *filename* argument specifies either the file object to wrap, or the name of the file to open (as a :class:`str`, - :class:`bytes` or :term:`path-like object` object). When wrapping an + :class:`bytes` or :term:`path-like ` object). When wrapping an existing file object, the wrapped file will not be closed when the :class:`LZMAFile` is closed. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 1ffc6effc7142d9..d65afc200411334 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -49,7 +49,7 @@ For format *version* lower than 3, recursive lists, sets and dictionaries cannot be written (see below). There are functions that read/write files as well as functions operating on -strings. +bytes-like objects. The module defines these functions: @@ -57,9 +57,7 @@ The module defines these functions: .. function:: dump(value, file[, version]) Write the value on the open file. The value must be a supported type. The - file must be an open file object such as ``sys.stdout`` or returned by - :func:`open` or :func:`os.popen`. It must be opened in binary mode (``'wb'`` - or ``'w+b'``). + file must be a writeable :term:`binary file`. If the value has (or contains an object that has) an unsupported type, a :exc:`ValueError` exception is raised --- but garbage data will also be written @@ -74,8 +72,7 @@ The module defines these functions: Read one value from the open file and return it. If no valid value is read (e.g. because the data has a different Python version's incompatible marshal format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be an open file object opened in binary mode (``'rb'`` or - ``'r+b'``). + file must be a readable :term:`binary file`. .. note:: @@ -85,7 +82,7 @@ The module defines these functions: .. function:: dumps(value[, version]) - Return the string that would be written to a file by ``dump(value, file)``. The + Return the bytes object that would be written to a file by ``dump(value, file)``. The value must be a supported type. Raise a :exc:`ValueError` exception if value has (or contains an object that has) an unsupported type. @@ -93,11 +90,11 @@ The module defines these functions: (see below). -.. function:: loads(string) +.. function:: loads(bytes) - Convert the string to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra characters in the - string are ignored. + Convert the :term:`bytes-like object` to a value. If no valid value is found, raise + :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the + input are ignored. In addition, the following constants are defined: diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 37fa2a2868dac9d..974ab2d481e2107 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2859,7 +2859,7 @@ These functions are all available on Linux only. :ref:`not following symlinks `. .. versionchanged:: 3.6 - Accepts a :term:`path-like object` fpr *path* and *attribute*. + Accepts a :term:`path-like object` for *path* and *attribute*. .. function:: listxattr(path=None, *, follow_symlinks=True) diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index ee1ce50b6bd5e11..d40abe167653b71 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -9,14 +9,15 @@ available for Python: .. seealso:: `PyGObject `_ - provides introspection bindings for C libraries using + PyGObject provides introspection bindings for C libraries using `GObject `_. One of these libraries is the `GTK+ 3 `_ widget set. GTK+ comes with many more widgets than Tkinter provides. An online `Python GTK+ 3 Tutorial `_ is available. - `PyGTK `_ provides bindings for an older version + `PyGTK `_ + PyGTK provides bindings for an older version of the library, GTK+ 2. It provides an object oriented interface that is slightly higher level than the C one. There are also bindings to `GNOME `_. An online `tutorial @@ -27,15 +28,10 @@ available for Python: extensive C++ GUI application development framework that is available for Unix, Windows and Mac OS X. :program:`sip` is a tool for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. The *PyQt3* bindings have a - book, `GUI Programming with Python: QT Edition - `_ by Boudewijn - Rempt. The *PyQt4* bindings also have a book, `Rapid GUI Programming - with Python and Qt `_, by Mark - Summerfield. + is specifically designed for Python. `PySide `_ - is a newer binding to the Qt toolkit, provided by Nokia. + PySide is a newer binding to the Qt toolkit, provided by Nokia. Compared to PyQt, its licensing scheme is friendlier to non-open source applications. @@ -49,9 +45,7 @@ available for Python: documentation and context sensitive help, printing, HTML viewing, low-level device context drawing, drag and drop, system clipboard access, an XML-based resource format and more, including an ever growing library - of user-contributed modules. wxPython has a book, `wxPython in Action - `_, by Noel Rappin and - Robin Dunn. + of user-contributed modules. PyGTK, PyQt, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 959d9b98a846918..b4b1479e2ceba4c 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -85,11 +85,11 @@ next line: ``Ordered by: standard name``, indicates that the text string in the far right column was used to sort the output. The column headings include: ncalls - for the number of calls, + for the number of calls. tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) + for the total time spent in the given function (and excluding time made in + calls to sub-functions) percall is the quotient of ``tottime`` divided by ``ncalls`` @@ -444,9 +444,10 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. significant entries. Initially, the list is taken to be the complete set of profiled functions. Each restriction is either an integer (to select a count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a regular expression (to pattern match - the standard name that is printed. If several restrictions are provided, - then they are applied sequentially. For example:: + select a percentage of lines), or a string that will interpreted as a + regular expression (to pattern match the standard name that is printed). + If several restrictions are provided, then they are applied sequentially. + For example:: print_stats(.1, 'foo:') diff --git a/Doc/library/re.rst b/Doc/library/re.rst index fe5ebcc67c840b6..944c6fb10d6c250 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -42,6 +42,12 @@ module-level functions and methods on that don't require you to compile a regex object first, but miss some fine-tuning parameters. +.. seealso:: + + The third-party `regex `_ module, + which has an API compatible with the standard library :mod:`re` module, + but offers additional functionality and a more thorough Unicode support. + .. _re-syntax: @@ -778,11 +784,22 @@ form. Unmatched groups are replaced with an empty string. -.. function:: escape(string) +.. function:: escape(pattern) - Escape all the characters in pattern except ASCII letters, numbers and ``'_'``. + Escape all the characters in *pattern* except ASCII letters, numbers and ``'_'``. This is useful if you want to match an arbitrary literal string that may - have regular expression metacharacters in it. + have regular expression metacharacters in it. For example:: + + >>> print(re.escape('python.exe')) + python\.exe + + >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" + >>> print('[%s]+' % re.escape(legal_chars)) + [abcdefghijklmnopqrstuvwxyz0123456789\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\:]+ + + >>> operators = ['+', '-', '*', '/', '**'] + >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) + \/|\-|\+|\*\*|\* .. versionchanged:: 3.3 The ``'_'`` character is no longer escaped. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f97118ebe057888..bd5442c6a27aa09 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -290,7 +290,7 @@ Edge and Level Trigger Polling (epoll) Objects | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | | | associated fd has an event. The default (if | | | this flag is not set) is to wake all epoll | - | | objects polling on on a fd. | + | | objects polling on a fd. | +-------------------------+-----------------------------------------------+ | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | | | down writing half of connection. | diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index a85bf339ae1bc8d..41e5bafa53d4680 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -153,7 +153,7 @@ Directory and files operations is true and *src* is a symbolic link, *dst* will be a copy of the file *src* refers to. - :func:`copy` copies the file data and the file's permission + :func:`~shutil.copy` copies the file data and the file's permission mode (see :func:`os.chmod`). Other metadata, like the file's creation and modification times, is not preserved. To preserve all file metadata from the original, use @@ -302,7 +302,7 @@ Directory and files operations *src* and *dst*, and will be used to copy *src* to *dest* if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the :func:`copy_function`. The - default *copy_function* is :func:`copy2`. Using :func:`copy` as the + default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index e383201aab1461c..85ee8a75cf77a9a 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -13,6 +13,12 @@ This module offers several classes to implement SMTP (email) servers. +.. seealso:: + + The `aiosmtpd `_ package is a recommended + replacement for this module. It is based on :mod:`asyncio` and provides a + more straightforward API. :mod:`smtpd` should be considered deprecated. + Several server implementations are present; one is a generic do-nothing implementation, which can be overridden, while the other two offer specific mail-sending strategies. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 5635577da0f9465..9fef7d7f03f1f00 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -640,6 +640,11 @@ Cursor Objects .. versionchanged:: 3.6 Added support for the ``REPLACE`` statement. + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + .. attribute:: description This read-only attribute provides the column names of the last query. To @@ -927,14 +932,6 @@ By default, the :mod:`sqlite3` module opens transactions implicitly before a Data Modification Language (DML) statement (i.e. ``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). -So if you are within a transaction and issue a command like ``CREATE TABLE -...``, ``VACUUM``, ``PRAGMA``, the :mod:`sqlite3` module will commit implicitly -before executing that command. There are two reasons for doing that. The first -is that some of these commands don't work within transactions. The other reason -is that sqlite3 needs to keep track of the transaction state (if a transaction -is active or not). The current transaction state is exposed through the -:attr:`Connection.in_transaction` attribute of the connection object. - You can control which kind of ``BEGIN`` statements sqlite3 implicitly executes (or none at all) via the *isolation_level* parameter to the :func:`connect` call, or via the :attr:`isolation_level` property of connections. @@ -945,6 +942,9 @@ Otherwise leave it at its default, which will result in a plain "BEGIN" statement, or set it to one of SQLite's supported isolation levels: "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". +The current transaction state is exposed through the +:attr:`Connection.in_transaction` attribute of the connection object. + .. versionchanged:: 3.6 :mod:`sqlite3` used to implicitly commit an open transaction before DDL statements. This is no longer the case. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 07b97715a985b54..bbb13745b203e6e 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -616,7 +616,7 @@ Constants .. data:: PROTOCOL_TLS_CLIENT - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, but only support client-side :class:`SSLSocket` connections. The protocol enables :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` by default. @@ -625,7 +625,7 @@ Constants .. data:: PROTOCOL_TLS_SERVER - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, but only support server-side :class:`SSLSocket` connections. .. versionadded:: 3.6 @@ -846,7 +846,7 @@ Constants The version string of the OpenSSL library loaded by the interpreter:: >>> ssl.OPENSSL_VERSION - 'OpenSSL 0.9.8k 25 Mar 2009' + 'OpenSSL 1.0.2k 26 Jan 2017' .. versionadded:: 3.2 @@ -856,7 +856,7 @@ Constants OpenSSL library:: >>> ssl.OPENSSL_VERSION_INFO - (0, 9, 8, 11, 15) + (1, 0, 2, 11, 15) .. versionadded:: 3.2 @@ -865,9 +865,9 @@ Constants The raw version number of the OpenSSL library, as a single integer:: >>> ssl.OPENSSL_VERSION_NUMBER - 9470143 + 268443839 >>> hex(ssl.OPENSSL_VERSION_NUMBER) - '0x9080bf' + '0x100020bf' .. versionadded:: 3.2 @@ -948,7 +948,7 @@ SSL Sockets :ref:`notes on non-blocking sockets `. Usually, :class:`SSLSocket` are not created directly, but using the - the :meth:`SSLContext.wrap_socket` method. + :meth:`SSLContext.wrap_socket` method. .. versionchanged:: 3.5 The :meth:`sendfile` method was added. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9a4f42caa408a58..6729a20a9c2ecca 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -108,11 +108,11 @@ Notes: (1) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`False`. + argument if the first one is false. (2) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`True`. + argument if the first one is true. (3) ``not`` has a lower priority than non-Boolean operators, so ``not a == b`` is diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ad2abe824536227..548e4a68cbf1d17 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -466,9 +466,13 @@ functions. The *pass_fds* parameter was added. If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. In particular, the function looks for - *executable* (or for the first item in *args*) relative to *cwd* if the - executable path is a relative path. + *cwd* before executing the child. *cwd* can be a :class:`str` and + :term:`path-like ` object. In particular, the function + looks for *executable* (or for the first item in *args*) relative to *cwd* + if the executable path is a relative path. + + .. versionchanged:: 3.6 + *cwd* parameter accepts a :term:`path-like object`. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 1edb0fbabb20235..dfe688a2f93e0cb 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -48,14 +48,14 @@ described below. .. exception:: NannyNag - Raised by :func:`tokeneater` if detecting an ambiguous indent. Captured and + Raised by :func:`process_tokens` if detecting an ambiguous indent. Captured and handled in :func:`check`. -.. function:: tokeneater(type, token, start, end, line) +.. function:: process_tokens(tokens) - This function is used by :func:`check` as a callback parameter to the function - :func:`tokenize.tokenize`. + This function is used by :func:`check` to process tokens generated by the + :mod:`tokenize` module. .. XXX document errprint, format_witnesses, Whitespace, check_equal, indents, reset_globals diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index d8f809753dfdd71..337c061107299c8 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -146,6 +146,10 @@ Some facts and figures: .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. class:: TarFile Class for reading and writing tar archives. Do not use this class directly: @@ -266,7 +270,8 @@ be finalized; only the internally used file object will be closed. See the All following arguments are optional and can be accessed as instance attributes as well. - *name* is the pathname of the archive. It can be omitted if *fileobj* is given. + *name* is the pathname of the archive. *name* may be a :term:`path-like object`. + It can be omitted if *fileobj* is given. In this case, the file object's :attr:`name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append @@ -319,6 +324,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. classmethod:: TarFile.open(...) Alternative constructor. The :func:`tarfile.open` function is actually a @@ -390,14 +399,17 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* may be a filename or a :class:`TarInfo` object. You can specify a different - directory using *path*. File attributes (owner, mtime, mode) are set unless - *set_attrs* is false. + directory using *path*. *path* may be a :term:`path-like object`. + File attributes (owner, mtime, mode) are set unless *set_attrs* is false. If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile are used to set the owner/group for the extracted files. Otherwise, the named @@ -418,6 +430,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + + .. method:: TarFile.extractfile(member) Extract a member from the archive as a file object. *member* may be a filename @@ -464,7 +480,8 @@ be finalized; only the internally used file object will be closed. See the Create a :class:`TarInfo` object from the result of :func:`os.stat` or equivalent on an existing file. The file is either named by *name*, or - specified as a :term:`file object` *fileobj* with a file descriptor. If + specified as a :term:`file object` *fileobj* with a file descriptor. + *name* may be a :term:`path-like object`. If given, *arcname* specifies an alternative name for the file in the archive, otherwise, the name is taken from *fileobj*’s :attr:`~io.FileIO.name` attribute, or the *name* argument. The name @@ -478,6 +495,9 @@ be finalized; only the internally used file object will be closed. See the The :attr:`~TarInfo.name` may also be modified, in which case *arcname* could be a dummy string. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + .. method:: TarFile.close() diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 665261ffb21f943..c59aca1e1890864 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -245,12 +245,12 @@ The module uses a global variable to store the name of the directory used for temporary files returned by :func:`gettempdir`. It can be set directly to override the selection process, but this is discouraged. All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommend approach. +to specify the directory and this is the recommended approach. .. data:: tempdir When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to all the functions defined in this + default value for the *dir* argument to the functions defined in this module. If ``tempdir`` is unset or ``None`` at any call to any of the above diff --git a/Doc/library/time.rst b/Doc/library/time.rst index ae17f6f4f0d236a..d2e2ac4410aba36 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -17,11 +17,23 @@ semantics of these functions varies among platforms. An explanation of some terminology and conventions is in order. +.. _epoch: + .. index:: single: epoch -* The :dfn:`epoch` is the point where the time starts. On January 1st of that - year, at 0 hours, the "time since the epoch" is zero. For Unix, the epoch is - 1970. To find out what the epoch is, look at ``gmtime(0)``. +* The :dfn:`epoch` is the point where the time starts, and is platform + dependent. For Unix, the epoch is January 1, 1970, 00:00:00 (UTC). + To find out what the epoch is on a given platform, look at + ``time.gmtime(0)``. + +.. _leap seconds: https://en.wikipedia.org/wiki/Leap_second + +.. index:: seconds since the epoch + +* The term :dfn:`seconds since the epoch` refers to the total number + of elapsed seconds since the epoch, typically excluding + `leap seconds`_. Leap seconds are excluded from this total on all + POSIX-compliant platforms. .. index:: single: Year 2038 @@ -467,7 +479,7 @@ The module defines the following functions and data items: (2) The range really is ``0`` to ``61``; value ``60`` is valid in - timestamps representing leap seconds and value ``61`` is supported + timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. (3) @@ -572,12 +584,28 @@ The module defines the following functions and data items: .. function:: time() - Return the time in seconds since the epoch as a floating point number. + Return the time in seconds since the epoch_ as a floating point + number. The specific date of the epoch and the handling of + `leap seconds`_ is platform dependent. + On Windows and most Unix systems, the epoch is January 1, 1970, + 00:00:00 (UTC) and leap seconds are not counted towards the time + in seconds since the epoch. This is commonly referred to as + `Unix time `_. + To find out what the epoch is on a given platform, look at + ``gmtime(0)``. + Note that even though the time is always returned as a floating point number, not all systems provide time with a better precision than 1 second. While this function normally returns non-decreasing values, it can return a - lower value than a previous call if the system clock has been set back between - the two calls. + lower value than a previous call if the system clock has been set back + between the two calls. + + The number returned by :func:`.time` may be converted into a more common + time format (i.e. year, month, day, hour, etc...) in UTC by passing it to + :func:`gmtime` function or in local time by passing it to the + :func:`localtime` function. In both cases a + :class:`struct_time` object is returned, from which the components + of the calendar date may be accessed as attributes. .. data:: timezone diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 3b772765aca2605..5793c54ae8d1a35 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -134,21 +134,21 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None) - Automatically determine how many times to call :meth:`.timeit`. + Automatically determine how many times to call :meth:`.timeit`. - This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual - (number of loops, time taken for that number of loops). It calls - :meth:`.timeit` with *number* set to successive powers of ten (10, - 100, 1000, ...) up to a maximum of one billion, until the time taken - is at least 0.2 second, or the maximum is reached. + This is a convenience function that calls :meth:`.timeit` repeatedly + so that the total time >= 0.2 second, returning the eventual + (number of loops, time taken for that number of loops). It calls + :meth:`.timeit` with *number* set to successive powers of ten (10, + 100, 1000, ...) up to a maximum of one billion, until the time taken + is at least 0.2 second, or the maximum is reached. - If *callback* is given and is not ``None``, it will be called after - each trial with two arguments: ``callback(number, time_taken)``. + If *callback* is given and is not ``None``, it will be called after + each trial with two arguments: ``callback(number, time_taken)``. - .. versionadded:: 3.6 + .. versionadded:: 3.6 .. method:: Timer.repeat(repeat=3, number=1000000) diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index b0ac81271c5c183..5cb7029adf5e9e3 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -13,6 +13,12 @@ annotated statement coverage listings, print caller/callee relationships and list functions executed during a program run. It can be used in another program or from the command line. +.. seealso:: + + `Coverage.py `_ + A popular third-party coverage tool that provides HTML + output along with advanced features such as branch coverage. + .. _trace-cli: Command-Line Usage diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 6c8982ba743526b..2cfc37f695ef2d4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -508,6 +508,14 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__float__``. +.. class:: SupportsComplex + + An ABC with one abstract method ``__complex__``. + +.. class:: SupportsBytes + + An ABC with one abstract method ``__bytes__``. + .. class:: SupportsAbs An ABC with one abstract method ``__abs__`` that is covariant @@ -574,6 +582,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.6.1 + .. class:: List(list, MutableSequence[T]) Generic version of :class:`list`. @@ -656,7 +666,19 @@ The module defines the following classes, functions and decorators: .. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - A generic version of :class:`collections.defaultdict` + A generic version of :class:`collections.defaultdict`. + +.. class:: Counter(collections.Counter, Dict[T, int]) + + A generic version of :class:`collections.Counter`. + + .. versionadded:: 3.6.1 + +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + + A generic version of :class:`collections.ChainMap`. + + .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -740,9 +762,12 @@ The module defines the following classes, functions and decorators: This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These representing the types of I/O streams such as returned by + These represent the types of I/O streams such as returned by :func:`open`. + These types are also accessible directly as ``typing.IO``, + ``typing.TextIO``, and ``typing.BinaryIO``. + .. class:: re Wrapper namespace for regular expression matching types. @@ -754,6 +779,9 @@ The module defines the following classes, functions and decorators: ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. + These types are also accessible directly as ``typing.Pattern`` + and ``typing.Match``. + .. class:: NamedTuple Typed version of namedtuple. @@ -780,10 +808,20 @@ The module defines the following classes, functions and decorators: Fields with a default value must come after any fields without a default. The resulting class has two extra attributes: ``_field_types``, - giving a dict mapping field names to types, and ``field_defaults``, a dict + giving a dict mapping field names to types, and ``_field_defaults``, a dict mapping field names to default values. (The field names are in the ``_fields`` attribute, which is part of the namedtuple API.) + ``NamedTuple`` subclasses can also have docstrings and methods:: + + class Employee(NamedTuple): + """Represents an employee.""" + name: str + id: int = 3 + + def __repr__(self) -> str: + return f'' + Backward-compatible usage:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) @@ -792,7 +830,7 @@ The module defines the following classes, functions and decorators: Added support for :pep:`526` variable annotation syntax. .. versionchanged:: 3.6.1 - Added support for default values. + Added support for default values, methods, and docstrings. .. function:: NewType(typ) @@ -970,9 +1008,9 @@ The module defines the following classes, functions and decorators: :data:`ClassVar` is not a class itself, and should not be used with :func:`isinstance` or :func:`issubclass`. - Note that :data:`ClassVar` does not change Python runtime behavior; - it can be used by 3rd party type checkers, so that the following - code might flagged as an error by those:: + :data:`ClassVar` does not change Python runtime behavior, but + it can be used by third-party type checkers. For example, a type checker + might flag the following code as an error:: enterprise_d = Starship(3000) enterprise_d.stats = {} # Error, setting class variable on instance diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index c6d0ec92b6e9ab4..a552cbfc70ad2e3 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -303,14 +303,14 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_once_with(*args, **kwargs) - Assert that the mock was called exactly once and with the specified - arguments. + Assert that the mock was called exactly once and that that call was + with the specified arguments. >>> mock = Mock(return_value=None) >>> mock('foo', bar='baz') >>> mock.assert_called_once_with('foo', bar='baz') - >>> mock('foo', bar='baz') - >>> mock.assert_called_once_with('foo', bar='baz') + >>> mock('other', bar='values') + >>> mock.assert_called_once_with('other', bar='values') Traceback (most recent call last): ... AssertionError: Expected 'mock' to be called once. Called 2 times. @@ -322,7 +322,8 @@ the *new_callable* argument to :func:`patch`. The assert passes if the mock has *ever* been called, unlike :meth:`assert_called_with` and :meth:`assert_called_once_with` that - only pass if the call is the most recent one. + only pass if the call is the most recent one, and in the case of + :meth:`assert_called_once_with` it must also be the only call. >>> mock = Mock(return_value=None) >>> mock(1, 2, arg='thing') diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index e289b971e7c269f..b02a006d733b6e2 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -166,8 +166,8 @@ Extension types can easily be made to support weak references; see performed by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakKeyDictionary` objects have the following additional methods. These -expose the internal references directly. The references are not guaranteed to +:class:`WeakKeyDictionary` objects have an additional method that +exposes the internal references directly. The references are not guaranteed to be "live" at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the keys around longer @@ -192,9 +192,9 @@ than needed. by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakValueDictionary` objects have the following additional methods. -These method have the same issues as the and :meth:`keyrefs` method of -:class:`WeakKeyDictionary` objects. +:class:`WeakValueDictionary` objects have an additional method that has the +same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` +objects. .. method:: WeakValueDictionary.valuerefs() diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5eb6f103380661d..a5d42118ba51768 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -132,8 +132,9 @@ ZipFile Objects .. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) - Open a ZIP file, where *file* can be either a path to a file (a string) or a - file-like object. The *mode* parameter should be ``'r'`` to read an existing + Open a ZIP file, where *file* can be a path to a file (a string), a + file-like object or a :term:`path-like object`. + The *mode* parameter should be ``'r'`` to read an existing file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an existing file, or ``'x'`` to exclusively create and write a new file. If *mode* is ``'x'`` and *file* refers to an existing file, @@ -183,6 +184,9 @@ ZipFile Objects Previously, a plain :exc:`RuntimeError` was raised for unrecognized compression values. + .. versionchanged:: 3.6.2 + The *file* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.close() @@ -284,6 +288,9 @@ ZipFile Objects Calling :meth:`extract` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.extractall(path=None, members=None, pwd=None) @@ -304,6 +311,9 @@ ZipFile Objects Calling :meth:`extractall` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.printdir() @@ -403,6 +413,9 @@ ZipFile Objects The following data attributes are also available: +.. attribute:: ZipFile.filename + + Name of the ZIP file. .. attribute:: ZipFile.debug @@ -451,12 +464,12 @@ The :class:`PyZipFile` constructor takes the same parameters as the added to the archive, compiling if necessary. If *pathname* is a file, the filename must end with :file:`.py`, and - just the (corresponding :file:`\*.py[co]`) file is added at the top level + just the (corresponding :file:`\*.pyc`) file is added at the top level (no path information). If *pathname* is a file that does not end with :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, and the directory is not a package directory, then all the files - :file:`\*.py[co]` are added at the top level. If the directory is a - package directory, then all :file:`\*.py[co]` are added under the package + :file:`\*.pyc` are added at the top level. If the directory is a + package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, all of these are added recursively. @@ -488,6 +501,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionadded:: 3.4 The *filterfunc* parameter. + .. versionchanged:: 3.6.2 + The *pathname* parameter accepts a :term:`path-like object`. + .. _zipinfo-objects: @@ -514,6 +530,10 @@ file: .. versionadded:: 3.6 + .. versionchanged:: 3.6.2 + The *filename* parameter accepts a :term:`path-like object`. + + Instances have the following methods and attributes: .. method:: ZipInfo.is_dir() diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 46b8c245f7b3fe2..eaae2bb04b74822 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -9,7 +9,7 @@ -------------- This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.py[co]`) and packages from ZIP-format archives. It is usually not +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not needed to use the :mod:`zipimport` module explicitly; it is automatically used by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths to ZIP archives. diff --git a/Doc/make.bat b/Doc/make.bat index da1f8765a4f3582..b1a3738f309d3b5 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -74,7 +74,7 @@ echo. Provided by this script: echo. clean, check, serve, htmlview echo. echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.txt in this directory or +echo.filenames to build or are ignored. See README.rst in this directory or echo.the documentation for your version of Sphinx for more exhaustive lists echo.of available targets and descriptions of each. echo. @@ -86,7 +86,7 @@ goto end :build if NOT "%PAPER%" == "" ( - set SPHINXOPTS=-D latex_paper_size=%PAPER% %SPHINXOPTS% + set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% ) cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 5f1ea92ed460f35..f0dbbd1edf63216 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -194,12 +194,6 @@ This means that the following code will print 42:: i = 42 f() -There are several cases where Python statements are illegal when used in -conjunction with nested scopes that contain free variables. - -If a variable is referenced in an enclosing scope, it is illegal to delete the -name. An error will be reported at compile time. - .. XXX from * also invalid with relative imports (at least currently) The :func:`eval` and :func:`exec` functions do not have access to the full diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3a4b80557caf67b..c4f6c55c7ca3aac 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -190,7 +190,7 @@ Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. A comprehension in an :keyword:`async def` function may consist of either a :keyword:`for` or :keyword:`async for` clause following the leading -expression, may contan additonal :keyword:`for` or :keyword:`async for` +expression, may contain additional :keyword:`for` or :keyword:`async for` clauses, and may also use :keyword:`await` expressions. If a comprehension contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an @@ -1317,7 +1317,7 @@ built-in types. * Sequences (instances of :class:`tuple`, :class:`list`, or :class:`range`) can be compared only within each of their types, with the restriction that ranges do not support order comparison. Equality comparison across these types - results in unequality, and ordering comparison across these types raises + results in inequality, and ordering comparison across these types raises :exc:`TypeError`. Sequences compare lexicographically using comparison of corresponding @@ -1355,7 +1355,7 @@ built-in types. true). * Mappings (instances of :class:`dict`) compare equal if and only if they have - equal `(key, value)` pairs. Equality comparison of the keys and elements + equal `(key, value)` pairs. Equality comparison of the keys and values enforces reflexivity. Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. @@ -1431,28 +1431,29 @@ Membership test operations -------------------------- The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in -s`` evaluates to true if *x* is a member of *s*, and false otherwise. ``x not -in s`` returns the negation of ``x in s``. All built-in sequences and set types -support this as well as dictionary, for which :keyword:`in` tests whether the -dictionary has a given key. For container types such as list, tuple, set, -frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent +s`` evaluates to ``True`` if *x* is a member of *s*, and ``False`` otherwise. +``x not in s`` returns the negation of ``x in s``. All built-in sequences and +set types support this as well as dictionary, for which :keyword:`in` tests +whether the dictionary has a given key. For container types such as list, tuple, +set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent to ``any(x is e or x == e for e in y)``. -For the string and bytes types, ``x in y`` is true if and only if *x* is a +For the string and bytes types, ``x in y`` is ``True`` if and only if *x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are always considered to be a substring of any other string, so ``"" in "abc"`` will return ``True``. For user-defined classes which define the :meth:`__contains__` method, ``x in -y`` is true if and only if ``y.__contains__(x)`` is true. +y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and +``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is true if some value ``z`` with ``x == z`` is +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is produced while iterating over ``y``. If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is true if and only if there is a non-negative +:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x == y[i]``, and all lower integer indices do not raise :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 5e2c1c8b0758d83..4dbd9d88c301a0f 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -431,7 +431,7 @@ on the module object. If the method returns ``None``, the import machinery will create the new module itself. .. versionadded:: 3.4 - The create_module() method of loaders. + The :meth:`~importlib.abc.Loader.create_module` method of loaders. .. versionchanged:: 3.4 The :meth:`~importlib.abc.Loader.load_module` method was replaced by @@ -464,8 +464,11 @@ import machinery will create the new module itself. .. versionchanged:: 3.5 A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. Starting in Python 3.6 it will be an error to not - define ``create_module()`` on a loader attached to a ModuleSpec. + ``create_module()`` is not. + +.. versionchanged:: 3.6 + An :exc:`ImportError` is raised when ``exec_module()`` is defined but + ``create_module()`` is not. Submodules ---------- diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index da7017afff0a3df..7f9c66481785c1f 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -696,6 +696,17 @@ a temporary variable. >>> f"newline: {newline}" 'newline: 10' +Formatted string literals cannot be used as docstrings, even if they do not +include expressions. + +:: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> foo.__doc__ is None + True + See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 273191bbd3c0257..865f2edee022056 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -21,6 +21,7 @@ from sphinx import addnodes from sphinx.builders import Builder +from sphinx.locale import translators from sphinx.util.nodes import split_explicit_title from sphinx.util.compat import Directive from sphinx.writers.html import HTMLTranslator @@ -34,7 +35,7 @@ ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://hg.python.org/cpython/file/3.6/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.6/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body @@ -79,7 +80,7 @@ def new_depart_literal_block(self, node): def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): issue = utils.unescape(text) - text = 'issue ' + issue + text = 'bpo-' + issue refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) return [refnode], [] @@ -103,16 +104,25 @@ class ImplementationDetail(Directive): optional_arguments = 1 final_argument_whitespace = True + # This text is copied to templates/dummy.html + label_text = 'CPython implementation detail:' + def run(self): pnode = nodes.compound(classes=['impl-detail']) + label = translators['sphinx'].gettext(self.label_text) content = self.content - add_text = nodes.strong('CPython implementation detail:', - 'CPython implementation detail:') + add_text = nodes.strong(label, label) if self.arguments: n, m = self.state.inline_text(self.arguments[0], self.lineno) pnode.append(nodes.paragraph('', '', *(n + m))) self.state.nested_parse(content, self.content_offset, pnode) if pnode.children and isinstance(pnode[0], nodes.paragraph): + content = nodes.inline(pnode[0].rawsource, translatable=True) + content.source = pnode[0].source + content.line = pnode[0].line + content += pnode[0].children + pnode[0].replace_self(nodes.paragraph('', '', content, + translatable=False)) pnode[0].insert(0, add_text) pnode[0].insert(1, nodes.Text(' ')) else: @@ -225,7 +235,7 @@ def run(self): # Support for including Misc/NEWS -issue_re = re.compile('([Ii])ssue #([0-9]+)') +issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)') whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") @@ -253,7 +263,7 @@ def run(self): text = 'The NEWS file is not available.' node = nodes.strong(text, text) return [node] - content = issue_re.sub(r'`\1ssue #\2 `__', + content = issue_re.sub(r'`bpo-\1 `__', content) content = whatsnew_re.sub(r'\1', content) # remove first 3 lines as they are the main heading diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index c6e03119ae97307..57b0cf12f537129 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -302,6 +302,8 @@ whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe: whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" whatsnew/3.2,,:location,zope9-location = ${zope9:location} whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf +library/re,,`,!#$%&'*+-.^_`|~: +library/re,,`,\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\: library/tarfile,,:xz,'x:xz' library/xml.etree.elementtree,,:sometag,prefix:sometag library/xml.etree.elementtree,,:fictional,">> addr6 = ipaddress.IPv6Address('::1') whatsnew/3.5,,:root,ERROR:root:exception whatsnew/3.5,,:exception,ERROR:root:exception whatsnew/changelog,,:version,import sys; I = version[:version.index(' ')] -whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't" -whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as" whatsnew/changelog,,`,"for readability (was ""`"")." diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 243d81077793610..71d0bba683074e3 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -3,8 +3,11 @@

{{ _('This Page') }}

{%- endif %} diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index de84ae3abc81c79..3a05cb697938fd3 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -42,7 +42,7 @@

Download Python {{ release }} Documentation

These archives contain all the content in the documentation.

HTML Help (.chm) files are made available in the "Windows" section -on the Python +on the Python download page.

diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html new file mode 100644 index 000000000000000..6e43be23230b544 --- /dev/null +++ b/Doc/tools/templates/dummy.html @@ -0,0 +1,6 @@ +This file is not an actual template, but used to add some +texts in extensions to sphinx.pot file. + +In extensions/pyspecific.py: + +{% trans %}CPython implementation detail:{% endtrans %} diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1076c1f51b7d194..d795c0a5586bc83 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -1,5 +1,12 @@ -{% extends "defindex.html" %} -{% block tables %} +{% extends "layout.html" %} +{%- block htmltitle -%} +{{ shorttitle }} +{%- endblock -%} +{% block body %} +

{{ docstitle|e }}

+

+ {% trans %}Welcome! This is the documentation for Python {{ release }}.{% endtrans %} +

{% trans %}Parts of the documentation:{% endtrans %}

diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index d43461886e79337..6a9bb4889ff857c 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -492,8 +492,7 @@ function like this:: for arg in arguments: print(arg) print("-" * 40) - keys = sorted(keywords.keys()) - for kw in keys: + for kw in keywords: print(kw, ":", keywords[kw]) It could be called like this:: @@ -513,13 +512,13 @@ and of course it would print: It's very runny, sir. It's really very, VERY runny, sir. ---------------------------------------- - client : John Cleese shopkeeper : Michael Palin + client : John Cleese sketch : Cheese Shop Sketch -Note that the list of keyword argument names is created by sorting the result -of the keywords dictionary's ``keys()`` method before printing its contents; -if this is not done, the order in which the arguments are printed is undefined. +Note that the order in which the keyword arguments are printed is guaranteed +to match the order in which they were provided in the function call. + .. _tut-arbitraryargs: diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 953a68b44af7bbc..6140ece046b9754 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -22,11 +22,11 @@ objects: Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. -.. method:: list.extend(L) +.. method:: list.extend(iterable) :noindex: - Extend the list by appending all the items in the given list. Equivalent to - ``a[len(a):] = L``. + Extend the list by appending all the items from the iterable. Equivalent to + ``a[len(a):] = iterable``. .. method:: list.insert(i, x) @@ -68,7 +68,7 @@ objects: The optional arguments *start* and *end* are interpreted as in the slice notation and are used to limit the search to a particular subsequence of - *x*. The returned index is computed relative to the beginning of the full + the list. The returned index is computed relative to the beginning of the full sequence rather than the *start* argument. diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 7e8ee3e5ea19bcd..52120a0b16e75f6 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -359,7 +359,7 @@ The built-in function :func:`len` returns the length of a string:: Information about string formatting with :meth:`str.format`. :ref:`old-string-formatting` - The old formatting operations invoked when strings and Unicode strings are + The old formatting operations invoked when strings are the left operand of the ``%`` operator are described in more detail here. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 195f63f0a319da9..08dc311d837be96 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -532,8 +532,8 @@ conflict. .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set to a non-empty string, Python won't try to write ``.pyc`` or - ``.pyo`` files on the import of source modules. This is equivalent to + If this is set to a non-empty string, Python won't try to write ``.pyc`` + files on the import of source modules. This is equivalent to specifying the :option:`-B` option. diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index edb74f043e0b648..a6ba5bbb720ca2f 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2327,11 +2327,12 @@ The :func:`inspect.getargspec` function is deprecated and scheduled to be removed in Python 3.6. (See :issue:`20438` for details.) The :mod:`inspect` :func:`~inspect.getfullargspec`, -:func:`~inspect.getargvalues`, :func:`~inspect.getcallargs`, -:func:`~inspect.getargvalues`, :func:`~inspect.formatargspec`, and -:func:`~inspect.formatargvalues` functions are deprecated in favor of -the :func:`inspect.signature` API. -(Contributed by Yury Selivanov in :issue:`20438`.) +:func:`~inspect.getcallargs`, and :func:`~inspect.formatargspec` functions are +deprecated in favor of the :func:`inspect.signature` API. (Contributed by Yury +Selivanov in :issue:`20438`.) + +:func:`~inspect.getargvalues` and :func:`~inspect.formatargvalues` functions +were inadvertently marked as deprecated with the release of Python 3.5.0. Use of :const:`re.LOCALE` flag with str patterns or :const:`re.ASCII` is now deprecated. (Contributed by Serhiy Storchaka in :issue:`22407`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 96fd256b991b762..a696af4a999c845 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2,8 +2,6 @@ What's New In Python 3.6 **************************** -:Release: |release| -:Date: |today| :Editors: Elvis Pranskevichus , Yury Selivanov .. Rules for maintenance: diff --git a/Include/ceval.h b/Include/ceval.h index 89c6062f11e17c2..1e482729a1cc981 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -216,6 +216,7 @@ PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); +PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #endif diff --git a/Include/fileutils.h b/Include/fileutils.h index b933e985392e73b..900c70faad719f2 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -22,7 +22,7 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #ifdef MS_WINDOWS struct _Py_stat_struct { unsigned long st_dev; - __int64 st_ino; + uint64_t st_ino; unsigned short st_mode; int st_nlink; int st_uid; diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 49930e855653499..a658e9f5840e1c5 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 -#define PY_MICRO_VERSION 0 +#define PY_MICRO_VERSION 1 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.0+" +#define PY_VERSION "3.6.1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 5a67666874da947..01abfa9fcd6fb7d 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -70,8 +70,8 @@ PyAPI_FUNC(const char *) Py_GetCopyright(void); PyAPI_FUNC(const char *) Py_GetCompiler(void); PyAPI_FUNC(const char *) Py_GetBuildInfo(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(const char *) _Py_hgidentifier(void); -PyAPI_FUNC(const char *) _Py_hgversion(void); +PyAPI_FUNC(const char *) _Py_gitidentifier(void); +PyAPI_FUNC(const char *) _Py_gitversion(void); #endif /* Internal -- various one-time initializations */ diff --git a/Include/pyport.h b/Include/pyport.h index 52a91a0d11d0568..426822a81f075ab 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -37,7 +37,7 @@ Used in: Py_SAFE_DOWNCAST * integral synonyms. Only define the ones we actually need. */ -// long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. +/* long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. */ #ifndef HAVE_LONG_LONG #define HAVE_LONG_LONG 1 #endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 587cf03e3697eb0..5b877185f258d38 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1609,50 +1609,41 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII( This codec uses mappings to encode and decode characters. - Decoding mappings must map single string characters to single - Unicode characters, integers (which are then interpreted as Unicode - ordinals) or None (meaning "undefined mapping" and causing an - error). - - Encoding mappings must map single Unicode characters to single - string characters, integers (which are then interpreted as Latin-1 - ordinals) or None (meaning "undefined mapping" and causing an - error). - - If a character lookup fails with a LookupError, the character is - copied as-is meaning that its ordinal value will be interpreted as - Unicode or Latin-1 ordinal resp. Because of this mappings only need - to contain those mappings which map characters to different code - points. + Decoding mappings must map byte ordinals (integers in the range from 0 to + 255) to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or None. Unmapped data bytes (ones which cause a LookupError) + as well as mapped to None, 0xFFFE or '\ufffe' are treated as "undefined + mapping" and cause an error. + + Encoding mappings must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or None. Unmapped character + ordinals (ones which cause a LookupError) as well as mapped to + None are treated as "undefined mapping" and cause an error. */ PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap( const char *string, /* Encoded string */ Py_ssize_t length, /* size of string */ - PyObject *mapping, /* character mapping - (char ordinal -> unicode ordinal) */ + PyObject *mapping, /* decoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString( PyObject *unicode, /* Unicode object */ - PyObject *mapping /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping /* encoding mapping */ ); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( PyObject *unicode, /* Unicode object */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); #endif @@ -1661,8 +1652,8 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and @@ -1960,8 +1951,8 @@ PyAPI_FUNC(PyObject*) PyUnicode_RSplit( /* Translate a string by applying a character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and @@ -2318,6 +2309,10 @@ PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy( PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); +#elif !defined(NDEBUG) +/* For asserts that call _PyUnicode_CheckConsistency(), which would + * otherwise be a problem when building with asserts but without Py_DEBUG. */ +#define _PyUnicode_CheckConsistency(op, check_content) PyUnicode_Check(op) #endif #ifndef Py_LIMITED_API diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index b172f3f360e6ef5..005d884572465f2 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -908,7 +908,8 @@ def index(self, value, start=0, stop=None): i = start while stop is None or i < stop: try: - if self[i] == value: + v = self[i] + if v is value or v == value: return i except IndexError: break @@ -917,7 +918,7 @@ def index(self, value, start=0, stop=None): def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' - return sum(1 for v in self if v == value) + return sum(1 for v in self if v is value or v == value) Sequence.register(tuple) Sequence.register(str) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index eadf06f20e2507f..e37852e2536c339 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -210,7 +210,7 @@ def _remove_universal_flags(_config_vars): # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub(r'-arch\s+\w+\s', ' ', flags, re.ASCII) + flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) flags = re.sub('-isysroot [^ \t]*', ' ', flags) _save_modified_value(_config_vars, cv, flags) diff --git a/Lib/aifc.py b/Lib/aifc.py index 692d0bfd272bf0e..13ad7dc5ca3d62d 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -303,6 +303,8 @@ class Aifc_read: # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk # _framesize -- size of one frame in the file + _file = None # Set here since __del__ checks it + def initfp(self, file): self._version = 0 self._convert = None @@ -344,9 +346,15 @@ def initfp(self, file): def __init__(self, f): if isinstance(f, str): - f = builtins.open(f, 'rb') - # else, assume it is an open file object already - self.initfp(f) + file_object = builtins.open(f, 'rb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + else: + # assume it is an open file object already + self.initfp(f) def __enter__(self): return self @@ -541,18 +549,23 @@ class Aifc_write: # _datalength -- the size of the audio samples written to the header # _datawritten -- the size of the audio samples actually written + _file = None # Set here since __del__ checks it + def __init__(self, f): if isinstance(f, str): - filename = f - f = builtins.open(f, 'wb') - else: - # else, assume it is an open file object already - filename = '???' - self.initfp(f) - if filename[-5:] == '.aiff': - self._aifc = 0 + file_object = builtins.open(f, 'wb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + + # treat .aiff file extensions as non-compressed audio + if f.endswith('.aiff'): + self._aifc = 0 else: - self._aifc = 1 + # assume it is an open file object already + self.initfp(f) def initfp(self, file): self._file = file diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 28a45fc3cc5aee5..e85634e588f5a5f 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -11,6 +11,7 @@ import functools import inspect +import os import reprlib import socket import subprocess @@ -611,6 +612,9 @@ def new_event_loop(self): # A TLS for the running event loop, used by _get_running_loop. class _RunningLoop(threading.local): _loop = None + _pid = None + + _running_loop = _RunningLoop() @@ -620,7 +624,9 @@ def _get_running_loop(): This is a low-level function intended to be used by event loops. This function is thread-specific. """ - return _running_loop._loop + running_loop = _running_loop._loop + if running_loop is not None and _running_loop._pid == os.getpid(): + return running_loop def _set_running_loop(loop): @@ -629,6 +635,7 @@ def _set_running_loop(loop): This is a low-level function intended to be used by event loops. This function is thread-specific. """ + _running_loop._pid = os.getpid() _running_loop._loop = loop diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 7ad28d6aa0089a6..ab7ff0bf93d076a 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -543,8 +543,10 @@ def eof_received(self): def _get_extra_info(self, name, default=None): if name in self._extra: return self._extra[name] - else: + elif self._transport is not None: return self._transport.get_extra_info(name, default) + else: + return default def _start_shutdown(self): if self._in_shutdown: diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index b2f5304f772121d..4c85466859f8f0f 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -24,6 +24,8 @@ def __init__(self, limit, loop): self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None + self._process_exited = False + self._pipe_fds = [] def __repr__(self): info = [self.__class__.__name__] @@ -43,12 +45,14 @@ def connection_made(self, transport): self.stdout = streams.StreamReader(limit=self._limit, loop=self._loop) self.stdout.set_transport(stdout_transport) + self._pipe_fds.append(1) stderr_transport = transport.get_pipe_transport(2) if stderr_transport is not None: self.stderr = streams.StreamReader(limit=self._limit, loop=self._loop) self.stderr.set_transport(stderr_transport) + self._pipe_fds.append(2) stdin_transport = transport.get_pipe_transport(0) if stdin_transport is not None: @@ -86,9 +90,18 @@ def pipe_connection_lost(self, fd, exc): else: reader.set_exception(exc) + if fd in self._pipe_fds: + self._pipe_fds.remove(fd) + self._maybe_close_transport() + def process_exited(self): - self._transport.close() - self._transport = None + self._process_exited = True + self._maybe_close_transport() + + def _maybe_close_transport(self): + if len(self._pipe_fds) == 0 and self._process_exited: + self._transport.close() + self._transport = None class Process: diff --git a/Lib/asyncio/test_utils.py b/Lib/asyncio/test_utils.py index 99e3839f456858a..b12d5db2a9755dc 100644 --- a/Lib/asyncio/test_utils.py +++ b/Lib/asyncio/test_utils.py @@ -449,12 +449,15 @@ def new_test_loop(self, gen=None): self.set_event_loop(loop) return loop + def unpatch_get_running_loop(self): + events._get_running_loop = self._get_running_loop + def setUp(self): self._get_running_loop = events._get_running_loop events._get_running_loop = lambda: None def tearDown(self): - events._get_running_loop = self._get_running_loop + self.unpatch_get_running_loop() events.set_event_loop(None) diff --git a/Lib/base64.py b/Lib/base64.py index 58f6ad6816ee2f8..eb8f258a2d19774 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -541,7 +541,8 @@ def encodebytes(s): def encodestring(s): """Legacy alias of encodebytes().""" import warnings - warnings.warn("encodestring() is a deprecated alias, use encodebytes()", + warnings.warn("encodestring() is a deprecated alias since 3.1, " + "use encodebytes()", DeprecationWarning, 2) return encodebytes(s) @@ -554,7 +555,8 @@ def decodebytes(s): def decodestring(s): """Legacy alias of decodebytes().""" import warnings - warnings.warn("decodestring() is a deprecated alias, use decodebytes()", + warnings.warn("decodestring() is a deprecated alias since Python 3.1, " + "use decodebytes()", DeprecationWarning, 2) return decodebytes(s) diff --git a/Lib/configparser.py b/Lib/configparser.py index af5aca1feae34a4..230ab2b017eade3 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -143,6 +143,7 @@ import functools import io import itertools +import os import re import sys import warnings @@ -687,7 +688,7 @@ def read(self, filenames, encoding=None): Return list of successfully read files. """ - if isinstance(filenames, str): + if isinstance(filenames, (str, os.PathLike)): filenames = [filenames] read_ok = [] for filename in filenames: @@ -696,6 +697,8 @@ def read(self, filenames, encoding=None): self._read(fp, filename) except OSError: continue + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) read_ok.append(filename) return read_ok diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 7d94a579c872b7c..5e47054954ba5a2 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -88,7 +88,7 @@ def __exit__(self, type, value, traceback): try: next(self.gen) except StopIteration: - return + return False else: raise RuntimeError("generator didn't stop") else: @@ -98,20 +98,19 @@ def __exit__(self, type, value, traceback): value = type() try: self.gen.throw(type, value, traceback) - raise RuntimeError("generator didn't stop after throw()") except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration # raised inside the "with" statement from being suppressed. return exc is not value except RuntimeError as exc: - # Don't re-raise the passed in exception. (issue27112) + # Don't re-raise the passed in exception. (issue27122) if exc is value: return False # Likewise, avoid suppressing if a StopIteration exception # was passed to throw() and later wrapped into a RuntimeError # (see PEP 479). - if exc.__cause__ is value: + if type is StopIteration and exc.__cause__ is value: return False raise except: @@ -122,8 +121,10 @@ def __exit__(self, type, value, traceback): # fixes the impedance mismatch between the throw() protocol # and the __exit__() protocol. # - if sys.exc_info()[1] is not value: - raise + if sys.exc_info()[1] is value: + return False + raise + raise RuntimeError("generator didn't stop after throw()") def contextmanager(func): diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/ctypes/test/test_callbacks.py index 8eac58f0262cafd..f622093df61da58 100644 --- a/Lib/ctypes/test/test_callbacks.py +++ b/Lib/ctypes/test/test_callbacks.py @@ -244,6 +244,7 @@ def callback(a, b, c, d, e): def test_callback_large_struct(self): class Check: pass + # This should mirror the structure in Modules/_ctypes/_ctypes_test.c class X(Structure): _fields_ = [ ('first', c_ulong), @@ -255,6 +256,11 @@ def callback(check, s): check.first = s.first check.second = s.second check.third = s.third + # See issue #29565. + # The structure should be passed by value, so + # any changes to it should not be reflected in + # the value passed + s.first = s.second = s.third = 0x0badf00d check = Check() s = X() @@ -275,6 +281,11 @@ def callback(check, s): self.assertEqual(check.first, 0xdeadbeef) self.assertEqual(check.second, 0xcafebabe) self.assertEqual(check.third, 0x0bad1dea) + # See issue #29565. + # Ensure that the original struct is unchanged. + self.assertEqual(s.first, check.first) + self.assertEqual(s.second, check.second) + self.assertEqual(s.third, check.third) ################################################################ diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 8f6fe5f25429e5c..2e778fb1b437407 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -2,7 +2,8 @@ from ctypes import * from ctypes.test import need_symbol from struct import calcsize -import _testcapi +import _ctypes_test +import test.support class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -201,7 +202,10 @@ class X(Structure): "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + @test.support.cpython_only + def test_packed_c_limits(self): # Issue 15989 + import _testcapi d = {"_fields_": [("a", c_byte)], "_pack_": _testcapi.INT_MAX + 1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) @@ -391,6 +395,28 @@ class Z(Y): (1, 0, 0, 0, 0, 0)) self.assertRaises(TypeError, lambda: Z(1, 2, 3, 4, 5, 6, 7)) + def test_pass_by_value(self): + # This should mirror the structure in Modules/_ctypes/_ctypes_test.c + class X(Structure): + _fields_ = [ + ('first', c_ulong), + ('second', c_ulong), + ('third', c_ulong), + ] + + s = X() + s.first = 0xdeadbeef + s.second = 0xcafebabe + s.third = 0x0bad1dea + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_large_struct_update_value + func.argtypes = (X,) + func.restype = None + func(s) + self.assertEqual(s.first, 0xdeadbeef) + self.assertEqual(s.second, 0xcafebabe) + self.assertEqual(s.third, 0x0bad1dea) + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Lib/getpass.py b/Lib/getpass.py index be511211585a484..36e17e4cb6965db 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -7,7 +7,6 @@ echoing of the password contents while reading. On Windows, the msvcrt module will be used. -On the Mac EasyDialogs.AskPassword is used, if available. """ diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index ffc03c4112f073c..f10cd345e886c87 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -90,7 +90,7 @@

Navigation

25.5. IDLE

-

Source code: Lib/idlelib/

+

Source code: Lib/idlelib/

IDLE is Python’s Integrated Development and Learning Environment.

IDLE has the following features:

diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index f3ee391ca006fc8..dd6c997abc587bb 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -1,5 +1,7 @@ #! /usr/bin/env python3 +import sys + try: from tkinter import * except ImportError: @@ -25,7 +27,6 @@ import re import socket import subprocess -import sys import threading import time import tokenize diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index b6a9f82e05f2681..8b11d22b024ffae 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -136,7 +136,7 @@ def reload(module): """ if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be module") + raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: diff --git a/Lib/inspect.py b/Lib/inspect.py index 4d56ef5d41b47f7..a2dcb888a0c6094 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -253,18 +253,24 @@ def iscode(object): """Return true if the object is a code object. Code objects provide these attributes: - co_argcount number of arguments (not including * or ** args) - co_code string of raw compiled bytecode - co_consts tuple of constants used in the bytecode - co_filename name of file in which this code object was created - co_firstlineno number of first line in Python source code - co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg - co_lnotab encoded mapping of line numbers to bytecode indices - co_name name with which this code object was defined - co_names tuple of names of local variables - co_nlocals number of local variables - co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_argcount number of arguments (not including *, ** args + or keyword only arguments) + co_code string of raw compiled bytecode + co_cellvars tuple of names of cell variables + co_consts tuple of constants used in the bytecode + co_filename name of file in which this code object was created + co_firstlineno number of first line in Python source code + co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg + | 16=nested | 32=generator | 64=nofree | 128=coroutine + | 256=iterable_coroutine | 512=async_generator + co_freevars tuple of names of free variables + co_kwonlyargcount number of keyword only arguments (not including ** arg) + co_lnotab encoded mapping of line numbers to bytecode indices + co_name name with which this code object was defined + co_names tuple of names of local variables + co_nlocals number of local variables + co_stacksize virtual machine stack space required + co_varnames tuple of names of arguments and local variables""" return isinstance(object, types.CodeType) def isbuiltin(object): diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60f7da89ea..fba0fa2c00dccd9 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index cf31a5411a855a7..0b9bee0ab42ea62 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,6 +72,28 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 + def testUnderscoresInNumbers(self): + # Integers + x = 1_0 + x = 123_456_7_89 + x = 0xabc_123_4_5 + x = 0X_abc_123 + x = 0B11_01 + x = 0b_11_01 + x = 0o45_67 + x = 0O_45_67 + + # Floats + x = 3_1.4 + x = 03_1.4 + x = 3_1. + x = .3_1 + x = 3.1_4 + x = 0_3.1_4 + x = 3e1_4 + x = 3_1e+4_1 + x = 3_1E-4_1 + def testFloats(self): x = 3.14 x = 314. diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 7d779734f37b0d2..2356f8d3d2d8ce9 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -815,7 +815,14 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), if isinstance(address, str): self.unixsocket = True - self._connect_unixsocket(address) + # Syslog server may be unavailable during handler initialisation. + # C's openlog() function also ignores connection errors. + # Moreover, we ignore these errors while logging, so it not worse + # to ignore it also here. + try: + self._connect_unixsocket(address) + except OSError: + pass else: self.unixsocket = False if socktype is None: diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ffdf42614d59eba..a545f3c1a189613 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -118,7 +118,7 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, try: result = (True, func(*args, **kwds)) except Exception as e: - if wrap_exception: + if wrap_exception and func is not _helper_reraises_exception: e = ExceptionWithTraceback(e, e.__traceback__) result = (False, e) try: @@ -128,9 +128,15 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) +def _helper_reraises_exception(ex): + 'Pickle-able helper function for use by _guarded_task_generation.' + raise ex + # # Class representing a process pool # @@ -275,6 +281,17 @@ def starmap_async(self, func, iterable, chunksize=None, callback=None, return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) + def _guarded_task_generation(self, result_job, func, iterable): + '''Provides a generator of tasks for imap and imap_unordered with + appropriate handling for iterables which throw exceptions during + iteration.''' + try: + i = -1 + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + except Exception as e: + yield (result_job, i+1, _helper_reraises_exception, (e,), {}) + def imap(self, func, iterable, chunksize=1): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. @@ -283,15 +300,23 @@ def imap(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def imap_unordered(self, func, iterable, chunksize=1): @@ -302,15 +327,23 @@ def imap_unordered(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def apply_async(self, func, args=(), kwds={}, callback=None, @@ -321,7 +354,7 @@ def apply_async(self, func, args=(), kwds={}, callback=None, if self._state != RUN: raise ValueError("Pool not running") result = ApplyResult(self._cache, callback, error_callback) - self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) + self._taskqueue.put(([(result._job, 0, func, args, kwds)], None)) return result def map_async(self, func, iterable, chunksize=None, callback=None, @@ -352,8 +385,14 @@ def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, task_batches = Pool._get_tasks(func, iterable, chunksize) result = MapResult(self._cache, chunksize, len(iterable), callback, error_callback=error_callback) - self._taskqueue.put((((result._job, i, mapper, (x,), {}) - for i, x in enumerate(task_batches)), None)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapper, + task_batches), + None + ) + ) return result @staticmethod @@ -375,37 +414,32 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): for taskseq, set_length in iter(taskqueue.get, None): task = None - i = -1 try: - for i, task in enumerate(taskseq): + # iterating taskseq cannot fail + for task in taskseq: if thread._state: util.debug('task handler found thread._state != RUN') break try: put(task) except Exception as e: - job, ind = task[:2] + job, idx = task[:2] try: - cache[job]._set(ind, (False, e)) + cache[job]._set(idx, (False, e)) except KeyError: pass else: if set_length: util.debug('doing set_length()') - set_length(i+1) + idx = task[1] if task else -1 + set_length(idx + 1) continue break - except Exception as ex: - job, ind = task[:2] if task else (0, 0) - if job in cache: - cache[job]._set(ind + 1, (False, ex)) - if set_length: - util.debug('doing set_length()') - set_length(i+1) + finally: + task = taskseq = job = None else: util.debug('task handler got sentinel') - try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') @@ -445,6 +479,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None while cache and thread._state != TERMINATE: try: @@ -461,6 +496,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 9f347216b183a95..70f5cba76ffa8ae 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1220,25 +1220,23 @@ def touch(self, mode=0o666, exist_ok=True): os.close(fd) def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ if self._closed: self._raise_closed() - if not parents: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - else: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - except OSError as e: - if e.errno != ENOENT or self.parent == self: - raise - self.parent.mkdir(parents=True) - self._accessor.mkdir(self, mode) + try: + self._accessor.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise def chmod(self, mode): """ diff --git a/Lib/platform.py b/Lib/platform.py index e48ad0b6e7bdea0..cc2db9870d84b69 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -110,7 +110,7 @@ """ -__version__ = '1.0.7' +__version__ = '1.0.8' import collections import sys, os, re, subprocess @@ -1198,7 +1198,9 @@ def _sys_version(sys_version=None): elif buildtime: builddate = builddate + ' ' + buildtime - if hasattr(sys, '_mercurial'): + if hasattr(sys, '_git'): + _, branch, revision = sys._git + elif hasattr(sys, '_mercurial'): _, branch, revision = sys._mercurial elif hasattr(sys, 'subversion'): # sys.subversion was added in Python 2.5 diff --git a/Lib/pstats.py b/Lib/pstats.py index d861413d4195f72..2c5bf981b85cf85 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -48,11 +48,14 @@ class Stats: printed. The sort_stats() method now processes some additional options (i.e., in - addition to the old -1, 0, 1, or 2). It takes an arbitrary number of - quoted strings to select the sort order. For example sort_stats('time', - 'name') sorts on the major key of 'internal function time', and on the - minor key of 'the name of the function'. Look at the two tables in - sort_stats() and get_sort_arg_defs(self) for more examples. + addition to the old -1, 0, 1, or 2 that are respectively interpreted as + 'stdname', 'calls', 'time', and 'cumulative'). It takes an arbitrary number + of quoted strings to select the sort order. + + For example sort_stats('time', 'name') sorts on the major key of 'internal + function time', and on the minor key of 'the name of the function'. Look at + the two tables in sort_stats() and get_sort_arg_defs(self) for more + examples. All methods return self, so you can string together commands like: Stats('foo', 'goo').strip_dirs().sort_stats('calls').\ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index c7fac3395b3be06..2caab630978fa07 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Dec 6 18:51:51 2016 -topics = {'assert': '\n' - 'The "assert" statement\n' +# Autogenerated by Sphinx on Sat Mar 4 12:14:44 2017 +topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' 'Assert statements are a convenient way to insert debugging ' @@ -39,8 +38,7 @@ 'Assignments to "__debug__" are illegal. The value for the ' 'built-in\n' 'variable is determined when the interpreter starts.\n', - 'assignment': '\n' - 'Assignment statements\n' + 'assignment': 'Assignment statements\n' '*********************\n' '\n' 'Assignment statements are used to (re)bind names to values and ' @@ -405,8 +403,7 @@ 'See also: **PEP 526** - Variable and attribute annotation ' 'syntax\n' ' **PEP 484** - Type hints\n', - 'atom-identifiers': '\n' - 'Identifiers (Names)\n' + 'atom-identifiers': 'Identifiers (Names)\n' '*******************\n' '\n' 'An identifier occurring as an atom is a name. See ' @@ -446,8 +443,7 @@ 'happen. If the class name consists only of underscores, ' 'no\n' 'transformation is done.\n', - 'atom-literals': '\n' - 'Literals\n' + 'atom-literals': 'Literals\n' '********\n' '\n' 'Python supports string and bytes literals and various ' @@ -476,8 +472,7 @@ 'may obtain\n' 'the same object or a different object with the same ' 'value.\n', - 'attribute-access': '\n' - 'Customizing attribute access\n' + 'attribute-access': 'Customizing attribute access\n' '****************************\n' '\n' 'The following methods can be defined to customize the ' @@ -851,8 +846,7 @@ '* *__class__* assignment works only if both classes have ' 'the same\n' ' *__slots__*.\n', - 'attribute-references': '\n' - 'Attribute references\n' + 'attribute-references': 'Attribute references\n' '********************\n' '\n' 'An attribute reference is a primary followed by a ' @@ -875,8 +869,7 @@ 'determined by the object. Multiple evaluations of ' 'the same attribute\n' 'reference may yield different objects.\n', - 'augassign': '\n' - 'Augmented assignment statements\n' + 'augassign': 'Augmented assignment statements\n' '*******************************\n' '\n' 'Augmented assignment is the combination, in a single statement, ' @@ -940,8 +933,7 @@ 'about\n' 'class and instance attributes applies as for regular ' 'assignments.\n', - 'binary': '\n' - 'Binary arithmetic operations\n' + 'binary': 'Binary arithmetic operations\n' '****************************\n' '\n' 'The binary arithmetic operations have the conventional priority\n' @@ -1029,8 +1021,7 @@ 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' 'The numeric arguments are first converted to a common type.\n', - 'bitwise': '\n' - 'Binary bitwise operations\n' + 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' 'Each of the three bitwise operations has a different priority ' @@ -1050,8 +1041,7 @@ 'The "|" operator yields the bitwise (inclusive) OR of its ' 'arguments,\n' 'which must be integers.\n', - 'bltin-code-objects': '\n' - 'Code Objects\n' + 'bltin-code-objects': 'Code Objects\n' '************\n' '\n' 'Code objects are used by the implementation to ' @@ -1074,8 +1064,7 @@ '\n' 'See The standard type hierarchy for more ' 'information.\n', - 'bltin-ellipsis-object': '\n' - 'The Ellipsis Object\n' + 'bltin-ellipsis-object': 'The Ellipsis Object\n' '*******************\n' '\n' 'This object is commonly used by slicing (see ' @@ -1087,8 +1076,7 @@ '"Ellipsis" singleton.\n' '\n' 'It is written as "Ellipsis" or "...".\n', - 'bltin-null-object': '\n' - 'The Null Object\n' + 'bltin-null-object': 'The Null Object\n' '***************\n' '\n' "This object is returned by functions that don't " @@ -1100,8 +1088,7 @@ 'same singleton.\n' '\n' 'It is written as "None".\n', - 'bltin-type-objects': '\n' - 'Type Objects\n' + 'bltin-type-objects': 'Type Objects\n' '************\n' '\n' 'Type objects represent the various object types. An ' @@ -1113,8 +1100,7 @@ 'all standard built-in types.\n' '\n' 'Types are written like this: "".\n', - 'booleans': '\n' - 'Boolean operations\n' + 'booleans': 'Boolean operations\n' '******************\n' '\n' ' or_test ::= and_test | or_test "or" and_test\n' @@ -1163,8 +1149,7 @@ 'its\n' 'argument (for example, "not \'foo\'" produces "False" rather ' 'than "\'\'".)\n', - 'break': '\n' - 'The "break" statement\n' + 'break': 'The "break" statement\n' '*********************\n' '\n' ' break_stmt ::= "break"\n' @@ -1185,8 +1170,7 @@ 'clause, that "finally" clause is executed before really leaving ' 'the\n' 'loop.\n', - 'callable-types': '\n' - 'Emulating callable objects\n' + 'callable-types': 'Emulating callable objects\n' '**************************\n' '\n' 'object.__call__(self[, args...])\n' @@ -1195,8 +1179,7 @@ 'this method\n' ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' ' "x.__call__(arg1, arg2, ...)".\n', - 'calls': '\n' - 'Calls\n' + 'calls': 'Calls\n' '*****\n' '\n' 'A call calls a callable object (e.g., a *function*) with a ' @@ -1217,7 +1200,8 @@ ' ("," "*" expression | "," ' 'keyword_item)*\n' ' keywords_arguments ::= (keyword_item | "**" expression)\n' - ' ("," keyword_item | "**" expression)*\n' + ' ("," keyword_item | "," "**" ' + 'expression)*\n' ' keyword_item ::= identifier "=" expression\n' '\n' 'An optional trailing comma may be present after the positional and\n' @@ -1382,8 +1366,7 @@ ' The class must define a "__call__()" method; the effect is then ' 'the\n' ' same as if that method was called.\n', - 'class': '\n' - 'Class definitions\n' + 'class': 'Class definitions\n' '*****************\n' '\n' 'A class definition defines a class object (see section The ' @@ -1469,8 +1452,7 @@ '\n' 'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n' ' Class Decorators\n', - 'comparisons': '\n' - 'Comparisons\n' + 'comparisons': 'Comparisons\n' '***********\n' '\n' 'Unlike C, all comparison operations in Python have the same ' @@ -1623,7 +1605,7 @@ 'restriction that\n' ' ranges do not support order comparison. Equality ' 'comparison across\n' - ' these types results in unequality, and ordering comparison ' + ' these types results in inequality, and ordering comparison ' 'across\n' ' these types raises "TypeError".\n' '\n' @@ -1762,6 +1744,12 @@ ' to sequences, but not to sets or mappings). See also the\n' ' "total_ordering()" decorator.\n' '\n' + '* The "hash()" result should be consistent with equality. ' + 'Objects\n' + ' that are equal should either have the same hash value, or ' + 'be marked\n' + ' as unhashable.\n' + '\n' 'Python does not enforce these consistency rules. In fact, ' 'the\n' 'not-a-number values are an example for not following these ' @@ -1833,8 +1821,7 @@ 'is determined using the "id()" function. "x is not y" yields ' 'the\n' 'inverse truth value. [4]\n', - 'compound': '\n' - 'Compound statements\n' + 'compound': 'Compound statements\n' '*******************\n' '\n' 'Compound statements contain (groups of) other statements; they ' @@ -2613,7 +2600,8 @@ 'functions, even if they do not contain "await" or "async" ' 'keywords.\n' '\n' - 'It is a "SyntaxError" to use "yield" expressions in "async def"\n' + 'It is a "SyntaxError" to use "yield from" expressions in "async ' + 'def"\n' 'coroutines.\n' '\n' 'An example of a coroutine function:\n' @@ -2724,8 +2712,7 @@ ' body is transformed into the namespace\'s "__doc__" item ' 'and\n' " therefore the class's *docstring*.\n", - 'context-managers': '\n' - 'With Statement Context Managers\n' + 'context-managers': 'With Statement Context Managers\n' '*******************************\n' '\n' 'A *context manager* is an object that defines the ' @@ -2787,8 +2774,7 @@ ' The specification, background, and examples for the ' 'Python "with"\n' ' statement.\n', - 'continue': '\n' - 'The "continue" statement\n' + 'continue': 'The "continue" statement\n' '************************\n' '\n' ' continue_stmt ::= "continue"\n' @@ -2805,8 +2791,7 @@ '"finally" clause, that "finally" clause is executed before ' 'really\n' 'starting the next loop cycle.\n', - 'conversions': '\n' - 'Arithmetic conversions\n' + 'conversions': 'Arithmetic conversions\n' '**********************\n' '\n' 'When a description of an arithmetic operator below uses the ' @@ -2832,8 +2817,7 @@ "left argument to the '%' operator). Extensions must define " 'their own\n' 'conversion behavior.\n', - 'customization': '\n' - 'Basic customization\n' + 'customization': 'Basic customization\n' '*******************\n' '\n' 'object.__new__(cls[, ...])\n' @@ -3152,15 +3136,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only ' - 'required property\n' + ' "__hash__()" should return an integer. The only required ' + 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive ' - 'or) the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the ' + 'components of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -3272,8 +3259,7 @@ ' neither "__len__()" nor "__bool__()", all its instances ' 'are\n' ' considered true.\n', - 'debugger': '\n' - '"pdb" --- The Python Debugger\n' + 'debugger': '"pdb" --- The Python Debugger\n' '*****************************\n' '\n' '**Source code:** Lib/pdb.py\n' @@ -3938,8 +3924,7 @@ '[1] Whether a frame is considered to originate in a certain ' 'module\n' ' is determined by the "__name__" in the frame globals.\n', - 'del': '\n' - 'The "del" statement\n' + 'del': 'The "del" statement\n' '*******************\n' '\n' ' del_stmt ::= "del" target_list\n' @@ -3968,8 +3953,7 @@ 'Changed in version 3.2: Previously it was illegal to delete a name\n' 'from the local namespace if it occurs as a free variable in a nested\n' 'block.\n', - 'dict': '\n' - 'Dictionary displays\n' + 'dict': 'Dictionary displays\n' '*******************\n' '\n' 'A dictionary display is a possibly empty series of key/datum pairs\n' @@ -4013,8 +3997,7 @@ 'should be *hashable*, which excludes all mutable objects.) Clashes\n' 'between duplicate keys are not detected; the last datum (textually\n' 'rightmost in the display) stored for a given key value prevails.\n', - 'dynamic-features': '\n' - 'Interaction with dynamic features\n' + 'dynamic-features': 'Interaction with dynamic features\n' '*********************************\n' '\n' 'Name resolution of free variables occurs at runtime, not ' @@ -4050,8 +4033,7 @@ 'override the global and local namespace. If only one ' 'namespace is\n' 'specified, it is used for both.\n', - 'else': '\n' - 'The "if" statement\n' + 'else': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -4068,8 +4050,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'exceptions': '\n' - 'Exceptions\n' + 'exceptions': 'Exceptions\n' '**********\n' '\n' 'Exceptions are a means of breaking out of the normal flow of ' @@ -4145,8 +4126,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'execmodel': '\n' - 'Execution model\n' + 'execmodel': 'Execution model\n' '***************\n' '\n' '\n' @@ -4477,8 +4457,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'exprlists': '\n' - 'Expression lists\n' + 'exprlists': 'Expression lists\n' '****************\n' '\n' ' expression_list ::= expression ( "," expression )* [","]\n' @@ -4515,8 +4494,7 @@ 'value of that expression. (To create an empty tuple, use an ' 'empty pair\n' 'of parentheses: "()".)\n', - 'floating': '\n' - 'Floating point literals\n' + 'floating': 'Floating point literals\n' '***********************\n' '\n' 'Floating point literals are described by the following lexical\n' @@ -4552,8 +4530,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'for': '\n' - 'The "for" statement\n' + 'for': 'The "for" statement\n' '*******************\n' '\n' 'The "for" statement is used to iterate over the elements of a ' @@ -4625,8 +4602,7 @@ '\n' ' for x in a[:]:\n' ' if x < 0: a.remove(x)\n', - 'formatstrings': '\n' - 'Format String Syntax\n' + 'formatstrings': 'Format String Syntax\n' '********************\n' '\n' 'The "str.format()" method and the "Formatter" class share ' @@ -5345,8 +5321,7 @@ ' 9 9 11 1001\n' ' 10 A 12 1010\n' ' 11 B 13 1011\n', - 'function': '\n' - 'Function definitions\n' + 'function': 'Function definitions\n' '********************\n' '\n' 'A function definition defines a user-defined function object ' @@ -5515,8 +5490,7 @@ '\n' ' **PEP 3107** - Function Annotations\n' ' The original specification for function annotations.\n', - 'global': '\n' - 'The "global" statement\n' + 'global': 'The "global" statement\n' '**********************\n' '\n' ' global_stmt ::= "global" identifier ("," identifier)*\n' @@ -5560,8 +5534,7 @@ 'code containing the function call. The same applies to the ' '"eval()"\n' 'and "compile()" functions.\n', - 'id-classes': '\n' - 'Reserved classes of identifiers\n' + 'id-classes': 'Reserved classes of identifiers\n' '*******************************\n' '\n' 'Certain classes of identifiers (besides keywords) have ' @@ -5609,8 +5582,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'identifiers': '\n' - 'Identifiers and keywords\n' + 'identifiers': 'Identifiers and keywords\n' '************************\n' '\n' 'Identifiers (also referred to as *names*) are described by ' @@ -5758,8 +5730,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'if': '\n' - 'The "if" statement\n' + 'if': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -5775,8 +5746,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'imaginary': '\n' - 'Imaginary literals\n' + 'imaginary': 'Imaginary literals\n' '******************\n' '\n' 'Imaginary literals are described by the following lexical ' @@ -5796,8 +5766,7 @@ '\n' ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j ' '3.14_15_93j\n', - 'import': '\n' - 'The "import" statement\n' + 'import': 'The "import" statement\n' '**********************\n' '\n' ' import_stmt ::= "import" module ["as" name] ( "," module ' @@ -6058,8 +6027,7 @@ '\n' ' **PEP 236** - Back to the __future__\n' ' The original proposal for the __future__ mechanism.\n', - 'in': '\n' - 'Membership test operations\n' + 'in': 'Membership test operations\n' '**************************\n' '\n' 'The operators "in" and "not in" test for membership. "x in s"\n' @@ -6094,8 +6062,7 @@ '\n' 'The operator "not in" is defined to have the inverse true value of\n' '"in".\n', - 'integers': '\n' - 'Integer literals\n' + 'integers': 'Integer literals\n' '****************\n' '\n' 'Integer literals are described by the following lexical ' @@ -6141,8 +6108,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'lambda': '\n' - 'Lambdas\n' + 'lambda': 'Lambdas\n' '*******\n' '\n' ' lambda_expr ::= "lambda" [parameter_list]: expression\n' @@ -6165,8 +6131,7 @@ 'Note that functions created with lambda expressions cannot ' 'contain\n' 'statements or annotations.\n', - 'lists': '\n' - 'List displays\n' + 'lists': 'List displays\n' '*************\n' '\n' 'A list display is a possibly empty series of expressions enclosed ' @@ -6183,8 +6148,7 @@ 'from left to right and placed into the list object in that order.\n' 'When a comprehension is supplied, the list is constructed from the\n' 'elements resulting from the comprehension.\n', - 'naming': '\n' - 'Naming and binding\n' + 'naming': 'Naming and binding\n' '******************\n' '\n' '\n' @@ -6397,8 +6361,7 @@ 'override the global and local namespace. If only one namespace ' 'is\n' 'specified, it is used for both.\n', - 'nonlocal': '\n' - 'The "nonlocal" statement\n' + 'nonlocal': 'The "nonlocal" statement\n' '************************\n' '\n' ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' @@ -6429,8 +6392,7 @@ '\n' ' **PEP 3104** - Access to Names in Outer Scopes\n' ' The specification for the "nonlocal" statement.\n', - 'numbers': '\n' - 'Numeric literals\n' + 'numbers': 'Numeric literals\n' '****************\n' '\n' 'There are three types of numeric literals: integers, floating ' @@ -6444,8 +6406,7 @@ 'is actually an expression composed of the unary operator \'"-"\' ' 'and the\n' 'literal "1".\n', - 'numeric-types': '\n' - 'Emulating numeric types\n' + 'numeric-types': 'Emulating numeric types\n' '***********************\n' '\n' 'The following methods can be defined to emulate numeric ' @@ -6621,8 +6582,7 @@ ' "__index__()" is defined "__int__()" should also be ' 'defined, and\n' ' both should return the same value.\n', - 'objects': '\n' - 'Objects, values and types\n' + 'objects': 'Objects, values and types\n' '*************************\n' '\n' "*Objects* are Python's abstraction for data. All data in a " @@ -6750,8 +6710,7 @@ 'created empty lists. (Note that "c = d = []" assigns the same ' 'object\n' 'to both "c" and "d".)\n', - 'operator-summary': '\n' - 'Operator precedence\n' + 'operator-summary': 'Operator precedence\n' '*******************\n' '\n' 'The following table summarizes the operator precedence ' @@ -6924,8 +6883,7 @@ 'arithmetic\n' ' or bitwise unary operator on its right, that is, ' '"2**-1" is "0.5".\n', - 'pass': '\n' - 'The "pass" statement\n' + 'pass': 'The "pass" statement\n' '********************\n' '\n' ' pass_stmt ::= "pass"\n' @@ -6938,8 +6896,7 @@ ' def f(arg): pass # a function that does nothing (yet)\n' '\n' ' class C: pass # a class with no methods (yet)\n', - 'power': '\n' - 'The power operator\n' + 'power': 'The power operator\n' '******************\n' '\n' 'The power operator binds more tightly than unary operators on its\n' @@ -6973,8 +6930,7 @@ 'Raising a negative number to a fractional power results in a ' '"complex"\n' 'number. (In earlier versions it raised a "ValueError".)\n', - 'raise': '\n' - 'The "raise" statement\n' + 'raise': 'The "raise" statement\n' '*********************\n' '\n' ' raise_stmt ::= "raise" [expression ["from" expression]]\n' @@ -7059,8 +7015,7 @@ 'Exceptions, and information about handling exceptions is in ' 'section\n' 'The try statement.\n', - 'return': '\n' - 'The "return" statement\n' + 'return': 'The "return" statement\n' '**********************\n' '\n' ' return_stmt ::= "return" [expression_list]\n' @@ -7087,9 +7042,15 @@ 'generator is done and will cause "StopIteration" to be raised. ' 'The\n' 'returned value (if any) is used as an argument to construct\n' - '"StopIteration" and becomes the "StopIteration.value" attribute.\n', - 'sequence-types': '\n' - 'Emulating container types\n' + '"StopIteration" and becomes the "StopIteration.value" attribute.\n' + '\n' + 'In an asynchronous generator function, an empty "return" ' + 'statement\n' + 'indicates that the asynchronous generator is done and will cause\n' + '"StopAsyncIteration" to be raised. A non-empty "return" statement ' + 'is\n' + 'a syntax error in an asynchronous generator function.\n', + 'sequence-types': 'Emulating container types\n' '*************************\n' '\n' 'The following methods can be defined to implement ' @@ -7310,8 +7271,7 @@ ' iteration protocol via "__getitem__()", see this ' 'section in the\n' ' language reference.\n', - 'shifting': '\n' - 'Shifting operations\n' + 'shifting': 'Shifting operations\n' '*******************\n' '\n' 'The shifting operations have lower priority than the arithmetic\n' @@ -7335,8 +7295,7 @@ 'operand is\n' ' larger than "sys.maxsize" an "OverflowError" exception is ' 'raised.\n', - 'slicings': '\n' - 'Slicings\n' + 'slicings': 'Slicings\n' '********\n' '\n' 'A slicing selects a range of items in a sequence object (e.g., ' @@ -7387,8 +7346,7 @@ 'as lower bound, upper bound and stride, respectively, ' 'substituting\n' '"None" for missing expressions.\n', - 'specialattrs': '\n' - 'Special Attributes\n' + 'specialattrs': 'Special Attributes\n' '******************\n' '\n' 'The implementation adds a few special read-only attributes ' @@ -7473,8 +7431,7 @@ '[5] To format only a tuple you should therefore provide a\n' ' singleton tuple whose only element is the tuple to be ' 'formatted.\n', - 'specialnames': '\n' - 'Special method names\n' + 'specialnames': 'Special method names\n' '********************\n' '\n' 'A class can implement certain operations that are invoked by ' @@ -7835,15 +7792,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only required ' + ' "__hash__()" should return an integer. The only required ' 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive or) ' - 'the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the components ' + 'of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -9262,8 +9222,7 @@ 'special method *must* be set on the class object itself in ' 'order to be\n' 'consistently invoked by the interpreter).\n', - 'string-methods': '\n' - 'String Methods\n' + 'string-methods': 'String Methods\n' '**************\n' '\n' 'Strings implement all of the common sequence operations, ' @@ -9500,12 +9459,11 @@ 'characters\n' ' and there is at least one character, false otherwise. ' 'Decimal\n' - ' characters are those from general category "Nd". This ' - 'category\n' - ' includes digit characters, and all characters that can ' - 'be used to\n' - ' form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC ' - 'DIGIT ZERO.\n' + ' characters are those that can be used to form numbers ' + 'in base 10,\n' + ' e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a ' + 'decimal character\n' + ' is a character in the Unicode General Category "Nd".\n' '\n' 'str.isdigit()\n' '\n' @@ -9515,10 +9473,13 @@ 'include decimal\n' ' characters and digits that need special handling, such ' 'as the\n' - ' compatibility superscript digits. Formally, a digit is ' - 'a character\n' - ' that has the property value Numeric_Type=Digit or\n' - ' Numeric_Type=Decimal.\n' + ' compatibility superscript digits. This covers digits ' + 'which cannot\n' + ' be used to form numbers in base 10, like the Kharosthi ' + 'numbers.\n' + ' Formally, a digit is a character that has the property ' + 'value\n' + ' Numeric_Type=Digit or Numeric_Type=Decimal.\n' '\n' 'str.isidentifier()\n' '\n' @@ -10064,8 +10025,7 @@ " '00042'\n" ' >>> "-42".zfill(5)\n' " '-0042'\n", - 'strings': '\n' - 'String and Bytes literals\n' + 'strings': 'String and Bytes literals\n' '*************************\n' '\n' 'String literals are described by the following lexical ' @@ -10299,8 +10259,7 @@ 'followed by a newline is interpreted as those two characters as ' 'part\n' 'of the literal, *not* as a line continuation.\n', - 'subscriptions': '\n' - 'Subscriptions\n' + 'subscriptions': 'Subscriptions\n' '*************\n' '\n' 'A subscription selects an item of a sequence (string, tuple ' @@ -10357,8 +10316,7 @@ "A string's items are characters. A character is not a " 'separate data\n' 'type but a string of exactly one character.\n', - 'truth': '\n' - 'Truth Value Testing\n' + 'truth': 'Truth Value Testing\n' '*******************\n' '\n' 'Any object can be tested for truth value, for use in an "if" or\n' @@ -10390,8 +10348,7 @@ 'otherwise stated. (Important exception: the Boolean operations ' '"or"\n' 'and "and" always return one of their operands.)\n', - 'try': '\n' - 'The "try" statement\n' + 'try': 'The "try" statement\n' '*******************\n' '\n' 'The "try" statement specifies exception handlers and/or cleanup code\n' @@ -10538,8 +10495,7 @@ 'Exceptions, and information on using the "raise" statement to ' 'generate\n' 'exceptions may be found in section The raise statement.\n', - 'types': '\n' - 'The standard type hierarchy\n' + 'types': 'The standard type hierarchy\n' '***************************\n' '\n' 'Below is a list of the types that are built into Python. ' @@ -11097,6 +11053,27 @@ 'statements.\n' ' See also the Coroutine Objects section.\n' '\n' + ' Asynchronous generator functions\n' + ' A function or method which is defined using "async def" and\n' + ' which uses the "yield" statement is called a *asynchronous\n' + ' generator function*. Such a function, when called, returns ' + 'an\n' + ' asynchronous iterator object which can be used in an "async ' + 'for"\n' + ' statement to execute the body of the function.\n' + '\n' + ' Calling the asynchronous iterator\'s "aiterator.__anext__()"\n' + ' method will return an *awaitable* which when awaited will\n' + ' execute until it provides a value using the "yield" ' + 'expression.\n' + ' When the function executes an empty "return" statement or ' + 'falls\n' + ' off the end, a "StopAsyncIteration" exception is raised and ' + 'the\n' + ' asynchronous iterator will have reached the end of the set ' + 'of\n' + ' values to be yielded.\n' + '\n' ' Built-in functions\n' ' A built-in function object is a wrapper around a C function.\n' ' Examples of built-in functions are "len()" and "math.sin()"\n' @@ -11233,14 +11210,14 @@ 'the\n' ' dictionary containing the class\'s namespace; "__bases__" is a ' 'tuple\n' - ' (possibly empty or a singleton) containing the base classes, in ' - 'the\n' - ' order of their occurrence in the base class list; "__doc__" is ' - 'the\n' - ' class\'s documentation string, or "None" if undefined;\n' - ' "__annotations__" (optional) is a dictionary containing ' - '*variable\n' - ' annotations* collected during class body execution.\n' + ' containing the base classes, in the order of their occurrence ' + 'in\n' + ' the base class list; "__doc__" is the class\'s documentation ' + 'string,\n' + ' or "None" if undefined; "__annotations__" (optional) is a\n' + ' dictionary containing *variable annotations* collected during ' + 'class\n' + ' body execution.\n' '\n' 'Class instances\n' ' A class instance is created by calling a class object (see ' @@ -11520,8 +11497,7 @@ ' under "User-defined methods". Class method objects are ' 'created\n' ' by the built-in "classmethod()" constructor.\n', - 'typesfunctions': '\n' - 'Functions\n' + 'typesfunctions': 'Functions\n' '*********\n' '\n' 'Function objects are created by function definitions. The ' @@ -11538,8 +11514,7 @@ 'different object types.\n' '\n' 'See Function definitions for more information.\n', - 'typesmapping': '\n' - 'Mapping Types --- "dict"\n' + 'typesmapping': 'Mapping Types --- "dict"\n' '************************\n' '\n' 'A *mapping* object maps *hashable* values to arbitrary ' @@ -11896,8 +11871,7 @@ " {'bacon'}\n" " >>> keys ^ {'sausage', 'juice'}\n" " {'juice', 'sausage', 'bacon', 'spam'}\n", - 'typesmethods': '\n' - 'Methods\n' + 'typesmethods': 'Methods\n' '*******\n' '\n' 'Methods are functions that are called using the attribute ' @@ -11954,8 +11928,7 @@ " 'my name is method'\n" '\n' 'See The standard type hierarchy for more information.\n', - 'typesmodules': '\n' - 'Modules\n' + 'typesmodules': 'Modules\n' '*******\n' '\n' 'The only special operation on a module is attribute access: ' @@ -11992,8 +11965,7 @@ 'written as\n' '"".\n', - 'typesseq': '\n' - 'Sequence Types --- "list", "tuple", "range"\n' + 'typesseq': 'Sequence Types --- "list", "tuple", "range"\n' '*******************************************\n' '\n' 'There are three basic sequence types: lists, tuples, and range\n' @@ -12141,9 +12113,9 @@ '\n' '3. If *i* or *j* is negative, the index is relative to the end ' 'of\n' - ' the string: "len(s) + i" or "len(s) + j" is substituted. But ' - 'note\n' - ' that "-0" is still "0".\n' + ' sequence *s*: "len(s) + i" or "len(s) + j" is substituted. ' + 'But\n' + ' note that "-0" is still "0".\n' '\n' '4. The slice of *s* from *i* to *j* is defined as the sequence ' 'of\n' @@ -12162,12 +12134,17 @@ ' (j-i)/k". In other words, the indices are "i", "i+k", ' '"i+2*k",\n' ' "i+3*k" and so on, stopping when *j* is reached (but never\n' - ' including *j*). If *i* or *j* is greater than "len(s)", use\n' - ' "len(s)". If *i* or *j* are omitted or "None", they become ' - '"end"\n' - ' values (which end depends on the sign of *k*). Note, *k* ' - 'cannot be\n' - ' zero. If *k* is "None", it is treated like "1".\n' + ' including *j*). When *k* is positive, *i* and *j* are ' + 'reduced to\n' + ' "len(s)" if they are greater. When *k* is negative, *i* and ' + '*j* are\n' + ' reduced to "len(s) - 1" if they are greater. If *i* or *j* ' + 'are\n' + ' omitted or "None", they become "end" values (which end ' + 'depends on\n' + ' the sign of *k*). Note, *k* cannot be zero. If *k* is ' + '"None", it\n' + ' is treated like "1".\n' '\n' '6. Concatenating immutable sequences always results in a new\n' ' object. This means that building up a sequence by repeated\n' @@ -12685,8 +12662,7 @@ ' * The linspace recipe shows how to implement a lazy version ' 'of\n' ' range that suitable for floating point applications.\n', - 'typesseq-mutable': '\n' - 'Mutable Sequence Types\n' + 'typesseq-mutable': 'Mutable Sequence Types\n' '**********************\n' '\n' 'The operations in the following table are defined on ' @@ -12826,8 +12802,7 @@ 'referenced multiple\n' ' times, as explained for "s * n" under Common Sequence ' 'Operations.\n', - 'unary': '\n' - 'Unary arithmetic and bitwise operations\n' + 'unary': 'Unary arithmetic and bitwise operations\n' '***************************************\n' '\n' 'All unary arithmetic and bitwise operations have the same ' @@ -12849,8 +12824,7 @@ 'In all three cases, if the argument does not have the proper type, ' 'a\n' '"TypeError" exception is raised.\n', - 'while': '\n' - 'The "while" statement\n' + 'while': 'The "while" statement\n' '*********************\n' '\n' 'The "while" statement is used for repeated execution as long as an\n' @@ -12874,8 +12848,7 @@ 'executed in the first suite skips the rest of the suite and goes ' 'back\n' 'to testing the expression.\n', - 'with': '\n' - 'The "with" statement\n' + 'with': 'The "with" statement\n' '********************\n' '\n' 'The "with" statement is used to wrap the execution of a block with\n' @@ -12948,8 +12921,7 @@ ' The specification, background, and examples for the Python ' '"with"\n' ' statement.\n', - 'yield': '\n' - 'The "yield" statement\n' + 'yield': 'The "yield" statement\n' '*********************\n' '\n' ' yield_stmt ::= yield_expression\n' diff --git a/Lib/sqlite3/test/transactions.py b/Lib/sqlite3/test/transactions.py index 45f1b04c69648f9..b8a13de55bc7209 100644 --- a/Lib/sqlite3/test/transactions.py +++ b/Lib/sqlite3/test/transactions.py @@ -179,6 +179,15 @@ def CheckDdlDoesNotAutostartTransaction(self): result = self.con.execute("select * from test").fetchall() self.assertEqual(result, []) + def CheckImmediateTransactionalDDL(self): + # You can achieve transactional DDL by issuing a BEGIN + # statement manually. + self.con.execute("begin immediate") + self.con.execute("create table test(i)") + self.con.rollback() + with self.assertRaises(sqlite.OperationalError): + self.con.execute("select * from test") + def CheckTransactionalDDL(self): # You can achieve transactional DDL by issuing a BEGIN # statement manually. diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 822ddb459e2615b..0db6571f291dae0 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -986,7 +986,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, int(not close_fds), creationflags, env, - cwd, + os.fspath(cwd) if cwd is not None else None, startupinfo) finally: # Child is launched. Close the parent's copy of those pipe diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 46e0f56a392ae62..bfb670c9027d769 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -59,7 +59,7 @@ def main(): class NannyNag(Exception): """ - Raised by tokeneater() if detecting an ambiguous indent. + Raised by process_tokens() if detecting an ambiguous indent. Captured and handled in check(). """ def __init__(self, lineno, msg, line): diff --git a/Lib/test/185test.db b/Lib/test/185test.db deleted file mode 100644 index 14cb5e258bc0961..000000000000000 Binary files a/Lib/test/185test.db and /dev/null differ diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b5f47825466de6c..57ada127465d758 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -18,6 +18,7 @@ import logging import struct import operator +import weakref import test.support import test.support.script_helper @@ -1738,14 +1739,30 @@ def raise_large_valuerror(wait): time.sleep(wait) raise ValueError("x" * 1024**2) +def identity(x): + return x + +class CountedObject(object): + n_instances = 0 + + def __new__(cls): + cls.n_instances += 1 + return object.__new__(cls) + + def __del__(self): + type(self).n_instances -= 1 + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): + if when == -1: + raise SayWhenError("Somebody said when") for i in range(total): if i == when: raise SayWhenError("Somebody said when") yield i + class _TestPool(BaseTestCase): @classmethod @@ -1818,6 +1835,32 @@ def test_map_chunksize(self): except multiprocessing.TimeoutError: self.fail("pool.map_async with chunksize stalled on null list") + def test_map_handle_iterable_exception(self): + if self.TYPE == 'manager': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + # SayWhenError seen at the very first of the iterable + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + # again, make sure it's reentrant + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(10, 3), 1) + + class SpecialIterable: + def __iter__(self): + return self + def __next__(self): + raise SayWhenError + def __len__(self): + return 1 + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + def test_async(self): res = self.pool.apply_async(sqr, (7, TIMEOUT1,)) get = TimingWrapper(res.get) @@ -1848,6 +1891,13 @@ def test_imap_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap(sqr, exception_throwing_generator(10, 3), 1) for i in range(3): self.assertEqual(next(it), i*i) @@ -1874,6 +1924,17 @@ def test_imap_unordered_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap_unordered(sqr, exception_throwing_generator(10, 3), 1) @@ -1955,7 +2016,7 @@ def test_traceback(self): except Exception as e: exc = e else: - raise AssertionError('expected RuntimeError') + self.fail('expected RuntimeError') self.assertIs(type(exc), RuntimeError) self.assertEqual(exc.args, (123,)) cause = exc.__cause__ @@ -1969,6 +2030,17 @@ def test_traceback(self): sys.excepthook(*sys.exc_info()) self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + # _helper_reraises_exception should not make the error + # a remote exception + with self.Pool(1) as p: + try: + p.map(sqr, exception_throwing_generator(1, -1), 1) + except Exception as e: + exc = e + else: + self.fail('expected SayWhenError') + self.assertIs(type(exc), SayWhenError) + self.assertIs(exc.__cause__, None) @classmethod def _test_wrapped_exception(cls): @@ -2000,6 +2072,20 @@ def test_map_no_failfast(self): # check that we indeed waited for all jobs self.assertGreater(time.time() - t_start, 0.9) + def test_release_task_refs(self): + # Issue #29861: task arguments and results should not be kept + # alive after we are done with them. + objs = [CountedObject() for i in range(10)] + refs = [weakref.ref(o) for o in objs] + self.pool.map(identity, objs) + + del objs + time.sleep(DELTA) # let threaded cleanup code run + self.assertEqual(set(wr() for wr in refs), {None}) + # With a process pool, copies of the objects are returned, check + # they were released too. + self.assertEqual(CountedObject.n_instances, 0) + def raising(): raise KeyError("key") diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2350125f6d6dbec..bccd97aa3c7f600 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -4313,6 +4313,11 @@ def test_replace(self): dt = dt.replace(fold=1, tzinfo=Eastern) self.assertEqual(t.replace(tzinfo=None).fold, 1) self.assertEqual(dt.replace(tzinfo=None).fold, 1) + # Out of bounds. + with self.assertRaises(ValueError): + t.replace(fold=2) + with self.assertRaises(ValueError): + dt.replace(fold=2) # Check that fold is a keyword-only argument with self.assertRaises(TypeError): t.replace(1, 1, 1, None, 1) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index d194e775eaeb8f3..c619b3d6c38a599 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -437,6 +437,8 @@ def test_select(self): self.stop_alarm() self.assertGreaterEqual(dt, self.sleep_time) + @unittest.skipIf(sys.platform == "darwin", + "poll may fail on macOS; see issue #28087") @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') def test_poll(self): poller = select.poll() diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 3b3d2458b1f2e09..e2e58bd872f094c 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -143,9 +143,14 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + abs_classes = filter(isabstract, abs_classes) + if 'typing' in sys.modules: + t = sys.modules['typing'] + # These classes require special treatment because they do not appear + # in direct subclasses of collections.abc classes + abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] + for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() obj._abc_cache.clear() diff --git a/Lib/test/mod_generics_cache.py b/Lib/test/mod_generics_cache.py new file mode 100644 index 000000000000000..d9a60b4b28c325a --- /dev/null +++ b/Lib/test/mod_generics_cache.py @@ -0,0 +1,14 @@ +"""Module for testing the behavior of generics across different modules.""" + +from typing import TypeVar, Generic + +T = TypeVar('T') + + +class A(Generic[T]): + pass + + +class B(Generic[T]): + class A(Generic[T]): + pass diff --git a/Lib/test/sndhdrdata/README b/Lib/test/sndhdrdata/README index 8a17c0041a463f9..b2cb664615b9c7b 100644 --- a/Lib/test/sndhdrdata/README +++ b/Lib/test/sndhdrdata/README @@ -3,10 +3,3 @@ following commands: dd if=/dev/zero of=sndhdr.raw bs=20 count=1 sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - -Sound file samples used by Lib/test/test_sndhdr.py and generated using the -following commands: - - dd if=/dev/zero of=sndhdr.raw bs=20 count=1 - sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 1bd1f89c8aa6099..a731a5136ba5fd9 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -1,5 +1,6 @@ -from test.support import findfile, TESTFN, unlink +from test.support import check_no_resource_warning, findfile, TESTFN, unlink import unittest +from unittest import mock from test import audiotests from audioop import byteswap import io @@ -149,6 +150,21 @@ def test_skipunknown(self): #This file contains chunk types aifc doesn't recognize. self.f = aifc.open(findfile('Sine-1000Hz-300ms.aif')) + def test_close_opened_files_on_error(self): + non_aifc_file = findfile('pluck-pcm8.wav', subdir='audiodata') + with check_no_resource_warning(self): + with self.assertRaises(aifc.Error): + # Try opening a non-AIFC file, with the expectation that + # `aifc.open` will fail (without raising a ResourceWarning) + self.f = aifc.open(non_aifc_file, 'rb') + + # Aifc_write.initfp() won't raise in normal case. But some errors + # (e.g. MemoryError, KeyboardInterrupt, etc..) can happen. + with mock.patch.object(aifc.Aifc_write, 'initfp', + side_effect=RuntimeError): + with self.assertRaises(RuntimeError): + self.fout = aifc.open(TESTFN, 'wb') + def test_params_added(self): f = self.f = aifc.open(TESTFN, 'wb') f.aiff() diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 1f8967ccff3e7c7..d67f9195eba04ce 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -14,14 +14,6 @@ import array from array import _array_reconstructor as array_reconstructor -try: - # Try to determine availability of long long independently - # of the array module under test - struct.calcsize('@q') - have_long_long = True -except struct.error: - have_long_long = False - sizeof_wchar = array.array('u').itemsize @@ -32,9 +24,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = "ubBhHiIlLfd" -if have_long_long: - typecodes += 'qQ' +typecodes = 'ubBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -1240,7 +1230,26 @@ def test_frombytearray(self): b = array.array(self.typecode, a) self.assertEqual(a, b) -class SignedNumberTest(NumberTest): +class IntegerNumberTest(NumberTest): + def test_type_error(self): + a = array.array(self.typecode) + a.append(42) + with self.assertRaises(TypeError): + a.append(42.0) + with self.assertRaises(TypeError): + a[0] = 42.0 + +class Intable: + def __init__(self, num): + self._num = num + def __int__(self): + return self._num + def __sub__(self, other): + return Intable(int(self) - int(other)) + def __add__(self, other): + return Intable(int(self) + int(other)) + +class SignedNumberTest(IntegerNumberTest): example = [-1, 0, 1, 42, 0x7f] smallerexample = [-1, 0, 1, 42, 0x7e] biggerexample = [-1, 0, 1, 43, 0x7f] @@ -1251,8 +1260,9 @@ def test_overflow(self): lower = -1 * int(pow(2, a.itemsize * 8 - 1)) upper = int(pow(2, a.itemsize * 8 - 1)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) -class UnsignedNumberTest(NumberTest): +class UnsignedNumberTest(IntegerNumberTest): example = [0, 1, 17, 23, 42, 0xff] smallerexample = [0, 1, 17, 23, 42, 0xfe] biggerexample = [0, 1, 17, 23, 43, 0xff] @@ -1263,6 +1273,7 @@ def test_overflow(self): lower = 0 upper = int(pow(2, a.itemsize * 8)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) def test_bytes_extend(self): s = bytes(self.example) @@ -1314,12 +1325,10 @@ class UnsignedLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'L' minitemsize = 4 -@unittest.skipIf(not have_long_long, 'need long long support') class LongLongTest(SignedNumberTest, unittest.TestCase): typecode = 'q' minitemsize = 8 -@unittest.skipIf(not have_long_long, 'need long long support') class UnsignedLongLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'Q' minitemsize = 8 diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 28d92a9f4e3eac2..802763bd11ff611 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1,6 +1,7 @@ """Tests for events.py.""" import collections.abc +import concurrent.futures import functools import gc import io @@ -57,6 +58,15 @@ def osx_tiger(): return version < (10, 5) +def _test_get_event_loop_new_process__sub_proc(): + async def doit(): + return 'hello' + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop.run_until_complete(doit()) + + ONLYCERT = data_file('ssl_cert.pem') ONLYKEY = data_file('ssl_key.pem') SIGNED_CERTFILE = data_file('keycert3.pem') @@ -2181,6 +2191,18 @@ def tearDown(self): asyncio.set_child_watcher(None) super().tearDown() + def test_get_event_loop_new_process(self): + async def main(): + pool = concurrent.futures.ProcessPoolExecutor() + return await self.loop.run_in_executor( + pool, _test_get_event_loop_new_process__sub_proc) + + self.unpatch_get_running_loop() + + self.assertEqual( + self.loop.run_until_complete(main()), + 'hello') + if hasattr(selectors, 'KqueueSelector'): class KqueueEventLoopTests(UnixEventLoopTestsMixin, SubprocessTestsMixin, diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 89afdcaff286658..99336f86ab824eb 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -569,6 +569,35 @@ def test_remove_done_callback(self): self.assertEqual(bag, [2]) self.assertEqual(f.result(), 'foo') + def test_remove_done_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_schedule_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + def mut(f): + f.remove_done_callback(str) + + fut = self._new_future() + fut.add_done_callback(mut) + fut.add_done_callback(str) + fut.add_done_callback(str) + fut.set_result(1) + test_utils.run_briefly(self.loop) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 59ff0f6967e5b5b..f1771c5561afea8 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -95,5 +95,17 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_get_extra_info_on_closed_connection(self): + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + default = object() + self.assertIs(ssl_proto._get_extra_info('socket', default), default) + self.connection_made(ssl_proto) + self.assertIsNotNone(ssl_proto._get_extra_info('socket')) + ssl_proto.connection_lost(None) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bba688bb5a53c7e..2e14a8a97355357 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -459,6 +459,30 @@ def test_popen_error(self): self.loop.run_until_complete(create) self.assertEqual(warns, []) + def test_read_stdout_after_process_exit(self): + @asyncio.coroutine + def execute(): + code = '\n'.join(['import sys', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + + fut = asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + loop=self.loop) + + process = yield from fut + while True: + data = yield from process.stdout.read(65536) + if data: + yield from asyncio.sleep(0.3, loop=self.loop) + else: + break + + self.loop.run_until_complete(execute()) + if sys.platform != 'win32': # Unix diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index a18d49ae3741128..4f05319457985ad 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1462,6 +1462,14 @@ def test_current_task(self): def coro(loop): self.assertTrue(Task.current_task(loop=loop) is task) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(loop) + try: + self.assertIs(Task.current_task(None), task) + self.assertIs(Task.current_task(), task) + finally: + asyncio.set_event_loop(None) + task = self.new_task(self.loop, coro(self.loop)) self.loop.run_until_complete(task) self.assertIsNone(Task.current_task(loop=self.loop)) @@ -1806,8 +1814,17 @@ def kill_me(loop): # schedule the task coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) + self.assertEqual(Task.all_tasks(loop=self.loop), {task}) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(self.loop) + try: + self.assertEqual(Task.all_tasks(), {task}) + self.assertEqual(Task.all_tasks(None), {task}) + finally: + asyncio.set_event_loop(None) + # execute the task so it waits for future self.loop._run_once() self.assertEqual(len(self.loop._ready), 0) diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 51c65737a441de5..d05462b7efce358 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -661,6 +661,9 @@ def test_handle_expt(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") + if sys.platform == "darwin" and self.use_poll: + self.skipTest("poll may fail on macOS; see issue #28087") + class TestClient(BaseClient): def handle_expt(self): self.socket.recv(1024, socket.MSG_OOB) diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 172bd25419cea73..c761076c4a0225a 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -143,6 +143,7 @@ def test_bound_methods(self): self.assertEqual(l, [5]) +@support.cpython_only class SubinterpreterTest(unittest.TestCase): def test_callbacks_leak(self): diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 4f86aaa0c0eb499..47547396b8cb54b 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -18,6 +18,14 @@ def check_type_errors(self, f): int_data = memoryview(b"1234").cast('I') self.assertRaises(TypeError, f, int_data) + def test_encodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.encodestring(b"www.python.org") + + def test_decodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.decodestring(b"d3d3LnB5dGhvbi5vcmc=\n") + def test_encodebytes(self): eq = self.assertEqual eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n") diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a103a7d39cae241..cd82fa64570cb5e 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -507,6 +507,11 @@ def test_mod(self): a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') self.assertIs(type(a), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b = b % b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_imod(self): b = self.type2test(b'hello, %b!') @@ -519,6 +524,11 @@ def test_imod(self): b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') self.assertIs(type(b), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b %= b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_rmod(self): with self.assertRaises(TypeError): diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e058ecd086df3ce..1587daf8f582dd0 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -572,6 +572,73 @@ def test_syntaxerror_indented_caret_position(self): self.assertNotIn("\f", text) self.assertIn("\n 1 + 1 = 2\n ^", text) + def test_consistent_sys_path_for_direct_execution(self): + # This test case ensures that the following all give the same + # sys.path configuration: + # + # ./python -s script_dir/__main__.py + # ./python -s script_dir + # ./python -I script_dir + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir, support.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from directly executing __main__.py + # We omit PYTHONPATH and user site to align with isolated mode + p = spawn_python("-Es", script_name, cwd=work_dir) + out_by_name = kill_python(p).decode().splitlines() + self.assertEqual(out_by_name[0], script_dir) + self.assertNotIn(work_dir, out_by_name) + # Directory execution should give the same output + p = spawn_python("-Es", script_dir, cwd=work_dir) + out_by_dir = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir, out_by_name) + # As should directory execution in isolated mode + p = spawn_python("-I", script_dir, cwd=work_dir) + out_by_dir_isolated = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir_isolated, out_by_dir, out_by_name) + + def test_consistent_sys_path_for_module_execution(self): + # This test case ensures that the following both give the same + # sys.path configuration: + # ./python -sm script_pkg.__main__ + # ./python -sm script_pkg + # + # And that this fails as unable to find the package: + # ./python -Im script_pkg + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from `-m script_pkg.__main__` + # We omit PYTHONPATH and user site to better align with the + # direct execution test cases + p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) + out_by_module = kill_python(p).decode().splitlines() + self.assertEqual(out_by_module[0], '') + self.assertNotIn(script_dir, out_by_module) + # Package execution should give the same output + p = spawn_python("-sm", "script_pkg", cwd=work_dir) + out_by_package = kill_python(p).decode().splitlines() + self.assertEqual(out_by_package, out_by_module) + # Isolated mode should fail with an import error + exitcode, stdout, stderr = assert_python_failure( + "-Im", "script_pkg", cwd=work_dir + ) + traceback_lines = stderr.decode().splitlines() + self.assertIn("No module named script_pkg", traceback_lines[-1]) def test_main(): support.run_unittest(CmdLineTest) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 87454cc6704cb07..47f756213ddf009 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1310,20 +1310,29 @@ def test_issue26915(self): class CustomEqualObject: def __eq__(self, other): return False - class CustomSequence(list): - def __contains__(self, value): - return Sequence.__contains__(self, value) + class CustomSequence(Sequence): + def __init__(self, seq): + self._seq = seq + def __getitem__(self, index): + return self._seq[index] + def __len__(self): + return len(self._seq) nan = float('nan') obj = CustomEqualObject() + seq = CustomSequence([nan, obj, nan]) containers = [ - CustomSequence([nan, obj]), + seq, ItemsView({1: nan, 2: obj}), ValuesView({1: nan, 2: obj}) ] for container in containers: for elem in container: self.assertIn(elem, container) + self.assertEqual(seq.index(nan), 0) + self.assertEqual(seq.index(obj), 1) + self.assertEqual(seq.count(nan), 2) + self.assertEqual(seq.count(obj), 1) def assertSameSet(self, s1, s2): # coerce both to a real set then check equality diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index c249ca724bf0cec..cee49343e268a23 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -387,6 +387,29 @@ def __complex__(self): self.assertAlmostEqual(complex(complex1(1j)), 2j) self.assertRaises(TypeError, complex, complex2(1j)) + @support.requires_IEEE_754 + def test_constructor_special_numbers(self): + class complex2(complex): + pass + for x in 0.0, -0.0, INF, -INF, NAN: + for y in 0.0, -0.0, INF, -INF, NAN: + with self.subTest(x=x, y=y): + z = complex(x, y) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(x, y) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex(complex2(x, y)) + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(complex(x, y)) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + def test_underscores(self): # check underscores for lit in VALID_UNDERSCORE_LITERALS: diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 0d06080da33c523..72c3f19fb41f569 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,7 @@ import configparser import io import os -import sys +import pathlib import textwrap import unittest import warnings @@ -721,6 +721,16 @@ def test_read_returns_file_list(self): parsed_files = cf.read(file1) self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we pass only a Path object: + cf = self.newconfig() + parsed_files = cf.read(pathlib.Path(file1)) + self.assertEqual(parsed_files, [file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we passed both a filename and a Path object: + cf = self.newconfig() + parsed_files = cf.read([pathlib.Path(file1), file1]) + self.assertEqual(parsed_files, [file1, file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: cf = self.newconfig() parsed_files = cf.read(["nonexistent-file"]) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index c04c804af570470..b1a467d952da0fb 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -152,6 +152,29 @@ def woohoo(): else: self.fail('StopIteration was suppressed') + def test_contextmanager_do_not_unchain_non_stopiteration_exceptions(self): + @contextmanager + def test_issue29692(): + try: + yield + except Exception as exc: + raise RuntimeError('issue29692:Chained') from exc + try: + with test_issue29692(): + raise ZeroDivisionError + except Exception as ex: + self.assertIs(type(ex), RuntimeError) + self.assertEqual(ex.args[0], 'issue29692:Chained') + self.assertIsInstance(ex.__cause__, ZeroDivisionError) + + try: + with test_issue29692(): + raise StopIteration('issue29692:Unchained') + except Exception as ex: + self.assertIs(type(ex), StopIteration) + self.assertEqual(ex.args[0], 'issue29692:Unchained') + self.assertIsNone(ex.__cause__) + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 78439a2acae2d8b..2b79a17ea703f53 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1103,6 +1103,21 @@ async def waiter(coro): "coroutine is being awaited already"): waiter(coro).send(None) + def test_await_16(self): + # See https://bugs.python.org/issue29600 for details. + + async def f(): + return ValueError() + + async def g(): + try: + raise KeyError + except: + return await f() + + _, result = run_async(g()) + self.assertIsNone(result.__context__) + def test_with_1(self): class Manager: def __init__(self, name): @@ -1680,6 +1695,44 @@ async def foo(): warnings.simplefilter("error") run_async(foo()) + def test_for_11(self): + class F: + def __aiter__(self): + return self + def __anext__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __anext__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + + def test_for_12(self): + class F: + def __aiter__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __aiter__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + def test_for_tuple(self): class Done(Exception): pass @@ -2064,6 +2117,7 @@ def wrap(gen): sys.set_coroutine_wrapper(None) +@support.cpython_only class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 48379222c376592..960fc0f1bf9486b 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1,5 +1,6 @@ # Python test set -- part 5, built-in exceptions +import copy import os import sys import unittest @@ -1119,6 +1120,25 @@ def test_non_str_argument(self): exc = ImportError(arg) self.assertEqual(str(arg), str(exc)) + def test_copy_pickle(self): + for kwargs in (dict(), + dict(name='somename'), + dict(path='somepath'), + dict(name='somename', path='somepath')): + orig = ImportError('test', **kwargs) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + exc = pickle.loads(pickle.dumps(orig, proto)) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + for c in copy.copy, copy.deepcopy: + exc = c(orig) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 3da210ae1997ad4..57a02656206ff4c 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,7 @@ from functools import wraps from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest, - make_bad_fd, cpython_only) + make_bad_fd, cpython_only, swap_attr) from collections import UserList import _io # C implementation of io @@ -176,6 +176,12 @@ def testReprNoCloseFD(self): finally: os.close(fd) + def testRecursiveRepr(self): + # Issue #25455 + with swap_attr(self.f, 'name', self.f): + with self.assertRaises(RuntimeError): + repr(self.f) # Should not crash + def testErrors(self): f = self.f self.assertFalse(f.isatty()) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index ac8473db503b76b..6491f458c3acc4d 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -119,15 +119,27 @@ def test_float_memoryview(self): self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3) def test_error_message(self): - testlist = ('\xbd', '123\xbd', ' 123 456 ') - for s in testlist: - try: + def check(s): + with self.assertRaises(ValueError, msg='float(%r)' % (s,)) as cm: float(s) - except ValueError as e: - self.assertIn(s.strip(), e.args[0]) - else: - self.fail("Expected int(%r) to raise a ValueError", s) - + self.assertEqual(str(cm.exception), + 'could not convert string to float: %r' % (s,)) + + check('\xbd') + check('123\xbd') + check(' 123 456 ') + check(b' 123 456 ') + + # non-ascii digits (error came from non-digit '!') + check('\u0663\u0661\u0664!') + # embedded NUL + check('123\x00') + check('123\x00 245') + check('123\x00245') + # byte string with embedded NUL + check(b'123\x00') + # non-UTF-8 byte string + check(b'123\xa0') @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') def test_float_with_comma(self): diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 83eb29faf88d6cd..b6ba2e566b295f6 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -315,10 +315,12 @@ def __bytes__(self): testcommon(b"%b", b"abc", b"abc") testcommon(b"%b", bytearray(b"def"), b"def") testcommon(b"%b", fb, b"123") + testcommon(b"%b", memoryview(b"abc"), b"abc") # # %s is an alias for %b -- should only be used for Py2/3 code testcommon(b"%s", b"abc", b"abc") testcommon(b"%s", bytearray(b"def"), b"def") testcommon(b"%s", fb, b"123") + testcommon(b"%s", memoryview(b"abc"), b"abc") # %a will give the equivalent of # repr(some_obj).encode('ascii', 'backslashreplace') testcommon(b"%a", 3.14, b"3.14") @@ -377,9 +379,11 @@ def test_exc(formatstr, args, exception, excmsg): test_exc(b"%c", 3.14, TypeError, "%c requires an integer in range(256) or a single byte") test_exc(b"%b", "Xc", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") test_exc(b"%s", "Wd", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 824549b80342ed8..cd4664cec087456 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -89,6 +89,15 @@ def func(a=10, b=20): p(b=7) self.assertEqual(d, {'a':3}) + def test_kwargs_copy(self): + # Issue #29532: Altering a kwarg dictionary passed to a constructor + # should not affect a partial object after creation + d = {'a': 3} + p = self.partial(capture, **d) + self.assertEqual(p(), ((), {'a': 3})) + d['a'] = 5 + self.assertEqual(p(), ((), {'a': 3})) + def test_arg_combinations(self): # exercise special code paths for zero args in either partial # object or the caller @@ -393,6 +402,32 @@ def test_attributes_unwritable(self): else: self.fail('partial object allowed __dict__ to be deleted') + def test_manually_adding_non_string_keyword(self): + p = self.partial(capture) + # Adding a non-string/unicode keyword to partial kwargs + p.keywords[1234] = 'value' + r = repr(p) + self.assertIn('1234', r) + self.assertIn("'value'", r) + with self.assertRaises(TypeError): + p() + + def test_keystr_replaces_value(self): + p = self.partial(capture) + + class MutatesYourDict(object): + def __str__(self): + p.keywords[self] = ['sth2'] + return 'astr' + + # Raplacing the value during key formatting should keep the original + # value alive (at least long enough). + p.keywords[MutatesYourDict()] = ['sth'] + r = repr(p) + self.assertIn('astr', r) + self.assertIn("['sth']", r) + + class TestPartialPy(TestPartial, unittest.TestCase): partial = py_functools.partial diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 88eaabe6765ab2b..cfea281c70db11a 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -387,6 +387,11 @@ def test_cleandoc(self): def test_getcomments(self): self.assertEqual(inspect.getcomments(mod), '# line 1\n') self.assertEqual(inspect.getcomments(mod.StupidGit), '# line 20\n') + # If the object source file is not available, return None. + co = compile('x=1', '_non_existing_filename.py', 'exec') + self.assertIsNone(inspect.getcomments(co)) + # If the object has been defined in C, return None. + self.assertIsNone(inspect.getcomments(list)) def test_getmodule(self): # Check actual module diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index aaa64eadffe1ea3..8f895fe0bfb885a 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1014,6 +1014,16 @@ def test_repr(self): raw.name = b"dummy" self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname) + def test_recursive_repr(self): + # Issue #25455 + raw = self.MockRawIO() + b = self.tp(raw) + with support.swap_attr(raw, 'name', b): + try: + repr(b) # Should not crash + except RuntimeError: + pass + def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush # and that flush() is called before file closed. @@ -2424,6 +2434,16 @@ def test_repr(self): t.buffer.detach() repr(t) # Should not raise an exception + def test_recursive_repr(self): + # Issue #25455 + raw = self.BytesIO() + t = self.TextIOWrapper(raw) + with support.swap_attr(raw, 'name', t): + try: + repr(t) # Should not crash + except RuntimeError: + pass + def test_line_buffering(self): r = self.BytesIO() b = self.BufferedWriter(r, 1000) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index ea1f57caade12b9..c431f0dc6e1d7f4 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1976,6 +1976,14 @@ def gen2(x): self.assertRaises(AssertionError, list, cycle(gen1())) self.assertEqual(hist, [0,1]) + def test_long_chain_of_empty_iterables(self): + # Make sure itertools.chain doesn't run into recursion limits when + # dealing with long chains of empty iterables. Even with a high + # number this would probably only fail in Py_DEBUG mode. + it = chain.from_iterable(() for unused in range(10000000)) + with self.assertRaises(StopIteration): + next(it) + class SubclassWithKwargsTest(unittest.TestCase): def test_keywords_in_subclass(self): # count is not subclassable... diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 15f373664e1278a..9d93f931ca39588 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -2,7 +2,7 @@ import sys import textwrap import unittest -import subprocess +from subprocess import Popen, PIPE from test import support from test.support.script_helper import assert_python_ok @@ -61,12 +61,11 @@ class TestTool(unittest.TestCase): """) def test_stdin_stdout(self): - with subprocess.Popen( - (sys.executable, '-m', 'json.tool'), - stdin=subprocess.PIPE, stdout=subprocess.PIPE) as proc: + args = sys.executable, '-m', 'json.tool' + with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: out, err = proc.communicate(self.data.encode()) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) - self.assertEqual(err, None) + self.assertEqual(err, b'') def _create_infile(self): infile = support.TESTFN diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 88a93e5802c1cbe..846f721e8d9abbd 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -8,6 +8,7 @@ import stat import tempfile import unittest +from unittest import mock from test import support android_not_root = support.android_not_root @@ -1776,6 +1777,11 @@ def test_mkdir_exist_ok_with_parent(self): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + @only_nt # XXX: not sure how to test this on POSIX def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': @@ -1811,6 +1817,35 @@ def test_mkdir_no_parents_file(self): p.mkdir(exist_ok=True) self.assertEqual(cm.exception.errno, errno.EEXIST) + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(BASE, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + os.mkdir(path, mode) # from another process + concurrently_created.add(path) + os.mkdir(path, mode) # our real call + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("pathlib._normal_accessor.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + @with_symlinks def test_symlink_to(self): P = self.cls(BASE) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 5f875eff6c7ede1..ea6752b61d8f77f 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -67,12 +67,12 @@ def test_processor(self): def setUp(self): self.save_version = sys.version - self.save_mercurial = sys._mercurial + self.save_git = sys._git self.save_platform = sys.platform def tearDown(self): sys.version = self.save_version - sys._mercurial = self.save_mercurial + sys._git = self.save_git sys.platform = self.save_platform def test_sys_version(self): @@ -102,7 +102,7 @@ def test_sys_version(self): ('CPython', '2.4.3', '', '', 'truncation', '', 'GCC')), ): # branch and revision are not "parsed", but fetched - # from sys._mercurial. Ignore them + # from sys._git. Ignore them (name, version, branch, revision, buildno, builddate, compiler) \ = platform._sys_version(input) self.assertEqual( @@ -149,10 +149,10 @@ def test_sys_version(self): sys_versions.items(): sys.version = version_tag if subversion is None: - if hasattr(sys, "_mercurial"): - del sys._mercurial + if hasattr(sys, "_git"): + del sys._git else: - sys._mercurial = subversion + sys._git = subversion if sys_platform is not None: sys.platform = sys_platform self.assertEqual(platform.python_implementation(), info[0]) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 9e11e518f61e1f5..679759ec6d676a5 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -99,20 +99,24 @@ def test_large_operands(self): x = range(10**20+10, 10**20, 3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20, 10**20+10, -3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20+10, 10**20, -3) self.assertEqual(len(x), 4) self.assertEqual(len(list(x)), 4) + self.assertTrue(x) # Now test range() with longs - self.assertEqual(list(range(-2**100)), []) - self.assertEqual(list(range(0, -2**100)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) + for x in [range(-2**100), + range(0, -2**100), + range(0, 2**100, -1)]: + self.assertEqual(list(x), []) + self.assertFalse(x) a = int(10 * sys.maxsize) b = int(100 * sys.maxsize) @@ -153,6 +157,7 @@ def _range_len(x): step = x[1] - x[0] length = 1 + ((x[-1] - x[0]) // step) return length + a = -sys.maxsize b = sys.maxsize expected_len = b - a @@ -160,6 +165,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -177,6 +183,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -195,6 +202,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -213,6 +221,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0620b242766057b..342ec9e43d5de30 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -501,15 +501,15 @@ def _create_underpth_exe(self, lines): print(line, file=f) return exe_file except: - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) raise @classmethod def _cleanup_underpth_exe(self, exe_file): _pth_file = os.path.splitext(exe_file)[0] + '._pth' - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) @classmethod def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 97dc3cd76b24a5e..2497e47c668bc3c 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -803,11 +803,6 @@ def testHostnameRes(self): self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) def test_host_resolution(self): - for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', - '1:1:1:1:1:1:1:1:1']: - self.assertRaises(OSError, socket.gethostbyname, addr) - self.assertRaises(OSError, socket.gethostbyaddr, addr) - for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: self.assertEqual(socket.gethostbyname(addr), addr) @@ -816,6 +811,21 @@ def test_host_resolution(self): for host in [support.HOST]: self.assertIn(host, socket.gethostbyaddr(host)[2]) + def test_host_resolution_bad_address(self): + # These are all malformed IP addresses and expected not to resolve to + # any result. But some ISPs, e.g. AWS, may successfully resolve these + # IPs. + explanation = ( + "resolving an invalid IP address did not raise OSError; " + "can be caused by a broken DNS server" + ) + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + with self.assertRaises(OSError): + socket.gethostbyname(addr) + with self.assertRaises(OSError, msg=explanation): + socket.gethostbyaddr(addr) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") def test_sethostname(self): @@ -896,6 +906,7 @@ def testNtoH(self): self.assertEqual(swapped & mask, mask) self.assertRaises(OverflowError, func, 1<<34) + @support.cpython_only def testNtoHErrors(self): good_values = [ 1, 2, 3, 1, 2, 3 ] bad_values = [ -1, -2, -3, -1, -2, -3 ] @@ -5469,7 +5480,7 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 3) # see test_aes_cbc + @support.requires_linux_version(4, 9) # see issue29324 def test_aead_aes_gcm(self): key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') @@ -5492,8 +5503,7 @@ def test_aead_aes_gcm(self): op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen, flags=socket.MSG_MORE) op.sendall(assoc, socket.MSG_MORE) - op.sendall(plain, socket.MSG_MORE) - op.sendall(b'\x00' * taglen) + op.sendall(plain) res = op.recv(assoclen + len(plain) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5501,7 +5511,7 @@ def test_aead_aes_gcm(self): # now with msg op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen) res = op.recv(assoclen + len(plain) + taglen) @@ -5512,7 +5522,7 @@ def test_aead_aes_gcm(self): pack_uint32 = struct.Struct('I').pack op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg( [msg], ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], @@ -5520,7 +5530,7 @@ def test_aead_aes_gcm(self): [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], ) ) - res = op.recv(len(msg)) + res = op.recv(len(msg) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5530,8 +5540,8 @@ def test_aead_aes_gcm(self): msg = assoc + expected_ct + expected_tag op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, assoclen=assoclen) - res = op.recv(len(msg)) - self.assertEqual(plain, res[assoclen:-taglen]) + res = op.recv(len(msg) - taglen) + self.assertEqual(plain, res[assoclen:]) @support.requires_linux_version(4, 3) # see test_aes_cbc def test_drbg_pr_sha256(self): diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index e63f9f254cd2b24..3c871dde5c532f4 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -347,6 +347,16 @@ def test_cwd(self): temp_dir = self._normalize_cwd(temp_dir) self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + def test_cwd_with_pathlike(self): + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + + class _PathLikeObj: + def __fspath__(self): + return temp_dir + + self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): # Check that Popen looks for args[0] relative to cwd if args[0] diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 619cbc03b28abc1..fc79055421161c9 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -4,6 +4,7 @@ from hashlib import md5 from contextlib import contextmanager from random import Random +import pathlib import unittest import unittest.mock @@ -440,6 +441,22 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) + def test_pathlike_name(self): + tarname = pathlib.Path(self.tarname) + with tarfile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with self.taropen(tarname) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with tarfile.TarFile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + if self.suffix == '': + with tarfile.TarFile(tarname, mode='r') as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + def test_illegal_mode_arg(self): with open(tmpname, 'wb'): pass @@ -582,6 +599,26 @@ def test_extract_directory(self): finally: support.rmtree(DIR) + def test_extractall_pathlike_name(self): + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + for tarinfo in directories: + path = DIR / tarinfo.name + self.assertEqual(os.path.getmtime(path), tarinfo.mtime) + + def test_extract_pathlike_name(self): + dirtype = "ustar/dirtype" + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + tarinfo = tar.getmember(dirtype) + tar.extract(tarinfo, path=DIR) + extracted = DIR / dirtype + self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) + def test_init_close_fobj(self): # Issue #7341: Close the internal file object in the TarFile # constructor in case of an error. For the test we rely on @@ -1092,6 +1129,17 @@ def test_directory_size(self): finally: support.rmdir(path) + def test_gettarinfo_pathlike_name(self): + with tarfile.open(tmpname, self.mode) as tar: + path = pathlib.Path(TEMPDIR) / "file" + with open(path, "wb") as fobj: + fobj.write(b"aaa") + tarinfo = tar.gettarinfo(path) + tarinfo2 = tar.gettarinfo(os.fspath(path)) + self.assertIsInstance(tarinfo.name, str) + self.assertEqual(tarinfo.name, tarinfo2.name) + self.assertEqual(tarinfo.size, 3) + @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1528,6 +1576,34 @@ def test_create_existing_taropen(self): self.assertEqual(len(names), 1) self.assertIn("spameggs42", names[0]) + def test_create_pathlike_name(self): + with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + def test_create_taropen_pathlike_name(self): + with self.taropen(pathlib.Path(tmpname), "x") as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + class GzipCreateTest(GzipTest, CreateTest): pass diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 2c2914fd6d89693..6b6c4d220a3bd09 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -170,6 +170,9 @@ def f(mutex): mutex.acquire() self.assertIn(tid, threading._active) self.assertIsInstance(threading._active[tid], threading._DummyThread) + #Issue 29376 + self.assertTrue(threading._active[tid].is_alive()) + self.assertRegex(repr(threading._active[tid]), '_DummyThread') del threading._active[tid] # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 790ab7ee606d609..742259b4396b986 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -865,6 +865,7 @@ def test_sys_xoptions_invalid(self): b'number of frames', stderr) + @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_pymem_alloc0(self): # Issue #21639: Check that PyMem_Malloc(0) with tracemalloc enabled # does not crash. diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fce6b5aaffdf932..f0070ec975791ac 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -190,6 +190,10 @@ def test_bound_errors(self): with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) + def test_no_bivariant(self): + with self.assertRaises(ValueError): + TypeVar('T', covariant=True, contravariant=True) + class UnionTests(BaseTestCase): @@ -254,6 +258,11 @@ def test_repr(self): self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) u = Union[int, Employee] self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) + T = TypeVar('T') + u = Union[T, int][int] + self.assertEqual(repr(u), repr(int)) + u = Union[List[int], int] + self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') def test_cannot_subclass(self): with self.assertRaises(TypeError): @@ -304,6 +313,15 @@ def test_union_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, Union[int, str]) + def test_no_eval_union(self): + u = Union[int, str] + def f(x: u): ... + self.assertIs(get_type_hints(f)['x'], u) + + def test_function_repr_union(self): + def fun() -> int: ... + self.assertEqual(repr(Union[fun, int]), 'typing.Union[fun, int]') + def test_union_str_pattern(self): # Shouldn't crash; see http://bugs.python.org/issue25390 A = Union[str, Pattern] @@ -401,6 +419,8 @@ def test_callable_wrong_forms(self): Callable[[()], int] with self.assertRaises(TypeError): Callable[[int, 1], 2] + with self.assertRaises(TypeError): + Callable[int] def test_callable_instance_works(self): def f(): @@ -546,15 +566,27 @@ def test_basics(self): Y[str] with self.assertRaises(TypeError): Y[str, str] + self.assertIsSubclass(SimpleMapping[str, int], SimpleMapping) def test_generic_errors(self): T = TypeVar('T') + S = TypeVar('S') with self.assertRaises(TypeError): Generic[T]() + with self.assertRaises(TypeError): + Generic[T][T] + with self.assertRaises(TypeError): + Generic[T][S] with self.assertRaises(TypeError): isinstance([], List[int]) with self.assertRaises(TypeError): issubclass(list, List[int]) + with self.assertRaises(TypeError): + class NewGeneric(Generic): ... + with self.assertRaises(TypeError): + class MyGeneric(Generic[T], Generic[S]): ... + with self.assertRaises(TypeError): + class MyGeneric(List[T], Generic[S]): ... def test_init(self): T = TypeVar('T') @@ -642,6 +674,41 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_subscripted_generics_as_proxies(self): + T = TypeVar('T') + class C(Generic[T]): + x = 'def' + self.assertEqual(C[int].x, 'def') + self.assertEqual(C[C[int]].x, 'def') + C[C[int]].x = 'changed' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[str].x, 'changed') + C[List[str]].z = 'new' + self.assertEqual(C.z, 'new') + self.assertEqual(C[Tuple[int]].z, 'new') + + self.assertEqual(C().x, 'changed') + self.assertEqual(C[Tuple[str]]().z, 'new') + + class D(C[T]): + pass + self.assertEqual(D[int].x, 'changed') + self.assertEqual(D.z, 'new') + D.z = 'from derived z' + D[int].x = 'from derived x' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[int].z, 'new') + self.assertEqual(D.x, 'from derived x') + self.assertEqual(D[str].z, 'from derived z') + + def test_abc_registry_kept(self): + T = TypeVar('T') + class C(Generic[T]): ... + C.register(int) + self.assertIsInstance(1, C) + C[int] + self.assertIsInstance(1, C) + def test_false_subclasses(self): class MyMapping(MutableMapping[str, str]): pass self.assertNotIsInstance({}, MyMapping) @@ -738,6 +805,53 @@ def test_subscript_meta(self): self.assertEqual(Union[T, int][GenericMeta], Union[GenericMeta, int]) self.assertEqual(Callable[..., GenericMeta].__args__, (Ellipsis, GenericMeta)) + def test_generic_hashes(self): + try: + from test import mod_generics_cache + except ImportError: # for Python 3.4 and previous versions + import mod_generics_cache + class A(Generic[T]): + ... + + class B(Generic[T]): + class A(Generic[T]): + ... + + self.assertEqual(A, A) + self.assertEqual(mod_generics_cache.A[str], mod_generics_cache.A[str]) + self.assertEqual(B.A, B.A) + self.assertEqual(mod_generics_cache.B.A[B.A[str]], + mod_generics_cache.B.A[B.A[str]]) + + self.assertNotEqual(A, B.A) + self.assertNotEqual(A, mod_generics_cache.A) + self.assertNotEqual(A, mod_generics_cache.B.A) + self.assertNotEqual(B.A, mod_generics_cache.A) + self.assertNotEqual(B.A, mod_generics_cache.B.A) + + self.assertNotEqual(A[str], B.A[str]) + self.assertNotEqual(A[List[Any]], B.A[List[Any]]) + self.assertNotEqual(A[str], mod_generics_cache.A[str]) + self.assertNotEqual(A[str], mod_generics_cache.B.A[str]) + self.assertNotEqual(B.A[int], mod_generics_cache.A[int]) + self.assertNotEqual(B.A[List[Any]], mod_generics_cache.B.A[List[Any]]) + + self.assertNotEqual(Tuple[A[str]], Tuple[B.A[str]]) + self.assertNotEqual(Tuple[A[List[Any]]], Tuple[B.A[List[Any]]]) + self.assertNotEqual(Union[str, A[str]], Union[str, mod_generics_cache.A[str]]) + self.assertNotEqual(Union[A[str], A[str]], + Union[A[str], mod_generics_cache.A[str]]) + self.assertNotEqual(typing.FrozenSet[A[str]], + typing.FrozenSet[mod_generics_cache.B.A[str]]) + + if sys.version_info[:2] > (3, 2): + self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) + self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) + .endswith('mod_generics_cache.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) + .endswith('mod_generics_cache.B.A[str]]')) + def test_extended_generic_rules_eq(self): T = TypeVar('T') U = TypeVar('U') @@ -835,6 +949,8 @@ def test_fail_with_bare_generic(self): Tuple[Generic[T]] with self.assertRaises(TypeError): List[typing._Protocol] + with self.assertRaises(TypeError): + isinstance(1, Generic) def test_type_erasure_special(self): T = TypeVar('T') @@ -853,6 +969,11 @@ class MyDict(typing.Dict[T, T]): ... class MyDef(typing.DefaultDict[str, T]): ... self.assertIs(MyDef[int]().__class__, MyDef) self.assertIs(MyDef[int]().__orig_class__, MyDef[int]) + # ChainMap was added in 3.3 + if sys.version_info >= (3, 3): + class MyChain(typing.ChainMap[str, T]): ... + self.assertIs(MyChain[int]().__class__, MyChain) + self.assertIs(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) @@ -1203,6 +1324,19 @@ def test_forwardref_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, fr) + def test_forwardref_subclass_type_error(self): + fr = typing._ForwardRef('int') + with self.assertRaises(TypeError): + issubclass(int, fr) + + def test_forward_equality(self): + fr = typing._ForwardRef('int') + self.assertEqual(fr, typing._ForwardRef('int')) + self.assertNotEqual(List['int'], List[int]) + + def test_forward_repr(self): + self.assertEqual(repr(List['int']), "typing.List[_ForwardRef('int')]") + def test_union_forward(self): def foo(a: Union['T']): @@ -1285,6 +1419,15 @@ def foo(a: 'whatevers') -> {}: ith = get_type_hints(C().foo) self.assertEqual(ith, {}) + def test_no_type_check_no_bases(self): + class C: + def meth(self, x: int): ... + @no_type_check + class D(C): + c = C + # verify that @no_type_check never affects bases + self.assertEqual(get_type_hints(C.meth), {'x': int}) + def test_meta_no_type_check(self): @no_type_check_decorator @@ -1401,11 +1544,16 @@ class A: class B(A): x: ClassVar[Optional['B']] = None y: int + b: int class CSub(B): z: ClassVar['CSub'] = B() class G(Generic[T]): lst: ClassVar[List[T]] = [] +class NoneAndForward: + parent: 'NoneAndForward' + meaning: None + class CoolEmployee(NamedTuple): name: str cool: int @@ -1419,10 +1567,13 @@ class XMeth(NamedTuple): def double(self): return 2 * self.x -class XMethBad(NamedTuple): +class XRepr(NamedTuple): x: int - def _fields(self): - return 'no chance for this' + y: int = 1 + def __str__(self): + return f'{self.x} -> {self.y}' + def __add__(self, other): + return 0 """ if PY36: @@ -1431,7 +1582,7 @@ def _fields(self): # fake names for the sake of static analysis ann_module = ann_module2 = ann_module3 = None A = B = CSub = G = CoolEmployee = CoolEmployeeWithDefault = object - XMeth = XMethBad = object + XMeth = XRepr = NoneAndForward = object gth = get_type_hints @@ -1466,6 +1617,8 @@ def test_get_type_hints_classes(self): {'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.S), {'x': str, 'y': str}) self.assertEqual(gth(ann_module.foo), {'x': int}) + self.assertEqual(gth(NoneAndForward, globals()), + {'parent': NoneAndForward, 'meaning': type(None)}) @skipUnless(PY36, 'Python 3.6 required') def test_respect_no_type_check(self): @@ -1482,17 +1635,22 @@ def meth(x: int): ... class Der(ABase): ... self.assertEqual(gth(ABase.meth), {'x': int}) - def test_get_type_hints_for_builins(self): + def test_get_type_hints_for_builtins(self): # Should not fail for built-in classes and functions. self.assertEqual(gth(int), {}) self.assertEqual(gth(type), {}) self.assertEqual(gth(dir), {}) self.assertEqual(gth(len), {}) + self.assertEqual(gth(object.__str__), {}) + self.assertEqual(gth(object().__str__), {}) + self.assertEqual(gth(str.join), {}) def test_previous_behavior(self): def testf(x, y): ... testf.__annotations__['x'] = 'int' self.assertEqual(gth(testf), {'x': int}) + def testg(x: None): ... + self.assertEqual(gth(testg), {'x': type(None)}) def test_get_type_hints_for_object_with_annotations(self): class A: ... @@ -1506,9 +1664,10 @@ def test_get_type_hints_ClassVar(self): self.assertEqual(gth(ann_module2.CV, ann_module2.__dict__), {'var': typing.ClassVar[ann_module2.CV]}) self.assertEqual(gth(B, globals()), - {'y': int, 'x': ClassVar[Optional[B]]}) + {'y': int, 'x': ClassVar[Optional[B]], 'b': int}) self.assertEqual(gth(CSub, globals()), - {'z': ClassVar[CSub], 'y': int, 'x': ClassVar[Optional[B]]}) + {'z': ClassVar[CSub], 'y': int, 'b': int, + 'x': ClassVar[Optional[B]]}) self.assertEqual(gth(G), {'lst': ClassVar[List[T]]}) @@ -1628,6 +1787,11 @@ def test_list(self): def test_deque(self): self.assertIsSubclass(collections.deque, typing.Deque) + class MyDeque(typing.Deque[int]): ... + self.assertIsInstance(MyDeque(), collections.deque) + + def test_counter(self): + self.assertIsSubclass(collections.Counter, typing.Counter) def test_set(self): self.assertIsSubclass(set, typing.Set) @@ -1680,13 +1844,10 @@ class MyDict(typing.Dict[str, int]): self.assertIsSubclass(MyDict, dict) self.assertNotIsSubclass(dict, MyDict) - def test_no_defaultdict_instantiation(self): - with self.assertRaises(TypeError): - typing.DefaultDict() - with self.assertRaises(TypeError): - typing.DefaultDict[KT, VT]() - with self.assertRaises(TypeError): - typing.DefaultDict[str, int]() + def test_defaultdict_instantiation(self): + self.assertIs(type(typing.DefaultDict()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[KT, VT]()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[str, int]()), collections.defaultdict) def test_defaultdict_subclass(self): @@ -1699,13 +1860,49 @@ class MyDefDict(typing.DefaultDict[str, int]): self.assertIsSubclass(MyDefDict, collections.defaultdict) self.assertNotIsSubclass(collections.defaultdict, MyDefDict) - def test_no_deque_instantiation(self): - with self.assertRaises(TypeError): - typing.Deque() - with self.assertRaises(TypeError): - typing.Deque[T]() - with self.assertRaises(TypeError): - typing.Deque[int]() + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_instantiation(self): + self.assertIs(type(typing.ChainMap()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[KT, VT]()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[str, int]()), collections.ChainMap) + class CM(typing.ChainMap[KT, VT]): ... + self.assertIs(type(CM[int, str]()), CM) + + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_subclass(self): + + class MyChainMap(typing.ChainMap[str, int]): + pass + + cm = MyChainMap() + self.assertIsInstance(cm, MyChainMap) + + self.assertIsSubclass(MyChainMap, collections.ChainMap) + self.assertNotIsSubclass(collections.ChainMap, MyChainMap) + + def test_deque_instantiation(self): + self.assertIs(type(typing.Deque()), collections.deque) + self.assertIs(type(typing.Deque[T]()), collections.deque) + self.assertIs(type(typing.Deque[int]()), collections.deque) + class D(typing.Deque[T]): ... + self.assertIs(type(D[int]()), D) + + def test_counter_instantiation(self): + self.assertIs(type(typing.Counter()), collections.Counter) + self.assertIs(type(typing.Counter[T]()), collections.Counter) + self.assertIs(type(typing.Counter[int]()), collections.Counter) + class C(typing.Counter[T]): ... + self.assertIs(type(C[int]()), C) + + def test_counter_subclass_instantiation(self): + + class MyCounter(typing.Counter[int]): + pass + + d = MyCounter() + self.assertIsInstance(d, MyCounter) + self.assertIsInstance(d, typing.Counter) + self.assertIsInstance(d, collections.Counter) def test_no_set_instantiation(self): with self.assertRaises(TypeError): @@ -2018,6 +2215,14 @@ def test_basics(self): collections.OrderedDict([('name', str), ('id', int)])) self.assertIs(Emp._field_types, Emp.__annotations__) + def test_namedtuple_pyversion(self): + if sys.version_info[:2] < (3, 6): + with self.assertRaises(TypeError): + NamedTuple('Name', one=int, other=str) + with self.assertRaises(TypeError): + class NotYet(NamedTuple): + whatever = 0 + @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage(self): tim = CoolEmployee('Tim', 9000) @@ -2055,10 +2260,18 @@ class NonDefaultAfterDefault(NamedTuple): @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage_with_methods(self): - self.assertEquals(XMeth(1).double(), 2) - self.assertEquals(XMeth(42).x, XMeth(42)[0]) - self.assertEquals(XMethBad(1)._fields, ('x',)) - self.assertEquals(XMethBad(1).__annotations__, {'x': int}) + self.assertEqual(XMeth(1).double(), 2) + self.assertEqual(XMeth(42).x, XMeth(42)[0]) + self.assertEqual(str(XRepr(42)), '42 -> 1') + self.assertEqual(XRepr(1, 2) + XRepr(3), 0) + + with self.assertRaises(AttributeError): + exec(""" +class XMethBad(NamedTuple): + x: int + def _fields(self): + return 'no chance for this' +""") @skipUnless(PY36, 'Python 3.6 required') def test_namedtuple_keyword_usage(self): @@ -2138,6 +2351,12 @@ def test_basics(self): Pattern[Union[str, bytes]] Match[Union[bytes, str]] + def test_alias_equality(self): + self.assertEqual(Pattern[str], Pattern[str]) + self.assertNotEqual(Pattern[str], Pattern[bytes]) + self.assertNotEqual(Pattern[str], Match[str]) + self.assertNotEqual(Pattern[str], str) + def test_errors(self): with self.assertRaises(TypeError): # Doesn't fit AnyStr. @@ -2152,6 +2371,9 @@ def test_errors(self): with self.assertRaises(TypeError): # We don't support isinstance(). isinstance(42, Pattern[str]) + with self.assertRaises(TypeError): + # We don't support issubclass(). + issubclass(Pattern[bytes], Pattern[str]) def test_repr(self): self.assertEqual(repr(Pattern), 'Pattern[~AnyStr]') diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 86ebd45e7b031da..2844bc5540c38b3 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1448,6 +1448,15 @@ def test_formatting_huge_precision(self): with self.assertRaises(ValueError): result = format_string % 2.34 + def test_issue28598_strsubclass_rhs(self): + # A subclass of str with an __rmod__ method should be able to hook + # into the % operator + class SubclassedStr(str): + def __rmod__(self, other): + return 'Success, self.__rmod__({!r}) was called'.format(other) + self.assertEqual('lhs %% %r' % SubclassedStr('rhs'), + "Success, self.__rmod__('lhs %% %r') was called") + @support.cpython_only def test_formatting_huge_precision_c_limits(self): from _testcapi import INT_MAX diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 5084486e5ab4798..fa3757cc94be52f 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -206,6 +206,7 @@ def test_iter(self): def test_relativelocalfile(self): self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + class ProxyTests(unittest.TestCase): def setUp(self): @@ -259,6 +260,7 @@ def test_proxy_bypass_environment_host_match(self): self.assertFalse(bypass('newdomain.com')) # no port self.assertFalse(bypass('newdomain.com:1235')) # wrong port + class ProxyTests_withOrderedEnv(unittest.TestCase): def setUp(self): @@ -294,6 +296,7 @@ def test_getproxies_environment_prefer_lowercase(self): proxies = urllib.request.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http']) + class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): """Test urlopen() opening a fake http connection.""" @@ -432,7 +435,6 @@ def test_ftp_cache_pruning(self): finally: self.unfakeftp() - def test_userpass_inurl(self): self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") try: @@ -476,6 +478,7 @@ def test_cafile_and_context(self): "https://localhost", cafile="/nonexistent/path", context=context ) + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -549,6 +552,7 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 34329f87162a625..876fcd4199fb923 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -141,44 +141,55 @@ def test_password_manager(self): mgr = urllib.request.HTTPPasswordMgr() add = mgr.add_password find_user_pass = mgr.find_user_password + add("Some Realm", "http://example.com/", "joe", "password") add("Some Realm", "http://example.com/ni", "ni", "ni") - add("c", "http://example.com/foo", "foo", "ni") - add("c", "http://example.com/bar", "bar", "nini") - add("b", "http://example.com/", "first", "blah") - add("b", "http://example.com/", "second", "spam") - add("a", "http://example.com", "1", "a") add("Some Realm", "http://c.example.com:3128", "3", "c") add("Some Realm", "d.example.com", "4", "d") add("Some Realm", "e.example.com:3128", "5", "e") + # For the same realm, password set the highest path is the winner. self.assertEqual(find_user_pass("Some Realm", "example.com"), ('joe', 'password')) - - #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), - # ('ni', 'ni')) - + self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), + ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com"), ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam"), - ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam/spam"), - ('joe', 'password')) + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam"), + ('joe', 'password')) + + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam/spam"), + ('joe', 'password')) + + # You can have different passwords for different paths. + + add("c", "http://example.com/foo", "foo", "ni") + add("c", "http://example.com/bar", "bar", "nini") + self.assertEqual(find_user_pass("c", "http://example.com/foo"), ('foo', 'ni')) + self.assertEqual(find_user_pass("c", "http://example.com/bar"), ('bar', 'nini')) + + # For the same path, newer password should be considered. + + add("b", "http://example.com/", "first", "blah") + add("b", "http://example.com/", "second", "spam") + self.assertEqual(find_user_pass("b", "http://example.com/"), ('second', 'spam')) # No special relationship between a.example.com and example.com: + add("a", "http://example.com", "1", "a") self.assertEqual(find_user_pass("a", "http://example.com/"), ('1', 'a')) + self.assertEqual(find_user_pass("a", "http://a.example.com/"), (None, None)) @@ -830,7 +841,6 @@ def test_file(self): for url, ftp in [ ("file://ftp.example.com//foo.txt", False), ("file://ftp.example.com///foo.txt", False), -# XXXX bug: fails with OSError, should be URLError ("file://ftp.example.com/foo.txt", False), ("file://somehost//foo/something.txt", False), ("file://localhost//foo/something.txt", False), @@ -838,8 +848,7 @@ def test_file(self): req = Request(url) try: h.file_open(req) - # XXXX remove OSError when bug fixed - except (urllib.error.URLError, OSError): + except urllib.error.URLError: self.assertFalse(ftp) else: self.assertIs(o.req, req) @@ -1414,7 +1423,6 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") - # TODO: This should be only for OSX @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { @@ -1690,7 +1698,6 @@ def test_invalid_closed(self): self.assertTrue(conn.fakesock.closed, "Connection not closed") - class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 949716c2b567e00..865a7f4f5fb172e 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -29,7 +29,7 @@ def testURLread(self): class urlopenNetworkTests(unittest.TestCase): - """Tests urllib.reqest.urlopen using the network. + """Tests urllib.request.urlopen using the network. These tests are not exhaustive. Assuming that testing using files does a good job overall of some of the basic interface features. There are no diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 99c5c033e30d20d..3c89ed928ff51fd 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -6,8 +6,8 @@ RFC3986_BASE = 'http://a/b/c/d;p?q' SIMPLE_BASE = 'http://a/b/c/d' -# A list of test cases. Each test case is a two-tuple that contains -# a string with the query and a dictionary with the expected result. +# Each parse_qsl testcase is a two-tuple that contains +# a string with the query and a list with the expected result. parse_qsl_test_cases = [ ("", []), @@ -42,6 +42,9 @@ (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), ] +# Each parse_qs testcase is a two-tuple that contains +# a string with the query and a dictionary with the expected result. + parse_qs_test_cases = [ ("", {}), ("&", {}), @@ -290,7 +293,6 @@ def test_RFC2368(self): def test_RFC2396(self): # cases from RFC 2396 - self.checkJoin(RFC2396_BASE, 'g:h', 'g:h') self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g') self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g') @@ -333,9 +335,7 @@ def test_RFC2396(self): # self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g') # self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g') - def test_RFC3986(self): - # Test cases from RFC3986 self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y') self.checkJoin(RFC3986_BASE, ';x', 'http://a/b/c/;x') self.checkJoin(RFC3986_BASE, 'g:h','g:h') @@ -363,7 +363,7 @@ def test_RFC3986(self): self.checkJoin(RFC3986_BASE, '../../g','http://a/g') self.checkJoin(RFC3986_BASE, '../../../g', 'http://a/g') - #Abnormal Examples + # Abnormal Examples # The 'abnormal scenarios' are incompatible with RFC2986 parsing # Tests are here for reference. diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c0144d1cb8febf6..dbdad23a7423f2f 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1879,6 +1879,118 @@ def test_recursive_repr(self): with self.assertRaises(RuntimeError): repr(e) # Should not crash + def test_element_get_text(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem.text + except NameError: + pass + + b = ET.TreeBuilder() + b.start('tag', {}) + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('tag') + + elem = b.close() + self.assertEqual(elem.text, 'ABCDEFGHIJKL') + + def test_element_get_tail(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem[0].tail + except NameError: + pass + + b = ET.TreeBuilder() + b.start('root', {}) + b.start('tag', {}) + b.end('tag') + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('root') + + elem = b.close() + self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') + + def test_element_iter(self): + # Issue #27863 + state = { + 'tag': 'tag', + '_children': [None], # non-Element + 'attrib': 'attr', + 'tail': 'tail', + 'text': 'text', + } + + e = ET.Element('tag') + try: + e.__setstate__(state) + except AttributeError: + e.__dict__ = state + + it = e.iter() + self.assertIs(next(it), e) + self.assertRaises(AttributeError, next, it) + + def test_subscr(self): + # Issue #27863 + class X: + def __index__(self): + del e[:] + return 1 + + e = ET.Element('elem') + e.append(ET.Element('child')) + e[:X()] # shouldn't crash + + e.append(ET.Element('child')) + e[0:10:X()] # shouldn't crash + + def test_ass_subscr(self): + # Issue #27863 + class X: + def __index__(self): + e[:] = [] + return 1 + + e = ET.Element('elem') + for _ in range(10): + e.insert(0, ET.Element('child')) + + e[0:10:X()] = [] # shouldn't crash + + def test_treebuilder_start(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.start, 'tag2', {}) + del b + gc_collect() + + def test_treebuilder_end(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.end, 'tag') + del b + gc_collect() + + class MutatingElementPath(str): def __new__(cls, elem, *args): self = str.__new__(cls, *args) diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index df9c79e3df06891..30025e388ddf2e8 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -343,6 +343,94 @@ def run_server(): self.assertEqual(p.method(), 5) self.assertEqual(p.method(), 5) + +class SimpleXMLRPCDispatcherTestCase(unittest.TestCase): + class DispatchExc(Exception): + """Raised inside the dispatched functions when checking for + chained exceptions""" + + def test_call_registered_func(self): + """Calls explicitly registered function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + def dispatched_func(*params): + raise self.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(dispatched_func) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_instance_func(self): + """Calls a registered instance attribute as a function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + class DispatchedClass: + def dispatched_func(self, *params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(DispatchedClass()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_dispatch_func(self): + """Calls the registered instance's `_dispatch` function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_method = 'method' + exp_params = 1, 2, 3 + + class TestInstance: + def _dispatch(self, method, params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc( + method, params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(TestInstance()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch(exp_method, exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_method, exp_params)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_registered_func_is_none(self): + """Calls explicitly registered function which is None""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(None, name='method') + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + def test_instance_has_no_func(self): + """Attempts to call nonexistent function on a registered instance""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(object()) + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + def test_cannot_locate_func(self): + """Calls a function that the dispatcher cannot locate""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + class HelperTestCase(unittest.TestCase): def test_escape(self): self.assertEqual(xmlrpclib.escape("a&b"), "a&b") @@ -1312,7 +1400,7 @@ def test_main(): KeepaliveServerTestCase1, KeepaliveServerTestCase2, GzipServerTestCase, GzipUtilTestCase, MultiPathServerTestCase, ServerProxyTestCase, FailingServerTestCase, - CGIHandlerTestCase) + CGIHandlerTestCase, SimpleXMLRPCDispatcherTestCase) if __name__ == "__main__": diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 0a43b20e2bcf0d5..d39f05fdb7788ec 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2,6 +2,7 @@ import io import os import importlib.util +import pathlib import posixpath import time import struct @@ -13,7 +14,7 @@ from random import randint, random, getrandbits from test.support import script_helper -from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, +from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, temp_cwd, requires_zlib, requires_bz2, requires_lzma, captured_stdout, check_warnings) @@ -148,6 +149,12 @@ def test_open(self): for f in get_files(self): self.zip_open_test(f, self.compression) + def test_open_with_pathlike(self): + path = pathlib.Path(TESTFN2) + self.zip_open_test(path, self.compression) + with zipfile.ZipFile(path, "r", self.compression) as zipfp: + self.assertIsInstance(zipfp.filename, str) + def zip_random_open_test(self, f, compression): self.make_test_archive(f, compression) @@ -906,22 +913,56 @@ def test_write_pyfile_bad_syntax(self): finally: rmtree(TESTFN2) + def test_write_pathlike(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w") as fp: + fp.write("print(42)\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + finally: + rmtree(TESTFN2) + class ExtractTests(unittest.TestCase): - def test_extract(self): + + def make_test_file(self): with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: for fpath, fdata in SMALL_TEST_DATA: zipfp.writestr(fpath, fdata) + def test_extract(self): + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + writtenfile = zipfp.extract(fpath) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.normpath(correctfile) + + self.assertEqual(writtenfile, correctfile) + + # make sure correct data is in correct file + with open(writtenfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(writtenfile) + + def _test_extract_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: for fpath, fdata in SMALL_TEST_DATA: - writtenfile = zipfp.extract(fpath) + writtenfile = zipfp.extract(fpath, target) # make sure it was written to the right place - correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.join(target, fpath) correctfile = os.path.normpath(correctfile) - - self.assertEqual(writtenfile, correctfile) + self.assertTrue(os.path.samefile(writtenfile, correctfile), (writtenfile, target)) # make sure correct data is in correct file with open(writtenfile, "rb") as f: @@ -929,26 +970,50 @@ def test_extract(self): unlink(writtenfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_with_target(self): + with temp_dir() as extdir: + self._test_extract_with_target(extdir) + + def test_extract_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_with_target(pathlib.Path(extdir)) def test_extract_all(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + zipfp.extractall() + for fpath, fdata in SMALL_TEST_DATA: + outfile = os.path.join(os.getcwd(), fpath) + + with open(outfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(outfile) + def _test_extract_all_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: - zipfp.extractall() + zipfp.extractall(target) for fpath, fdata in SMALL_TEST_DATA: - outfile = os.path.join(os.getcwd(), fpath) + outfile = os.path.join(target, fpath) with open(outfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) unlink(outfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_all_with_target(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(extdir) + + def test_extract_all_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(pathlib.Path(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1188,6 +1253,8 @@ def test_is_zip_erroneous_file(self): with open(TESTFN, "w") as fp: fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a path-like object + self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2033,6 +2100,26 @@ def test_from_file(self): zi = zipfile.ZipInfo.from_file(__file__) self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_pathlike(self): + zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_bytes(self): + zi = zipfile.ZipInfo.from_file(os.fsencode(__file__), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_fileno(self): + with open(__file__, 'rb') as f: + zi = zipfile.ZipInfo.from_file(f.fileno(), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_dir(self): dirpath = os.path.dirname(os.path.abspath(__file__)) diff --git a/Lib/threading.py b/Lib/threading.py index 4829ff426e0bfd8..95978d310a2f3aa 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1217,6 +1217,10 @@ def __init__(self): def _stop(self): pass + def is_alive(self): + assert not self._is_stopped and self._started.is_set() + return True + def join(self, timeout=None): assert False, "cannot join a dummy thread" diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100644 new mode 100755 index 2770efa35a01601..8eea766b4c344b3 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -208,7 +208,7 @@ def repeat(self, repeat=default_repeat, number=default_number): return r def autorange(self, callback=None): - """Return the number of loops so that total time >= 0.2. + """Return the number of loops and time taken so that total time >= 0.2. Calls the timeit method with *number* set to successive powers of ten (10, 100, 1000, ...) up to a maximum of one billion, until diff --git a/Lib/typing.py b/Lib/typing.py index c9e341753769e01..9a0f49099a31145 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -10,6 +10,12 @@ import collections.abc as collections_abc except ImportError: import collections as collections_abc # Fallback for PY3.2. +try: + from types import SlotWrapperType, MethodWrapperType, MethodDescriptorType +except ImportError: + SlotWrapperType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) # Please keep __all__ alphabetized within each category. @@ -62,6 +68,7 @@ 'SupportsRound', # Concrete collection types. + 'Counter', 'Deque', 'Dict', 'DefaultDict', @@ -849,19 +856,6 @@ def _next_in_mro(cls): return next_in_mro -def _valid_for_check(cls): - """An internal helper to prohibit isinstance([1], List[str]) etc.""" - if cls is Generic: - raise TypeError("Class %r cannot be used with class " - "or instance checks" % cls) - if ( - cls.__origin__ is not None and - sys._getframe(3).f_globals['__name__'] not in ['abc', 'functools'] - ): - raise TypeError("Parameterized generics cannot be used with class " - "or instance checks") - - def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates the associated __extra__ class in subclass checks performed @@ -872,7 +866,6 @@ def _make_subclasshook(cls): # Registered classes need not be checked here because # cls and its extra share the same _abc_registry. def __extrahook__(subclass): - _valid_for_check(cls) res = cls.__extra__.__subclasshook__(subclass) if res is not NotImplemented: return res @@ -887,7 +880,6 @@ def __extrahook__(subclass): else: # For non-ABC extras we'll just call issubclass(). def __extrahook__(subclass): - _valid_for_check(cls) if cls.__extra__ and issubclass(subclass, cls.__extra__): return True return NotImplemented @@ -974,6 +966,7 @@ def __new__(cls, name, bases, namespace, # remove bare Generic from bases if there are other generic bases if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): bases = tuple(b for b in bases if b is not Generic) + namespace.update({'__origin__': origin, '__extra__': extra}) self = super().__new__(cls, name, bases, namespace, _root=True) self.__parameters__ = tvars @@ -982,8 +975,6 @@ def __new__(cls, name, bases, namespace, self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in args) if args else None - self.__origin__ = origin - self.__extra__ = extra # Speed hack (https://github.com/python/typing/issues/196). self.__next_in_mro__ = _next_in_mro(self) # Preserve base classes on subclassing (__bases__ are type erased now). @@ -994,20 +985,56 @@ def __new__(cls, name, bases, namespace, # with issubclass() and isinstance() in the same way as their # collections.abc counterparts (e.g., isinstance([], Iterable)). if ( - # allow overriding '__subclasshook__' not in namespace and extra or - hasattr(self.__subclasshook__, '__name__') and - self.__subclasshook__.__name__ == '__extrahook__' + # allow overriding + getattr(self.__subclasshook__, '__name__', '') == '__extrahook__' ): self.__subclasshook__ = _make_subclasshook(self) if isinstance(extra, abc.ABCMeta): self._abc_registry = extra._abc_registry + self._abc_cache = extra._abc_cache + elif origin is not None: + self._abc_registry = origin._abc_registry + self._abc_cache = origin._abc_cache if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. self.__qualname__ = origin.__qualname__ - self.__tree_hash__ = hash(self._subs_tree()) if origin else hash((self.__name__,)) + self.__tree_hash__ = (hash(self._subs_tree()) if origin else + super(GenericMeta, self).__hash__()) return self + # _abc_negative_cache and _abc_negative_cache_version + # realised as descriptors, since GenClass[t1, t2, ...] always + # share subclass info with GenClass. + # This is an important memory optimization. + @property + def _abc_negative_cache(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache + return _gorg(self)._abc_generic_negative_cache + + @_abc_negative_cache.setter + def _abc_negative_cache(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + else: + self._abc_generic_negative_cache = value + + @property + def _abc_negative_cache_version(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache_version + return _gorg(self)._abc_generic_negative_cache_version + + @_abc_negative_cache_version.setter + def _abc_negative_cache_version(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + else: + self._abc_generic_negative_cache_version = value + def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: _get_type_vars(self.__parameters__, tvars) @@ -1095,8 +1122,10 @@ def __getitem__(self, params): _check_generic(self, params) tvars = _type_vars(params) args = params + + prepend = (self,) if self.__origin__ is None else () return self.__class__(self.__name__, - self.__bases__, + prepend + self.__bases__, _no_slots_copy(self.__dict__), tvars=tvars, args=args, @@ -1104,6 +1133,17 @@ def __getitem__(self, params): extra=self.__extra__, orig_bases=self.__orig_bases__) + def __subclasscheck__(self, cls): + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if self is Generic: + raise TypeError("Class %r cannot be used with class " + "or instance checks" % self) + return super().__subclasscheck__(cls) + def __instancecheck__(self, instance): # Since we extend ABC.__subclasscheck__ and # ABC.__instancecheck__ inlines the cache checking done by the @@ -1118,6 +1158,16 @@ def __copy__(self): self.__parameters__, self.__args__, self.__origin__, self.__extra__, self.__orig_bases__) + def __setattr__(self, attr, value): + # We consider all the subscripted genrics as proxies for original class + if ( + attr.startswith('__') and attr.endswith('__') or + attr.startswith('_abc_') + ): + super(GenericMeta, self).__setattr__(attr, value) + else: + super(GenericMeta, _gorg(self)).__setattr__(attr, value) + # Prevent checks for Generic to crash when defining Generic. Generic = None @@ -1398,6 +1448,11 @@ def _get_defaults(func): return res +_allowed_types = (types.FunctionType, types.BuiltinFunctionType, + types.MethodType, types.ModuleType, + SlotWrapperType, MethodWrapperType, MethodDescriptorType) + + def get_type_hints(obj, globalns=None, localns=None): """Return type hints for an object. @@ -1452,12 +1507,7 @@ def get_type_hints(obj, globalns=None, localns=None): hints = getattr(obj, '__annotations__', None) if hints is None: # Return empty annotations for something that _could_ have them. - if ( - isinstance(obj, types.FunctionType) or - isinstance(obj, types.BuiltinFunctionType) or - isinstance(obj, types.MethodType) or - isinstance(obj, types.ModuleType) - ): + if isinstance(obj, _allowed_types): return {} else: raise TypeError('{!r} is not a module, class, method, ' @@ -1824,8 +1874,7 @@ class Deque(collections.deque, MutableSequence[T], extra=collections.deque): def __new__(cls, *args, **kwds): if _geqv(cls, Deque): - raise TypeError("Type Deque cannot be instantiated; " - "use deque() instead") + return collections.deque(*args, **kwds) return _generic_new(collections.deque, cls, *args, **kwds) @@ -1894,11 +1943,35 @@ class DefaultDict(collections.defaultdict, MutableMapping[KT, VT], def __new__(cls, *args, **kwds): if _geqv(cls, DefaultDict): - raise TypeError("Type DefaultDict cannot be instantiated; " - "use collections.defaultdict() instead") + return collections.defaultdict(*args, **kwds) return _generic_new(collections.defaultdict, cls, *args, **kwds) +class Counter(collections.Counter, Dict[T, int], extra=collections.Counter): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if _geqv(cls, Counter): + return collections.Counter(*args, **kwds) + return _generic_new(collections.Counter, cls, *args, **kwds) + + +if hasattr(collections, 'ChainMap'): + # ChainMap only exists in 3.3+ + __all__.append('ChainMap') + + class ChainMap(collections.ChainMap, MutableMapping[KT, VT], + extra=collections.ChainMap): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if _geqv(cls, ChainMap): + return collections.ChainMap(*args, **kwds) + return _generic_new(collections.ChainMap, cls, *args, **kwds) + + # Determine what base class to use for Generator. if hasattr(collections_abc, 'Generator'): # Sufficiently recent versions of 3.5 have a Generator ABC. @@ -1975,6 +2048,13 @@ def _make_nmtuple(name, types): _PY36 = sys.version_info[:2] >= (3, 6) +# attributes prohibited to set in NamedTuple class syntax +_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', + '_fields', '_field_defaults', '_field_types', + '_make', '_replace', '_asdict') + +_special = ('__module__', '__name__', '__qualname__', '__annotations__') + class NamedTupleMeta(type): @@ -2002,7 +2082,9 @@ def __new__(cls, typename, bases, ns): nm_tpl._field_defaults = defaults_dict # update from user namespace without overriding special namedtuple attributes for key in ns: - if not hasattr(nm_tpl, key): + if key in _prohibited: + raise AttributeError("Cannot overwrite NamedTuple attribute " + key) + elif key not in _special and key not in nm_tpl._fields: setattr(nm_tpl, key, ns[key]) return nm_tpl diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index b6b38362341f16d..5f97728de28c595 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -815,8 +815,8 @@ def _error_message(): def assert_called_once_with(_mock_self, *args, **kwargs): - """assert that the mock was called exactly once and with the specified - arguments.""" + """assert that the mock was called exactly once and that that call was + with the specified arguments.""" self = _mock_self if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times." % diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py index d5f9e7c1d692044..d2202a7b4132e99 100644 --- a/Lib/unittest/test/testmock/testhelpers.py +++ b/Lib/unittest/test/testmock/testhelpers.py @@ -307,18 +307,9 @@ def test_two_args_call(self): self.assertEqual(args, other_args) def test_call_with_name(self): - self.assertEqual( - 'foo', - _Call((), 'foo')[0], - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), ), )[0] - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), {'hello': 'world'}), )[0] - ) + self.assertEqual(_Call((), 'foo')[0], 'foo') + self.assertEqual(_Call((('bar', 'barz'),),)[0], '') + self.assertEqual(_Call((('bar', 'barz'), {'hello': 'world'}),)[0], '') class SpecSignatureTest(unittest.TestCase): diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index c5b675d16188b85..851515bc307d625 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -16,10 +16,6 @@ __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] -# do these error classes make sense? -# make sure all of the OSError stuff is overridden. we just want to be -# subtypes. - class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of # the implementation. need to override __init__ and __str__. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 958767a08d7e77e..32de25b374da930 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -612,6 +612,7 @@ def unquote(string, encoding='utf-8', errors='replace'): append(bits[i + 1]) return ''.join(res) + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. @@ -633,6 +634,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. + + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, @@ -644,28 +647,29 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result[name] = [value] return parsed_result + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. - Arguments: + Arguments: - qs: percent-encoded query string to be parsed + qs: percent-encoded query string to be parsed - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. + keep_blank_values: flag indicating whether blank values in + percent-encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. + encoding and errors: specify how to decode percent-encoded sequences + into Unicode characters, as accepted by the bytes.decode() method. - Returns a list, as G-d intended. + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index b6690c3d0e4bf5a..96921440edea11a 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1488,7 +1488,6 @@ def open_local_file(self, req): origurl = 'file://' + filename return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - # users shouldn't expect OSErrors coming from urlopen() raise URLError(exp) raise URLError('file not on local host') @@ -1658,14 +1657,10 @@ def pathname2url(pathname): of the 'file' scheme; not recommended for general use.""" return quote(pathname) -# This really consists of two pieces: -# (1) a class which handles opening of all sorts of URLs -# (plus assorted utilities etc.) -# (2) a set of functions for parsing URLs -# XXX Should these be separated out into different modules? - ftpcache = {} + + class URLopener: """Class to open URLs. This is a class rather than just a subroutine because we may need diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 849bfddd84816f2..6faa2d6f8fa3192 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -386,31 +386,36 @@ def _dispatch(self, method, params): not be called. """ - func = None try: - # check to see if a matching function has been registered + # call the matching registered function func = self.funcs[method] except KeyError: - if self.instance is not None: - # check for a _dispatch method - if hasattr(self.instance, '_dispatch'): - return self.instance._dispatch(method, params) - else: - # call instance method directly - try: - func = resolve_dotted_attribute( - self.instance, - method, - self.allow_dotted_names - ) - except AttributeError: - pass - - if func is not None: - return func(*params) + pass else: + if func is not None: + return func(*params) raise Exception('method "%s" is not supported' % method) + if self.instance is not None: + if hasattr(self.instance, '_dispatch'): + # call the `_dispatch` method on the instance + return self.instance._dispatch(method, params) + + # call the instance's method directly + try: + func = resolve_dotted_attribute( + self.instance, + method, + self.allow_dotted_names + ) + except AttributeError: + pass + else: + if func is not None: + return func(*params) + + raise Exception('method "%s" is not supported' % method) + class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): """Simple XML-RPC request handler class. diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 7f2b43ce1cc848b..d7f5beba996e6e6 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -479,6 +479,8 @@ def from_file(cls, filename, arcname=None): this will be the same as filename, but without a drive letter and with leading path separators removed). """ + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) @@ -1070,6 +1072,8 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): self._comment = b'' # Check if we were passed a file-like object + if isinstance(file, os.PathLike): + file = os.fspath(file) if isinstance(file, str): # No, it's a filename self._filePassed = 0 @@ -1470,11 +1474,10 @@ def extract(self, member, path=None, pwd=None): as possible. `member' may be a filename or a ZipInfo object. You can specify a different directory using `path'. """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - if path is None: path = os.getcwd() + else: + path = os.fspath(path) return self._extract_member(member, path, pwd) @@ -1487,8 +1490,13 @@ def extractall(self, path=None, members=None, pwd=None): if members is None: members = self.namelist() + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + for zipinfo in members: - self.extract(zipinfo, path, pwd) + self._extract_member(zipinfo, path, pwd) @classmethod def _sanitize_windows_name(cls, arcname, pathsep): @@ -1509,6 +1517,9 @@ def _extract_member(self, member, targetpath, pwd): """Extract the ZipInfo object 'member' to a physical file on the path targetpath. """ + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + # build the destination pathname, replacing # forward slashes to platform specific separators. arcname = member.filename.replace('/', os.path.sep) @@ -1801,6 +1812,7 @@ def writepy(self, pathname, basename="", filterfunc=None): If filterfunc(pathname) is given, it is called with every argument. When it is False, the file or directory is skipped. """ + pathname = os.fspath(pathname) if filterfunc and not filterfunc(pathname): if self.debug: label = 'path' if os.path.isdir(pathname) else 'file' diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 8dfd092d08b5123..f59ae34df2549a5 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -13,7 +13,7 @@ Python 2.6. In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed version of hg and a third-party version of +requires an installed version of git and a third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to @@ -23,7 +23,7 @@ 32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates), hg, and for the documentation +(2.5 is needed for Python parser updates), git, and for the documentation build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). Usage: see USAGE variable in the script. @@ -213,9 +213,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2j", - url="https://www.openssl.org/source/openssl-1.0.2j.tar.gz", - checksum='96322138f0b69e61b7212bc53d5e912b', + name="OpenSSL 1.0.2k", + url="https://www.openssl.org/source/openssl-1.0.2k.tar.gz", + checksum='f965fc0bf01bf882b31314b61391ae65', patches=[ "openssl_sdk_makedepend.patch", ], @@ -635,9 +635,9 @@ def checkEnvironment(): base_path = base_path + ':' + OLD_DEVELOPER_TOOLS os.environ['PATH'] = base_path print("Setting default PATH: %s"%(os.environ['PATH'])) - # Ensure ws have access to hg and to sphinx-build. + # Ensure ws have access to git and to sphinx-build. # You may have to create links in /usr/bin for them. - runCommand('hg --version') + runCommand('git --version') runCommand('sphinx-build --version') def parseOptions(args=None): @@ -1142,8 +1142,9 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) - print("Running make touch") - runCommand("make touch") + # bpo-29550: avoid using make touch until it is fixed for git + # print("Running make touch") + # runCommand("make touch") print("Running make") runCommand("make") diff --git a/Mac/BuildScript/openssl_sdk_makedepend.patch b/Mac/BuildScript/openssl_sdk_makedepend.patch index e22d67e4e45143b..0caac0a64c1ec6c 100644 --- a/Mac/BuildScript/openssl_sdk_makedepend.patch +++ b/Mac/BuildScript/openssl_sdk_makedepend.patch @@ -1,6 +1,6 @@ # HG changeset patch # -# using openssl 1.0.2j +# using openssl 1.0.2k # # - support building with an OS X SDK diff --git a/Makefile.pre.in b/Makefile.pre.in index 93aae9124726bd5..8f27d7320c1d69d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,9 +41,9 @@ RANLIB= @RANLIB@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ -HGVERSION= @HGVERSION@ -HGTAG= @HGTAG@ -HGBRANCH= @HGBRANCH@ +GITVERSION= @GITVERSION@ +GITTAG= @GITTAG@ +GITBRANCH= @GITBRANCH@ PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ @@ -740,9 +740,9 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ $(MODOBJS) \ $(srcdir)/Modules/getbuildinfo.c $(CC) -c $(PY_CORE_CFLAGS) \ - -DHGVERSION="\"`LC_ALL=C $(HGVERSION)`\"" \ - -DHGTAG="\"`LC_ALL=C $(HGTAG)`\"" \ - -DHGBRANCH="\"`LC_ALL=C $(HGBRANCH)`\"" \ + -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ + -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ -o $@ $(srcdir)/Modules/getbuildinfo.c Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile diff --git a/Misc/ACKS b/Misc/ACKS index a6b3df12ff230ca..72070e19f65701b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -269,6 +269,7 @@ Albert Chin-A-Young Adal Chiriliuc Matt Chisholm Lita Cho +Sayan Chowdhury Anders Chrigström Tom Christiansen Renee Chu @@ -347,6 +348,7 @@ A. Jesse Jiryu Davis Merlijn van Deen John DeGood Ned Deily +Jim DeLaHunt Vincent Delft Arnaud Delobelle Konrad Delong @@ -768,11 +770,13 @@ Lawrence Kesteloot Vivek Khera Dhiru Kholia Akshit Khurana +Sanyam Khurana Mads Kiilerich Jason Killen Jan Kim Taek Joo Kim Sam Kimbrel +Tomohiko Kinebuchi James King W. Trevor King Paul Kippes @@ -1369,6 +1373,7 @@ Federico Schwindt Barry Scott Steven Scott Nick Seidenman +Michael Seifert Žiga Seilnacht Yury Selivanov Fred Sells @@ -1706,6 +1711,7 @@ Artur Zaprzala Mike Zarnstorff Yury V. Zaytsev Siebren van der Zee +Christophe Zeitouny Nickolai Zeldovich Yuxiao Zeng Uwe Zessin diff --git a/Misc/NEWS b/Misc/NEWS index d79ff2450defaa7..e623457cdd18ce0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,7 +2,7 @@ Python News +++++++++++ -What's New in Python 3.6.1 release candidate 1? +What's New in Python 3.6.2 release candidate 1? =============================================== *Release date: XXXX-XX-XX* @@ -10,6 +10,152 @@ What's New in Python 3.6.1 release candidate 1? Core and Builtins ----------------- +- bpo-29949: Fix memory usage regression of set and frozenset object. + +- bpo-29935: Fixed error messages in the index() method of tuple, list and deque + when pass indices of wrong type. + +- bpo-29859: Show correct error messages when any of the pthread_* calls in + thread_pthread.h fails. + +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + +- bpo-29600: Fix wrapping coroutine return values in StopIteration. + +- bpo-28856: Fix an oversight that %b format for bytes should support objects + follow the buffer protocol. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +Library +------- + +- bpo-29694: Fixed race condition in pathlib mkdir with flags + parents=True. Patch by Armin Rigo. + +- bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in + contextlib.contextmanager. + Patch by Siddharth Velankar. + +- bpo-29998: Pickling and copying ImportError now preserves name and path + attributes. + +- bpo-29953: Fixed memory leaks in the replace() method of datetime and time + objects when pass out of bound fold argument. + +- bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering + long runs of empty iterables. + +- bpo-27863: Fixed multiple crashes in ElementTree caused by race conditions + and wrong types. + +- bpo-28699: Fixed a bug in pools in multiprocessing.pool that raising an + exception at the very first of an iterable may swallow the exception or + make the program hang. Patch by Davin Potts and Xiang Zhang. + +- bpo-25803: Avoid incorrect errors raised by Path.mkdir(exist_ok=True) + when the OS gives priority to errors such as EACCES over EEXIST. + +- bpo-29861: Release references to tasks, their arguments and their results + as soon as they are finished in multiprocessing.Pool. + +- bpo-29884: faulthandler: Restore the old sigaltstack during teardown. + Patch by Christophe Zeitouny. + +- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. + +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + +Build +----- + +- bpo-29643: Fix ``--enable-optimization`` didn't work. + +Documentation +------------- + +- bpo-26985: Add missing info of code object in inspect documentation. + + +What's New in Python 3.6.1? +=========================== + +*Release date: 2017-03-21* + +Core and Builtins +----------------- + +- bpo-29723: The ``sys.path[0]`` initialization change for bpo-29139 caused a + regression by revealing an inconsistency in how sys.path is initialized when + executing ``__main__`` from a zipfile, directory, or other import location. + The interpreter now consistently avoids ever adding the import location's + parent directory to ``sys.path``, and ensures no other ``sys.path`` entries + are inadvertently modified when inserting the import location named on the + command line. + +Build +----- + +- bpo-27593: fix format of git information used in sys.version + +- Fix incompatible comment in python.h + + +What's New in Python 3.6.1 release candidate 1 +============================================== + +*Release date: 2017-03-04* + +Core and Builtins +----------------- + +- bpo-28893: Set correct __cause__ for errors about invalid awaitables + returned from __aiter__ and __anext__. + +- bpo-29683: Fixes to memory allocation in _PyCode_SetExtra. Patch by + Brian Coleman. + +- bpo-29684: Fix minor regression of PyEval_CallObjectWithKeywords. + It should raise TypeError when kwargs is not a dict. But it might + cause segv when args=NULL and kwargs is not a dict. + +- Issue #28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + +- bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. + Patch by Matthieu Dartiailh. + +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + +- bpo-29347: Fixed possibly dereferencing undefined pointers + when creating weakref objects. + +- bpo-29438: Fixed use-after-free problem in key sharing dict. + - Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - Issue #29337: Fixed possible BytesWarning when compare the code objects. @@ -57,6 +203,39 @@ Extension Modules Library ------- +- bpo-29623: Allow use of path-like object as a single argument in + ConfigParser.read(). Patch by David Ellis. + +- bpo-28963: Fix out of bound iteration in asyncio.Future.remove_done_callback + implemented in C. + +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes + before all pipes are closed. + +- bpo-29271: Fix Task.current_task and Task.all_tasks implemented in C + to accept None argument as their pure Python implementation. + +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + +- bpo-28624: Add a test that checks that cwd parameter of Popen() accepts + PathLike objects. Patch by Sayan Chowdhury. + +- bpo-28518: Start a transaction implicitly before a DML statement. + Patch by Aviv Palivoda. + +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + +- Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, + improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, + Manuel Krebber, and Łukasz Langa. + - Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python 3.6.0: check minimum and maximum years. @@ -126,7 +305,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #29055: Neaten-up empty population error on random.choice() by suppressing the upstream exception. @@ -154,6 +333,8 @@ Library Windows ------- +- bpo-29579: Removes readme.txt from the installer + - Issue #29326: Ignores blank lines in ._pth files (Patch by Alexey Izbyshev) - Issue #28164: Correctly handle special console filenames (patch by Eryk Sun) @@ -186,6 +367,11 @@ C API Documentation ------------- +- bpo-28929: Link the documentation to its source file on GitHub. + +- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to + aiosmtpd, a third-party asyncio-based replacement. + - Issue #26355: Add canonical header link on each page to corresponding major version of the documentation. Patch by Matthias Bussonnier. @@ -194,6 +380,15 @@ Documentation Tests ----- +- bpo-28087: Skip test_asyncore and test_eintr poll failures on macOS. + Skip some tests of select.poll when running on macOS due to unresolved + issues with the underlying system poll function on some macOS versions. + +- Issue #29571: to match the behaviour of the ``re.LOCALE`` flag, + test_re.test_locale_flag now uses ``locale.getpreferredencoding(False)`` to + determine the candidate encoding for the test regex (allowing it to correctly + skip the test when the default locale encoding is a multi-byte encoding) + - Issue #28950: Disallow -j0 to be combined with -T/-l in regrtest command line arguments. @@ -206,6 +401,12 @@ Tests Build ----- +- bpo-27593: sys.version and the platform module python_build(), + python_branch(), and python_revision() functions now use + git information rather than hg when building from a repo. + +- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. + - Issue #26851: Set Android compilation and link flags. - Issue #28768: Fix implicit declaration of function _setmode. Patch by @@ -872,7 +1073,7 @@ Core and Builtins alpha releases, where backslashes are allowed anywhere in an f-string. Also, require that expressions inside f-strings be enclosed within literal braces, and not escapes like - f'\x7b"hi"\x7d'. + ``f'\x7b"hi"\x7d'``. - Issue #28046: Remove platform-specific directories from sys.path. @@ -1091,7 +1292,7 @@ Library - Issue #24277: The new email API is no longer provisional, and the docs have been reorganized and rewritten to emphasize the new API. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -1110,9 +1311,9 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27331: The email.mime classes now all accept an optional policy keyword. +- Issue #27331: The email.mime classes now all accept an optional policy keyword. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #16113: Add SHA-3 and SHAKE support to hashlib module. @@ -2220,7 +2421,7 @@ Core and Builtins in ``def f(): 1.0``. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and + (``*``) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. - Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used @@ -2778,7 +2979,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -2914,7 +3115,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -3549,7 +3750,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #28871: Fixed a crash when deallocate deep ElementTree. @@ -3672,7 +3873,7 @@ Library - Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). -- Issue #19003:m email.generator now replaces only \r and/or \n line +- Issue #19003:m email.generator now replaces only ``\r`` and/or ``\n`` line endings, per the RFC, instead of all unicode line endings. - Issue #28019: itertools.count() no longer rounds non-integer step in range @@ -3695,7 +3896,7 @@ Library - Issue #27445: Don't pass str(_charset) to MIMEText.set_payload(). Patch by Claude Paroz. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -3710,7 +3911,7 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #27691: Fix ssl module's parsing of GEN_RID subject alternative name fields in X.509 certs. @@ -4161,7 +4362,7 @@ Core and Builtins cookie names. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and + (``*``) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. - Issue #27138: Fix the doc comment for FileFinder.find_spec(). @@ -4541,7 +4742,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -4951,7 +5152,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -6494,7 +6695,7 @@ Library - Issue #23521: Corrected pure python implementation of timedelta division. - * Eliminated OverflowError from timedelta * float for some floats; + * Eliminated OverflowError from ``timedelta * float`` for some floats; * Corrected rounding in timedlta true division. - Issue #21619: Popen objects no longer leave a zombie after exit in the with @@ -7293,7 +7494,7 @@ Library character instead of truncating it. Based on patch by Victor Stinner. - Issue #13968: The glob module now supports recursive search in - subdirectories using the "**" pattern. + subdirectories using the ``**`` pattern. - Issue #21951: Fixed a crash in Tkinter on AIX when called Tcl command with empty string or tuple argument. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index fff90468a5927c1..a77ff9689e4705e 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -521,7 +521,7 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) return NULL; } - for (i = 0; i < len; i++) { + for (i = 0; i < PyList_GET_SIZE(self->fut_callbacks); i++) { int ret; PyObject *item = PyList_GET_ITEM(self->fut_callbacks, i); @@ -1413,7 +1413,7 @@ TaskObj_get_fut_waiter(TaskObj *task) @classmethod _asyncio.Task.current_task - loop: 'O' = NULL + loop: 'O' = None Return the currently running task in an event loop or None. @@ -1424,12 +1424,12 @@ None is returned when called not in the context of a Task. static PyObject * _asyncio_Task_current_task_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=99fbe7332c516e03 input=cd784537f02cf833]*/ +/*[clinic end generated code: output=99fbe7332c516e03 input=a0d6cdf2e3b243e1]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } @@ -1500,7 +1500,7 @@ task_all_tasks(PyObject *loop) @classmethod _asyncio.Task.all_tasks - loop: 'O' = NULL + loop: 'O' = None Return a set of all tasks for an event loop. @@ -1509,12 +1509,12 @@ By default all tasks for the current event loop are returned. static PyObject * _asyncio_Task_all_tasks_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=11f9b20749ccca5d input=cd64aa5f88bd5c49]*/ +/*[clinic end generated code: output=11f9b20749ccca5d input=c6f5b53bd487488f]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index e6111c64e77de2d..30157701d70b47b 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1051,8 +1051,8 @@ deque_index(dequeobject *deque, PyObject *args) int cmp; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(deque); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index df3aedec6eeab54..12234e254e1a7a0 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4273,11 +4273,10 @@ Array_subscript(PyObject *myself, PyObject *item) PyObject *np; Py_ssize_t start, stop, step, slicelen, cur, i; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for array object instances */ @@ -4414,11 +4413,10 @@ Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, otherlen, i, cur; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); if ((step < 0 && start < stop) || (step > 0 && start > stop)) stop = start; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 629ddf66bc42122..6119ecdaf90dc6e 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -44,6 +44,19 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) func(in); } +/* + * See issue 29565. Update a structure passed by value; + * the caller should not see any change. + */ + +EXPORT(void) +_testfunc_large_struct_update_value(Test in) +{ + ((volatile Test *)&in)->first = 0x0badf00d; + ((volatile Test *)&in)->second = 0x0badf00d; + ((volatile Test *)&in)->third = 0x0badf00d; +} + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", diff --git a/Modules/_ctypes/libffi_msvc/ffi.c b/Modules/_ctypes/libffi_msvc/ffi.c index 1d82929f5302205..91a27dce3f25753 100644 --- a/Modules/_ctypes/libffi_msvc/ffi.c +++ b/Modules/_ctypes/libffi_msvc/ffi.c @@ -239,6 +239,16 @@ ffi_call(/*@dependent@*/ ffi_cif *cif, break; #else case FFI_SYSV: + /* If a single argument takes more than 8 bytes, + then a copy is passed by reference. */ + for (unsigned i = 0; i < cif->nargs; i++) { + size_t z = cif->arg_types[i]->size; + if (z > 8) { + void *temp = alloca(z); + memcpy(temp, avalue[i], z); + avalue[i] = temp; + } + } /*@-usedef@*/ return ffi_call_AMD64(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, fn); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 3bf2ca7ce15392d..d88d06e78220ece 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -2082,7 +2082,7 @@ static PyGetSetDef PyCursesWindow_getsets[] = { PyTypeObject PyCursesWindow_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_curses.curses window", /*tp_name*/ + "_curses.window", /*tp_name*/ sizeof(PyCursesWindowObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c784d0f4a909999..c2ad9a203e96527 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3926,16 +3926,16 @@ time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) time_kws, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = time_new(Py_TYPE(self), tuple, NULL); if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } TIME_SET_FOLD(clone, fold); } Py_DECREF(tuple); @@ -5019,17 +5019,16 @@ datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) &y, &m, &d, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } DATE_SET_FOLD(clone, fold); } Py_DECREF(tuple); diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 2cda98e61127d6d..bef702ebe69c933 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -131,7 +131,7 @@ elementtree_free(void *m) LOCAL(PyObject*) list_join(PyObject* list) { - /* join list elements (destroying the list in the process) */ + /* join list elements */ PyObject* joiner; PyObject* result; @@ -140,8 +140,6 @@ list_join(PyObject* list) return NULL; result = PyUnicode_Join(joiner, list); Py_DECREF(joiner); - if (result) - Py_DECREF(list); return result; } @@ -508,15 +506,17 @@ element_get_text(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->text; + PyObject *res = self->text; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->text = res; + self->text = tmp; + Py_DECREF(res); + res = tmp; } } @@ -528,15 +528,17 @@ element_get_tail(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->tail; + PyObject *res = self->tail; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->tail = res; + self->tail = tmp; + Py_DECREF(res); + res = tmp; } } @@ -1710,11 +1712,11 @@ element_subscr(PyObject* self_, PyObject* item) if (!self->extra) return PyList_New(0); - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (slicelen <= 0) return PyList_New(0); @@ -1766,11 +1768,11 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) return -1; } - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (value == NULL) { /* Delete slice */ @@ -2147,6 +2149,12 @@ elementiter_next(ElementIterObject *it) continue; } + if (!PyObject_TypeCheck(extra->children[child_index], &Element_Type)) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute 'iter'", + Py_TYPE(extra->children[child_index])->tp_name); + return NULL; + } elem = (ElementObject *)extra->children[child_index]; item->child_index++; Py_INCREF(elem); @@ -2396,40 +2404,51 @@ treebuilder_dealloc(TreeBuilderObject *self) /* helpers for handling of arbitrary element-like objects */ static int -treebuilder_set_element_text_or_tail(PyObject *element, PyObject *data, +treebuilder_set_element_text_or_tail(PyObject *element, PyObject **data, PyObject **dest, _Py_Identifier *name) { if (Element_CheckExact(element)) { - Py_DECREF(JOIN_OBJ(*dest)); - *dest = JOIN_SET(data, PyList_CheckExact(data)); + PyObject *tmp = JOIN_OBJ(*dest); + *dest = JOIN_SET(*data, PyList_CheckExact(*data)); + *data = NULL; + Py_DECREF(tmp); return 0; } else { - PyObject *joined = list_join(data); + PyObject *joined = list_join(*data); int r; if (joined == NULL) return -1; r = _PyObject_SetAttrId(element, name, joined); Py_DECREF(joined); - return r; + if (r < 0) + return -1; + Py_CLEAR(*data); + return 0; } } -/* These two functions steal a reference to data */ -static int -treebuilder_set_element_text(PyObject *element, PyObject *data) +LOCAL(int) +treebuilder_flush_data(TreeBuilderObject* self) { - _Py_IDENTIFIER(text); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->text, &PyId_text); -} + PyObject *element = self->last; -static int -treebuilder_set_element_tail(PyObject *element, PyObject *data) -{ - _Py_IDENTIFIER(tail); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->tail, &PyId_tail); + if (!self->data) { + return 0; + } + + if (self->this == element) { + _Py_IDENTIFIER(text); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->text, &PyId_text); + } + else { + _Py_IDENTIFIER(tail); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->tail, &PyId_tail); + } } static int @@ -2479,16 +2498,8 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, PyObject* this; elementtreestate *st = ET_STATE_GLOBAL; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } - else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (!self->element_factory || self->element_factory == Py_None) { @@ -2591,15 +2602,8 @@ treebuilder_handle_end(TreeBuilderObject* self, PyObject* tag) { PyObject* item; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (self->index == 0) { diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index f785a7260e6a2e0..1bcf16a7e00f539 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -88,10 +88,13 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (kw == NULL) { pto->kw = PyDict_New(); } - else { + else if (Py_REFCNT(kw) == 1) { Py_INCREF(kw); pto->kw = kw; } + else { + pto->kw = PyDict_Copy(kw); + } } else { pto->kw = PyDict_Copy(pkw); @@ -247,8 +250,11 @@ partial_repr(partialobject *pto) /* Pack keyword arguments */ assert (PyDict_Check(pto->kw)); for (i = 0; PyDict_Next(pto->kw, &i, &key, &value);) { - Py_SETREF(arglist, PyUnicode_FromFormat("%U, %U=%R", arglist, + /* Prevent key.__str__ from deleting the value. */ + Py_INCREF(value); + Py_SETREF(arglist, PyUnicode_FromFormat("%U, %S=%R", arglist, key, value)); + Py_DECREF(value); if (arglist == NULL) goto done; } diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index cbe7425eaefa655..efc7d05b7d01b9a 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1416,8 +1416,18 @@ buffered_repr(buffered *self) res = PyUnicode_FromFormat("<%s>", Py_TYPE(self)->tp_name); } else { - res = PyUnicode_FromFormat("<%s name=%R>", - Py_TYPE(self)->tp_name, nameobj); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat("<%s name=%R>", + Py_TYPE(self)->tp_name, nameobj); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 7f3bcab9625ef51..833ea8e7b50894c 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -1082,9 +1082,19 @@ fileio_repr(fileio *self) self->fd, mode_string(self), self->closefd ? "True" : "False"); } else { - res = PyUnicode_FromFormat( - "<_io.FileIO name=%R mode='%s' closefd=%s>", - nameobj, mode_string(self), self->closefd ? "True" : "False"); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat( + "<_io.FileIO name=%R mode='%s' closefd=%s>", + nameobj, mode_string(self), self->closefd ? "True" : "False"); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 4df55626d4bc239..bc8d11efa5258cc 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2483,6 +2483,7 @@ static PyObject * textiowrapper_repr(textio *self) { PyObject *nameobj, *modeobj, *res, *s; + int status; CHECK_INITIALIZED(self); @@ -2490,6 +2491,15 @@ textiowrapper_repr(textio *self) if (res == NULL) return NULL; + status = Py_ReprEnter((PyObject *)self); + if (status != 0) { + if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } + goto error; + } nameobj = _PyObject_GetAttrId((PyObject *) self, &PyId_name); if (nameobj == NULL) { if (PyErr_ExceptionMatches(PyExc_Exception)) @@ -2504,7 +2514,7 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } modeobj = _PyObject_GetAttrId((PyObject *) self, &PyId_mode); if (modeobj == NULL) { @@ -2520,14 +2530,21 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } s = PyUnicode_FromFormat("%U encoding=%R>", res, self->encoding); Py_DECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return s; -error: + + error: Py_XDECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return NULL; } diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0d3282db8f2810c..e5dd2c961663849 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -2,7 +2,7 @@ /* ------------------------------------------------------------------ The code in this module was based on a download from: - http://www.math.keio.ac.jp/~matumoto/MT2002/emt19937ar.html + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html It was modified in 2002 by Raymond Hettinger as follows: @@ -60,8 +60,8 @@ Any feedback is very welcome. - http://www.math.keio.ac.jp/matumoto/emt.html - email: matumoto@math.keio.ac.jp + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html + email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) */ /* ---------------------------------------------------------------*/ diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 39f7a6508c2aef1..8341fb8480172c9 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -511,10 +511,9 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* pysqlite_statement_reset(self->statement); pysqlite_statement_mark_dirty(self->statement); - /* For backwards compatibility reasons, do not start a transaction if a - DDL statement is encountered. If anybody wants transactional DDL, - they can issue a BEGIN statement manually. */ - if (self->connection->begin_statement && !sqlite3_stmt_readonly(self->statement->st) && !self->statement->is_ddl) { + /* We start a transaction implicitly before a DML statement. + SELECT is the only exception. See #9924. */ + if (self->connection->begin_statement && self->statement->is_dml) { if (sqlite3_get_autocommit(self->connection->db)) { result = _pysqlite_connection_begin(self->connection); if (!result) { @@ -609,7 +608,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* } } - if (!sqlite3_stmt_readonly(self->statement->st)) { + if (self->statement->is_dml) { self->rowcount += (long)sqlite3_changes(self->connection->db); } else { self->rowcount= -1L; diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 0df661b9c7ad315..087375be9b63d88 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -73,8 +73,9 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con Py_INCREF(sql); self->sql = sql; - /* determine if the statement is a DDL statement */ - self->is_ddl = 0; + /* Determine if the statement is a DML statement. + SELECT is the only exception. See #9924. */ + self->is_dml = 0; for (p = sql_cstr; *p != 0; p++) { switch (*p) { case ' ': @@ -84,9 +85,10 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con continue; } - self->is_ddl = (PyOS_strnicmp(p, "create ", 7) == 0) - || (PyOS_strnicmp(p, "drop ", 5) == 0) - || (PyOS_strnicmp(p, "reindex ", 8) == 0); + self->is_dml = (PyOS_strnicmp(p, "insert ", 7) == 0) + || (PyOS_strnicmp(p, "update ", 7) == 0) + || (PyOS_strnicmp(p, "delete ", 7) == 0) + || (PyOS_strnicmp(p, "replace ", 8) == 0); break; } diff --git a/Modules/_sqlite/statement.h b/Modules/_sqlite/statement.h index 6eef16857f7b859..8db10f6649ce806 100644 --- a/Modules/_sqlite/statement.h +++ b/Modules/_sqlite/statement.h @@ -38,7 +38,7 @@ typedef struct sqlite3_stmt* st; PyObject* sql; int in_use; - int is_ddl; + int is_dml; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Statement; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b1988570604db15..c0a7b8e1052c337 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2729,12 +2729,12 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) #endif -#ifndef OPENSSL_NO_ECDH +#if !defined(OPENSSL_NO_ECDH) && !defined(OPENSSL_VERSION_1_1) /* Allow automatic ECDH curve selection (on OpenSSL 1.0.2+), or use prime256v1 by default. This is Apache mod_ssl's initialization policy, so we should be safe. OpenSSL 1.1 has it enabled by default. */ -#if defined(SSL_CTX_set_ecdh_auto) && !defined(OPENSSL_VERSION_1_1) +#if defined(SSL_CTX_set_ecdh_auto) SSL_CTX_set_ecdh_auto(self->ctx, 1); #else { diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 4e1ce6851a5eb6e..6b8ab34d931cd3f 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1715,10 +1715,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { @@ -1935,9 +1935,10 @@ slice_indices(PyObject *self, PyObject *args) "first argument must be a slice object"); return NULL; } - if (PySlice_GetIndicesEx(key, len, &s[0], &s[1], &s[2], &s[3]) < 0) { + if (PySlice_Unpack(key, &s[0], &s[1], &s[2]) < 0) { return NULL; } + s[3] = PySlice_AdjustIndices(len, &s[0], &s[1], s[2]); ret = PyTuple_New(4); if (ret == NULL) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f09205f63cc5abc..b25f4434ffa35c4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3812,7 +3812,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (_PyTime_AsTimeval(t, &tv, round) < 0) return NULL; - seconds = PyLong_FromLong((long long)tv.tv_sec); + seconds = PyLong_FromLongLong(tv.tv_sec); if (seconds == NULL) return NULL; return Py_BuildValue("Nl", seconds, tv.tv_usec); diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2caa8ee5a8e8928..64e0f172fd7d821 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -331,35 +331,51 @@ II_getitem(arrayobject *ap, Py_ssize_t i) (unsigned long) ((unsigned int *)ap->ob_item)[i]); } +static PyObject * +get_int_unless_float(PyObject *v) +{ + if (PyFloat_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "array item must be integer"); + return NULL; + } + return (PyObject *)_PyLong_FromNbInt(v); +} + static int II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned int is less than minimum"); - return -1; + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long)y; - + return -1; } if (x > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, - "unsigned int is greater than maximum"); + "unsigned int is greater than maximum"); + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -390,31 +406,28 @@ static int LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) - return -1; - } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is less than minimum"); + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; } - x = (unsigned long)y; - + do_decref = 1; } - if (x > ULONG_MAX) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is greater than maximum"); + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -446,25 +459,28 @@ static int QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLongLong(v); - if (x == (unsigned long long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long long y; - if (!PyArg_Parse(v, "L;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long long is less than minimum"); - return -1; + x = PyLong_AsUnsignedLongLong(v); + if (x == (unsigned long long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long long)y; + return -1; } - if (i >= 0) ((unsigned long long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -2281,10 +2297,11 @@ array_subscr(arrayobject* self, PyObject* item) arrayobject* ar; int itemsize = self->ob_descr->itemsize; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return newarrayobject(&Arraytype, 0, self->ob_descr); @@ -2352,11 +2369,11 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) return (*self->ob_descr->setitem)(self, i, value); } else if (PySlice_Check(item)) { - if (PySlice_GetIndicesEx(item, - Py_SIZE(self), &start, &stop, - &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); } else { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 052d252331f6a7a..41f11f4cac5494c 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -278,7 +278,7 @@ _asyncio_Task_current_task(PyTypeObject *type, PyObject **args, Py_ssize_t nargs PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:current_task", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -310,7 +310,7 @@ _asyncio_Task_all_tasks(PyTypeObject *type, PyObject **args, Py_ssize_t nargs, P PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:all_tasks", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -517,4 +517,4 @@ _asyncio_Task__wakeup(TaskObj *self, PyObject **args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=8f036321bb083066 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=40ca6c9da517da73 input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1c1e4fb7d177600..2f8b624fd160d9d 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -124,6 +124,7 @@ static const size_t faulthandler_nsignals = \ #ifdef HAVE_SIGALTSTACK static stack_t stack; +static stack_t old_stack; #endif @@ -1310,7 +1311,7 @@ int _PyFaulthandler_Init(void) stack.ss_size = SIGSTKSZ; stack.ss_sp = PyMem_Malloc(stack.ss_size); if (stack.ss_sp != NULL) { - err = sigaltstack(&stack, NULL); + err = sigaltstack(&stack, &old_stack); if (err) { PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; @@ -1366,6 +1367,20 @@ void _PyFaulthandler_Fini(void) faulthandler_disable(); #ifdef HAVE_SIGALTSTACK if (stack.ss_sp != NULL) { + /* Fetch the current alt stack */ + stack_t current_stack; + if (sigaltstack(NULL, ¤t_stack) == 0) { + if (current_stack.ss_sp == stack.ss_sp) { + /* The current alt stack is the one that we installed. + It is safe to restore the old stack that we found when + we installed ours */ + sigaltstack(&old_stack, NULL); + } else { + /* Someone switched to a different alt stack and didn't + restore ours when they were done (if they're done). + There's not much we can do in this unlikely case */ + } + } PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; } diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 0971a64fccbadc9..5f941a26e1d54c5 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -21,47 +21,47 @@ #endif /* XXX Only unix build process has been tested */ -#ifndef HGVERSION -#define HGVERSION "" +#ifndef GITVERSION +#define GITVERSION "" #endif -#ifndef HGTAG -#define HGTAG "" +#ifndef GITTAG +#define GITTAG "" #endif -#ifndef HGBRANCH -#define HGBRANCH "" +#ifndef GITBRANCH +#define GITBRANCH "" #endif const char * Py_GetBuildInfo(void) { - static char buildinfo[50 + sizeof(HGVERSION) + - ((sizeof(HGTAG) > sizeof(HGBRANCH)) ? - sizeof(HGTAG) : sizeof(HGBRANCH))]; - const char *revision = _Py_hgversion(); + static char buildinfo[50 + sizeof(GITVERSION) + + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ? + sizeof(GITTAG) : sizeof(GITBRANCH))]; + const char *revision = _Py_gitversion(); const char *sep = *revision ? ":" : ""; - const char *hgid = _Py_hgidentifier(); - if (!(*hgid)) - hgid = "default"; + const char *gitid = _Py_gitidentifier(); + if (!(*gitid)) + gitid = "default"; PyOS_snprintf(buildinfo, sizeof(buildinfo), - "%s%s%s, %.20s, %.9s", hgid, sep, revision, + "%s%s%s, %.20s, %.9s", gitid, sep, revision, DATE, TIME); return buildinfo; } const char * -_Py_hgversion(void) +_Py_gitversion(void) { - return HGVERSION; + return GITVERSION; } const char * -_Py_hgidentifier(void) +_Py_gitidentifier(void) { - const char *hgtag, *hgid; - hgtag = HGTAG; - if ((*hgtag) && strcmp(hgtag, "tip") != 0) - hgid = hgtag; + const char *gittag, *gitid; + gittag = GITTAG; + if ((*gittag) && strcmp(gittag, "undefined") != 0) + gitid = gittag; else - hgid = HGBRANCH; - return hgid; + gitid = GITBRANCH; + return gitid; } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6bf04cbee31c35c..8e22ec9c4b7cc19 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1864,33 +1864,37 @@ chain_next(chainobject *lz) { PyObject *item; - if (lz->source == NULL) - return NULL; /* already stopped */ - - if (lz->active == NULL) { - PyObject *iterable = PyIter_Next(lz->source); - if (iterable == NULL) { - Py_CLEAR(lz->source); - return NULL; /* no more input sources */ - } - lz->active = PyObject_GetIter(iterable); - Py_DECREF(iterable); + /* lz->source is the iterator of iterables. If it's NULL, we've already + * consumed them all. lz->active is the current iterator. If it's NULL, + * we should grab a new one from lz->source. */ + while (lz->source != NULL) { if (lz->active == NULL) { - Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + PyObject *iterable = PyIter_Next(lz->source); + if (iterable == NULL) { + Py_CLEAR(lz->source); + return NULL; /* no more input sources */ + } + lz->active = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (lz->active == NULL) { + Py_CLEAR(lz->source); + return NULL; /* input not iterable */ + } } + item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); + if (item != NULL) + return item; + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else + return NULL; /* input raised an exception */ + } + /* lz->active is consumed, try with the next iterable. */ + Py_CLEAR(lz->active); } - item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); - if (item != NULL) - return item; - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - return NULL; /* input raised an exception */ - } - Py_CLEAR(lz->active); - return chain_next(lz); /* recurse and use next active */ + /* Everything had been consumed already. */ + return NULL; } static PyObject * diff --git a/Modules/main.c b/Modules/main.c index 2e6a60b1673f886..475a2fdc36db9bb 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,7 +53,7 @@ static const char usage_1[] = "\ Options and arguments (and corresponding environment variables):\n\ -b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ and comparing bytes/bytearray with str. (-bb: issue errors)\n\ --B : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x\n\ +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : debug output from parser; also PYTHONDEBUG=x\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ @@ -225,55 +225,60 @@ static int RunModule(wchar_t *modname, int set_argv0) return 0; } -static int -RunMainFromImporter(wchar_t *filename) +static PyObject * +AsImportPathEntry(wchar_t *filename) { - PyObject *argv0 = NULL, *importer, *sys_path, *sys_path0; - int sts; + PyObject *sys_path0 = NULL, *importer; - argv0 = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (argv0 == NULL) + sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); + if (sys_path0 == NULL) goto error; - importer = PyImport_GetImporter(argv0); + importer = PyImport_GetImporter(sys_path0); if (importer == NULL) goto error; if (importer == Py_None) { - Py_DECREF(argv0); + Py_DECREF(sys_path0); Py_DECREF(importer); - return -1; + return NULL; } Py_DECREF(importer); + return sys_path0; + +error: + Py_XDECREF(sys_path0); + PySys_WriteStderr("Failed checking if argv[0] is an import path entry\n"); + PyErr_Print(); + PyErr_Clear(); + return NULL; +} + + +static int +RunMainFromImporter(PyObject *sys_path0) +{ + PyObject *sys_path; + int sts; - /* argv0 is usable as an import source, so put it in sys.path[0] - and import __main__ */ + /* Assume sys_path0 has already been checked by AsImportPathEntry, + * so put it in sys.path[0] and import __main__ */ sys_path = PySys_GetObject("path"); if (sys_path == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); goto error; } - sys_path0 = PyList_GetItem(sys_path, 0); - sts = 0; - if (!sys_path0) { - PyErr_Clear(); - sts = PyList_Append(sys_path, argv0); - } else if (PyObject_IsTrue(sys_path0)) { - sts = PyList_Insert(sys_path, 0, argv0); - } else { - sts = PyList_SetItem(sys_path, 0, argv0); - } + sts = PyList_Insert(sys_path, 0, sys_path0); if (sts) { - argv0 = NULL; + sys_path0 = NULL; goto error; } - Py_INCREF(argv0); sts = RunModule(L"__main__", 0); return sts != 0; error: - Py_XDECREF(argv0); + Py_XDECREF(sys_path0); PyErr_Print(); return 1; } @@ -358,6 +363,7 @@ Py_Main(int argc, wchar_t **argv) int saw_unbuffered_flag = 0; char *opt; PyCompilerFlags cf; + PyObject *main_importer_path = NULL; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -714,7 +720,17 @@ Py_Main(int argc, wchar_t **argv) argv[_PyOS_optind] = L"-m"; } - PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + if (filename != NULL) { + main_importer_path = AsImportPathEntry(filename); + } + + if (main_importer_path != NULL) { + /* Let RunMainFromImporter adjust sys.path[0] later */ + PySys_SetArgvEx(argc-_PyOS_optind, argv+_PyOS_optind, 0); + } else { + /* Use config settings to decide whether or not to update sys.path[0] */ + PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + } if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) && isatty(fileno(stdin)) && @@ -744,11 +760,11 @@ Py_Main(int argc, wchar_t **argv) sts = -1; /* keep track of whether we've already run __main__ */ - if (filename != NULL) { - sts = RunMainFromImporter(filename); + if (main_importer_path != NULL) { + sts = RunMainFromImporter(main_importer_path); } - if (sts==-1 && filename!=NULL) { + if (sts==-1 && filename != NULL) { fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5f1615ff82992c9..426b7cacebb573e 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -806,10 +806,10 @@ mmap_subscript(mmap_object *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->size, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); @@ -932,11 +932,10 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) Py_ssize_t start, stop, step, slicelen; Py_buffer vbuf; - if (PySlice_GetIndicesEx(item, - self->size, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "mmap object doesn't support slice deletion"); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8f8ba255ec48f6f..2ea5e2def9acff8 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1932,11 +1932,13 @@ _pystat_fromstructstat(STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); -#ifdef HAVE_LARGEFILE_SUPPORT +#if defined(HAVE_LARGEFILE_SUPPORT) || defined(MS_WINDOWS) + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(st->st_ino)); PyStructSequence_SET_ITEM(v, 1, - PyLong_FromLongLong((long long)st->st_ino)); + PyLong_FromUnsignedLongLong(st->st_ino)); #else - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long)st->st_ino)); + Py_BUILD_ASSERT(sizeof(unsigned long) >= sizeof(st->st_ino)); + PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLong(st->st_ino)); #endif #ifdef MS_WINDOWS PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); @@ -11156,7 +11158,7 @@ typedef struct { PyObject *lstat; #ifdef MS_WINDOWS struct _Py_stat_struct win32_lstat; - __int64 win32_file_index; + uint64_t win32_file_index; int got_file_index; #else /* POSIX */ #ifdef HAVE_DIRENT_D_TYPE @@ -11419,7 +11421,8 @@ DirEntry_inode(DirEntry *self) self->win32_file_index = stat.st_ino; self->got_file_index = 1; } - return PyLong_FromLongLong((long long)self->win32_file_index); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->win32_file_index)); + return PyLong_FromUnsignedLongLong(self->win32_file_index); #else /* POSIX */ #ifdef HAVE_LARGEFILE_SUPPORT return PyLong_FromLongLong((long long)self->d_ino); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b3259d57aa114c6..47f70a27f158461 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1191,7 +1191,7 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec Py_DECREF(self); return NULL; } -#if ((XML_MAJOR_VERSION >= 2) && (XML_MINOR_VERSION >= 1)) || defined(XML_HAS_SET_HASH_SALT) +#if XML_COMBINED_VERSION >= 20100 || defined(XML_HAS_SET_HASH_SALT) /* This feature was added upstream in libexpat 2.1.0. Our expat copy * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT * to indicate that we can still use it. */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f4edc062fd65397..f3654c97e76ce91 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -288,36 +288,6 @@ if_indextoname(index) -- return the corresponding interface name\n\ #include #endif -#ifdef HAVE_SOCKADDR_ALG -#include -#ifndef AF_ALG -#define AF_ALG 38 -#endif -#ifndef SOL_ALG -#define SOL_ALG 279 -#endif - -/* Linux 3.19 */ -#ifndef ALG_SET_AEAD_ASSOCLEN -#define ALG_SET_AEAD_ASSOCLEN 4 -#endif -#ifndef ALG_SET_AEAD_AUTHSIZE -#define ALG_SET_AEAD_AUTHSIZE 5 -#endif -/* Linux 4.8 */ -#ifndef ALG_SET_PUBKEY -#define ALG_SET_PUBKEY 6 -#endif - -#ifndef ALG_OP_SIGN -#define ALG_OP_SIGN 2 -#endif -#ifndef ALG_OP_VERIFY -#define ALG_OP_VERIFY 3 -#endif - -#endif /* HAVE_SOCKADDR_ALG */ - /* Generic socket object definitions and includes */ #define PySocket_BUILDING_SOCKET #include "socketmodule.h" diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 3cce927e0b35f01..03f982b91083f3d 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -98,6 +98,37 @@ typedef int socklen_t; #include #endif +#ifdef HAVE_SOCKADDR_ALG +#include +#ifndef AF_ALG +#define AF_ALG 38 +#endif +#ifndef SOL_ALG +#define SOL_ALG 279 +#endif + +/* Linux 3.19 */ +#ifndef ALG_SET_AEAD_ASSOCLEN +#define ALG_SET_AEAD_ASSOCLEN 4 +#endif +#ifndef ALG_SET_AEAD_AUTHSIZE +#define ALG_SET_AEAD_AUTHSIZE 5 +#endif +/* Linux 4.8 */ +#ifndef ALG_SET_PUBKEY +#define ALG_SET_PUBKEY 6 +#endif + +#ifndef ALG_OP_SIGN +#define ALG_OP_SIGN 2 +#endif +#ifndef ALG_OP_VERIFY +#define ALG_OP_VERIFY 3 +#endif + +#endif /* HAVE_SOCKADDR_ALG */ + + #ifndef Py__SOCKET_H #define Py__SOCKET_H #ifdef __cplusplus @@ -159,6 +190,9 @@ typedef union sock_addr { #ifdef HAVE_SYS_KERN_CONTROL_H struct sockaddr_ctl ctl; #endif +#ifdef HAVE_SOCKADDR_ALG + struct sockaddr_alg alg; +#endif } sock_addr_t; /* The object holding a socket. It holds some extra information, diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 59046aaae41f512..cccc033d2d9faba 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -1102,7 +1102,7 @@ get_decompress_func(void) _Py_IDENTIFIER(decompress); if (importing_zlib != 0) - /* Someone has a zlib.py[co] in their Zip file; + /* Someone has a zlib.pyc in their Zip file; let's avoid a stack overflow. */ return NULL; importing_zlib = 1; @@ -1261,7 +1261,7 @@ eq_mtime(time_t t1, time_t t2) return d <= 1; } -/* Given the contents of a .py[co] file in a buffer, unmarshal the data +/* Given the contents of a .pyc file in a buffer, unmarshal the data and return the code object. Return None if it the magic word doesn't match (we do this instead of raising an exception as we fall back to .py if available and we don't want to mask other errors). @@ -1403,7 +1403,7 @@ get_mtime_of_source(ZipImporter *self, PyObject *path) PyObject *toc_entry, *stripped; time_t mtime; - /* strip 'c' or 'o' from *.py[co] */ + /* strip 'c' from *.pyc */ if (PyUnicode_READY(path) == -1) return (time_t)-1; stripped = PyUnicode_FromKindAndData(PyUnicode_KIND(path), diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a8d698025087361..a9c8ca6f1f35a83 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -254,7 +254,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } @@ -400,11 +400,11 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) } else if (PySlice_Check(index)) { Py_ssize_t start, stop, step, slicelength, cur, i; - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), + &start, &stop, step); if (slicelength <= 0) return PyByteArray_FromStringAndSize("", 0); @@ -630,11 +630,11 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu } } else if (PySlice_Check(index)) { - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), &start, + &stop, step); } else { PyErr_Format(PyExc_TypeError, diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index d5c4fe6346fc53a..625e242d563d893 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -542,7 +542,11 @@ _Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_index__doc__, "B.index(sub[, start[, end]]) -> int\n\ \n\ -Like B.find() but raise ValueError when the subsection is not found."); +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the subsection is not found."); PyObject * _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) @@ -579,7 +583,11 @@ _Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_rindex__doc__, "B.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like B.rfind() but raise ValueError when the subsection is not found."); +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raise ValueError when the subsection is not found."); PyObject * _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) @@ -811,4 +819,3 @@ PyDoc_STRVAR_shared(_Py_zfill__doc__, "\n" "Pad a numeric string B with zeros on the left, to fill a field\n" "of the specified width. B is never truncated."); - diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 5d4844096008528..4c55294d9d8c8a1 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -528,6 +528,8 @@ byte_converter(PyObject *arg, char *p) return 0; } +static PyObject *_PyBytes_FromBuffer(PyObject *x); + static PyObject * format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) { @@ -564,8 +566,19 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) *plen = PyBytes_GET_SIZE(result); return result; } + /* does it support buffer protocol? */ + if (PyObject_CheckBuffer(v)) { + /* maybe we can avoid making a copy of the buffer object here? */ + result = _PyBytes_FromBuffer(v); + if (result == NULL) + return NULL; + *pbuf = PyBytes_AS_STRING(result); + *plen = PyBytes_GET_SIZE(result); + return result; + } PyErr_Format(PyExc_TypeError, - "%%b requires bytes, or an object that implements __bytes__, not '%.100s'", + "%%b requires a bytes-like object, " + "or an object that implements __bytes__, not '%.100s'", Py_TYPE(v)->tp_name); return NULL; } @@ -619,11 +632,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, Py_ssize_t len; char *pos; - pos = strchr(fmt + 1, '%'); + pos = (char *)memchr(fmt + 1, '%', fmtcnt); if (pos != NULL) len = pos - fmt; else - len = format_len - (fmt - format); + len = fmtcnt + 1; assert(len != 0); memcpy(res, fmt, len); @@ -1425,7 +1438,7 @@ bytes_concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } @@ -1670,11 +1683,11 @@ bytes_subscript(PyBytesObject* self, PyObject* item) char* result_buf; PyObject* result; - if (PySlice_GetIndicesEx(item, - PyBytes_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyBytes_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyBytes_FromStringAndSize("", 0); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d8a675f9fbfc69..df8b9538fe9ed5d 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -856,16 +856,15 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra *) o->co_extra; if (co_extra == NULL) { - o->co_extra = (_PyCodeObjectExtra*) PyMem_Malloc( - sizeof(_PyCodeObjectExtra)); - if (o->co_extra == NULL) { + co_extra = PyMem_Malloc(sizeof(_PyCodeObjectExtra)); + if (co_extra == NULL) { return -1; } - co_extra = (_PyCodeObjectExtra *) o->co_extra; co_extra->ce_extras = PyMem_Malloc( tstate->co_extra_user_count * sizeof(void*)); if (co_extra->ce_extras == NULL) { + PyMem_Free(co_extra); return -1; } @@ -874,20 +873,25 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { co_extra->ce_extras[i] = NULL; } + + o->co_extra = co_extra; } else if (co_extra->ce_size <= index) { - co_extra->ce_extras = PyMem_Realloc( + void** ce_extras = PyMem_Realloc( co_extra->ce_extras, tstate->co_extra_user_count * sizeof(void*)); - if (co_extra->ce_extras == NULL) { + if (ce_extras == NULL) { return -1; } - co_extra->ce_size = tstate->co_extra_user_count; - - for (Py_ssize_t i = co_extra->ce_size; i < co_extra->ce_size; i++) { - co_extra->ce_extras[i] = NULL; + for (Py_ssize_t i = co_extra->ce_size; + i < tstate->co_extra_user_count; + i++) { + ce_extras[i] = NULL; } + + co_extra->ce_extras = ce_extras; + co_extra->ce_size = tstate->co_extra_user_count; } co_extra->ce_extras[index] = extra; diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 31e12784cc34f94..cfaba688c684f6e 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1025,11 +1025,11 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } cr.real = PyFloat_AsDouble(tmp); - cr.imag = 0.0; /* Shut up compiler warning */ + cr.imag = 0.0; Py_DECREF(tmp); } if (i == NULL) { - ci.real = 0.0; + ci.real = cr.imag; } else if (PyComplex_Check(i)) { ci = ((PyComplexObject*)i)->cval; @@ -1051,7 +1051,7 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ci_is_complex) { cr.real -= ci.imag; } - if (cr_is_complex) { + if (cr_is_complex && i != NULL) { ci.real += cr.imag; } return complex_subtype_from_doubles(type, cr.real, ci.real); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7b403bcecc5da2..566d1a5ac8d8936 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -436,7 +436,7 @@ static PyObject *empty_values[1] = { NULL }; /* #define DEBUG_PYDICT */ -#ifdef Py_DEBUG +#ifndef NDEBUG static int _PyDict_CheckConsistency(PyDictObject *mp) { @@ -676,7 +676,8 @@ Christian Tismer. lookdict() is general-purpose, and may return DKIX_ERROR if (and only if) a comparison raises an exception. lookdict_unicode() below is specialized to string keys, comparison of which can -never raise an exception; that function can never return DKIX_ERROR. +never raise an exception; that function can never return DKIX_ERROR when key +is string. Otherwise, it falls back to lookdict(). lookdict_unicode_nodummy is further specialized for string keys that cannot be the value. For both, when the key isn't found a DKIX_EMPTY is returned. hashpos returns @@ -1928,7 +1929,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, ESTIMATE_SIZE(Py_SIZE(iterable)))) { + if (dictresize(mp, ESTIMATE_SIZE(((PyDictObject *)iterable)->ma_used))) { Py_DECREF(d); return NULL; } @@ -4376,15 +4377,19 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, } if (value == NULL) { res = PyDict_DelItem(dict, key); - if (cached != ((PyDictObject *)dict)->ma_keys) { + // Since key sharing dict doesn't allow deletion, PyDict_DelItem() + // always converts dict to combined form. + if ((cached = CACHED_KEYS(tp)) != NULL) { CACHED_KEYS(tp) = NULL; DK_DECREF(cached); } } else { - int was_shared = cached == ((PyDictObject *)dict)->ma_keys; + int was_shared = (cached == ((PyDictObject *)dict)->ma_keys); res = PyDict_SetItem(dict, key, value); - if (was_shared && cached != ((PyDictObject *)dict)->ma_keys) { + if (was_shared && + (cached = CACHED_KEYS(tp)) != NULL && + cached != ((PyDictObject *)dict)->ma_keys) { /* PyDict_SetItem() may call dictresize and convert split table * into combined table. In such case, convert it to split * table again and update type's shared key only when this is diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f63f06a145bcb15..d158b9768d45f50 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -686,6 +686,53 @@ ImportError_str(PyImportErrorObject *self) } } +static PyObject * +ImportError_getstate(PyImportErrorObject *self) +{ + PyObject *dict = ((PyBaseExceptionObject *)self)->dict; + if (self->name || self->path) { + _Py_IDENTIFIER(name); + _Py_IDENTIFIER(path); + dict = dict ? PyDict_Copy(dict) : PyDict_New(); + if (dict == NULL) + return NULL; + if (self->name && _PyDict_SetItemId(dict, &PyId_name, self->name) < 0) { + Py_DECREF(dict); + return NULL; + } + if (self->path && _PyDict_SetItemId(dict, &PyId_path, self->path) < 0) { + Py_DECREF(dict); + return NULL; + } + return dict; + } + else if (dict) { + Py_INCREF(dict); + return dict; + } + else { + Py_RETURN_NONE; + } +} + +/* Pickling support */ +static PyObject * +ImportError_reduce(PyImportErrorObject *self) +{ + PyObject *res; + PyObject *args; + PyObject *state = ImportError_getstate(self); + if (state == NULL) + return NULL; + args = ((PyBaseExceptionObject *)self)->args; + if (state == Py_None) + res = PyTuple_Pack(2, Py_TYPE(self), args); + else + res = PyTuple_Pack(3, Py_TYPE(self), args, state); + Py_DECREF(state); + return res; +} + static PyMemberDef ImportError_members[] = { {"msg", T_OBJECT, offsetof(PyImportErrorObject, msg), 0, PyDoc_STR("exception message")}, @@ -697,6 +744,7 @@ static PyMemberDef ImportError_members[] = { }; static PyMethodDef ImportError_methods[] = { + {"__reduce__", (PyCFunction)ImportError_reduce, METH_NOARGS}, {NULL} }; diff --git a/Objects/genobject.c b/Objects/genobject.c index 2680ab0e129da4b..1c29e296afe2a8d 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -575,8 +575,7 @@ _PyGen_SetStopIterationValue(PyObject *value) PyObject *e; if (value == NULL || - (!PyTuple_Check(value) && - !PyObject_TypeCheck(value, (PyTypeObject *) PyExc_StopIteration))) + (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) { /* Delay exception instantiation if we can */ PyErr_SetObject(PyExc_StopIteration, value); diff --git a/Objects/listobject.c b/Objects/listobject.c index dcd7b5efe5b0fe7..547bdf0b95f1ee8 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2153,8 +2153,8 @@ listindex(PyListObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); @@ -2420,10 +2420,11 @@ list_subscript(PyListObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return PyList_New(0); @@ -2469,10 +2470,11 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (step == 1) return list_ass_slice(self, start, stop, value); diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 515375772e63171..3dab2b98661695a 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,17 +1,18 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes -disguised as a Python string. It is used to map bytecode offsets to source code -line #s for tracebacks and to identify line number boundaries for line tracing. +disguised as a Python bytes object. It is used to map bytecode offsets to +source code line #s for tracebacks and to identify line number boundaries for +line tracing. The array is conceptually a compressed list of (bytecode offset increment, line number increment) pairs. The details are important and delicate, best illustrated by example: byte code offset source code line number - 0 1 - 6 2 - 50 7 + 0 1 + 6 2 + 50 7 350 207 361 208 @@ -24,7 +25,8 @@ look like: The above doesn't really work, but it's a start. An unsigned byte (byte code offset) can't hold negative values, or values larger than 255, a signed byte (line number) can't hold values larger than 127 or less than -128, and the -above example contains two such values. So we make two tweaks: +above example contains two such values. (Note that before 3.6, line number +was also encoded by an unsigned byte.) So we make two tweaks: (a) there's a deep assumption that byte code offsets increase monotonically, and @@ -52,7 +54,7 @@ the example above, assemble_lnotab in compile.c should not (as was actually done until 2.2) expand 300, 200 to 255, 255, 45, 45, but to - 255, 0, 45, 128, 0, 72. + 255, 0, 45, 127, 0, 73. The above is sufficient to reconstruct line numbers for tracebacks, but not for line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c @@ -83,30 +85,34 @@ Consider this code: 1: def f(a): 2: while a: -3: print 1, +3: print(1) 4: break 5: else: -6: print 2, +6: print(2) which compiles to this: - 2 0 SETUP_LOOP 19 (to 22) - >> 3 LOAD_FAST 0 (a) - 6 POP_JUMP_IF_FALSE 17 + 2 0 SETUP_LOOP 26 (to 28) + >> 2 LOAD_FAST 0 (a) + 4 POP_JUMP_IF_FALSE 18 - 3 9 LOAD_CONST 1 (1) - 12 PRINT_ITEM + 3 6 LOAD_GLOBAL 0 (print) + 8 LOAD_CONST 1 (1) + 10 CALL_FUNCTION 1 + 12 POP_TOP - 4 13 BREAK_LOOP - 14 JUMP_ABSOLUTE 3 - >> 17 POP_BLOCK + 4 14 BREAK_LOOP + 16 JUMP_ABSOLUTE 2 + >> 18 POP_BLOCK - 6 18 LOAD_CONST 2 (2) - 21 PRINT_ITEM - >> 22 LOAD_CONST 0 (None) - 25 RETURN_VALUE + 6 20 LOAD_GLOBAL 0 (print) + 22 LOAD_CONST 2 (2) + 24 CALL_FUNCTION 1 + 26 POP_TOP + >> 28 LOAD_CONST 0 (None) + 30 RETURN_VALUE -If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 17 +If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 18 and the co_lnotab will claim that execution has moved to line 4, which is wrong. In this case, we could instead associate the POP_BLOCK with line 5, but that would break jumps around loops without else clauses. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index b1798a2073d108c..e1ac7281783f38f 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2285,10 +2285,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index a1142f3b09ad9e0..32e7ecbe1e04368 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1227,7 +1227,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) _Py_AllocatedBlocks++; - assert(nelem <= PY_SSIZE_T_MAX / elsize); + assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; #ifdef WITH_VALGRIND diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 8449fc7a24d3944..8f5fc434bd49550 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -668,6 +668,16 @@ static PyMappingMethods range_as_mapping = { (objobjargproc)0, /* mp_ass_subscript */ }; +static int +range_bool(rangeobject* self) +{ + return PyObject_IsTrue(self->length); +} + +static PyNumberMethods range_as_number = { + .nb_bool = (inquiry)range_bool, +}; + static PyObject * range_iter(PyObject *seq); static PyObject * range_reverse(PyObject *seq); @@ -707,7 +717,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_setattr */ 0, /* tp_reserved */ (reprfunc)range_repr, /* tp_repr */ - 0, /* tp_as_number */ + &range_as_number, /* tp_as_number */ &range_as_sequence, /* tp_as_sequence */ &range_as_mapping, /* tp_as_mapping */ (hashfunc)range_hash, /* tp_hash */ diff --git a/Objects/setobject.c b/Objects/setobject.c index fdb9d3600d9fad2..9fe28138c0012b0 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -236,7 +236,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); found_active: Py_DECREF(key); @@ -304,7 +304,6 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry small_copy[PySet_MINSIZE]; assert(minused >= 0); - minused = (minused > 50000) ? minused * 2 : minused * 4; /* Find the smallest table size > minused. */ /* XXX speed-up with intrinsics */ @@ -646,8 +645,8 @@ set_merge(PySetObject *so, PyObject *otherset) * that there will be no (or few) overlapping keys. */ if ((so->fill + other->used)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + other->used) != 0) - return -1; + if (set_table_resize(so, (so->used + other->used)*2) != 0) + return -1; } so_entry = so->table; other_entry = other->table; @@ -990,7 +989,7 @@ set_update_internal(PySetObject *so, PyObject *other) if (dictsize < 0) return -1; if ((so->fill + dictsize)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + dictsize) != 0) + if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { @@ -1511,7 +1510,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /* If more than 1/4th are dummies, then resize them away. */ if ((size_t)(so->fill - so->used) <= (size_t)so->mask / 4) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } static PyObject * diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c0ff499e72c7dda..0dada74dc7c9862 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -522,8 +522,8 @@ tupleindex(PyTupleObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); @@ -720,11 +720,11 @@ tuplesubscript(PyTupleObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, - PyTuple_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyTuple_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyTuple_New(0); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a5ae454b495dee2..7871043681533da 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -11701,7 +11701,11 @@ unicode_hash(PyObject *self) PyDoc_STRVAR(index__doc__, "S.index(sub[, start[, end]]) -> int\n\ \n\ -Like S.find() but raise ValueError when the substring is not found."); +Return the lowest index in S where substring sub is found, \n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_index(PyObject *self, PyObject *args) @@ -12758,7 +12762,11 @@ unicode_rfind(PyObject *self, PyObject *args) PyDoc_STRVAR(rindex__doc__, "S.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like S.rfind() but raise ValueError when the substring is not found."); +Return the highest index in S where substring sub is found,\n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_rindex(PyObject *self, PyObject *args) @@ -13915,10 +13923,11 @@ unicode_subscript(PyObject* self, PyObject* item) int src_kind, dest_kind; Py_UCS4 ch, max_char, kind_limit; - if (PySlice_GetIndicesEx(item, PyUnicode_GET_LENGTH(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyUnicode_GET_LENGTH(self), + &start, &stop, step); if (slicelength <= 0) { _Py_RETURN_UNICODE_EMPTY(); diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index ab6b23525552b83..9ca386da2563cb7 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -24,6 +24,8 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) { self->hash = -1; self->wr_object = ob; + self->wr_prev = NULL; + self->wr_next = NULL; Py_XINCREF(callback); self->wr_callback = callback; } diff --git a/PC/getpathp.c b/PC/getpathp.c index 1eeebfe9c19451c..e7be704a9a78dd8 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -185,7 +185,7 @@ exists(wchar_t *filename) may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/.pyo too */ +ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ { size_t n; @@ -196,7 +196,7 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/ n = wcsnlen_s(filename, MAXPATHLEN+1); if (n < MAXPATHLEN) { int exist = 0; - filename[n] = Py_OptimizeFlag ? L'o' : L'c'; + filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); if (!update_filename) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 70ab340905e4c95..f7f2858d7d9d99c 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -105,9 +105,9 @@ if "%platf%"=="x64" ( ) ) -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if exist "%HG%" set HGProperty=/p:HG="%HG%" -if not exist "%HG%" echo Cannot find Mercurial on PATH & set HGProperty= +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" +if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= rem Setup the environment call "%dir%env.bat" %vs_platf% >nul @@ -145,7 +145,7 @@ msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:IncludeExternals=%IncludeExternals%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ - /p:UseTestMarker=%UseTestMarker% %HGProperty%^ + /p:UseTestMarker=%UseTestMarker% %GITProperty%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index a5185becee5a96d..3a2656e961ef7f7 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2j +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k set libraries=%libraries% sqlite-3.14.2.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 @@ -66,7 +66,7 @@ for %%e in (%libraries%) do ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - svn export %SVNROOT%%%e + svn export -q %SVNROOT%%%e ) ) diff --git a/PCbuild/python.props b/PCbuild/python.props index dde94f733a13ae7..d6bfd0877e09df5 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -45,7 +45,7 @@ $(ExternalsDir)sqlite-3.14.2.0\ $(ExternalsDir)bzip2-1.0.6\ $(ExternalsDir)xz-5.2.2\ - $(ExternalsDir)openssl-1.0.2j\ + $(ExternalsDir)openssl-1.0.2k\ $(opensslDir)include32 $(opensslDir)include64 $(ExternalsDir)\nasm-2.11.06\ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 6b23d8ecacce3b9..6ea184877f99cd5 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -407,24 +407,24 @@ - hg - <_HG>$(HG) - <_HG Condition="$(HG.Contains(` `))">"$(HG)" + git + <_GIT>$(GIT) + <_GIT Condition="$(GIT.Contains(` `))">"$(GIT)" - + - - - + + + - $([System.IO.File]::ReadAllText('$(IntDir)hgbranch.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)hgversion.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)hgtag.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gittag.txt').Trim()) - + - HGVERSION="$(HgVersion)";HGTAG="$(HgTag)";HGBRANCH="$(HgBranch)";%(PreprocessorDefinitions) + GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions) diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c04ba4e46b9894e..8924b3fcfa2e800 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -169,7 +169,7 @@ _lzma Homepage: http://tukaani.org/xz/ _ssl - Python wrapper for version 1.0.2j of the OpenSSL secure sockets + Python wrapper for version 1.0.2k of the OpenSSL secure sockets library, which is built by ssl.vcxproj Homepage: http://www.openssl.org/ diff --git a/PCbuild/rmpyc.py b/PCbuild/rmpyc.py index a1e75bb7ae5b4ba..0b58f687729f7e7 100644 --- a/PCbuild/rmpyc.py +++ b/PCbuild/rmpyc.py @@ -1,25 +1,19 @@ -# Remove all the .pyc and .pyo files under ../Lib. +# Remove all the .pyc files under ../Lib. def deltree(root): import os from os.path import join - npyc = npyo = 0 + npyc = 0 for root, dirs, files in os.walk(root): for name in files: - delete = False - if name.endswith('.pyc'): - delete = True + # to be thorough + if name.endswith(('.pyc', '.pyo')): npyc += 1 - elif name.endswith('.pyo'): - delete = True - npyo += 1 - - if delete: os.remove(join(root, name)) - return npyc, npyo + return npyc -npyc, npyo = deltree("../Lib") -print(npyc, ".pyc deleted,", npyo, ".pyo deleted") +npyc = deltree("../Lib") +print(npyc, ".pyc deleted") diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 35826727f3fa279..e73ac0457646e9a 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -4,8 +4,8 @@ rem Usage: rt [-d] [-O] [-q] [-x64] regrtest_args rem -d Run Debug build (python_d.exe). Else release build. rem -O Run python.exe or python_d.exe (see -d) with -O. rem -q "quick" -- normally the tests are run twice, the first time -rem after deleting all the .py[co] files reachable from Lib/. -rem -q runs the tests just once, and without deleting .py[co] files. +rem after deleting all the .pyc files reachable from Lib/. +rem -q runs the tests just once, and without deleting .pyc files. rem -x64 Run the 64-bit build of python (or python_d if -d was specified) rem from the 'amd64' dir instead of the 32-bit build in this dir. rem All leading instances of these switches are shifted off, and @@ -45,7 +45,7 @@ set exe=%prefix%python%suffix%.exe set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% if defined qmode goto Qmode -echo Deleting .pyc/.pyo files ... +echo Deleting .pyc files ... "%exe%" "%pcbuild%rmpyc.py" echo Cleaning _pth files ... @@ -55,7 +55,7 @@ echo on %cmd% @echo off -echo About to run again without deleting .pyc/.pyo first: +echo About to run again without deleting .pyc first: pause :Qmode diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ef5a34cae900b64..597e26ec69a95a9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1927,12 +1927,15 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *result; size_t len; + /* stdin is a text stream, so it must have an encoding. */ stdin_encoding = _PyObject_GetAttrId(fin, &PyId_encoding); stdin_errors = _PyObject_GetAttrId(fin, &PyId_errors); - if (!stdin_encoding || !stdin_errors) - /* stdin is a text stream, so it must have an - encoding. */ + if (!stdin_encoding || !stdin_errors || + !PyUnicode_Check(stdin_encoding) || + !PyUnicode_Check(stdin_errors)) { + tty = 0; goto _readline_errors; + } stdin_encoding_str = PyUnicode_AsUTF8(stdin_encoding); stdin_errors_str = PyUnicode_AsUTF8(stdin_errors); if (!stdin_encoding_str || !stdin_errors_str) @@ -1948,8 +1951,12 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *stringpo; stdout_encoding = _PyObject_GetAttrId(fout, &PyId_encoding); stdout_errors = _PyObject_GetAttrId(fout, &PyId_errors); - if (!stdout_encoding || !stdout_errors) + if (!stdout_encoding || !stdout_errors || + !PyUnicode_Check(stdout_encoding) || + !PyUnicode_Check(stdout_errors)) { + tty = 0; goto _readline_errors; + } stdout_encoding_str = PyUnicode_AsUTF8(stdout_encoding); stdout_errors_str = PyUnicode_AsUTF8(stdout_errors); if (!stdout_encoding_str || !stdout_errors_str) @@ -2003,13 +2010,17 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_XDECREF(po); PyMem_FREE(s); return result; + _readline_errors: Py_XDECREF(stdin_encoding); Py_XDECREF(stdout_encoding); Py_XDECREF(stdin_errors); Py_XDECREF(stdout_errors); Py_XDECREF(po); - return NULL; + if (tty) + return NULL; + + PyErr_Clear(); } /* Fallback if we're not interactive */ diff --git a/Python/ceval.c b/Python/ceval.c index d5172b9631f43d7..bce86ab12cf2fbe 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1409,9 +1409,15 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(BINARY_MODULO) { PyObject *divisor = POP(); PyObject *dividend = TOP(); - PyObject *res = PyUnicode_CheckExact(dividend) ? - PyUnicode_Format(dividend, divisor) : - PyNumber_Remainder(dividend, divisor); + PyObject *res; + if (PyUnicode_CheckExact(dividend) && ( + !PyUnicode_Check(divisor) || PyUnicode_CheckExact(divisor))) { + // fast path; string formatting, but not if the RHS is a str subclass + // (see issue28598) + res = PyUnicode_Format(dividend, divisor); + } else { + res = PyNumber_Remainder(dividend, divisor); + } Py_DECREF(divisor); Py_DECREF(dividend); SET_TOP(res); @@ -1898,13 +1904,13 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) awaitable = _PyCoro_GetAwaitableIter(iter); if (awaitable == NULL) { - SET_TOP(NULL); - PyErr_Format( + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __aiter__: %.100s", Py_TYPE(iter)->tp_name); + SET_TOP(NULL); Py_DECREF(iter); goto error; } else { @@ -1963,7 +1969,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { - PyErr_Format( + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", @@ -2851,13 +2857,16 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(IMPORT_STAR) { PyObject *from = POP(), *locals; int err; - if (PyFrame_FastToLocalsWithError(f) < 0) + if (PyFrame_FastToLocalsWithError(f) < 0) { + Py_DECREF(from); goto error; + } locals = f->f_locals; if (locals == NULL) { PyErr_SetString(PyExc_SystemError, "no locals found during 'import *'"); + Py_DECREF(from); goto error; } err = import_all_from(locals, from); @@ -4690,11 +4699,7 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) assert(!PyErr_Occurred()); #endif - if (args == NULL) { - return _PyObject_FastCallDict(func, NULL, 0, kwargs); - } - - if (!PyTuple_Check(args)) { + if (args != NULL && !PyTuple_Check(args)) { PyErr_SetString(PyExc_TypeError, "argument list must be a tuple"); return NULL; @@ -4706,7 +4711,12 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) return NULL; } - return PyObject_Call(func, args, kwargs); + if (args == NULL) { + return _PyObject_FastCallDict(func, NULL, 0, kwargs); + } + else { + return PyObject_Call(func, args, kwargs); + } } const char * @@ -5064,14 +5074,10 @@ do_call_core(PyObject *func, PyObject *callargs, PyObject *kwdict) and silently boost values less than -PY_SSIZE_T_MAX-1 to -PY_SSIZE_T_MAX-1. Return 0 on error, 1 on success. */ -/* Note: If v is NULL, return success without storing into *pi. This - is because_PyEval_SliceIndex() is called by apply_slice(), which can be - called by the SLICE opcode with v and/or w equal to NULL. -*/ int _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) { - if (v != NULL) { + if (v != Py_None) { Py_ssize_t x; if (PyIndex_Check(v)) { x = PyNumber_AsSsize_t(v, NULL); @@ -5089,6 +5095,26 @@ _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) return 1; } +int +_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) +{ + Py_ssize_t x; + if (PyIndex_Check(v)) { + x = PyNumber_AsSsize_t(v, NULL); + if (x == -1 && PyErr_Occurred()) + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + return 0; + } + *pi = x; + return 1; +} + + #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" diff --git a/Python/compile.c b/Python/compile.c index 0e1607585245b9e..6255ec7d47f5a20 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1043,7 +1043,7 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) case CALL_FUNCTION_KW: return -oparg-1; case CALL_FUNCTION_EX: - return - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0); + return -1 - ((oparg & 0x01) != 0); case MAKE_FUNCTION: return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) - ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0); diff --git a/Python/fileutils.c b/Python/fileutils.c index e84d66e99a473d5..f3764e4b3c5a2d0 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -583,7 +583,7 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); result->st_nlink = info->nNumberOfLinks; - result->st_ino = (((__int64)info->nFileIndexHigh)<<32) + info->nFileIndexLow; + result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; if (reparse_tag == IO_REPARSE_TAG_SYMLINK) { /* first clear the S_IFMT bits */ result->st_mode ^= (result->st_mode & S_IFMT); @@ -653,7 +653,7 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) _Py_attribute_data_to_stat(&info, 0, status); /* specific to fstat() */ - status->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; + status->st_ino = (((uint64_t)info.nFileIndexHigh) << 32) + info.nFileIndexLow; return 0; #else return fstat(fd, status); diff --git a/Python/importlib.h b/Python/importlib.h index 195fbfd13b62271..4607c40de60e7cd 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -392,7 +392,7 @@ const unsigned char _Py_M__importlib[] = { 111,118,101,100,197,0,0,0,115,2,0,0,0,0,8,114, 58,0,0,0,114,33,0,0,0,41,1,218,9,118,101,114, 98,111,115,105,116,121,99,1,0,0,0,1,0,0,0,3, - 0,0,0,5,0,0,0,71,0,0,0,115,54,0,0,0, + 0,0,0,4,0,0,0,71,0,0,0,115,54,0,0,0, 116,0,106,1,106,2,124,1,107,5,114,50,124,0,106,3, 100,6,131,1,115,30,100,3,124,0,23,0,125,0,116,4, 124,0,106,5,124,2,142,0,116,0,106,6,100,4,141,2, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 2f40978fb78e139..886aa68009ec03c 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -578,7 +578,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,17,95,102,105,110,100,95,109,111,100,117,108,101,95,115, 104,105,109,157,1,0,0,115,10,0,0,0,0,10,14,1, 16,1,4,1,22,1,114,127,0,0,0,99,4,0,0,0, - 0,0,0,0,11,0,0,0,22,0,0,0,67,0,0,0, + 0,0,0,0,11,0,0,0,19,0,0,0,67,0,0,0, 115,136,1,0,0,105,0,125,4,124,2,100,1,107,9,114, 22,124,2,124,4,100,2,60,0,110,4,100,3,125,2,124, 3,100,1,107,9,114,42,124,3,124,4,100,4,60,0,124, diff --git a/Python/marshal.c b/Python/marshal.c index 87a4b240a41c877..7b12ab7510dcfb2 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -549,7 +549,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_lnotab, p); } else if (PyObject_CheckBuffer(v)) { - /* Write unknown bytes-like objects as a byte string */ + /* Write unknown bytes-like objects as a bytes object */ Py_buffer view; if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) { w_byte(TYPE_UNKNOWN, p); @@ -1086,7 +1086,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (bytes object size out of range)"); break; } v = PyBytes_FromStringAndSize((char *)NULL, n); @@ -1110,7 +1110,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } goto _read_ascii; @@ -1150,7 +1150,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } if (n != 0) { @@ -1612,7 +1612,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) if (wf.ptr - base > PY_SSIZE_T_MAX) { Py_DECREF(wf.str); PyErr_SetString(PyExc_OverflowError, - "too much marshal data for a string"); + "too much marshal data for a bytes object"); return NULL; } if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) @@ -1658,8 +1658,7 @@ PyDoc_STRVAR(dump_doc, "dump(value, file[, version])\n\ \n\ Write the value on the open file. The value must be a supported type.\n\ -The file must be an open file object such as sys.stdout or returned by\n\ -open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\ +The file must be a writeable binary file.\n\ \n\ If the value has (or contains an object that has) an unsupported type, a\n\ ValueError exception is raised - but garbage data will also be written\n\ @@ -1715,8 +1714,7 @@ PyDoc_STRVAR(load_doc, Read one value from the open file and return it. If no valid value is\n\ read (e.g. because the data has a different Python version's\n\ incompatible marshal format), raise EOFError, ValueError or TypeError.\n\ -The file must be an open file object opened in binary mode ('rb' or\n\ -'r+b').\n\ +The file must be a readable binary file.\n\ \n\ Note: If an object containing an unsupported type was marshalled with\n\ dump(), load() will substitute None for the unmarshallable type."); @@ -1735,7 +1733,7 @@ marshal_dumps(PyObject *self, PyObject *args) PyDoc_STRVAR(dumps_doc, "dumps(value[, version])\n\ \n\ -Return the string that would be written to a file by dump(value, file).\n\ +Return the bytes object that would be written to a file by dump(value, file).\n\ The value must be a supported type. Raise a ValueError exception if\n\ value has (or contains an object that has) an unsupported type.\n\ \n\ @@ -1771,8 +1769,8 @@ marshal_loads(PyObject *self, PyObject *args) PyDoc_STRVAR(loads_doc, "loads(bytes)\n\ \n\ -Convert the bytes object to a value. If no valid value is found, raise\n\ -EOFError, ValueError or TypeError. Extra characters in the input are\n\ +Convert the bytes-like object to a value. If no valid value is found,\n\ +raise EOFError, ValueError or TypeError. Extra bytes in the input are\n\ ignored."); static PyMethodDef marshal_methods[] = { @@ -1810,8 +1808,8 @@ Functions:\n\ \n\ dump() -- write value to a file\n\ load() -- read value from a file\n\ -dumps() -- write value to a string\n\ -loads() -- read value from a string"); +dumps() -- marshal value as a bytes object\n\ +loads() -- read value from a bytes-like object"); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a4f7f823bc695dd..af96d6d5fcb63c8 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -88,7 +88,7 @@ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ -int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ +int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.pyc) */ int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 52034ff7fb5ddf6..99e6b5ea018ea1e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1942,9 +1942,9 @@ _PySys_Init(void) PyUnicode_FromString(Py_GetVersion())); SET_SYS_FROM_STRING("hexversion", PyLong_FromLong(PY_VERSION_HEX)); - SET_SYS_FROM_STRING("_mercurial", - Py_BuildValue("(szz)", "CPython", _Py_hgidentifier(), - _Py_hgversion())); + SET_SYS_FROM_STRING("_git", + Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(), + _Py_gitversion())); SET_SYS_FROM_STRING("dont_write_bytecode", PyBool_FromLong(Py_DontWriteBytecodeFlag)); SET_SYS_FROM_STRING("api_version", diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 27e0dc84bcb52ed..ba7393f03de6aed 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -143,6 +143,8 @@ typedef struct { } pthread_lock; #define CHECK_STATUS(name) if (status != 0) { perror(name); error = 1; } +#define CHECK_STATUS_PTHREAD(name) if (status != 0) { fprintf(stderr, \ + "%s: %s\n", name, strerror(status)); error = 1; } /* * Initialization. @@ -417,7 +419,7 @@ PyThread_allocate_lock(void) status = pthread_mutex_init(&lock->mut, pthread_mutexattr_default); - CHECK_STATUS("pthread_mutex_init"); + CHECK_STATUS_PTHREAD("pthread_mutex_init"); /* Mark the pthread mutex underlying a Python mutex as pure happens-before. We can't simply mark the Python-level mutex as a mutex because it can be @@ -427,7 +429,7 @@ PyThread_allocate_lock(void) status = pthread_cond_init(&lock->lock_released, pthread_condattr_default); - CHECK_STATUS("pthread_cond_init"); + CHECK_STATUS_PTHREAD("pthread_cond_init"); if (error) { PyMem_RawFree((void *)lock); @@ -452,10 +454,10 @@ PyThread_free_lock(PyThread_type_lock lock) * and must have the cond destroyed first. */ status = pthread_cond_destroy( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_destroy"); + CHECK_STATUS_PTHREAD("pthread_cond_destroy"); status = pthread_mutex_destroy( &thelock->mut ); - CHECK_STATUS("pthread_mutex_destroy"); + CHECK_STATUS_PTHREAD("pthread_mutex_destroy"); PyMem_RawFree((void *)thelock); } @@ -472,7 +474,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, lock, microseconds, intr_flag)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); if (thelock->locked == 0) { success = PY_LOCK_ACQUIRED; @@ -494,13 +496,13 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, &thelock->mut, &ts); if (status == ETIMEDOUT) break; - CHECK_STATUS("pthread_cond_timed_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); } else { status = pthread_cond_wait( &thelock->lock_released, &thelock->mut); - CHECK_STATUS("pthread_cond_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_wait"); } if (intr_flag && status == 0 && thelock->locked) { @@ -518,7 +520,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, } if (success == PY_LOCK_ACQUIRED) thelock->locked = 1; status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); if (error) success = PY_LOCK_FAILURE; dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", @@ -536,16 +538,16 @@ PyThread_release_lock(PyThread_type_lock lock) dprintf(("PyThread_release_lock(%p) called\n", lock)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[3]"); thelock->locked = 0; /* wake up someone (anyone, if any) waiting on the lock */ status = pthread_cond_signal( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_signal"); + CHECK_STATUS_PTHREAD("pthread_cond_signal"); status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[3]"); } #endif /* USE_SEMAPHORES */ diff --git a/README b/README deleted file mode 100644 index 4fc97e6013063e2..000000000000000 --- a/README +++ /dev/null @@ -1,234 +0,0 @@ -This is Python version 3.6.1 release candidate 1 -================================================ - -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation. All rights -reserved. - -Python 3.x is a new version of the language, which is incompatible with the -2.x line of releases. The language is mostly the same, but many details, -especially how built-in objects like dictionaries and strings work, -have changed considerably, and a lot of deprecated features have finally -been removed. - - -Build Instructions ------------------- - -On Unix, Linux, BSD, OSX, and Cygwin: - - ./configure - make - make test - sudo make install - -This will install Python as python3. - -You can pass many options to the configure script; run "./configure --help" to -find out more. On OSX and Cygwin, the executable is called python.exe; -elsewhere it's just python. - -On Mac OS X, if you have configured Python with --enable-framework, you should -use "make frameworkinstall" to do the installation. Note that this installs the -Python executable in a place that is not normally on your PATH, you may want to -set up a symlink in /usr/local/bin. - -On Windows, see PCbuild/readme.txt. - -If you wish, you can create a subdirectory and invoke configure from there. -For example: - - mkdir debug - cd debug - ../configure --with-pydebug - make - make test - -(This will fail if you *also* built at the top-level directory. -You should do a "make clean" at the toplevel first.) - -To get an optimized build of Python, "configure --enable-optimizations" before -you run make. This sets the default make targets up to enable Profile Guided -Optimization (PGO) and may be used to auto-enable Link Time Optimization (LTO) -on some platforms. For more details, see the sections bellow. - - -Profile Guided Optimization ---------------------------- - -PGO takes advantage of recent versions of the GCC or Clang compilers. -If ran, "make profile-opt" will do several steps. - -First, the entire Python directory is cleaned of temporary files that -may have resulted in a previous compilation. - -Then, an instrumented version of the interpreter is built, using suitable -compiler flags for each flavour. Note that this is just an intermediary -step and the binary resulted after this step is not good for real life -workloads, as it has profiling instructions embedded inside. - -After this instrumented version of the interpreter is built, the Makefile -will automatically run a training workload. This is necessary in order to -profile the interpreter execution. Note also that any output, both stdout -and stderr, that may appear at this step is suppressed. - -Finally, the last step is to rebuild the interpreter, using the information -collected in the previous one. The end result will be a Python binary -that is optimized and suitable for distribution or production installation. - - -Link Time Optimization ----------------------- - -Enabled via configure's --with-lto flag. LTO takes advantages of recent -compiler toolchains ability to optimize across the otherwise arbitrary .o file -boundary when building final executables or shared libraries for additional -performance gains. - - -What's New ----------- - -We have a comprehensive overview of the changes in the "What's New in -Python 3.6" document, found at - - https://docs.python.org/3.6/whatsnew/3.6.html - -For a more detailed change log, read Misc/NEWS (though this file, too, -is incomplete, and also doesn't list anything merged in from the 2.7 -release under development). - -If you want to install multiple versions of Python see the section below -entitled "Installing multiple versions". - - -Documentation -------------- - -Documentation for Python 3.6 is online, updated daily: - - https://docs.python.org/3.6/ - -It can also be downloaded in many formats for faster access. The documentation -is downloadable in HTML, PDF, and reStructuredText formats; the latter version -is primarily for documentation authors, translators, and people with special -formatting requirements. - -If you would like to contribute to the development of Python, relevant -documentation is available at: - - https://docs.python.org/devguide/ - -For information about building Python's documentation, refer to Doc/README.txt. - - -Converting From Python 2.x to 3.x ---------------------------------- - -Python starting with 2.6 contains features to help locating code that needs to -be changed, such as optional warnings when deprecated features are used, and -backported versions of certain key Python 3.x features. - -A source-to-source translation tool, "2to3", can take care of the mundane task -of converting large amounts of source code. It is not a complete solution but -is complemented by the deprecation warnings in 2.6. See -https://docs.python.org/3.6/library/2to3.html for more information. - - -Testing -------- - -To test the interpreter, type "make test" in the top-level directory. -The test set produces some output. You can generally ignore the messages -about skipped tests due to optional features which can't be imported. -If a message is printed about a failed test or a traceback or core dump -is produced, something is wrong. - -By default, tests are prevented from overusing resources like disk space and -memory. To enable these tests, run "make testall". - -IMPORTANT: If the tests fail and you decide to mail a bug report, *don't* -include the output of "make test". It is useless. Run the failing test -manually, as follows: - - ./python -m test -v test_whatever - -(substituting the top of the source tree for '.' if you built in a different -directory). This runs the test in verbose mode. - - -Installing multiple versions ----------------------------- - -On Unix and Mac systems if you intend to install multiple versions of Python -using the same installation prefix (--prefix argument to the configure script) -you must take care that your primary python executable is not overwritten by the -installation of a different version. All files and directories installed using -"make altinstall" contain the major and minor version and can thus live -side-by-side. "make install" also creates ${prefix}/bin/python3 which refers to -${prefix}/bin/pythonX.Y. If you intend to install multiple versions using the -same prefix you must decide which version (if any) is your "primary" version. -Install that version using "make install". Install all other versions using -"make altinstall". - -For example, if you want to install Python 2.7, 3.5, and 3.6 with 3.6 being the -primary version, you would execute "make install" in your 3.6 build directory -and "make altinstall" in the others. - - -Issue Tracker and Mailing List ------------------------------- - -We're soliciting bug reports about all aspects of the language. Fixes are also -welcome, preferably in unified diff format. Please use the issue tracker: - - https://bugs.python.org/ - -If you're not sure whether you're dealing with a bug or a feature, use the -mailing list: - - python-dev@python.org - -To subscribe to the list, use the mailman form: - - https://mail.python.org/mailman/listinfo/python-dev/ - - -Proposals for enhancement -------------------------- - -If you have a proposal to change Python, you may want to send an email to the -comp.lang.python or python-ideas mailing lists for initial feedback. A Python -Enhancement Proposal (PEP) may be submitted if your idea gains ground. All -current PEPs, as well as guidelines for submitting a new PEP, are listed at -https://www.python.org/dev/peps/. - - -Release Schedule ----------------- - -See PEP 494 for release details: https://www.python.org/dev/peps/pep-0494/ - - -Copyright and License Information ---------------------------------- - -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016 Python Software Foundation. All rights reserved. - -Copyright (c) 2000 BeOpen.com. All rights reserved. - -Copyright (c) 1995-2001 Corporation for National Research Initiatives. All -rights reserved. - -Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved. - -See the file "LICENSE" for information on the history of this software, -terms & conditions for usage, and a DISCLAIMER OF ALL WARRANTIES. - -This Python distribution contains *no* GNU General Public License (GPL) code, -so it may be used in proprietary projects. There are interfaces to some GNU -code but these are entirely optional. - -All trademarks referenced herein are property of their respective holders. - diff --git a/README.rst b/README.rst new file mode 100644 index 000000000000000..b0ce13599ad653b --- /dev/null +++ b/README.rst @@ -0,0 +1,251 @@ +This is Python version 3.6.1+ +============================= + +.. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 + :alt: CPython build status on Travis CI + :target: https://travis-ci.org/python/cpython + +.. image:: https://ci.appveyor.com/api/projects/status/4mew1a93xdkbf5ua/branch/3.6?svg=true + :alt: CPython build status on Appveyor + :target: https://ci.appveyor.com/project/python/cpython/branch/3.6 + +.. image:: https://codecov.io/gh/python/cpython/branch/3.6/graph/badge.svg + :alt: CPython code coverage on Codecov + :target: https://codecov.io/gh/python/cpython + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation. All rights +reserved. + +See the end of this file for further copyright and license information. + +General Information +------------------- + +- Website: https://www.python.org +- Source code: https://github.com/python/cpython +- Issue tracker: https://bugs.python.org +- Documentation: https://docs.python.org +- Developer's Guide: https://docs.python.org/devguide/ + +Contributing to CPython +----------------------- + +For more complete instructions on contributing to CPython development, +see the `Developer Guide`_. + +.. _Developer Guide: https://docs.python.org/devguide/ + +Using Python +------------ + +Installable Python kits, and information about using Python, are available at +`python.org`_. + +.. _python.org: https://www.python.org/ + + +Build Instructions +------------------ + +On Unix, Linux, BSD, macOS, and Cygwin:: + + ./configure + make + make test + sudo make install + +This will install Python as python3. + +You can pass many options to the configure script; run ``./configure --help`` +to find out more. On macOS and Cygwin, the executable is called ``python.exe``; +elsewhere it's just ``python``. + +On macOS, if you have configured Python with ``--enable-framework``, you +should use ``make frameworkinstall`` to do the installation. Note that this +installs the Python executable in a place that is not normally on your PATH, +you may want to set up a symlink in ``/usr/local/bin``. + +On Windows, see `PCbuild/readme.txt +`_. + +If you wish, you can create a subdirectory and invoke configure from there. +For example:: + + mkdir debug + cd debug + ../configure --with-pydebug + make + make test + +(This will fail if you *also* built at the top-level directory. You should do +a ``make clean`` at the toplevel first.) + +To get an optimized build of Python, ``configure --enable-optimizations`` +before you run ``make``. This sets the default make targets up to enable +Profile Guided Optimization (PGO) and may be used to auto-enable Link Time +Optimization (LTO) on some platforms. For more details, see the sections +below. + + +Profile Guided Optimization +--------------------------- + +PGO takes advantage of recent versions of the GCC or Clang compilers. If ran, +``make profile-opt`` will do several steps. + +First, the entire Python directory is cleaned of temporary files that may have +resulted in a previous compilation. + +Then, an instrumented version of the interpreter is built, using suitable +compiler flags for each flavour. Note that this is just an intermediary step +and the binary resulted after this step is not good for real life workloads, as +it has profiling instructions embedded inside. + +After this instrumented version of the interpreter is built, the Makefile will +automatically run a training workload. This is necessary in order to profile +the interpreter execution. Note also that any output, both stdout and stderr, +that may appear at this step is suppressed. + +Finally, the last step is to rebuild the interpreter, using the information +collected in the previous one. The end result will be a Python binary that is +optimized and suitable for distribution or production installation. + + +Link Time Optimization +---------------------- + +Enabled via configure's ``--with-lto`` flag. LTO takes advantage of the +ability of recent compiler toolchains to optimize across the otherwise +arbitrary ``.o`` file boundary when building final executables or shared +libraries for additional performance gains. + + +What's New +---------- + +We have a comprehensive overview of the changes in the `What's New in Python +3.6 `_ document. For a more +detailed change log, read `Misc/NEWS +`_, but a full +accounting of changes can only be gleaned from the `commit history +`_. + +If you want to install multiple versions of Python see the section below +entitled "Installing multiple versions". + + +Documentation +------------- + +`Documentation for Python 3.6 `_ is online, +updated daily. + +It can also be downloaded in many formats for faster access. The documentation +is downloadable in HTML, PDF, and reStructuredText formats; the latter version +is primarily for documentation authors, translators, and people with special +formatting requirements. + +For information about building Python's documentation, refer to `Doc/README.rst +`_. + + +Converting From Python 2.x to 3.x +--------------------------------- + +Significant backward incompatible changes were made for the release of Python +3.0, which may cause programs written for Python 2 to fail when run with Python +3. For more information about porting your code from Python 2 to Python 3, see +the `Porting HOWTO `_. + + +Testing +------- + +To test the interpreter, type ``make test`` in the top-level directory. The +test set produces some output. You can generally ignore the messages about +skipped tests due to optional features which can't be imported. If a message +is printed about a failed test or a traceback or core dump is produced, +something is wrong. + +By default, tests are prevented from overusing resources like disk space and +memory. To enable these tests, run ``make testall``. + +If any tests fail, you can re-run the failing test(s) in verbose mode:: + + make test TESTOPTS="-v test_that_failed" + +If the failure persists and appears to be a problem with Python rather than +your environment, you can `file a bug report `_ and +include relevant output from that command to show the issue. + + +Installing multiple versions +---------------------------- + +On Unix and Mac systems if you intend to install multiple versions of Python +using the same installation prefix (``--prefix`` argument to the configure +script) you must take care that your primary python executable is not +overwritten by the installation of a different version. All files and +directories installed using ``make altinstall`` contain the major and minor +version and can thus live side-by-side. ``make install`` also creates +``${prefix}/bin/python3`` which refers to ``${prefix}/bin/pythonX.Y``. If you +intend to install multiple versions using the same prefix you must decide which +version (if any) is your "primary" version. Install that version using ``make +install``. Install all other versions using ``make altinstall``. + +For example, if you want to install Python 2.7, 3.5, and 3.6 with 3.6 being the +primary version, you would execute ``make install`` in your 3.6 build directory +and ``make altinstall`` in the others. + + +Issue Tracker and Mailing List +------------------------------ + +Bug reports are welcome! You can use the `issue tracker +`_ to report bugs, and/or submit pull requests `on +GitHub `_. + +You can also follow development discussion on the `python-dev mailing list +`_. + + +Proposals for enhancement +------------------------- + +If you have a proposal to change Python, you may want to send an email to the +comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A +Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. +All current PEPs, as well as guidelines for submitting a new PEP, are listed at +`python.org/dev/peps/ `_. + +.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ + + +Release Schedule +---------------- + +See :pep:`494` for Python 3.6 release details. + + +Copyright and License Information +--------------------------------- + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +2012, 2013, 2014, 2015, 2016 Python Software Foundation. All rights reserved. + +Copyright (c) 2000 BeOpen.com. All rights reserved. + +Copyright (c) 1995-2001 Corporation for National Research Initiatives. All +rights reserved. + +Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved. + +See the file "LICENSE" for information on the history of this software, terms & +conditions for usage, and a DISCLAIMER OF ALL WARRANTIES. + +This Python distribution contains *no* GNU General Public License (GPL) code, +so it may be used in proprietary projects. There are interfaces to some GNU +code but these are entirely optional. + +All trademarks referenced herein are property of their respective holders. diff --git a/Tools/README b/Tools/README index 0d961de23d6cb8e..edbf4fb83321ba5 100644 --- a/Tools/README +++ b/Tools/README @@ -45,4 +45,4 @@ unittestgui A Tkinter based GUI test runner for unittest, with test discovery. -(*) A generic benchmark suite is maintained separately at http://hg.python.org/benchmarks/ +(*) A generic benchmark suite is maintained separately at https://github.com/python/performance diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index cc1afbe16d01d17..d28851f9fd7f276 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -714,7 +714,7 @@ def _get_entries(self, keys): try: # <= Python 3.5 return keys['dk_entries'], dk_size - except gdb.error: + except RuntimeError: # >= Python 3.6 pass diff --git a/Tools/importbench/README b/Tools/importbench/README index 81a5544a383b4c8..6ba386c2b608d5d 100644 --- a/Tools/importbench/README +++ b/Tools/importbench/README @@ -3,4 +3,4 @@ Importbench is a set of micro-benchmarks for various import scenarios. It should not be used as an overall benchmark of import performance, but rather an easy way to measure impact of possible code changes. For a real-world benchmark of import, use the normal_startup benchmark from -hg.python.org/benchmarks. +https://github.com/python/performance diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 4659a32b838cc17..81a3f86cab36b03 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -65,8 +65,8 @@ if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1 if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if not exist "%HG%" echo Cannot find Mercurial on PATH && exit /B 1 +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 call "%D%get_externals.bat" diff --git a/Tools/msi/exe/exe_files.wxs b/Tools/msi/exe/exe_files.wxs index 01385874fa90b73..e675c21c8975ef1 100644 --- a/Tools/msi/exe/exe_files.wxs +++ b/Tools/msi/exe/exe_files.wxs @@ -8,9 +8,6 @@ - - - diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj index f78e6ffa28fb72d..b3588b7a0bae902 100644 --- a/Tools/msi/make_zip.proj +++ b/Tools/msi/make_zip.proj @@ -17,15 +17,12 @@ rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) - set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) - + diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 4e319ce4c98ccb1..670836be8c1c8b3 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -9,6 +9,8 @@ set USER= set TARGET= set DRYRUN=false set NOGPG= +set PURGE_OPTION=/p:Purge=true +set NOTEST= :CheckOpts if "%1" EQU "-h" goto Help @@ -19,7 +21,11 @@ if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts -if "%1" EQU "--no-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts +if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && godo CheckOpts +if "%1" EQU "--skip-test" (set NOTEST=true) && shift && godo CheckOpts +if "%1" EQU "-T" (set NOTEST=true) && shift && godo CheckOpts +if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 if not defined PLINK where plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" if not defined PLINK where /R "%ProgramFiles(x86)%\PuTTY" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" @@ -35,7 +41,7 @@ echo Found pscp.exe at %PSCP% if defined NOGPG ( set GPG= - echo Skipping GPG signature generation because of --no-gpg + echo Skipping GPG signature generation because of --skip-gpg ) else ( if not defined GPG where gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" if not defined GPG where /R "%PCBUILD%..\externals\windows-installer" gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" @@ -45,8 +51,12 @@ if defined NOGPG ( call "%PCBUILD%env.bat" > nul 2> nul pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +if not defined NOTEST ( + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 +) msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false popd @@ -55,9 +65,12 @@ exit /B 0 :Help echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] echo. -echo --host (-o) Specify the upload host (required) -echo --user (-u) Specify the user on the host (required) -echo --target (-t) Specify the target directory on the host -echo --dry-run Display commands and filenames without executing them -echo -h Display this help information +echo --host (-o) Specify the upload host (required) +echo --user (-u) Specify the user on the host (required) +echo --target (-t) Specify the target directory on the host +echo --dry-run Display commands and filenames without executing them +echo --skip-gpg Does not generate GPG signatures before uploading +echo --skip-purge Does not perform CDN purge after uploading +echo --skip-test (-T) Does not perform post-upload tests +echo -h Display this help information echo. diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 0d472dea5428a09..75840f2f851ecda 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -8,7 +8,9 @@ $(TARGET) /srv/www.python.org/ftp/python true + true false + false @@ -64,7 +66,37 @@ echo. echo." /> - + + + + + + + + + + $(TEMP)\%(Filename)_source + $(TEMP)\%(Filename)_source\%(Filename)%(Extension) + $(TEMP)\%(Filename)_layout + $(OutputPath)\%(Filename)_layoutlog + $(OutputPath)\%(Filename)_layoutlog\%(Filename).log + + + + + + + + + + + + + + + + diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj index d7e932cee54d3c1..464ef0456af47a7 100644 --- a/Tools/nuget/make_pkg.proj +++ b/Tools/nuget/make_pkg.proj @@ -34,9 +34,8 @@ $(NugetArguments) -Version "$(NuspecVersion)" $(NugetArguments) -NoPackageAnalysis -NonInteractive - setlocal -set DOC_FILENAME=python$(PythonVersion).chm -set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140.CRT + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) @@ -45,8 +44,7 @@ set VCREDIST_PATH=$(VS140COMNTOOLS)\..\..\VC\redist\$(Platform)\Microsoft.VC140. - + diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 58b081a9c53ff5f..33a9fead879325c 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -12,7 +12,6 @@ SRCDIR = sysconfig.get_config_var('srcdir') - def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -46,27 +45,76 @@ def mq_patches_applied(): return st.returncode == 0 and bstdout +def get_git_branch(): + """Get the symbolic name for the current git branch""" + cmd = "git rev-parse --abbrev-ref HEAD".split() + try: + return subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return None + + +def get_git_upstream_remote(): + """Get the remote name to use for upstream branches + + Uses "upstream" if it exists, "origin" otherwise + """ + cmd = "git remote get-url upstream".split() + try: + subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return "origin" + return "upstream" + + +@status("Getting base branch for PR", + info=lambda x: x if x is not None else "not a PR branch") +def get_base_branch(): + if not os.path.exists(os.path.join(SRCDIR, '.git')): + # Not a git checkout, so there's no base branch + return None + version = sys.version_info + if version.releaselevel == 'alpha': + base_branch = "master" + else: + base_branch = "{0.major}.{0.minor}".format(version) + this_branch = get_git_branch() + if this_branch is None or this_branch == base_branch: + # Not on a git PR branch, so there's no base branch + return None + upstream_remote = get_git_upstream_remote() + return upstream_remote + "/" + base_branch + + @status("Getting the list of files that have been added/changed", info=lambda x: n_files_str(len(x))) -def changed_files(): +def changed_files(base_branch=None): """Get the list of changed or added files from Mercurial or git.""" if os.path.isdir(os.path.join(SRCDIR, '.hg')): + if base_branch is not None: + sys.exit('need a git checkout to check PR status') cmd = 'hg status --added --modified --no-status' if mq_patches_applied(): cmd += ' --rev qparent' with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] - elif os.path.isdir(os.path.join(SRCDIR, '.git')): - cmd = 'git status --porcelain' + elif os.path.exists(os.path.join(SRCDIR, '.git')): + # We just use an existence check here as: + # directory = normal git checkout/clone + # file = git worktree directory + if base_branch: + cmd = 'git diff --name-status ' + base_branch + else: + cmd = 'git status --porcelain' filenames = [] with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: for line in st.stdout: line = line.decode().rstrip() - status = set(line[:2]) + status_text, filename = line.split(maxsplit=1) + status = set(status_text) # modified, added or unmerged files if not status.intersection('MAU'): continue - filename = line[3:] if ' -> ' in filename: # file is renamed filename = filename.split(' -> ', 2)[1].strip() @@ -165,7 +213,8 @@ def regenerated_pyconfig_h_in(file_paths): return "not needed" def main(): - file_paths = changed_files() + base_branch = get_base_branch() + file_paths = changed_files(base_branch) python_files = [fn for fn in file_paths if fn.endswith('.py')] c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] doc_files = [fn for fn in file_paths if fn.startswith('Doc') and diff --git a/configure b/configure index 1500cea4a8292b6..8bdb82ba4a7bce6 100755 --- a/configure +++ b/configure @@ -760,10 +760,10 @@ build_os build_vendor build_cpu build -HAS_HG -HGBRANCH -HGTAG -HGVERSION +HAS_GIT +GITBRANCH +GITTAG +GITVERSION BASECPPFLAGS target_alias host_alias @@ -2698,17 +2698,17 @@ fi -if test -e $srcdir/.hg/dirstate +if test -e $srcdir/.git/HEAD then -# Extract the first word of "hg", so it can be a program name with args. -set dummy hg; ac_word=$2 +# Extract the first word of "git", so it can be a program name with args. +set dummy git; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_HAS_HG+:} false; then : +if ${ac_cv_prog_HAS_GIT+:} false; then : $as_echo_n "(cached) " >&6 else - if test -n "$HAS_HG"; then - ac_cv_prog_HAS_HG="$HAS_HG" # Let the user override the test. + if test -n "$HAS_GIT"; then + ac_cv_prog_HAS_GIT="$HAS_GIT" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH @@ -2717,7 +2717,7 @@ do test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_HAS_HG="found" + ac_cv_prog_HAS_GIT="found" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -2725,13 +2725,13 @@ done done IFS=$as_save_IFS - test -z "$ac_cv_prog_HAS_HG" && ac_cv_prog_HAS_HG="not-found" + test -z "$ac_cv_prog_HAS_GIT" && ac_cv_prog_HAS_GIT="not-found" fi fi -HAS_HG=$ac_cv_prog_HAS_HG -if test -n "$HAS_HG"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_HG" >&5 -$as_echo "$HAS_HG" >&6; } +HAS_GIT=$ac_cv_prog_HAS_GIT +if test -n "$HAS_GIT"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_GIT" >&5 +$as_echo "$HAS_GIT" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } @@ -2739,17 +2739,17 @@ fi else -HAS_HG=no-repository +HAS_GIT=no-repository fi -if test $HAS_HG = found +if test $HAS_GIT = found then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" + GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else - HGVERSION="" - HGTAG="" - HGBRANCH="" + GITVERSION="" + GITTAG="" + GITBRANCH="" fi @@ -6521,7 +6521,7 @@ $as_echo_n "checking for --enable-optimizations... " >&6; } # Check whether --enable-optimizations was given. if test "${enable_optimizations+set}" = set; then : enableval=$enable_optimizations; -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 diff --git a/configure.ac b/configure.ac index 64f12f28790a7d9..6b331d5742745f6 100644 --- a/configure.ac +++ b/configure.ac @@ -25,25 +25,25 @@ else BASECPPFLAGS="" fi -AC_SUBST(HGVERSION) -AC_SUBST(HGTAG) -AC_SUBST(HGBRANCH) +AC_SUBST(GITVERSION) +AC_SUBST(GITTAG) +AC_SUBST(GITBRANCH) -if test -e $srcdir/.hg/dirstate +if test -e $srcdir/.git/HEAD then -AC_CHECK_PROG(HAS_HG, hg, found, not-found) +AC_CHECK_PROG(HAS_GIT, git, found, not-found) else -HAS_HG=no-repository +HAS_GIT=no-repository fi -if test $HAS_HG = found +if test $HAS_GIT = found then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" + GITVERSION="git -C \$(srcdir) rev-parse --short HEAD" + GITTAG="git -C \$(srcdir) describe --all --always --dirty" + GITBRANCH="git -C \$(srcdir) name-rev --name-only HEAD" else - HGVERSION="" - HGTAG="" - HGBRANCH="" + GITVERSION="" + GITTAG="" + GITBRANCH="" fi AC_CONFIG_SRCDIR([Include/object.h]) @@ -1274,7 +1274,7 @@ Py_OPT='false' AC_MSG_CHECKING(for --enable-optimizations) AC_ARG_ENABLE(optimizations, AS_HELP_STRING([--enable-optimizations], [Enable expensive optimizations (PGO, etc). Disabled by default.]), [ -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' AC_MSG_RESULT(yes);