From 42e62d6d22458e4759dc06ae36fb7d02a0a741d6 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Tue, 29 Sep 2026 16:27:31 +0100 Subject: [PATCH 001/240] Fix typo in `cross-build-linux` job (#158368) --- .github/CODEOWNERS | 10 +++++----- .github/workflows/build.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0a0100fcb2ebb5..6b8a900f49a23a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -63,14 +63,14 @@ .azure-pipelines/ @AA-Turner # GitHub & related scripts -.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz @itamaro @JacobCoffee -Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz -Lib/test/test_tools/test_compute_changes.py @AA-Turner @hugovk @webknjaz +.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz @itamaro @JacobCoffee @StanFromIreland +Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz @StanFromIreland +Lib/test/test_tools/test_compute_changes.py @AA-Turner @hugovk @webknjaz @StanFromIreland Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg # Pre-commit -.pre-commit-config.yaml @hugovk @JacobCoffee -.ruff.toml @hugovk @AlexWaygood @AA-Turner @JacobCoffee +.pre-commit-config.yaml @hugovk @JacobCoffee @StanFromIreland +.ruff.toml @hugovk @AlexWaygood @AA-Turner @JacobCoffee @StanFromIreland # Patchcheck Tools/patchcheck/ @AA-Turner @itamaro diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 90686b83d4a636..9ab0473f4b9c59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -586,7 +586,7 @@ jobs: - name: Display build info run: | "$BUILD_DIR/cross-python/bin/python3" -m test.pythoninfo - - name: Run test subset with host build + - name: Run test subset with cross build run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed From 5c20517a4fc56683efe63a7751020db9573f538d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Tue, 29 Sep 2026 16:34:03 +0100 Subject: [PATCH 002/240] gh-157579: Fix race condition in the cleanup of `tempfile.TemporaryDirectory` (GH-157580) Co-authored-by: Petr Viktorin --- Doc/library/tempfile.rst | 9 ++ Lib/shutil.py | 23 ++-- Lib/tempfile.py | 101 ++++++++++++++++-- Lib/test/test_shutil.py | 31 ++++++ Lib/test/test_tempfile.py | 84 +++++++++++++++ ...-08-14-11-55-00.gh-issue-157579.Kq3Vt2.rst | 6 ++ 6 files changed, 235 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-08-14-11-55-00.gh-issue-157579.Kq3Vt2.rst diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 316d82775cf626..8bc1cdb81d65c6 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -234,6 +234,15 @@ The module defines the following user-callable items: debugging or when you need your cleanup behavior to be conditional based on other logic. + .. warning:: + + Cleanup is not robust against the tree being modified while it is removed. + Files outside of the tree may have their permissions and file flags reset. + + On systems where :data:`shutil.rmtree.avoids_symlink_attacks` is + false, manipulating symbolic links during cleanup + may cause files outside of the tree to be removed. + .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory .. versionadded:: 3.2 diff --git a/Lib/shutil.py b/Lib/shutil.py index ab75ba9da8894b..d6d7806802bd17 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -761,6 +761,7 @@ def _rmtree_safe_fd_step(stack, onexc): # save a call to os.lstat() when walking subdirectories. func, dirfd, path, orig_entry = stack.pop() name = path if orig_entry is None else orig_entry.name + parent_fd = None if func is os.close else dirfd try: if func is os.close: os.close(dirfd) @@ -808,14 +809,14 @@ def _rmtree_safe_fd_step(stack, onexc): except FileNotFoundError: continue except OSError as err: - onexc(os.unlink, fullname, err) + onexc(os.unlink, fullname, err, direntry=entry, dir_fd=topfd) except FileNotFoundError as err: if orig_entry is None or func is os.close: err.filename = path - onexc(func, path, err) + onexc(func, path, err, direntry=orig_entry, dir_fd=parent_fd) except OSError as err: err.filename = path - onexc(func, path, err) + onexc(func, path, err, direntry=orig_entry, dir_fd=parent_fd) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and @@ -823,7 +824,8 @@ def _rmtree_safe_fd_step(stack, onexc): os.stat in os.supports_follow_symlinks) _rmtree_impl = _rmtree_safe_fd if _use_fd_functions else _rmtree_unsafe -def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): +def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None, + _onexc_kwargs=False): """Recursively delete a directory tree. If dir_fd is not None, it should be a file descriptor open to a directory; @@ -846,24 +848,29 @@ def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): sys.audit("shutil.rmtree", path, dir_fd) if ignore_errors: - def onexc(*args): + def onexc(*args, **kwargs): pass elif onerror is None and onexc is None: - def onexc(*args): + def onexc(*args, **kwargs): raise elif onexc is None: if onerror is None: - def onexc(*args): + def onexc(*args, **kwargs): raise else: # delegate to onerror - def onexc(*args): + def onexc(*args, **kwargs): func, path, exc = args if exc is None: exc_info = None, None, None else: exc_info = type(exc), exc, exc.__traceback__ return onerror(func, path, exc_info) + elif not _onexc_kwargs: + # Only the internal caller in tempfile asks for the extra arguments. + _onexc = onexc + def onexc(func, path, err, **kwargs): + return _onexc(func, path, err) _rmtree_impl(path, dir_fd, onexc) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index b5f799f8d68554..ad81c5d3417b4c 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -44,6 +44,7 @@ import shutil as _shutil import errno as _errno from random import Random as _Random +import stat as _stat import sys as _sys import types as _types import weakref as _weakref @@ -274,15 +275,68 @@ def _dont_follow_symlinks(func, path, *args): elif not _os.path.islink(path): func(path, *args) -def _resetperms(path): +def _resetflags(path): try: chflags = _os.chflags except AttributeError: pass else: _dont_follow_symlinks(chflags, path, 0) + +def _resetperms(path): + _resetflags(path) _dont_follow_symlinks(_os.chmod, path, 0o700) +# True if TemporaryDirectory._rmtree() can work relative to open directories +# instead of resolving paths again. +_rmtree_use_dir_fd = ( + {_os.chmod, _os.unlink, _os.lstat} <= _os.supports_dir_fd + and _os.chmod in _os.supports_fd +) + +def _resetperms_fd(dir_fd, path): + # Same as _resetperms(), but for the directory referred to by dir_fd. + if dir_fd is None: + _resetperms(path) + return + _resetflags(path) + _os.chmod(dir_fd, 0o700) + +try: + _nofollow_mode = _os.O_RDONLY | _os.O_NONBLOCK | _os.O_NOFOLLOW +except AttributeError: + _nofollow_mode = None + +def _resetperms_at(name, dir_fd, path): + # Same as _resetperms(), but name is resolved relative to the directory + # file descriptor dir_fd. path is only used for os.chflags(), which + # doesn't support dir_fd or file descriptors. + if dir_fd is None: + _resetperms(path) + return + _resetflags(path) + if _os.chmod in _os.supports_follow_symlinks: + _os.chmod(name, 0o700, dir_fd=dir_fd, follow_symlinks=False) + else: + # dir_fd & follow_symlinks is not supported on this platform. + # Try chmod opening the file with O_NOFOLLOW. + if _nofollow_mode is not None: + try: + fd = _os.open(name, _nofollow_mode, dir_fd=dir_fd) + except OSError: + pass + else: + try: + _os.chmod(fd, 0o700) + finally: + _os.close(fd) + return + # If that did not work, we change by name, which is subject to a race + # condition. + stat = _os.lstat(name, dir_fd=dir_fd) + if not _stat.S_ISLNK(stat.st_mode): + _os.chmod(name, 0o700, dir_fd=dir_fd) + # User visible interfaces. @@ -927,8 +981,12 @@ def __init__(self, suffix=None, prefix=None, dir=None, ignore_errors=self._ignore_cleanup_errors, delete=self._delete) @classmethod - def _rmtree(cls, name, ignore_errors=False, repeated=False): - def onexc(func, path, exc): + def _rmtree(cls, name, ignore_errors=False, repeated=False, dir_fd=None, + fullname=None): + if fullname is None: + fullname = name + + def onexc(func, path, exc, direntry=None, dir_fd=None): # On DragonFly BSD, UF_NOUNLINK removal fails with EISDIR, not EPERM. if isinstance(exc, (PermissionError, IsADirectoryError)): if repeated and path == name: @@ -936,15 +994,30 @@ def onexc(func, path, exc): return raise + # fullpath is path as seen from the working directory + fullpath = fullname + path[len(name):] + # base is path relative to dir_fd, the directory rmtree() + # reached it through, or the whole path when there is none + if dir_fd is None or not _rmtree_use_dir_fd: + base, dir_fd = path, None + elif direntry is None: + base = path + else: + base = direntry.name + try: if path != name: - _resetperms(_os.path.dirname(path)) - _resetperms(path) + # The parent directory of path is the one referred to + # by dir_fd. + _resetperms_fd(dir_fd, _os.path.dirname(fullpath)) + _resetperms_at(base, dir_fd, fullpath) try: - _os.unlink(path) + _os.unlink(base, dir_fd=dir_fd) except IsADirectoryError: - cls._rmtree(path, ignore_errors=ignore_errors) + cls._rmtree(base, ignore_errors=ignore_errors, + repeated=(path == name), + dir_fd=dir_fd, fullname=fullpath) except PermissionError: # The PermissionError handler was originally added for # FreeBSD in directories, but it seems that it is raised @@ -953,21 +1026,27 @@ def onexc(func, path, exc): # raise NotADirectoryError and mask the PermissionError. # So we must re-raise the current PermissionError if # path is not a directory. - if not _os.path.isdir(path) or _os.path.isjunction(path): + if (not _os.path.isdir(fullpath) + or _os.path.isjunction(fullpath)): if ignore_errors: return raise - cls._rmtree(path, ignore_errors=ignore_errors, - repeated=(path == name)) + cls._rmtree(base, ignore_errors=ignore_errors, + repeated=(path == name), + dir_fd=dir_fd, fullname=fullpath) except FileNotFoundError: pass + except OSError: + if ignore_errors: + return + raise elif isinstance(exc, FileNotFoundError): pass else: if not ignore_errors: raise - _shutil.rmtree(name, onexc=onexc) + _shutil.rmtree(name, onexc=onexc, dir_fd=dir_fd, _onexc_kwargs=True) @classmethod def _cleanup(cls, name, warn_message, ignore_errors=False, delete=True): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index a7825adf923c19..87b6453fba8024 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -495,6 +495,37 @@ def check_args_to_onexc(self, func, arg, exc): self.assertTrue(isinstance(exc, OSError)) self.errorState = 3 + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, + 'requires the fd based implementation of rmtree()') + def test_on_exc_kwargs(self): + os.mkdir(TESTFN) + self.addCleanup(shutil.rmtree, TESTFN) + + child_dir_path = os.path.join(TESTFN, 'b') + child_file_path = os.path.join(child_dir_path, 'a') + os.mkdir(child_dir_path) + os_helper.create_empty_file(child_file_path) + old_child_dir_mode = os.stat(child_dir_path).st_mode + # Make unwritable. + new_mode = stat.S_IREAD|stat.S_IEXEC + os.chmod(child_dir_path, new_mode) + + self.addCleanup(os.chmod, child_dir_path, old_child_dir_mode) + + calls = [] + def onexc(func, path, err, direntry=None, dir_fd=None): + calls.append((func, path, err)) + if func is os.unlink: + self.assertEqual(direntry.name, os.path.basename(path)) + self.assertTrue(os.path.samestat( + os.stat(path), os.stat(direntry.name, dir_fd=dir_fd))) + + shutil.rmtree(TESTFN, onexc=onexc, _onexc_kwargs=True) + self.assertIn((os.unlink, child_file_path), + [(func, path) for func, path, err in calls]) + @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index e33cc65e090e3b..cd960ed99117b6 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -14,6 +14,7 @@ import gc import shutil import subprocess +import sysconfig from unittest import mock import unittest @@ -1861,6 +1862,54 @@ def test(target, target_is_directory): new_flags = os.stat(dir1).st_flags self.assertEqual(new_flags, old_flags) + @os_helper.skip_unless_symlink + @os_helper.skip_unless_working_chmod + @support.requires_non_root_user + @unittest.skipIf(support.is_emscripten, 'Fails due to Emscripten bug:' + 'emscripten-core/emscripten#27761') + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, + 'requires the fd based implementation of rmtree()') + def test_cleanup_with_symlink_race(self): + # cleanup() should not operate on files outside of the temporary + # directory when a directory is replaced with a symlink while it + # recovers from a PermissionError (CVE-2026-12345). + with self.do_create(recurse=0) as target: + target_file = os.path.join(target, 'file1') + open(target_file, 'wb').close() + target_mode = os.stat(target_file).st_mode + + d1 = self.do_create(recurse=0) + dir1 = os.path.join(d1.name, 'dir1') + os.mkdir(dir1) + open(os.path.join(dir1, 'file1'), 'wb').close() + # Removing contents of dir1 fails with a PermissionError, and + # dir1 is replaced with a symlink to target at the very moment + # cleanup() starts to recover from that error. + os.chmod(dir1, 0o500) + unlink = os.unlink + def hook(path, *, dir_fd=None): + try: + return unlink(path, dir_fd=dir_fd) + except PermissionError: + if not os.path.islink(dir1): + os.chmod(dir1, 0o700) + os.rename(dir1, dir1 + '_moved') + os.symlink(target, dir1) + raise + try: + with mock.patch('os.unlink', hook): + with contextlib.suppress(OSError): + d1.cleanup() + finally: + if os.path.islink(dir1): + os.unlink(dir1) + os.rename(dir1 + '_moved', dir1) + os.chmod(dir1, 0o700) + d1.cleanup() + + self.assertTrue(os.path.exists(target_file)) + self.assertEqual(os.stat(target_file).st_mode, target_mode) + @support.cpython_only def test_del_on_collection(self): # A TemporaryDirectory is deleted when garbage collected @@ -2033,6 +2082,29 @@ def test_modes(self): d.cleanup() self.assertFalse(os.path.exists(d.name)) + @support.subTests('ignore_errors', (True, False)) + def test_parent_mode_preserved(self, ignore_errors): + # Test that cleanup does not touch the parent directory, + # even if that prevents removal. + for mode in range(8): + mode <<= 6 + with self.subTest(mode=format(mode, '03o')): + outer = self.do_create() + with outer: + d = self.do_create(dir=outer.name, dirs=2, files=2, + ignore_cleanup_errors=ignore_errors) + with d: + os.chmod(outer.name, mode) + orig_mode = os.stat(outer.name).st_mode + try: + d.cleanup() + except PermissionError: + if ignore_errors: + raise + self.assertEqual(os.stat(outer.name).st_mode, orig_mode) + outer.cleanup() + self.assertFalse(os.path.exists(outer.name)) + def check_flags(self, flags): # skip the test if these flags are not supported (ex: FreeBSD 13) filename = os_helper.TESTFN @@ -2070,5 +2142,17 @@ def test_delete_false(self): self.assertTrue(os.path.exists(working_dir)) shutil.rmtree(working_dir) + @unittest.skipUnless( + sysconfig.get_config_var('PY_SUPPORT_TIER') + and sysconfig.get_config_var('PY_SUPPORT_TIER') <= 3, + 'regression test for supported platforms') + @unittest.skipIf(support.MS_WINDOWS, 'dirfd not used on Windows') + @unittest.skipIf(support.is_wasi, 'WASI has no chmod') + def test_cleanup_safe(self): + """Verify that cleanup uses the safer code path""" + # This is a regression test. Feel free to add exceptions for new + # platforms, but don't forget to update the docs. + self.assertTrue(tempfile._rmtree_use_dir_fd) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Security/2026-08-14-11-55-00.gh-issue-157579.Kq3Vt2.rst b/Misc/NEWS.d/next/Security/2026-08-14-11-55-00.gh-issue-157579.Kq3Vt2.rst new file mode 100644 index 00000000000000..46e87575dba983 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-08-14-11-55-00.gh-issue-157579.Kq3Vt2.rst @@ -0,0 +1,6 @@ +Fix a race condition in the cleanup of :class:`tempfile.TemporaryDirectory`. +When working around file system permission errors, files are now removed +relative to open directory file descriptors instead of resolving their path +again, so that replacing a directory of the tree with a symbolic link can no +longer make the cleanup delete files outside of the temporary directory. +This addresses :cve:`2026-12345`. From 568dd3dfe581d21e311a6fb429d74c3d638eaaa8 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Tue, 29 Sep 2026 16:46:32 +0100 Subject: [PATCH 003/240] Retire the `add-issue-header.yml` workflow (#158013) --- .github/ISSUE_TEMPLATE/bug.yml | 2 +- .github/ISSUE_TEMPLATE/crash.yml | 3 +- .github/ISSUE_TEMPLATE/feature.yml | 2 +- .github/workflows/add-issue-header.yml | 56 -------------------------- 4 files changed, 4 insertions(+), 59 deletions(-) delete mode 100644 .github/workflows/add-issue-header.yml diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 177615621f6b8c..fe1273804a5acb 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -16,7 +16,7 @@ body: Make sure to also search the [CPython issue tracker](https://github.com/python/cpython/issues?q=is%3Aissue+sort%3Acreated-desc) to check that the bug has not already been reported. - type: textarea attributes: - label: "Bug description:" + label: "Bug report" description: > Give a clear and concise description of what happened. Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index 81ae91e5b0af97..690fe7721fdb8e 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -10,8 +10,9 @@ body: The CPython interpreter is written in a different programming language, C. A "CPython crash" is when Python itself fails, leading to a traceback in the C stack. - type: textarea attributes: - label: What happened? + label: Crash report description: > + Give a clear and concise description of what happened. Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016), and put any code blocks inside triple backticks. diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index 4361ab2bf827fb..3ecae7112170e0 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -12,7 +12,7 @@ body: Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form. - type: textarea attributes: - label: "Proposal:" + label: "Feature or enhancement" description: > Explain your proposal, why it should be implemented, and how it would be used. Add examples, if applicable. diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml deleted file mode 100644 index 55095f17bb4cce..00000000000000 --- a/.github/workflows/add-issue-header.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Add issue header -# Automatically edits an issue's descriptions with a header, -# one of: -# -# - Bug report -# - Crash report -# - Feature or enhancement - -on: - issues: - types: - # Only ever run once - - opened - -permissions: - contents: read - -jobs: - add-header: - runs-on: ubuntu-slim - permissions: - issues: write - timeout-minutes: 5 - steps: - - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - # language=JavaScript - script: | - // https://devguide.python.org/triage/labels/#type-labels - const HEADERS = new Map([ - ['type-bug', 'Bug report'], - ['type-crash', 'Crash report'], - ['type-feature', 'Feature or enhancement'], - ]); - let issue_data = await github.rest.issues.get({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo - }).then(issue => issue.data); - let header = ''; - for (const label_data of issue_data.labels) { - const label_name = (typeof label_data === 'string') ? label_data : label_data.name; - if (HEADERS.has(label_name)) { - header = HEADERS.get(label_name); - break; - } - } - if (header !== '') { - console.log(`Setting new header: ${header}`); - await github.rest.issues.update({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `# ${header}\n\n${issue_data.body.replaceAll('\r', '')}` - }); - } From a4ad141c55326e5a45a5674adb5a034f5d03c120 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Tue, 29 Sep 2026 12:49:44 -0300 Subject: [PATCH 004/240] Fix unnumbered list item's text alignment in 'Thread safety for set objects' (#158261) --- Doc/builtins/threadsafety.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/builtins/threadsafety.rst b/Doc/builtins/threadsafety.rst index a529f7803affbc..32600a35ef6eea 100644 --- a/Doc/builtins/threadsafety.rst +++ b/Doc/builtins/threadsafety.rst @@ -408,9 +408,9 @@ as operands and always lock both objects: iterate through all the passed iterables and do the following: * :meth:`set.update` and :meth:`set.union` lock both objects only when - the other operand is a :class:`set`, :class:`frozenset`, or :class:`dict`. + the other operand is a :class:`set`, :class:`frozenset`, or :class:`dict`. * :meth:`set.intersection` and :meth:`set.difference` always try to lock - all objects. + all objects. :meth:`set.symmetric_difference` tries to lock both objects. @@ -418,9 +418,9 @@ The update variants of the above methods also have some differences between them: * :meth:`set.difference_update` and :meth:`set.intersection_update` try - to lock all objects one-by-one. + to lock all objects one-by-one. * :meth:`set.symmetric_difference_update` only locks the arguments if it is - of type :class:`set`, :class:`frozenset`, or :class:`dict`. + of type :class:`set`, :class:`frozenset`, or :class:`dict`. The following methods always try to lock both objects: From 115c297fdcc7d0fab41ed0e5c08f1b9cbec35976 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Tue, 29 Sep 2026 11:23:46 -0500 Subject: [PATCH 005/240] gh-158010: Update Windows builds to use OpenSSL 3.5.9 (GH-158432) --- .../2026-06-09-11-40-48.gh-issue-151159.JKVfme.rst | 1 - .../2026-08-28-10-51-35.gh-issue-156369.MPcRL5.rst | 1 - .../2026-09-29-09-56-14.gh-issue-158010.MPQX4w.rst | 1 + Misc/externals.spdx.json | 8 ++++---- PCbuild/get_externals.bat | 4 ++-- PCbuild/python.props | 4 ++-- 6 files changed, 9 insertions(+), 10 deletions(-) delete mode 100644 Misc/NEWS.d/next/Windows/2026-06-09-11-40-48.gh-issue-151159.JKVfme.rst delete mode 100644 Misc/NEWS.d/next/Windows/2026-08-28-10-51-35.gh-issue-156369.MPcRL5.rst create mode 100644 Misc/NEWS.d/next/Windows/2026-09-29-09-56-14.gh-issue-158010.MPQX4w.rst diff --git a/Misc/NEWS.d/next/Windows/2026-06-09-11-40-48.gh-issue-151159.JKVfme.rst b/Misc/NEWS.d/next/Windows/2026-06-09-11-40-48.gh-issue-151159.JKVfme.rst deleted file mode 100644 index ad1be115db5ce8..00000000000000 --- a/Misc/NEWS.d/next/Windows/2026-06-09-11-40-48.gh-issue-151159.JKVfme.rst +++ /dev/null @@ -1 +0,0 @@ -Updated bundled version of OpenSSL to 3.5.7. diff --git a/Misc/NEWS.d/next/Windows/2026-08-28-10-51-35.gh-issue-156369.MPcRL5.rst b/Misc/NEWS.d/next/Windows/2026-08-28-10-51-35.gh-issue-156369.MPcRL5.rst deleted file mode 100644 index 33c02132cfdaae..00000000000000 --- a/Misc/NEWS.d/next/Windows/2026-08-28-10-51-35.gh-issue-156369.MPcRL5.rst +++ /dev/null @@ -1 +0,0 @@ -Updated Windows builds to use OpenSSL 3.5.8. diff --git a/Misc/NEWS.d/next/Windows/2026-09-29-09-56-14.gh-issue-158010.MPQX4w.rst b/Misc/NEWS.d/next/Windows/2026-09-29-09-56-14.gh-issue-158010.MPQX4w.rst new file mode 100644 index 00000000000000..827173387f8e8a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-09-29-09-56-14.gh-issue-158010.MPQX4w.rst @@ -0,0 +1 @@ +Updated Windows builds to use OpenSSL 3.5.9. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 1b658b7d4b7929..0800347e34016d 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,21 +70,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "36af13403a6f34251c875946d1d524c228cbf7b82fd54dd13d9019fbc37da87e" + "checksumValue": "d6cf0d9a651f25fe506f6b3dc133a80b9b79a0bc565f570dcf8fdbd597551485" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.5.8.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.5.9.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.5.8:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.5.9:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.5.8" + "versionInfo": "3.5.9" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 812949dc8370db..b5f070ca2d858d 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.8 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.9 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.53.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.4.0 @@ -79,7 +79,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.8 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.9 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.4.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 diff --git a/PCbuild/python.props b/PCbuild/python.props index 97f04456505cd5..4c49decd7490d1 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -105,8 +105,8 @@ $(libffiDir)$(ArchName)\ $(libffiOutDir)include $(ExternalsDir)\mpdecimal-4.0.0\ - $(ExternalsDir)openssl-3.5.8\ - $(ExternalsDir)openssl-bin-3.5.8\$(ArchName)\ + $(ExternalsDir)openssl-3.5.9\ + $(ExternalsDir)openssl-bin-3.5.9\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.3.1\ From a0491c893956668ec09308e6334f7817b18efbe9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2026 22:03:16 +0200 Subject: [PATCH 006/240] gh-158439: Add _PyUnicodeWriter.recheck_maxchar (#158442) If PyUnicodeWriter_WriteUTF8() or PyUnicodeWriter_Format() fails, PyUnicodeWriter_Finish() now checks if the string uses the most efficient storage. Previously, PyUnicodeWriter_Finish() could return a string with an inefficient storage. * Add _PyUnicodeWriter.recheck_maxchar. PyUnicodeWriter_WriteUTF8() and PyUnicodeWriter_Format() now set recheck_maxchar to 1 on error. * Move error handling from caller to _PyUnicode_DecodeUTF8Writer(). * PyUnicodeWriter_DecodeUTF8Stateful(): rename length parameter to size and rename string parameter to str. --- Doc/c-api/unicode.rst | 2 +- Include/cpython/unicodeobject.h | 7 +- Lib/test/test_capi/test_unicode.py | 38 ++++++++-- ...-09-29-19-01-56.gh-issue-158439.ybKo_6.rst | 4 + Objects/unicode_writer.c | 76 +++++++++++-------- Objects/unicodeobject.c | 47 +++++++++--- 6 files changed, 121 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-29-19-01-56.gh-issue-158439.ybKo_6.rst diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 3b635fa7fa3744..5f918b447950d9 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1945,7 +1945,7 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. -.. c:function:: int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, const char *string, Py_ssize_t length, const char *errors, Py_ssize_t *consumed) +.. c:function:: int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, const char *str, Py_ssize_t size, const char *errors, Py_ssize_t *consumed) Decode the string *str* from UTF-8 with *errors* error handler and write the output into *writer*. diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index ea91f4158eb392..3da18a6ad35db3 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -516,8 +516,8 @@ PyAPI_FUNC(int) PyUnicodeWriter_Format( ...); PyAPI_FUNC(int) PyUnicodeWriter_DecodeUTF8Stateful( PyUnicodeWriter *writer, - const char *string, /* UTF-8 encoded string */ - Py_ssize_t length, /* size of string */ + const char *str, /* UTF-8 encoded string */ + Py_ssize_t size, /* size of string */ const char *errors, /* error handling */ Py_ssize_t *consumed); /* bytes consumed */ @@ -538,6 +538,9 @@ typedef struct { /* minimum character (default: 127, ASCII) */ Py_UCS4 min_char; + // If non-zero, _PyUnicodeWriter_Finish() needs to check maxchar. + int recheck_maxchar; + /* If non-zero, overallocate the buffer (default: 0). */ unsigned char overallocate; diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 032b910a280083..fc2bec2124a39b 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1976,7 +1976,11 @@ def test_invalid_utf8(self): writer.write_utf8(b"invalid=\xFF", -1) def test_recover_utf8_error(self): - # test recovering from PyUnicodeWriter_WriteUTF8() error + # Recover from PyUnicodeWriter_WriteUTF8() errors. A temporary write + # changes the buffer kind to UCS-2 before raising UnicodeDecodeError. + # Then, PyUnicodeWriter_Finish() has to change the buffer kind back to + # ASCII. + writer = self.create_writer(0) writer.write_utf8(b"value=", -1) @@ -1986,12 +1990,13 @@ def test_recover_utf8_error(self): with self.assertRaises(UnicodeDecodeError): s = "truncated\u20AC".encode() writer.write_utf8(s, len(s) - 1) + with self.assertRaises(UnicodeDecodeError): + # Change buffer kind to UCS-2 then raise UnicodeDecodeError + s = "\u20AC\u20AC".encode() + writer.write_utf8(s, len(s) - 1) - # retry write with a valid string writer.write_utf8(b"valid", -1) - - self.assertEqual(writer.finish(), - "value=valid") + self.assertEqual(writer.finish(), "value=valid") def test_decode_utf8(self): # test PyUnicodeWriter_DecodeUTF8Stateful() @@ -2299,6 +2304,29 @@ def test_recover_error(self): self.assertEqual(writer.finish(), 'Hello World.') + def test_recheck_maxchar(self): + # PyUnicodeWriter_Format() changes buffer kind to UCS-2 before raising + # an exception. Then, PyUnicodeWriter_Finish() has to change the buffer + # kind back to ASCII. + from ctypes import py_object + + class StrError: + def __str__(self): + raise RuntimeError("bug") + + writer = self.create_writer(0) + # Allocate ASCII buffer + writer.write_str('ascii') + + obj = StrError() + ucs2_utf8 = '\u20ac'.encode() + with self.assertRaises(RuntimeError): + # Change buffer kind to UCS-2, but then raise RuntimeError + self.writer_format(writer, b"%s%S", ucs2_utf8, py_object(obj)) + + writer.write_str('.') + self.assertEqual(writer.finish(), 'ascii.') + def test_readonly_optim(self): # Read-only optimization: if the first and only write is a Python str # object and no buffer was allocated yet, return the object unchanged diff --git a/Misc/NEWS.d/next/C_API/2026-09-29-19-01-56.gh-issue-158439.ybKo_6.rst b/Misc/NEWS.d/next/C_API/2026-09-29-19-01-56.gh-issue-158439.ybKo_6.rst new file mode 100644 index 00000000000000..59739c7a5e1609 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-29-19-01-56.gh-issue-158439.ybKo_6.rst @@ -0,0 +1,4 @@ +If :c:func:`PyUnicodeWriter_WriteUTF8` or :c:func:`PyUnicodeWriter_Format` +fails, :c:func:`PyUnicodeWriter_Finish` now checks if the string uses the most +efficient storage. Previously, :c:func:`PyUnicodeWriter_Finish` could return a +string with an inefficient storage. Patch by Victor Stinner. diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index c1a2af4d9ac1fe..26deffa6baac63 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -160,7 +160,8 @@ PyUnicodeWriter_Create(Py_ssize_t length) } -void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +void +PyUnicodeWriter_Discard(PyUnicodeWriter *writer) { if (writer == NULL) { return; @@ -511,8 +512,7 @@ PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, assert(writer != NULL); _Py_AssertHoldsTstate(); - _PyUnicodeWriter *priv_writer = (_PyUnicodeWriter*)writer; - return _PyUnicodeWriter_WriteASCIIString(priv_writer, str, size); + return _PyUnicodeWriter_WriteASCIIString((_PyUnicodeWriter*)writer, str, size); } @@ -525,40 +525,25 @@ PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, size = strlen(str); } - _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; - Py_ssize_t old_pos = _writer->pos; - int res = _PyUnicode_DecodeUTF8Writer(_writer, str, size, - _Py_ERROR_STRICT, NULL, NULL); - if (res < 0) { - _writer->pos = old_pos; - } - return res; + return _PyUnicode_DecodeUTF8Writer((_PyUnicodeWriter*)writer, str, size, + _Py_ERROR_STRICT, NULL, NULL); } int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, - const char *string, - Py_ssize_t length, + const char *str, + Py_ssize_t size, const char *errors, Py_ssize_t *consumed) { - if (length < 0) { - length = strlen(string); - } - - _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; - Py_ssize_t old_pos = _writer->pos; - int res = _PyUnicode_DecodeUTF8Writer(_writer, string, length, - _Py_ERROR_UNKNOWN, errors, - consumed); - if (res < 0) { - _writer->pos = old_pos; - if (consumed) { - *consumed = 0; - } + if (size < 0) { + size = strlen(str); } - return res; + + return _PyUnicode_DecodeUTF8Writer((_PyUnicodeWriter*)writer, str, size, + _Py_ERROR_UNKNOWN, errors, + consumed); } @@ -600,30 +585,55 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) Py_ssize_t final_size = writer->pos; if (final_size == 0) { + // Get the empty string singleton PyObject *empty = _PyUnicode_GetEmpty(); Py_XDECREF(str); // writer->buffer can be NULL if the position is 0 return empty; } - Py_ssize_t length = PyUnicode_GET_LENGTH(str); + if (writer->readonly) { + assert(final_size == PyUnicode_GET_LENGTH(str)); + goto done; + } + assert(final_size <= PyUnicode_GET_LENGTH(str)); + if (final_size == 1 && PyUnicode_KIND(str) == PyUnicode_1BYTE_KIND) { - assert(length >= 1); + // Get the single character singleton + assert(PyUnicode_GET_LENGTH(str) >= 1); const Py_UCS1 *data = PyUnicode_1BYTE_DATA(str); Py_UCS1 ch = data[0]; - PyObject *latin1_char = _Py_LATIN1_CHR(ch); Py_DECREF(str); - return latin1_char; + str = _Py_LATIN1_CHR(ch); + goto done; + } + + if (writer->recheck_maxchar) { + Py_UCS4 maxchar = _PyUnicode_FindMaxChar(str, 0, final_size); + if (maxchar != writer->maxchar) { + // Adjust the string kind + PyObject *str2 = PyUnicode_New(final_size, maxchar); + if (str2 == NULL) { + Py_DECREF(str); + return NULL; + } + _PyUnicode_FastCopyCharacters(str2, 0, str, 0, final_size); + Py_SETREF(str, str2); + goto done; + } } - if (!writer->readonly && length != final_size) { + if (PyUnicode_GET_LENGTH(str) != final_size) { + // Truncate the string PyObject *str2 = _PyUnicode_ResizeCompact(str, final_size); if (str2 == NULL) { Py_DECREF(str); return NULL; } str = str2; + goto done; } +done: assert(_PyUnicode_CheckConsistency(str, 1)); return str; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8446fdbfcb64a9..893621f041c9ad 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3166,16 +3166,21 @@ PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) } int -_PyUnicodeWriter_FormatV(PyUnicodeWriter *writer, const char *format, +_PyUnicodeWriter_FormatV(PyUnicodeWriter *pub_writer, const char *format, va_list vargs) { - _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; - Py_ssize_t old_pos = _writer->pos; + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)pub_writer; + Py_ssize_t old_pos = writer->pos; + Py_UCS4 old_maxchar = writer->maxchar; - int res = unicode_from_format(_writer, format, vargs); + int res = unicode_from_format(writer, format, vargs); if (res < 0) { - _writer->pos = old_pos; + writer->pos = old_pos; + if (writer->maxchar > old_maxchar) { + // _PyUnicodeWriter_Finish() will check maxchar + writer->recheck_maxchar = 1; + } } return res; } @@ -5364,7 +5369,7 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, } -// Used by PyUnicodeWriter_WriteUTF8() implementation +// Used by PyUnicodeWriter_WriteUTF8() and PyUnicodeWriter_DecodeUTF8Stateful() int _PyUnicode_DecodeUTF8Writer(_PyUnicodeWriter *writer, const char *s, Py_ssize_t size, @@ -5378,18 +5383,20 @@ _PyUnicode_DecodeUTF8Writer(_PyUnicodeWriter *writer, return 0; } + Py_ssize_t old_pos = writer->pos; + Py_UCS4 old_maxchar = writer->maxchar; + // fast path: try ASCII string. if (_PyUnicodeWriter_Prepare(writer, size, 127) < 0) { - return -1; + goto error; } assert(_PyUnicodeWriter_CanWrite(writer)); const char *starts = s; const char *end = s + size; - Py_ssize_t decoded = 0; - Py_UCS1 *dest = (Py_UCS1*)writer->data + writer->pos * writer->kind; if (writer->kind == PyUnicode_1BYTE_KIND) { - decoded = ascii_decode(s, end, dest); + Py_UCS1 *dest = (Py_UCS1*)writer->data + writer->pos * writer->kind; + Py_ssize_t decoded = ascii_decode(s, end, dest); writer->pos += decoded; if (decoded == size) { @@ -5401,8 +5408,24 @@ _PyUnicode_DecodeUTF8Writer(_PyUnicodeWriter *writer, s += decoded; } - return unicode_decode_utf8_impl(writer, starts, s, end, - error_handler, errors, consumed); + int res = unicode_decode_utf8_impl(writer, starts, s, end, + error_handler, errors, consumed); + if (res < 0) { + goto error; + } + return 0; + +error: + // Restore the writer to its previous state + writer->pos = old_pos; + if (writer->maxchar > old_maxchar) { + // _PyUnicodeWriter_Finish() will check maxchar + writer->recheck_maxchar = 1; + } + if (consumed) { + *consumed = 0; + } + return -1; } From 39e5ed7fea8364dc30efd3fa5d68644606b0eac8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2026 22:15:41 +0200 Subject: [PATCH 007/240] gh-157710: Avoid copy for large int in PyUnicodeWriter_WriteStr() (#158448) Enable the PyUnicodeWriter read-only optimization for large integer (more than 1000 digits) in PyUnicodeWriter_WriteStr() and PyUnicodeWriter_WriteRepr(). --- Lib/test/test_capi/test_unicode.py | 29 +++++++++++++++++++++++++++++ Objects/longobject.c | 4 ---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index fc2bec2124a39b..36ecca48230ddd 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -2267,6 +2267,35 @@ def __repr__(self): self.assertEqual(writer.get_buffer(), expected) self.assertIs(writer.finish(), unique_string) + def test_readonly_optim_large_int(self): + # Read-only optimization in _PyLong_FormatWriter() for large integer: + # use _pylong.int_to_decimal_string() result as a read-only string. + # See pylong_int_to_decimal_string(). + + self.addCleanup(sys.set_int_max_str_digits, + sys.get_int_max_str_digits()) + sys.set_int_max_str_digits(0) + + # _PyLong_FormatWriter() calls _pylong.int_to_decimal_string() for + # integer with Py_SIZE() > 1000. + large_int = 1 << (sys.int_info.bits_per_digit * 1020) + large_int_str = str(large_int) + expected = (len(large_int_str), 127, True) + + for size in (0, 123): + with self.subTest(size=size): + # Test PyUnicodeWriter_WriteStr() + writer = self.create_writer(size) + writer.write_str(large_int) + self.assertEqual(writer.get_buffer(), expected) + self.assertEqual(writer.finish(), large_int_str) + + # Test PyUnicodeWriter_WriteRepr() + writer = self.create_writer(size) + writer.write_repr(large_int) + self.assertEqual(writer.get_buffer(), expected) + self.assertEqual(writer.finish(), large_int_str) + # Test PyUnicodeWriter_Format() @unittest.skipIf(ctypes is None, 'need ctypes') diff --git a/Objects/longobject.c b/Objects/longobject.c index 1dac70820d142b..b9f00ca6fb471b 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2045,10 +2045,6 @@ pylong_int_to_decimal_string(PyObject *aa, goto error; } if (writer) { - Py_ssize_t size = PyUnicode_GET_LENGTH(s); - if (_PyUnicodeWriter_Prepare(writer, size, '9') == -1) { - goto error; - } if (_PyUnicodeWriter_WriteStr(writer, s) < 0) { goto error; } From 596d92387ab0930f88cfa9da595135abeb26595e Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Tue, 29 Sep 2026 16:50:14 -0500 Subject: [PATCH 008/240] gh-158010: Update Android and iOS build scripts to use OpenSSL 3.5.9 (GH-158417) --- .../next/Build/2026-08-26-16-10-15.gh-issue-156369.55jGEj.rst | 1 - .../next/Build/2026-09-29-09-40-25.gh-issue-158010.H4Top_.rst | 1 + .../Security/2026-06-09-10-23-57.gh-issue-151159.91GpWQ.rst | 1 - Platforms/Android/__main__.py | 2 +- Platforms/Apple/__main__.py | 2 +- 5 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 Misc/NEWS.d/next/Build/2026-08-26-16-10-15.gh-issue-156369.55jGEj.rst create mode 100644 Misc/NEWS.d/next/Build/2026-09-29-09-40-25.gh-issue-158010.H4Top_.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-06-09-10-23-57.gh-issue-151159.91GpWQ.rst diff --git a/Misc/NEWS.d/next/Build/2026-08-26-16-10-15.gh-issue-156369.55jGEj.rst b/Misc/NEWS.d/next/Build/2026-08-26-16-10-15.gh-issue-156369.55jGEj.rst deleted file mode 100644 index 22c0bb4514f98d..00000000000000 --- a/Misc/NEWS.d/next/Build/2026-08-26-16-10-15.gh-issue-156369.55jGEj.rst +++ /dev/null @@ -1 +0,0 @@ -Update Android and iOS build scripts to use OpenSSL 3.5.8. diff --git a/Misc/NEWS.d/next/Build/2026-09-29-09-40-25.gh-issue-158010.H4Top_.rst b/Misc/NEWS.d/next/Build/2026-09-29-09-40-25.gh-issue-158010.H4Top_.rst new file mode 100644 index 00000000000000..76e81855af90b0 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-09-29-09-40-25.gh-issue-158010.H4Top_.rst @@ -0,0 +1 @@ +Update Android and iOS build scripts to use OpenSSL 3.5.9. diff --git a/Misc/NEWS.d/next/Security/2026-06-09-10-23-57.gh-issue-151159.91GpWQ.rst b/Misc/NEWS.d/next/Security/2026-06-09-10-23-57.gh-issue-151159.91GpWQ.rst deleted file mode 100644 index 735164c1a65ec3..00000000000000 --- a/Misc/NEWS.d/next/Security/2026-06-09-10-23-57.gh-issue-151159.91GpWQ.rst +++ /dev/null @@ -1 +0,0 @@ -Update Android and iOS installers to use OpenSSL 3.5.7. diff --git a/Platforms/Android/__main__.py b/Platforms/Android/__main__.py index e153bc86811be6..27de5d01c1f19b 100755 --- a/Platforms/Android/__main__.py +++ b/Platforms/Android/__main__.py @@ -224,7 +224,7 @@ def unpack_deps(host, prefix_dir, cache_dir): for name_ver in [ "bzip2-1.0.8-3", "libffi-3.4.4-3", - "openssl-3.5.8-0", + "openssl-3.5.9-0", "sqlite-3.53.4-0", "xz-5.4.6-1", "zstd-1.5.7-2" diff --git a/Platforms/Apple/__main__.py b/Platforms/Apple/__main__.py index 8be0d8719f5b1b..4d29c240baac5a 100644 --- a/Platforms/Apple/__main__.py +++ b/Platforms/Apple/__main__.py @@ -328,7 +328,7 @@ def unpack_deps( for name_ver in [ "BZip2-1.0.8-2", "libFFI-3.4.7-2", - "OpenSSL-3.5.8-1", + "OpenSSL-3.5.9-1", "XZ-5.6.4-2", "mpdecimal-4.0.0-2", "zstd-1.5.7-1", From 540274c649e29a88f56cf55d5634a41f68ead430 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Tue, 29 Sep 2026 18:19:28 -0500 Subject: [PATCH 009/240] gh-158010: Update macOS installer to use OpenSSL 3.5.9 (#158415) --- Mac/BuildScript/build-installer.py | 6 +++--- .../Security/2026-06-09-23-38-08.gh-issue-151159.ds-9f8.rst | 1 - .../macOS/2026-08-27-03-19-21.gh-issue-156369.FKlSGY.rst | 1 - .../macOS/2026-09-29-09-29-17.gh-issue-158010.2zJnoj.rst | 1 + 4 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 Misc/NEWS.d/next/Security/2026-06-09-23-38-08.gh-issue-151159.ds-9f8.rst delete mode 100644 Misc/NEWS.d/next/macOS/2026-08-27-03-19-21.gh-issue-156369.FKlSGY.rst create mode 100644 Misc/NEWS.d/next/macOS/2026-09-29-09-29-17.gh-issue-158010.2zJnoj.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index f22256c1f506cb..f54a95b22e118f 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.5.8", - url="https://github.com/openssl/openssl/releases/download/openssl-3.5.8/openssl-3.5.8.tar.gz", - checksum="a8f84a39918ec6415ce765d9b429d313ba97b8143169c172e734b9514464f5b2", + name="OpenSSL 3.5.9", + url="https://github.com/openssl/openssl/releases/download/openssl-3.5.9/openssl-3.5.9.tar.gz", + checksum="603f5602e2eef00d77fbd429d34dcd5822bb301757a1bc9cdb24c670f1eb859a", buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/Security/2026-06-09-23-38-08.gh-issue-151159.ds-9f8.rst b/Misc/NEWS.d/next/Security/2026-06-09-23-38-08.gh-issue-151159.ds-9f8.rst deleted file mode 100644 index d9251a93b40b2c..00000000000000 --- a/Misc/NEWS.d/next/Security/2026-06-09-23-38-08.gh-issue-151159.ds-9f8.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use OpenSSL 3.5.7. diff --git a/Misc/NEWS.d/next/macOS/2026-08-27-03-19-21.gh-issue-156369.FKlSGY.rst b/Misc/NEWS.d/next/macOS/2026-08-27-03-19-21.gh-issue-156369.FKlSGY.rst deleted file mode 100644 index e953a92f5a106d..00000000000000 --- a/Misc/NEWS.d/next/macOS/2026-08-27-03-19-21.gh-issue-156369.FKlSGY.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use OpenSSL 3.5.8. diff --git a/Misc/NEWS.d/next/macOS/2026-09-29-09-29-17.gh-issue-158010.2zJnoj.rst b/Misc/NEWS.d/next/macOS/2026-09-29-09-29-17.gh-issue-158010.2zJnoj.rst new file mode 100644 index 00000000000000..cf9c417eec7fe6 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2026-09-29-09-29-17.gh-issue-158010.2zJnoj.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.5.9. From 00307b0e69341ed3325012e6a83a810c397f3649 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2026 02:47:46 +0200 Subject: [PATCH 010/240] gh-158451: Add _PyUnicodeWriter_SetBuffer() helper function (#158452) Replace _PyUnicodeWriter_Update() with _PyUnicodeWriter_SetBuffer(). Add _PyUnicodeWriter_SetReadOnly() helper function. Add _PyUnicodeWriter_FinishWithSize() for PyUnicode_DecodeUTF7Stateful(). --- Include/internal/pycore_unicodeobject.h | 53 ++++++++++------ Objects/unicode_writer.c | 81 +++++++++++++++---------- Objects/unicodeobject.c | 13 ++-- 3 files changed, 90 insertions(+), 57 deletions(-) diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index c091aa94371a75..1ee4fc4c4517ff 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -130,22 +130,39 @@ _PyUnicodeWriter_CanWrite(_PyUnicodeWriter *writer) #endif static inline void -_PyUnicodeWriter_Update(_PyUnicodeWriter *writer) +_PyUnicodeWriter_SetBuffer(_PyUnicodeWriter *writer, PyObject *buffer) { - PyObject *buffer = writer->buffer; - writer->maxchar = PyUnicode_MAX_CHAR_VALUE(buffer); + assert(writer->pos <= PyUnicode_GET_LENGTH(buffer)); + + // Py_DECREF() the previous buffer (if any) + Py_XSETREF(writer->buffer, buffer); writer->data = PyUnicode_DATA(buffer); writer->kind = PyUnicode_KIND(buffer); + writer->maxchar = PyUnicode_MAX_CHAR_VALUE(buffer); + writer->size = PyUnicode_GET_LENGTH(buffer); + writer->readonly = 0; +} - if (!writer->readonly) { - writer->size = PyUnicode_GET_LENGTH(buffer); - } - else { - /* Copy-on-write mode: set buffer size to 0 so - * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on - * next write. */ - writer->size = 0; - } +static inline void +_PyUnicodeWriter_SetReadOnly(_PyUnicodeWriter *writer, PyObject *obj, + Py_ssize_t length) +{ + assert(writer->buffer == NULL); + assert(writer->pos == 0); + // Micro-optimization: pass length as a parameter, as it's usually known + // by the caller + assert(length == PyUnicode_GET_LENGTH(obj)); + + writer->buffer = obj; + writer->data = NULL; + /* Set kind and size to 0 to make sure that the next + * _PyUnicodeWriter_Prepare() call allocates a new buffer and copies + * characters. */ + writer->kind = 0; + writer->maxchar = PyUnicode_MAX_CHAR_VALUE(obj); + writer->size = 0; + writer->pos = length; + writer->readonly = 1; } static inline int @@ -156,11 +173,9 @@ _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) // If the first write is a Latin1 character, use the singleton // as a read-only object PyObject *obj = _Py_LATIN1_CHR(ch); - writer->readonly = 1; - writer->buffer = obj; // Py_NewRef() is not need on immortal object - _PyUnicodeWriter_Update(writer); - assert(writer->pos == 0); - writer->pos = 1; + // Py_NewRef() is not need on immortal object + _PyUnicodeWriter_SetReadOnly(writer, obj, 1); + // The next write will create a new buffer and copy the string return 0; } @@ -176,6 +191,10 @@ _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) return 0; } +extern PyObject* _PyUnicodeWriter_FinishWithSize( + _PyUnicodeWriter *writer, + Py_ssize_t size); + /* --- Unicode API -------------------------------------------------------- */ // Export for '_json' shared extension diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index 26deffa6baac63..8637be921e2454 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -178,8 +178,7 @@ _PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) assert(PyUnstable_Object_IsUniquelyReferenced(buffer)); memset(writer, 0, sizeof(*writer)); - writer->buffer = buffer; - _PyUnicodeWriter_Update(writer); + _PyUnicodeWriter_SetBuffer(writer, buffer); writer->min_length = writer->size; assert(_PyUnicodeWriter_CanWrite(writer)); } @@ -204,16 +203,19 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, maxchar = Py_MAX(maxchar, writer->min_char); - PyObject *newbuffer; + PyObject *new_buffer; if (writer->buffer == NULL) { assert(!writer->readonly); + // Do not overallocate at the first allocation, but use min_length - if (alloc < writer->min_length) + if (alloc < writer->min_length) { alloc = writer->min_length; + } - writer->buffer = PyUnicode_New(alloc, maxchar); - if (writer->buffer == NULL) + new_buffer = PyUnicode_New(alloc, maxchar); + if (new_buffer == NULL) { return -1; + } } else if (alloc > writer->size) { // Do not overallocate at the first allocation, but use min_length @@ -223,38 +225,42 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, /* overallocate to limit the number of realloc() */ alloc += alloc / OVERALLOCATE_FACTOR; } - if (alloc < writer->min_length) + if (alloc < writer->min_length) { alloc = writer->min_length; + } if (maxchar > writer->maxchar || writer->readonly) { /* resize + widen */ maxchar = Py_MAX(maxchar, writer->maxchar); - newbuffer = PyUnicode_New(alloc, maxchar); - if (newbuffer == NULL) + new_buffer = PyUnicode_New(alloc, maxchar); + if (new_buffer == NULL) { return -1; - _PyUnicode_FastCopyCharacters(newbuffer, 0, + } + _PyUnicode_FastCopyCharacters(new_buffer, 0, writer->buffer, 0, writer->pos); - writer->readonly = 0; - Py_DECREF(writer->buffer); - writer->buffer = newbuffer; } else { - newbuffer = _PyUnicode_ResizeCompact(writer->buffer, alloc); - if (newbuffer == NULL) + new_buffer = _PyUnicode_ResizeCompact(writer->buffer, alloc); + if (new_buffer == NULL) { return -1; - writer->buffer = newbuffer; + } + // Do not DECREF the old buffer + writer->buffer = NULL; } } - else if (maxchar > writer->maxchar) { + else { + assert(maxchar > writer->maxchar); assert(!writer->readonly); - newbuffer = PyUnicode_New(writer->size, maxchar); - if (newbuffer == NULL) + + new_buffer = PyUnicode_New(writer->size, maxchar); + if (new_buffer == NULL) { return -1; - _PyUnicode_FastCopyCharacters(newbuffer, 0, + } + _PyUnicode_FastCopyCharacters(new_buffer, 0, writer->buffer, 0, writer->pos); - Py_SETREF(writer->buffer, newbuffer); } - _PyUnicodeWriter_Update(writer); + + _PyUnicodeWriter_SetBuffer(writer, new_buffer); return 0; #undef OVERALLOCATE_FACTOR @@ -316,11 +322,7 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) if (maxchar > writer->maxchar || len > writer->size - writer->pos) { if (writer->buffer == NULL && PyUnicode_CheckExact(str)) { assert(_PyUnicode_CheckConsistency(str, 1)); - writer->readonly = 1; - writer->buffer = Py_NewRef(str); - _PyUnicodeWriter_Update(writer); - writer->pos += len; - // The next write will create a new buffer and copy the string + _PyUnicodeWriter_SetReadOnly(writer, Py_NewRef(str), len); return 0; } if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1) @@ -457,10 +459,7 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, if (str == NULL) return -1; - writer->readonly = 1; - writer->buffer = str; - _PyUnicodeWriter_Update(writer); - writer->pos += len; + _PyUnicodeWriter_SetReadOnly(writer, str, len); return 0; } @@ -639,6 +638,26 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) } +PyObject * +_PyUnicodeWriter_FinishWithSize(_PyUnicodeWriter *writer, Py_ssize_t size) +{ + assert(0 <= size); + if (writer->buffer != NULL) { + assert(size <= writer->pos); + assert(size <= PyUnicode_GET_LENGTH(writer->buffer)); + if (size < writer->pos) { + // Truncate the string: we may need to adjust the string kind + writer->recheck_maxchar = 1; + } + } + else { + assert(size == 0); + } + writer->pos = size; + return _PyUnicodeWriter_Finish(writer); +} + + PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 893621f041c9ad..43b720e7cde218 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4761,15 +4761,10 @@ PyUnicode_DecodeUTF7Stateful(const char *s, if (consumed) { if (inShift) { *consumed = startinpos; - if (writer.pos != shiftOutStart && writer.maxchar > 127) { - PyObject *result = PyUnicode_FromKindAndData( - writer.kind, writer.data, shiftOutStart); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - _PyUnicodeWriter_Dealloc(&writer); - return result; - } - writer.pos = shiftOutStart; /* back off output */ + + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return _PyUnicodeWriter_FinishWithSize(&writer, shiftOutStart); } else { *consumed = s-starts; From 26d3f5dd6a9c6699f430138db0e92d224f958747 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2026 03:53:38 +0200 Subject: [PATCH 011/240] gh-158451: Add _PyUnicodeWriter_WriteLatin1String() test (#158464) Fix _PyUnicodeWriter_WriteLatin1String() when the writer buffer kind is UCS-2 or UCS-4. --- Lib/test/test_capi/test_unicode.py | 18 ++++ ...-09-30-03-19-34.gh-issue-158451.pUb2BJ.rst | 2 + Modules/_testcapi/unicode.c | 27 ++++++ Objects/unicode_writer.c | 91 +++++++------------ 4 files changed, 81 insertions(+), 57 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-30-03-19-34.gh-issue-158451.pUb2BJ.rst diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 36ecca48230ddd..c511ffc0fda13c 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1970,6 +1970,24 @@ def test_ascii(self): writer.write_ascii(b"Python! ", 6) self.assertEqual(writer.finish(), "Hello Python") + def test_write_latin1(self): + # Test _PyUnicodeWriter_WriteLatin1String() + writer = self.create_writer(0) + # Start with ASCII buffer + writer.write_latin1(b"abc IGNORED", 3) + writer.write_latin1(b"IGNORED", 0) + # Change buffer kind to UCS-1 + writer.write_latin1(b"\xe9", 1) + # Change buffer kind to UCS-2 + writer.write_str('[\u20ac]') + writer.write_latin1(b"def\xa0", 4) + # Change buffer kind to UCS-4 + writer.write_str('[\U0010ffff]') + writer.write_latin1(b"ghi\xff.", 5) + writer.write_latin1(b"IGNORED", 0) + self.assertEqual(writer.finish(), + "abc\xe9[\u20ac]def\xa0[\U0010ffff]ghi\xff.") + def test_invalid_utf8(self): writer = self.create_writer(0) with self.assertRaises(UnicodeDecodeError): diff --git a/Misc/NEWS.d/next/C_API/2026-09-30-03-19-34.gh-issue-158451.pUb2BJ.rst b/Misc/NEWS.d/next/C_API/2026-09-30-03-19-34.gh-issue-158451.pUb2BJ.rst new file mode 100644 index 00000000000000..a16dd024287076 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-30-03-19-34.gh-issue-158451.pUb2BJ.rst @@ -0,0 +1,2 @@ +Fix :c:func:`!_PyUnicodeWriter_WriteLatin1String` when the writer buffer +kind is UCS-2 or UCS-4. Patch by Victor Stinner. diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index c62813f4f3768d..ba9b205e07ef8b 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -685,6 +685,32 @@ writer_write_substring(PyObject *self_raw, PyObject *args) } +static PyObject* +writer_write_latin1(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + const char *str; + Py_ssize_t bsize, size; + if (!PyArg_ParseTuple(args, "z#n", &str, &bsize, &size)) { + return NULL; + } + + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)self->writer; +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + if (_PyUnicodeWriter_WriteLatin1String(writer, str, size) < 0) { + return NULL; + } +_Py_COMP_DIAG_POP + + Py_RETURN_NONE; +} + + static PyObject* writer_decodeutf8stateful(PyObject *self_raw, PyObject *args) { @@ -778,6 +804,7 @@ static PyMethodDef writer_methods[] = { {"write_str", _PyCFunction_CAST(writer_write_str), METH_O}, {"write_repr", _PyCFunction_CAST(writer_write_repr), METH_O}, {"write_substring", _PyCFunction_CAST(writer_write_substring), METH_VARARGS}, + {"write_latin1", _PyCFunction_CAST(writer_write_latin1), METH_VARARGS}, {"decodeutf8stateful", _PyCFunction_CAST(writer_decodeutf8stateful), METH_VARARGS}, {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS}, {"get_buffer", _PyCFunction_CAST(writer_get_buffer), METH_VARARGS}, diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index 8637be921e2454..0949e45d51cbad 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -62,59 +62,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "stringlib/undef.h" -/* Copy an ASCII or latin1 char* string into a Python Unicode string. - - WARNING: The function doesn't copy the terminating null character and - doesn't check the maximum character (may write a latin1 character in an - ASCII string). */ -static void -unicode_write_cstr(PyObject *unicode, Py_ssize_t index, - const char *str, Py_ssize_t len) -{ - int kind = PyUnicode_KIND(unicode); - const void *data = PyUnicode_DATA(unicode); - const char *end = str + len; - - assert(index + len <= PyUnicode_GET_LENGTH(unicode)); - switch (kind) { - case PyUnicode_1BYTE_KIND: { -#ifdef Py_DEBUG - if (PyUnicode_IS_ASCII(unicode)) { - Py_UCS4 maxchar = ucs1lib_find_max_char( - (const Py_UCS1*)str, - (const Py_UCS1*)str + len); - assert(maxchar < 128); - } -#endif - memcpy((char *) data + index, str, len); - break; - } - case PyUnicode_2BYTE_KIND: { - Py_UCS2 *start = (Py_UCS2 *)data + index; - Py_UCS2 *ucs2 = start; - - for (; str < end; ++ucs2, ++str) - *ucs2 = (Py_UCS2)*str; - - assert((ucs2 - start) <= PyUnicode_GET_LENGTH(unicode)); - break; - } - case PyUnicode_4BYTE_KIND: { - Py_UCS4 *start = (Py_UCS4 *)data + index; - Py_UCS4 *ucs4 = start; - - for (; str < end; ++ucs4, ++str) - *ucs4 = (Py_UCS4)*str; - - assert((ucs4 - start) <= PyUnicode_GET_LENGTH(unicode)); - break; - } - default: - Py_UNREACHABLE(); - } -} - - void _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) { @@ -550,13 +497,43 @@ int _PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, const char *str, Py_ssize_t len) { - Py_UCS4 maxchar; + if (len == 0) { + return 0; + } - maxchar = ucs1lib_find_max_char((const Py_UCS1*)str, (const Py_UCS1*)str + len); - if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1) + const Py_UCS1 *ucs1 = (const Py_UCS1 *)str; + Py_UCS4 maxchar = ucs1lib_find_max_char(ucs1, ucs1 + len); + if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) { return -1; + } assert(_PyUnicodeWriter_CanWrite(writer)); - unicode_write_cstr(writer->buffer, writer->pos, str, len); + + Py_ssize_t index = writer->pos; + switch (writer->kind) { + case PyUnicode_1BYTE_KIND: { + memcpy((Py_UCS1 *)writer->data + index, ucs1, len); + break; + } + case PyUnicode_2BYTE_KIND: { + Py_UCS2 *ucs2 = (Py_UCS2 *)writer->data + index; + const Py_UCS1 *end = ucs1 + len; + for (; ucs1 < end; ++ucs2, ++ucs1) { + *ucs2 = (Py_UCS2)*ucs1; + } + break; + } + case PyUnicode_4BYTE_KIND: { + Py_UCS4 *ucs4 = (Py_UCS4 *)writer->data + index; + const Py_UCS1 *end = ucs1 + len; + for (; ucs1 < end; ++ucs4, ++ucs1) { + *ucs4 = (Py_UCS4)*ucs1; + } + break; + } + default: + Py_UNREACHABLE(); + } + writer->pos += len; return 0; } From c3cb46870bdb18d104557d8b69d451a692c64c8a Mon Sep 17 00:00:00 2001 From: Himesh Rupchandani Date: Wed, 30 Sep 2026 07:54:24 +0530 Subject: [PATCH 012/240] gh-158364: Don't report other interpreters' threads in `sys._current_frames` (GH-158369) --- Lib/test/test_sys.py | 65 ++++++++++++ ...-09-28-18-58-27.gh-issue-158364.jBqmTs.rst | 3 + Python/pystate.c | 98 +++++++++---------- 3 files changed, 114 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-28-18-58-27.gh-issue-158364.jBqmTs.rst diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index da1bd381dd182e..1ccdcf5a187bbb 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -563,6 +563,38 @@ def g456(): leave_g.set() t.join() + @support.cpython_only + @requires_subinterpreters + @threading_helper.requires_working_threading() + def test_current_frames_other_interpreters(self): + # gh-158364: sys._current_frames() would access frames of another + # interpreter and crash + import threading + + entered = threading.Event() + left = threading.Event() + + def park(): + entered.set() + left.wait() + + t = threading.Thread(target=park) + with threading_helper.start_threads([t], unlock=left.set): + entered.wait() + interp = interpreters.create() + try: + interp.exec(f"""if True: + import sys + import threading + + frames = sys._current_frames() + assert threading.get_ident() in frames, frames + assert frames[threading.get_ident()].f_globals is globals() + assert {t.ident} not in frames, frames + """) + finally: + interp.close() + @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_current_exceptions(self): @@ -629,6 +661,39 @@ def g456(): leave_g.set() t.join() + @support.cpython_only + @requires_subinterpreters + @threading_helper.requires_working_threading() + def test_current_exceptions_other_interpreters(self): + # gh-158364: sys._current_exceptions() would hand out exceptions of + # another interpreter and crash + import threading + + entered = threading.Event() + left = threading.Event() + + def hold(): + # The thread has to be handling an exception, otherwise + # sys._current_exceptions() has nothing to report for it. + try: + raise ValueError + except ValueError: + entered.set() + left.wait() + + t = threading.Thread(target=hold) + with threading_helper.start_threads([t], unlock=left.set): + entered.wait() + interp = interpreters.create() + try: + interp.exec(f"""if True: + import sys + + assert {t.ident} not in sys._current_exceptions() + """) + finally: + interp.close() + def test_attributes(self): self.assertIsInstance(sys.api_version, int) self.assertIsInstance(sys.argv, list) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-28-18-58-27.gh-issue-158364.jBqmTs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-28-18-58-27.gh-issue-158364.jBqmTs.rst new file mode 100644 index 00000000000000..14ab4ef12becae --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-28-18-58-27.gh-issue-158364.jBqmTs.rst @@ -0,0 +1,3 @@ +Fix crash when :func:`sys._current_frames` or +:func:`sys._current_exceptions` is called while another interpreter is +running. diff --git a/Python/pystate.c b/Python/pystate.c index 737bf0f216bd34..76ad958478a3ca 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2801,36 +2801,33 @@ _PyThread_CurrentFrames(void) return NULL; } - /* for i in all interpreters: - * for t in all of i's thread states: - * if t's frame isn't NULL, map t's id to its frame + /* for t in all of the current interpreter's thread states: + * if t's frame isn't NULL, map t's id to its frame * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ - _PyEval_StopTheWorldAll(runtime); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); HEAD_LOCK(runtime); - PyInterpreterState *i; - for (i = runtime->interpreters.head; i != NULL; i = i->next) { - _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { - _PyInterpreterFrame *frame = t->current_frame; - frame = _PyFrame_GetFirstComplete(frame); - if (frame == NULL) { - continue; - } - PyObject *id = PyLong_FromUnsignedLong(t->thread_id); - if (id == NULL) { - goto fail; - } - PyObject *frameobj = (PyObject *)_PyFrame_GetFrameObject(frame); - if (frameobj == NULL) { - Py_DECREF(id); - goto fail; - } - int stat = PyDict_SetItem(result, id, frameobj); + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) { + _PyInterpreterFrame *frame = t->current_frame; + frame = _PyFrame_GetFirstComplete(frame); + if (frame == NULL) { + continue; + } + PyObject *id = PyLong_FromUnsignedLong(t->thread_id); + if (id == NULL) { + goto fail; + } + PyObject *frameobj = (PyObject *)_PyFrame_GetFrameObject(frame); + if (frameobj == NULL) { Py_DECREF(id); - if (stat < 0) { - goto fail; - } + goto fail; + } + int stat = PyDict_SetItem(result, id, frameobj); + Py_DECREF(id); + if (stat < 0) { + goto fail; } } goto done; @@ -2840,7 +2837,7 @@ _PyThread_CurrentFrames(void) done: HEAD_UNLOCK(runtime); - _PyEval_StartTheWorldAll(runtime); + _PyEval_StartTheWorld(interp); return result; } @@ -2866,35 +2863,32 @@ _PyThread_CurrentExceptions(void) return NULL; } - /* for i in all interpreters: - * for t in all of i's thread states: - * if t's frame isn't NULL, map t's id to its frame + /* for t in all of the current interpreter's thread states: + * if t's frame isn't NULL, map t's id to its exception * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ - _PyEval_StopTheWorldAll(runtime); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); HEAD_LOCK(runtime); - PyInterpreterState *i; - for (i = runtime->interpreters.head; i != NULL; i = i->next) { - _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { - _PyErr_StackItem *err_info = _PyErr_GetTopmostException(t); - if (err_info == NULL) { - continue; - } - PyObject *id = PyLong_FromUnsignedLong(t->thread_id); - if (id == NULL) { - goto fail; - } - PyObject *exc = err_info->exc_value; - assert(exc == NULL || - exc == Py_None || - PyExceptionInstance_Check(exc)); - - int stat = PyDict_SetItem(result, id, exc == NULL ? Py_None : exc); - Py_DECREF(id); - if (stat < 0) { - goto fail; - } + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) { + _PyErr_StackItem *err_info = _PyErr_GetTopmostException(t); + if (err_info == NULL) { + continue; + } + PyObject *id = PyLong_FromUnsignedLong(t->thread_id); + if (id == NULL) { + goto fail; + } + PyObject *exc = err_info->exc_value; + assert(exc == NULL || + exc == Py_None || + PyExceptionInstance_Check(exc)); + + int stat = PyDict_SetItem(result, id, exc == NULL ? Py_None : exc); + Py_DECREF(id); + if (stat < 0) { + goto fail; } } goto done; @@ -2904,7 +2898,7 @@ _PyThread_CurrentExceptions(void) done: HEAD_UNLOCK(runtime); - _PyEval_StartTheWorldAll(runtime); + _PyEval_StartTheWorld(interp); return result; } From 82a89b38aa48688c593f7888964b153015d38462 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2026 05:30:22 +0200 Subject: [PATCH 013/240] gh-158451: Reuse PyUnicodeWriter_WriteStr() in PyUnicode_Format() (#158467) Remove special case for int in PyUnicode_Format(). Instead, reuse PyUnicodeWriter_WriteStr() and PyUnicodeWriter_WriteRepr() in PyUnicode_Format(). Add PyUnicode_Format() tests on integer formats. --- Lib/test/test_format.py | 15 +++++++++++++++ Objects/unicode_format.c | 38 ++++++++++++++++++++++---------------- Objects/unicode_writer.c | 18 ++++++++++-------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 5d322cb444cfb6..a556faae252fa5 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -240,6 +240,8 @@ def test_common_format(self): testcommon("%d", 42, "42") testcommon("%d", -42, "-42") testcommon("%d", 42.0, "42") + testcommon("%#d", 42, "42") + testcommon("%#d", -42, "-42") testcommon("%#x", 1, "0x1") testcommon("%#X", 1, "0X1") testcommon("%#o", 1, "0o1") @@ -250,8 +252,12 @@ def test_common_format(self): testcommon("%#X", 0, "0X0") testcommon("%x", 0x42, "42") testcommon("%x", -0x42, "-42") + testcommon("%#x", 0x42, "0x42") + testcommon("%#x", -0x42, "-0x42") testcommon("%o", 0o42, "42") testcommon("%o", -0o42, "-42") + testcommon("%#o", 0o42, "0o42") + testcommon("%#o", -0o42, "-0o42") # alternate float formatting testcommon('%g', 1.1, '1.1') testcommon('%#g', 1.1, '1.10000') @@ -344,12 +350,21 @@ def test_common_format(self): "format argument 1: %g requires a real number, not str") def test_str_format(self): + testformat("%s", "abc", "abc") testformat("%r", "\u0378", "'\\u0378'") # non printable testformat("%a", "\u0378", "'\\u0378'") # non printable testformat("%r", "\u0374", "'\u0374'") # printable testformat("%a", "\u0374", "'\\u0374'") # printable testformat('%(x)r', {'x': 1}, '1') + # Some small ints + for fmt in ('s', 'r', 'a'): + with self.subTest(fmt=fmt): + testformat("%" + fmt, 42, "42") + testformat("%#" + fmt, 42, "42") + testformat("%" + fmt, -42, "-42") + testformat("%#" + fmt, -42, "-42") + # Test exception for unknown format characters, etc. if verbose: print('Testing exceptions') diff --git a/Objects/unicode_format.c b/Objects/unicode_format.c index 1d6f3f7d9a6f6a..caadf18f6a5d30 100644 --- a/Objects/unicode_format.c +++ b/Objects/unicode_format.c @@ -571,6 +571,7 @@ unicode_format_arg_parse(struct unicode_formatter_t *ctx, arg->ch = FORMAT_READ(ctx); ctx->fmtpos++; } + assert(arg->width >= 0); } else if (arg->ch >= '0' && arg->ch <= '9') { arg->width = arg->ch - '0'; @@ -590,6 +591,7 @@ unicode_format_arg_parse(struct unicode_formatter_t *ctx, } arg->width = arg->width*10 + (arg->ch - '0'); } + assert(arg->width >= 0); } /* Parse precision. Example: "%.3f" => prec=3 */ @@ -645,6 +647,7 @@ unicode_format_arg_parse(struct unicode_formatter_t *ctx, arg->prec = arg->prec*10 + (arg->ch - '0'); } } + assert(arg->prec >= 0); } /* Ignore "h", "l" and "L" format prefix (ex: "%hi" or "%ls") */ @@ -701,24 +704,27 @@ unicode_format_arg_format(struct unicode_formatter_t *ctx, case 's': case 'r': case 'a': - if (PyLong_CheckExact(v) && arg->width == -1 && arg->prec == -1) { - /* Fast path */ - if (_PyLong_FormatWriter(writer, v, 10, arg->flags & F_ALT) == -1) - return -1; - return 1; + if (arg->width < 0 && arg->prec < 0) { + if (arg->ch == 's') { + if (PyUnicodeWriter_WriteStr((PyUnicodeWriter*)writer, v) < 0) { + return -1; + } + return 1; + } + else if (arg->ch == 'r') { + if (PyUnicodeWriter_WriteRepr((PyUnicodeWriter*)writer, v) < 0) { + return -1; + } + return 1; + } } - if (PyUnicode_CheckExact(v) && arg->ch == 's') { - *p_str = Py_NewRef(v); - } - else { - if (arg->ch == 's') - *p_str = PyObject_Str(v); - else if (arg->ch == 'r') - *p_str = PyObject_Repr(v); - else - *p_str = PyObject_ASCII(v); - } + if (arg->ch == 's') + *p_str = PyObject_Str(v); + else if (arg->ch == 'r') + *p_str = PyObject_Repr(v); + else + *p_str = PyObject_ASCII(v); break; case 'i': diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index 0949e45d51cbad..be3cfb2538b780 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -285,15 +285,16 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) int -PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +PyUnicodeWriter_WriteStr(PyUnicodeWriter *pub_writer, PyObject *obj) { + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)pub_writer; PyTypeObject *type = Py_TYPE(obj); if (type == &PyUnicode_Type) { - return _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, obj); + return _PyUnicodeWriter_WriteStr(writer, obj); } if (type == &PyLong_Type) { - return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + return _PyLong_FormatWriter(writer, obj, 10, 0); } PyObject *str = PyObject_Str(obj); @@ -301,21 +302,22 @@ PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) return -1; } - int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + int res = _PyUnicodeWriter_WriteStr(writer, str); Py_DECREF(str); return res; } int -PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *pub_writer, PyObject *obj) { + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)pub_writer; if (obj == NULL) { - return _PyUnicodeWriter_WriteASCIIString((_PyUnicodeWriter*)writer, "", 6); + return _PyUnicodeWriter_WriteASCIIString(writer, "", 6); } if (Py_TYPE(obj) == &PyLong_Type) { - return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + return _PyLong_FormatWriter(writer, obj, 10, 0); } PyObject *repr = PyObject_Repr(obj); @@ -323,7 +325,7 @@ PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) return -1; } - int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, repr); + int res = _PyUnicodeWriter_WriteStr(writer, repr); Py_DECREF(repr); return res; } From 70e6f3cfe1da77859f1d497f6354b8a87048ed65 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2026 05:31:36 +0200 Subject: [PATCH 014/240] gh-158451: Convert _PyUnicodeWriter_Prepare() to static inline function (#158466) Add assertion to _PyUnicodeWriter_Prepare() to check that length is not negative. Add some assertions to check that _PyUnicodeWriter_Prepare() is called with len >= 1. --- Include/cpython/unicodeobject.h | 36 +++++++++++++++---------- Include/internal/pycore_unicodeobject.h | 6 ++--- Lib/test/test_capi/test_unicode.py | 8 ++++++ Objects/longobject.c | 7 ++--- Objects/unicode_formatter.c | 3 +++ Objects/unicode_writer.c | 12 +++++---- Objects/unicodeobject.c | 6 ++--- 7 files changed, 49 insertions(+), 29 deletions(-) diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 3da18a6ad35db3..54defdb2061aa9 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -557,25 +557,33 @@ typedef struct { _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(void) _PyUnicodeWriter_Init( _PyUnicodeWriter *writer); -/* Prepare the buffer to write 'length' characters - with the specified maximum character. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ - (((MAXCHAR) <= (WRITER)->maxchar \ - && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ - ? 0 \ - : (((LENGTH) == 0) \ - ? 0 \ - : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) - -/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro - instead. */ +// Don't call this function directly, use _PyUnicodeWriter_Prepare() instead. _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_PrepareInternal( _PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar); +// Prepare the buffer to write 'length' characters +// with the specified maximum character. +// +// Return 0 on success. Set an exception and return -1 on error. +_Py_DEPRECATED_EXTERNALLY(3.14) static inline int +_PyUnicodeWriter_Prepare(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar) +{ + assert(0 <= length); + if (maxchar <= writer->maxchar && length <= (writer->size - writer->pos)) { + return 0; + } + if (length == 0) { + return 0; + } +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + return _PyUnicodeWriter_PrepareInternal(writer, length, maxchar); +_Py_COMP_DIAG_POP +} + /* Prepare the buffer to have at least the kind KIND. For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will support characters in range U+000-U+FFFF. diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 1ee4fc4c4517ff..3b930e857e3704 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -168,15 +168,13 @@ _PyUnicodeWriter_SetReadOnly(_PyUnicodeWriter *writer, PyObject *obj, static inline int _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) { - if (ch > writer->maxchar || 1 > writer->size - writer->pos) { + if (ch > writer->maxchar || 1 > (writer->size - writer->pos)) { if (writer->buffer == NULL && ch <= 255) { // If the first write is a Latin1 character, use the singleton // as a read-only object PyObject *obj = _Py_LATIN1_CHR(ch); - // Py_NewRef() is not need on immortal object + // Py_NewRef() is not needed on immortal object _PyUnicodeWriter_SetReadOnly(writer, obj, 1); - - // The next write will create a new buffer and copy the string return 0; } diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index c511ffc0fda13c..6965b9dc11b57d 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -461,6 +461,14 @@ def check_format(expected, format, *args): check_format('%abc', b'%%%s', b'abc') + # test "%s" with empty string + check_format('x=', + b'x=%s', b'') + check_format('x=', + b'x=%0s', b'') + check_format('x=', + b'x=%.3s', b'') + # truncated string check_format('abc', b'%.3s', b'abcdef') diff --git a/Objects/longobject.c b/Objects/longobject.c index b9f00ca6fb471b..9577c8ef4d6ed0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2212,7 +2212,7 @@ long_to_decimal_string_internal(PyObject *aa, } } if (writer) { - if (_PyUnicodeWriter_Prepare(writer, strlen, '9') == -1) { + if (_PyUnicodeWriter_Prepare(writer, strlen, 127) == -1) { Py_DECREF(scratch); return -1; } @@ -2227,7 +2227,7 @@ long_to_decimal_string_internal(PyObject *aa, } } else { - str = PyUnicode_New(strlen, '9'); + str = PyUnicode_New(strlen, 127); if (str == NULL) { Py_DECREF(scratch); return -1; @@ -2385,9 +2385,10 @@ long_format_binary(PyObject *aa, int base, int alternate, /* 2 characters for prefix */ sz += 2; } + assert(sz >= 1); if (writer) { - if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) { + if (_PyUnicodeWriter_Prepare(writer, sz, 127) == -1) { return -1; } assert(_PyUnicodeWriter_CanWrite(writer)); diff --git a/Objects/unicode_formatter.c b/Objects/unicode_formatter.c index 2b7681f7ff3ce9..8cbc774584fa68 100644 --- a/Objects/unicode_formatter.c +++ b/Objects/unicode_formatter.c @@ -1349,6 +1349,7 @@ format_long_internal(PyObject *value, const InternalFormatSpec *format, if (n_total == -1) { goto done; } + assert(n_total >= 1); /* Allocate the memory. */ if (_PyUnicodeWriter_Prepare(writer, n_total, maxchar) == -1) @@ -1503,6 +1504,7 @@ format_float_internal(PyObject *value, if (n_total == -1) { goto done; } + assert(n_total >= 1); /* Allocate the memory. */ if (_PyUnicodeWriter_Prepare(writer, n_total, maxchar) == -1) @@ -1714,6 +1716,7 @@ format_complex_internal(PyObject *value, /* Add 1 for the 'j', and optionally 2 for parens. */ calc_padding(n_re_total + n_im_total + 1 + add_parens * 2, format->width, format->align, &lpad, &rpad, &total); + assert(total >= 1); if (lpad || rpad) maxchar = Py_MAX(maxchar, format->fill_char); diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index be3cfb2538b780..6d921ed24de298 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -138,9 +138,10 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, assert(length >= 0); assert(maxchar <= _Py_MAX_UNICODE); - /* ensure that the _PyUnicodeWriter_Prepare macro was used */ - assert((maxchar > writer->maxchar && length >= 0) - || length > 0); + // Check that _PyUnicodeWriter_Prepare() or _PyUnicodeWriter_PrepareKind() + // was used + assert(maxchar > writer->maxchar + || (length > (writer->size - writer->pos) && length >= 1)); if (length > PY_SSIZE_T_MAX - writer->pos) { PyErr_NoMemory(); @@ -336,11 +337,12 @@ _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) { assert(0 <= start); - assert(end <= PyUnicode_GET_LENGTH(str)); assert(start <= end); + assert(end <= PyUnicode_GET_LENGTH(str)); - if (start == 0 && end == PyUnicode_GET_LENGTH(str)) + if (start == 0 && end == PyUnicode_GET_LENGTH(str)) { return _PyUnicodeWriter_WriteStr(writer, str); + } Py_ssize_t len = end - start; if (len == 0) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 43b720e7cde218..68ce13afd9bf37 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -2545,9 +2545,9 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, Py_UCS4 maxchar; length = PyUnicode_GET_LENGTH(str); - if ((precision == -1 || precision >= length) - && width <= length) + if ((precision == -1 || precision >= length) && width <= length) { return _PyUnicodeWriter_WriteStr(writer, str); + } if (precision != -1) length = Py_MIN(precision, length); @@ -2837,7 +2837,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, #undef SPRINT #undef DO_SPRINTS - assert(len >= 0); + assert(len >= 1); int sign = (buffer[0] == '-'); len -= sign; From b7b4f3ecf2fce4451ee1a8e8c1b6e92dea60ce3d Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Tue, 29 Sep 2026 22:22:37 -0700 Subject: [PATCH 015/240] gh-158446: Reject float format precision near INT_MAX (GH-158474) Formatting a float or complex with a precision within about 1000 of INT_MAX could crash or produce incorrect output. PyOS_double_to_string() now raises ValueError("precision too big") for such precisions, as the format string parsers already do for precisions above INT_MAX. The limit applies regardless of presentation type or value, so a few calls that previously succeeded (inf, nan, or 'g' with such a precision) now raise as well. --- Lib/test/test_format.py | 22 ++++++++++++++++++ ...-09-29-16-32-46.gh-issue-158446.dToaPr.rst | 5 ++++ Python/dtoa.c | 16 ++++++++++++- Python/pystrtod.c | 23 +++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2026-09-29-16-32-46.gh-issue-158446.dToaPr.rst diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index a556faae252fa5..a69a69537d568d 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -654,6 +654,28 @@ def test_precision_c_limits(self): with self.assertRaises(ValueError) as cm: format(c, ".%sf" % (INT_MAX + 1)) + @support.cpython_only + def test_precision_near_int_max(self): + # gh-158446: Precisions just below INT_MAX are rejected before any + # output buffer size is computed from them. + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX + + f = 1e300 + c = complex(f) + for prec in (INT_MAX, INT_MAX - 1023): + for code in "feg": + spec = ".%d%s" % (prec, code) + with self.subTest(spec=spec): + with self.assertRaises(ValueError): + format(f, spec) + with self.assertRaises(ValueError): + format(c, spec) + with self.assertRaises(ValueError): + ("%" + spec) % f + with self.assertRaises(ValueError): + ("%" + spec).encode() % f + def test_g_format_has_no_trailing_zeros(self): # regression test for bugs.python.org/issue40780 self.assertEqual("%.3g" % 1505.0, "1.5e+03") diff --git a/Misc/NEWS.d/next/Security/2026-09-29-16-32-46.gh-issue-158446.dToaPr.rst b/Misc/NEWS.d/next/Security/2026-09-29-16-32-46.gh-issue-158446.dToaPr.rst new file mode 100644 index 00000000000000..f17a3f68b93e85 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-09-29-16-32-46.gh-issue-158446.dToaPr.rst @@ -0,0 +1,5 @@ +Fix a crash or incorrect output that could occur when formatting a +:class:`float` or :class:`complex` with a precision close to the platform's +``INT_MAX``. :c:func:`PyOS_double_to_string` now raises :exc:`ValueError` for +any precision of that magnitude, regardless of presentation type or value, as +the format string parsers already did for precisions above ``INT_MAX``. diff --git a/Python/dtoa.c b/Python/dtoa.c index 89fadd33391cb4..f412278765c8e3 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -67,6 +67,10 @@ * 8. A corner case where _Py_dg_dtoa didn't strip trailing zeros has been * fixed. (bugs.python.org/issue40780) * + * 9. _Py_dg_dtoa clamps ndigits in modes 3 and 5 so that its buffer size + * arithmetic cannot exceed the int range, and rv_alloc's size doubling + * uses size_t. (gh-158446) + * ***************************************************************/ /* Please send bug reports for the original dtoa.c code to David M. Gay (dmg @@ -2108,7 +2112,8 @@ _Py_dg_strtod(const char *s00, char **se) static char * rv_alloc(int i) { - int j, k, *r; + int k, *r; + size_t j; /* size_t so that j <<= 1 cannot overflow for i near INT_MAX */ j = sizeof(ULong); for(k = 0; @@ -2372,6 +2377,15 @@ _Py_dg_dtoa(double dd, int mode, int ndigits, leftright = 0; _Py_FALLTHROUGH; case 5: + /* -330 < k < 330 for any finite nonzero double. Clamp ndigits so + that ndigits + k + 1 stays within int range; no double has + anywhere near this many decimal digits so the digits returned + are unaffected (*decpt saturates in the no_digits case). Same + bound as DOUBLE_TO_STRING_PRECISION_MAX in pystrtod.c. */ + if (ndigits > INT_MAX - 1024) + ndigits = INT_MAX - 1024; + else if (ndigits < -(INT_MAX - 1024)) + ndigits = -(INT_MAX - 1024); i = ndigits + k + 1; ilim = i; ilim1 = i - 1; diff --git a/Python/pystrtod.c b/Python/pystrtod.c index e8aca939d1fb98..7753d1732cf78c 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -401,6 +401,15 @@ _Py_string_to_number_with_underscores( return NULL; } +/* Largest precision magnitude accepted by PyOS_double_to_string(). The + output buffer sizes computed below and within _Py_dg_dtoa() use int and + Py_ssize_t arithmetic on roughly precision + (digits before the point, at + most DBL_MAX_10_EXP + 1 == 309) + a few bytes of sign, point and exponent. + Staying this far inside the int range keeps all of those sums in range. + (Only C callers can pass a negative precision.) _Py_dg_dtoa() applies + the same bound to its ndigits argument. */ +#define DOUBLE_TO_STRING_PRECISION_MAX (INT_MAX - 1024) + #if _PY_SHORT_FLOAT_REPR == 0 /* Given a string that may have a decimal point in the current @@ -766,6 +775,13 @@ char * PyOS_double_to_string(double val, int t, exp; int upper = 0; + if (precision > DOUBLE_TO_STRING_PRECISION_MAX + || precision < -DOUBLE_TO_STRING_PRECISION_MAX) + { + PyErr_SetString(PyExc_ValueError, "precision too big"); + return NULL; + } + /* Validate format_code, and map upper and lower case */ switch (format_code) { case 'e': /* exponent */ @@ -1227,6 +1243,13 @@ char * PyOS_double_to_string(double val, const char * const *float_strings = lc_float_strings; int mode; + if (precision > DOUBLE_TO_STRING_PRECISION_MAX + || precision < -DOUBLE_TO_STRING_PRECISION_MAX) + { + PyErr_SetString(PyExc_ValueError, "precision too big"); + return NULL; + } + /* Validate format_code, and map upper and lower case. Compute the mode and make any adjustments as needed. */ switch (format_code) { From 77c06751571d9cff9a5d763c9bca87db5e939134 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2026 09:26:32 +0200 Subject: [PATCH 016/240] gh-158451: Add _PyUnicodeWriter_FinishWithSize() test (#158481) --- Include/internal/pycore_unicodeobject.h | 3 ++- Lib/test/test_capi/test_unicode.py | 13 +++++++++++++ Modules/_testcapi/unicode.c | 26 +++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 3b930e857e3704..112337d90968b3 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -189,7 +189,8 @@ _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) return 0; } -extern PyObject* _PyUnicodeWriter_FinishWithSize( +// Export for '_testcapi' shared extension +PyAPI_FUNC(PyObject*) _PyUnicodeWriter_FinishWithSize( _PyUnicodeWriter *writer, Py_ssize_t size); diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 6965b9dc11b57d..9c91d6eb512d2a 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -2322,6 +2322,19 @@ def test_readonly_optim_large_int(self): self.assertEqual(writer.get_buffer(), expected) self.assertEqual(writer.finish(), large_int_str) + def test_finish_with_size(self): + # Test _PyUnicodeWriter_FinishWithSize(). Truncate text requires + # to change the buffer kind. + text = 'a\xff\u20ac\U0010ffff' + expected = (0x10ffff, False) # do not test the buffer size + for size in range(len(text) + 1): + writer = self.create_writer(0) + writer.write_utf8(text.encode(), -1) + self.assertEqual(writer.get_buffer()[1:], expected) + self.assertEqual(writer.finish_with_size(size), text[:size]) + + # CRASHES writer.finish_with_size(len(text) + 1) + # Test PyUnicodeWriter_Format() @unittest.skipIf(ctypes is None, 'need ctypes') diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index ba9b205e07ef8b..5138fbab1f4ed6 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -795,6 +795,31 @@ writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) } +static PyObject* +writer_finish_with_size(PyObject *self_raw, PyObject *args) +{ + PyAPI_FUNC(PyObject*) _PyUnicodeWriter_FinishWithSize( + _PyUnicodeWriter *writer, + Py_ssize_t size); + + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "n", &size)) { + return NULL; + } + + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)self->writer; + PyObject *str = _PyUnicodeWriter_FinishWithSize(writer, size); + PyUnicodeWriter_Discard(self->writer); + self->writer = NULL; + return str; +} + + static PyMethodDef writer_methods[] = { {"write_char", _PyCFunction_CAST(writer_write_char), METH_VARARGS}, {"write_utf8", _PyCFunction_CAST(writer_write_utf8), METH_VARARGS}, @@ -809,6 +834,7 @@ static PyMethodDef writer_methods[] = { {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS}, {"get_buffer", _PyCFunction_CAST(writer_get_buffer), METH_VARARGS}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {"finish_with_size", _PyCFunction_CAST(writer_finish_with_size), METH_VARARGS}, {NULL, NULL} /* sentinel */ }; From 7eada7c2c63189fbe753cf5e84b77baf4099a4a1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2026 14:12:59 +0200 Subject: [PATCH 017/240] gh-158451: Fix python-gdb.py for invalid Unicode strings (#158475) python-gdb.py is now able to format an invalid Unicode string: render invalid characters as "\Uhhhhhhhh" instead of raising an exception. The PyUnicodeWriter API fills a UCS-4 buffer with 0xff byte pattern to detect usage of uninitialized characters. It produces invalid characters '\Uffffffff'. PyBytesObjectPtr now iterates on bytes (list of integers), instead of creating a temporary Unicode string. Add more bytes tests in test_gdb pretty printer. Remove the unused 'ch2' variable. --- Lib/test/test_gdb/test_pretty_print.py | 5 +- ...-09-30-06-55-37.gh-issue-158451.pZE56z.rst | 3 + Tools/gdb/libpython.py | 79 ++++++++++--------- 3 files changed, 49 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2026-09-30-06-55-37.gh-issue-158451.pZE56z.rst diff --git a/Lib/test/test_gdb/test_pretty_print.py b/Lib/test/test_gdb/test_pretty_print.py index edb79ef86d31db..ea047a12b802e8 100644 --- a/Lib/test/test_gdb/test_pretty_print.py +++ b/Lib/test/test_gdb/test_pretty_print.py @@ -103,13 +103,16 @@ def test_bytes(self): self.assertGdbRepr(b'And now for something hopefully the same') self.assertGdbRepr(b'string with embedded NUL here \0 and then some more text') self.assertGdbRepr(b'this is a tab:\t' + b' this is a slash:\\' b' this is a slash-N:\n' b' this is a slash-R:\r' ) + # Test double quotes (") + self.assertGdbRepr(b"it's a quote") self.assertGdbRepr(b'this is byte 255:\xff and byte 128:\x80') - self.assertGdbRepr(bytes([b for b in range(255)])) + self.assertGdbRepr(bytes([b for b in range(256)])) @support.requires_resource('cpu') def test_strings(self): diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-09-30-06-55-37.gh-issue-158451.pZE56z.rst b/Misc/NEWS.d/next/Tools-Demos/2026-09-30-06-55-37.gh-issue-158451.pZE56z.rst new file mode 100644 index 00000000000000..2d84d6f0a1393f --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-09-30-06-55-37.gh-issue-158451.pZE56z.rst @@ -0,0 +1,3 @@ +``python-gdb.py`` is now able to format an invalid Unicode string: render +invalid characters as ``\Uhhhhhhhh`` instead of raising an exception. Patch by +Victor Stinner. diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 422e4f605920a0..a7f89442dd21db 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -46,6 +46,9 @@ import sys +MAX_UNICODE = 0x10_ffff + + # Look up the gdb.Type for some standard types: # Those need to be refreshed as types (pointer sizes) may change when # gdb loads different executables @@ -1378,11 +1381,15 @@ def write_repr(self, out, visited): class PyBytesObjectPtr(PyObjectPtr): _typename = 'PyBytesObject' - def __str__(self): + def get_bytes(self): field_ob_size = self.field('ob_size') field_ob_sval = self.field('ob_sval') char_ptr = field_ob_sval.address.cast(_type_unsigned_char_ptr()) - return ''.join([chr(char_ptr[i]) for i in safe_range(field_ob_size)]) + return [char_ptr[i] for i in safe_range(field_ob_size)] + + def __str__(self): + as_bytes = self.get_bytes() + return ''.join([chr(byte) for byte in as_bytes]) def proxyval(self, visited): return str(self) @@ -1390,17 +1397,17 @@ def proxyval(self, visited): def write_repr(self, out, visited): # Write this out as a Python bytes literal, i.e. with a "b" prefix - # Get a PyStringObject* within the Python gdb process: - proxy = self.proxyval(visited) + as_bytes = self.get_bytes() # Transliteration of Python's Objects/bytesobject.c:PyBytes_Repr # to Python code: quote = "'" - if "'" in proxy and not '"' in proxy: + if ord("'") in as_bytes and ord('"') not in as_bytes: quote = '"' out.write('b') out.write(quote) - for byte in proxy: + for value in as_bytes: + byte = chr(value) if byte == quote or byte == '\\': out.write('\\') out.write(byte) @@ -1410,10 +1417,10 @@ def write_repr(self, out, visited): out.write('\\n') elif byte == '\r': out.write('\\r') - elif byte < ' ' or ord(byte) >= 0x7f: + elif value < ord(' ') or value >= 0x7f: out.write('\\x') - out.write(hexdigits[(ord(byte) & 0xf0) >> 4]) - out.write(hexdigits[ord(byte) & 0xf]) + out.write(hexdigits[(value & 0xf0) >> 4]) + out.write(hexdigits[value & 0xf]) else: out.write(byte) out.write(quote) @@ -1466,10 +1473,17 @@ def _unichr_is_printable(char): return unicodedata.category(char) not in ("C", "Z") +def safe_chr(i): + if i <= MAX_UNICODE: + return chr(i) + else: + return f'\\U{i:08x}' + + class PyUnicodeObjectPtr(PyObjectPtr): _typename = 'PyUnicodeObject' - def proxyval(self, visited): + def get_code_points(self): compact = self.field('_base') ascii = compact['_base'] state = ascii['state'] @@ -1491,11 +1505,13 @@ def proxyval(self, visited): # Gather a list of ints from the code point array; these are either # UCS-1, UCS-2 or UCS-4 code points: - code_points = [int(field_str[i]) for i in safe_range(field_length)] + return [int(field_str[i]) for i in safe_range(field_length)] + def proxyval(self, visited): + code_points = self.get_code_points() # Convert the int code points to unicode characters, and generate a # local unicode instance. - result = ''.join(map(chr, code_points)) + result = ''.join(map(safe_chr, code_points)) return result def write_repr(self, out, visited): @@ -1506,20 +1522,18 @@ def write_repr(self, out, visited): encoding = gdb.host_charset() # Get a PyUnicodeObject* within the Python gdb process: - proxy = self.proxyval(visited) + code_points = self.get_code_points() # Transliteration of Python's Object/unicodeobject.c:unicode_repr # to Python: - if "'" in proxy and '"' not in proxy: + if ord("'") in code_points and ord('"') not in code_points: quote = '"' else: quote = "'" out.write(quote) - i = 0 - while i < len(proxy): - ch = proxy[i] - i += 1 + for code_point in code_points: + ch = safe_chr(code_point) # Escape quotes and backslashes if ch == quote or ch == '\\': @@ -1535,24 +1549,24 @@ def write_repr(self, out, visited): out.write('\\r') # Map non-printable US ASCII to '\xhh' */ - elif ch < ' ' or ord(ch) == 0x7F: + elif ch < ' ' or code_point == 0x7F: out.write('\\x') - out.write(hexdigits[(ord(ch) >> 4) & 0x000F]) - out.write(hexdigits[ord(ch) & 0x000F]) + out.write(hexdigits[(code_point >> 4) & 0x000F]) + out.write(hexdigits[code_point & 0x000F]) # Copy ASCII characters as-is - elif ord(ch) < 0x7F: + elif code_point < 0x7F: out.write(ch) # Non-ASCII characters else: - ucs = ch - ch2 = None - - printable = ucs.isprintable() + if code_point <= MAX_UNICODE: + printable = ch.isprintable() + else: + printable = False if printable: try: - ucs.encode(encoding) + ch.encode(encoding) # LookupError or ValueError if the host charset is unknown # or invalid. except (UnicodeEncodeError, LookupError, ValueError): @@ -1561,14 +1575,7 @@ def write_repr(self, out, visited): # Map Unicode whitespace and control characters # (categories Z* and C* except ASCII space) if not printable: - if ch2 is not None: - # Match Python's representation of non-printable - # wide characters. - code = (ord(ch) & 0x03FF) << 10 - code |= ord(ch2) & 0x03FF - code += 0x00010000 - else: - code = ord(ucs) + code = code_point # Map 8-bit characters to '\\xhh' if code <= 0xff: @@ -1596,8 +1603,6 @@ def write_repr(self, out, visited): else: # Copy characters as-is out.write(ch) - if ch2 is not None: - out.write(ch2) out.write(quote) From b98c065756ea336707c30885831e4afcb85aac84 Mon Sep 17 00:00:00 2001 From: Hai Zhu Date: Wed, 30 Sep 2026 21:18:13 +0800 Subject: [PATCH 018/240] gh-158380: Fix the stack operand used by JIT dict access guards (GH-158381) --- Include/internal/pycore_uop_ids.h | 10 ++-- Include/internal/pycore_uop_metadata.h | 32 ++++++------ Lib/test/test_capi/test_opt.py | 13 ++++- ...-09-29-07-28-20.gh-issue-158380.F0J0PO.rst | 1 + Python/bytecodes.c | 6 +-- Python/executor_cases.c.h | 52 +++++++++---------- Python/optimizer_bytecodes.c | 6 +-- Python/optimizer_cases.c.h | 8 +-- 8 files changed, 70 insertions(+), 58 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-29-07-28-20.gh-issue-158380.F0J0PO.rst diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 625c33a6640b85..840a32f0e4481f 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -184,6 +184,7 @@ enum { _GUARD_NOS_NULL, _GUARD_NOS_OVERFLOWED, _GUARD_NOS_TUPLE, + _GUARD_NOS_TYPE, _GUARD_NOS_TYPE_VERSION, _GUARD_NOS_UNICODE, _GUARD_NOT_EXHAUSTED_LIST, @@ -207,7 +208,6 @@ enum { _GUARD_TOS_TUPLE, _GUARD_TOS_UNICODE, _GUARD_TYPE, - _GUARD_TYPE_ITER, _GUARD_TYPE_VERSION, _GUARD_TYPE_VERSION_LOCKED, _HANDLE_PENDING_AND_DEOPT, @@ -891,6 +891,10 @@ enum { _GUARD_NOS_TUPLE_r12, _GUARD_NOS_TUPLE_r22, _GUARD_NOS_TUPLE_r33, + _GUARD_NOS_TYPE_r02, + _GUARD_NOS_TYPE_r12, + _GUARD_NOS_TYPE_r22, + _GUARD_NOS_TYPE_r33, _GUARD_NOS_TYPE_VERSION_r02, _GUARD_NOS_TYPE_VERSION_r12, _GUARD_NOS_TYPE_VERSION_r22, @@ -983,10 +987,6 @@ enum { _GUARD_TYPE_r11, _GUARD_TYPE_r22, _GUARD_TYPE_r33, - _GUARD_TYPE_ITER_r02, - _GUARD_TYPE_ITER_r12, - _GUARD_TYPE_ITER_r22, - _GUARD_TYPE_ITER_r33, _GUARD_TYPE_VERSION_r01, _GUARD_TYPE_VERSION_r11, _GUARD_TYPE_VERSION_r22, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f464effe23bc0b..1c440fe9f1e978 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -265,7 +265,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_PUSH_TAGGED_ZERO] = 0, [_GET_ITER_TRAD] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_ITER] = HAS_EXIT_FLAG, + [_GUARD_NOS_TYPE] = HAS_EXIT_FLAG, [_ITER_NEXT_INLINE] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_ITER_VIRTUAL] = HAS_EXIT_FLAG, [_GUARD_TOS_NOT_NULL] = HAS_EXIT_FLAG, @@ -2515,13 +2515,13 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_GUARD_TYPE_ITER] = { + [_GUARD_NOS_TYPE] = { .best = { 0, 1, 2, 3 }, .entries = { - { 2, 0, _GUARD_TYPE_ITER_r02 }, - { 2, 1, _GUARD_TYPE_ITER_r12 }, - { 2, 2, _GUARD_TYPE_ITER_r22 }, - { 3, 3, _GUARD_TYPE_ITER_r33 }, + { 2, 0, _GUARD_NOS_TYPE_r02 }, + { 2, 1, _GUARD_NOS_TYPE_r12 }, + { 2, 2, _GUARD_NOS_TYPE_r22 }, + { 3, 3, _GUARD_NOS_TYPE_r33 }, }, }, [_ITER_NEXT_INLINE] = { @@ -4446,10 +4446,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_PUSH_TAGGED_ZERO_r23] = _PUSH_TAGGED_ZERO, [_GET_ITER_TRAD_r12] = _GET_ITER_TRAD, [_FOR_ITER_TIER_TWO_r23] = _FOR_ITER_TIER_TWO, - [_GUARD_TYPE_ITER_r02] = _GUARD_TYPE_ITER, - [_GUARD_TYPE_ITER_r12] = _GUARD_TYPE_ITER, - [_GUARD_TYPE_ITER_r22] = _GUARD_TYPE_ITER, - [_GUARD_TYPE_ITER_r33] = _GUARD_TYPE_ITER, + [_GUARD_NOS_TYPE_r02] = _GUARD_NOS_TYPE, + [_GUARD_NOS_TYPE_r12] = _GUARD_NOS_TYPE, + [_GUARD_NOS_TYPE_r22] = _GUARD_NOS_TYPE, + [_GUARD_NOS_TYPE_r33] = _GUARD_NOS_TYPE, [_ITER_NEXT_INLINE_r23] = _ITER_NEXT_INLINE, [_GUARD_NOS_ITER_VIRTUAL_r02] = _GUARD_NOS_ITER_VIRTUAL, [_GUARD_NOS_ITER_VIRTUAL_r12] = _GUARD_NOS_ITER_VIRTUAL, @@ -5475,6 +5475,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOS_TUPLE_r12] = "_GUARD_NOS_TUPLE_r12", [_GUARD_NOS_TUPLE_r22] = "_GUARD_NOS_TUPLE_r22", [_GUARD_NOS_TUPLE_r33] = "_GUARD_NOS_TUPLE_r33", + [_GUARD_NOS_TYPE] = "_GUARD_NOS_TYPE", + [_GUARD_NOS_TYPE_r02] = "_GUARD_NOS_TYPE_r02", + [_GUARD_NOS_TYPE_r12] = "_GUARD_NOS_TYPE_r12", + [_GUARD_NOS_TYPE_r22] = "_GUARD_NOS_TYPE_r22", + [_GUARD_NOS_TYPE_r33] = "_GUARD_NOS_TYPE_r33", [_GUARD_NOS_TYPE_VERSION] = "_GUARD_NOS_TYPE_VERSION", [_GUARD_NOS_TYPE_VERSION_r02] = "_GUARD_NOS_TYPE_VERSION_r02", [_GUARD_NOS_TYPE_VERSION_r12] = "_GUARD_NOS_TYPE_VERSION_r12", @@ -5590,11 +5595,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TYPE_r11] = "_GUARD_TYPE_r11", [_GUARD_TYPE_r22] = "_GUARD_TYPE_r22", [_GUARD_TYPE_r33] = "_GUARD_TYPE_r33", - [_GUARD_TYPE_ITER] = "_GUARD_TYPE_ITER", - [_GUARD_TYPE_ITER_r02] = "_GUARD_TYPE_ITER_r02", - [_GUARD_TYPE_ITER_r12] = "_GUARD_TYPE_ITER_r12", - [_GUARD_TYPE_ITER_r22] = "_GUARD_TYPE_ITER_r22", - [_GUARD_TYPE_ITER_r33] = "_GUARD_TYPE_ITER_r33", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_GUARD_TYPE_VERSION_r01] = "_GUARD_TYPE_VERSION_r01", [_GUARD_TYPE_VERSION_r11] = "_GUARD_TYPE_VERSION_r11", @@ -6626,7 +6626,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _FOR_ITER_TIER_TWO: return 0; - case _GUARD_TYPE_ITER: + case _GUARD_NOS_TYPE: return 0; case _ITER_NEXT_INLINE: return 0; diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index ca0545dba264f2..31440c8e609213 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2445,7 +2445,18 @@ def f(n): self.assertEqual(uops.count("_STORE_SUBSCR_DICT_KNOWN_HASH"), 1) self.assertEqual(uops.count("_GUARD_NOS_DICT_SUBSCRIPT"), 0) self.assertEqual(uops.count("_GUARD_NOS_DICT_STORE_SUBSCRIPT"), 0) - self.assertEqual(uops.count("_GUARD_TYPE"), 1) + self.assertEqual(uops.count("_GUARD_NOS_TYPE"), 1) + + def test_dict_subscr_probable_type(self): + def f(d): + for _ in range(TIER2_THRESHOLD): + value = d["key"] + return value + + res, ex = self._run_with_optimizer(f, {"key": 1}) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + self.assertIn("_GUARD_NOS_TYPE", get_opnames(ex)) def test_dict_subclass_subscr_with_override(self): class MyDict(dict): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-29-07-28-20.gh-issue-158380.F0J0PO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-29-07-28-20.gh-issue-158380.F0J0PO.rst new file mode 100644 index 00000000000000..2b3f1d9301e733 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-29-07-28-20.gh-issue-158380.F0J0PO.rst @@ -0,0 +1 @@ +Fix incorrect type guards for ``dict`` reads and writes in the experimental JIT. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fdfd5a0e609046..2ca52766c110b9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3846,9 +3846,9 @@ dummy_func( next = item; } - tier2 op(_GUARD_TYPE_ITER, (expected_type/4, iter, null_or_index -- iter, null_or_index)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - EXIT_IF(Py_TYPE(iter_o) != (PyTypeObject *)expected_type); + tier2 op(_GUARD_NOS_TYPE, (expected_type/4, nos, unused -- nos, unused)) { + PyObject *nos_o = PyStackRef_AsPyObjectBorrow(nos); + EXIT_IF(Py_TYPE(nos_o) != (PyTypeObject *)expected_type); } tier2 op(_ITER_NEXT_INLINE, (iternext_fn/4, iter, null_or_index -- iter, null_or_index, next)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e3dc2e781a3beb..0045296f1bc868 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14816,20 +14816,20 @@ break; } - case _GUARD_TYPE_ITER_r02: { + case _GUARD_NOS_TYPE_r02: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef iter; - iter = stack_pointer[-2]; + _PyStackRef nos; + nos = stack_pointer[-2]; PyObject *expected_type = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != (PyTypeObject *)expected_type) { + PyObject *nos_o = PyStackRef_AsPyObjectBorrow(nos); + if (Py_TYPE(nos_o) != (PyTypeObject *)expected_type) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = iter; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -14837,22 +14837,22 @@ break; } - case _GUARD_TYPE_ITER_r12: { + case _GUARD_NOS_TYPE_r12: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef iter; + _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; - iter = stack_pointer[-1]; + nos = stack_pointer[-1]; PyObject *expected_type = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != (PyTypeObject *)expected_type) { + PyObject *nos_o = PyStackRef_AsPyObjectBorrow(nos); + if (Py_TYPE(nos_o) != (PyTypeObject *)expected_type) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } _tos_cache1 = _stack_item_0; - _tos_cache0 = iter; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -14860,49 +14860,49 @@ break; } - case _GUARD_TYPE_ITER_r22: { + case _GUARD_NOS_TYPE_r22: { CHECK_CURRENT_CACHED_VALUES(2); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef iter; + _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - iter = _stack_item_0; + nos = _stack_item_0; PyObject *expected_type = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != (PyTypeObject *)expected_type) { + PyObject *nos_o = PyStackRef_AsPyObjectBorrow(nos); + if (Py_TYPE(nos_o) != (PyTypeObject *)expected_type) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; + _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _GUARD_TYPE_ITER_r33: { + case _GUARD_NOS_TYPE_r33: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef iter; + _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - iter = _stack_item_1; + nos = _stack_item_1; PyObject *expected_type = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != (PyTypeObject *)expected_type) { + PyObject *nos_o = PyStackRef_AsPyObjectBorrow(nos); + if (Py_TYPE(nos_o) != (PyTypeObject *)expected_type) { UOP_STAT_INC(uopcode, miss); _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; + _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; + _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 5246e50633461b..54b01e1ffbd322 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1472,7 +1472,7 @@ dummy_func(void) { sym_set_type(iter, type); assert((this_instr - 1)->opcode == _RECORD_NOS_TYPE); int32_t orig_target = (this_instr - 1)->target; - ADD_OP(_GUARD_TYPE_ITER, 0, (uintptr_t)type); + ADD_OP(_GUARD_NOS_TYPE, 0, (uintptr_t)type); uop_buffer_last(&ctx->out_buffer)->target = orig_target; } ADD_OP(_ITER_NEXT_INLINE, 0, (uintptr_t)type->tp_iternext); @@ -2216,7 +2216,7 @@ dummy_func(void) { ADD_OP(_NOP, 0, 0); } else { - ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp); + ADD_OP(_GUARD_NOS_TYPE, 0, (uintptr_t)tp); sym_set_type(nos, tp); } PyType_Watch(TYPE_WATCHER_ID, (PyObject *)tp); @@ -2237,7 +2237,7 @@ dummy_func(void) { ADD_OP(_NOP, 0, 0); } else { - ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp); + ADD_OP(_GUARD_NOS_TYPE, 0, (uintptr_t)tp); sym_set_type(nos, tp); } PyType_Watch(TYPE_WATCHER_ID, (PyObject *)tp); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 21f275f27cafe0..73a86bfe86dacd 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1500,7 +1500,7 @@ ADD_OP(_NOP, 0, 0); } else { - ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp); + ADD_OP(_GUARD_NOS_TYPE, 0, (uintptr_t)tp); sym_set_type(nos, tp); } PyType_Watch(TYPE_WATCHER_ID, (PyObject *)tp); @@ -1524,7 +1524,7 @@ ADD_OP(_NOP, 0, 0); } else { - ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp); + ADD_OP(_GUARD_NOS_TYPE, 0, (uintptr_t)tp); sym_set_type(nos, tp); } PyType_Watch(TYPE_WATCHER_ID, (PyObject *)tp); @@ -3705,7 +3705,7 @@ sym_set_type(iter, type); assert((this_instr - 1)->opcode == _RECORD_NOS_TYPE); int32_t orig_target = (this_instr - 1)->target; - ADD_OP(_GUARD_TYPE_ITER, 0, (uintptr_t)type); + ADD_OP(_GUARD_NOS_TYPE, 0, (uintptr_t)type); uop_buffer_last(&ctx->out_buffer)->target = orig_target; } ADD_OP(_ITER_NEXT_INLINE, 0, (uintptr_t)type->tp_iternext); @@ -3718,7 +3718,7 @@ break; } - case _GUARD_TYPE_ITER: { + case _GUARD_NOS_TYPE: { break; } From 83de667b25f9e0f3dd7a37b6f3d136a444cd6a0d Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Wed, 30 Sep 2026 14:32:18 +0100 Subject: [PATCH 019/240] gh-158072: Fix JIT crash with omitted default arguments (GH-158128) --- Lib/test/test_capi/test_opt.py | 31 +++++++++++++++++++ ...-09-24-18-44-40.gh-issue-158072.mHveWw.rst | 1 + Python/optimizer_symbols.c | 20 +++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-44-40.gh-issue-158072.mHveWw.rst diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 31440c8e609213..5f2843e56f46ad 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1812,6 +1812,37 @@ def testfunc(n): # __init__ resolution allows promotion of range to constant self.assertNotIn("_LOAD_GLOBAL_BUILTINS", uops) + # See https://github.com/python/cpython/issues/158072 + def test_init_with_default_argument(self): + script_helper.assert_python_ok("-c", textwrap.dedent(f"""\ + sentinel = object() + + class WithDefault: + def __init__(self, value=sentinel): + if value is not sentinel: + pass + + for _ in range({TIER2_THRESHOLD * 3}): + WithDefault() + """), PYTHON_JIT="1") + + def test_init_with_changed_code_argcount(self): + class C: + def __init__(self, value): + self.value = False + + def varargs_init(self, *args): + self.value = isinstance(args, tuple) + + def testfunc(n): + for _ in range(n): + result = C(42).value + return result + + self.assertFalse(testfunc(100)) + C.__init__.__code__ = C.varargs_init.__code__ + self.assertTrue(testfunc(TIER2_THRESHOLD * 3)) + def test_init_guards_removed(self): class MyPoint: def __init__(self, x, y): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-44-40.gh-issue-158072.mHveWw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-44-40.gh-issue-158072.mHveWw.rst new file mode 100644 index 00000000000000..faff8fff4aec5c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-44-40.gh-issue-158072.mHveWw.rst @@ -0,0 +1 @@ +Fix a JIT crash when a class is instantiated with their default arguments diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 79f81482d247e3..3147e9c5065edd 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -1388,17 +1388,27 @@ _Py_uop_frame_new( } // Initialize with the initial state of all local variables + arg_len = Py_MIN(arg_len, co->co_argcount); for (int i = 0; i < arg_len; i++) { frame->locals[i] = PyJitRef_RemoveUnique(args[i]); } - // If the args are known, then it's safe to just initialize - // every other non-set local to null symbol. - bool default_null = args != NULL; + // When args is available, missing parameters get defaults or *args/**kwargs. + // Other locals start as NULL. When args is NULL, treat all locals as unknown. + int parameter_count = co->co_argcount + co->co_kwonlyargcount; + parameter_count += (co->co_flags & CO_VARARGS) != 0; + parameter_count += (co->co_flags & CO_VARKEYWORDS) != 0; for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptRef local = default_null ? _Py_uop_sym_new_null(ctx) : _Py_uop_sym_new_unknown(ctx); - frame->locals[i] = local; + if (args == NULL) { + frame->locals[i] = _Py_uop_sym_new_unknown(ctx); + } + else if (i < parameter_count) { + frame->locals[i] = _Py_uop_sym_new_not_null(ctx); + } + else { + frame->locals[i] = _Py_uop_sym_new_null(ctx); + } } frame->callable = _Py_uop_sym_new_not_null(ctx); From 069c74a6b73505b58ffc9b2d7b753b3aac481410 Mon Sep 17 00:00:00 2001 From: Jun Luo <2368403+overcat@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:37:24 +0800 Subject: [PATCH 020/240] gh-157833: Fix specialized C calls with additional method flags (GH-157834) --- Include/internal/pycore_call.h | 5 + Lib/test/test_capi/test_opt.py | 180 ++++++++++++++++++ Lib/test/test_opcache.py | 83 ++++++++ ...-09-20-08-59-48.gh-issue-157833.3b0bac.rst | 2 + Modules/_testcapimodule.c | 5 + Modules/_testinternalcapi/test_cases.c.h | 21 +- Objects/descrobject.c | 3 +- Objects/methodobject.c | 3 +- Python/bytecodes.c | 22 ++- Python/executor_cases.c.h | 21 +- Python/generated_cases.c.h | 21 +- Python/optimizer_analysis.c | 1 + Python/optimizer_bytecodes.c | 34 ++-- Python/optimizer_cases.c.h | 33 ++-- Python/specialize.c | 9 +- 15 files changed, 383 insertions(+), 60 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-20-08-59-48.gh-issue-157833.3b0bac.rst diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index a9db8860e91c06..477b3c20e53f6e 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -12,6 +12,11 @@ extern "C" { #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_stats.h" +/* Flags that determine the C calling convention. */ +#define _Py_METH_CALL_FLAGS \ + (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | \ + METH_KEYWORDS | METH_METHOD) + /* Suggested size (number of positional arguments) for arrays of PyObject* allocated on a C stack to avoid allocating memory on the heap memory. Such array is used to pass positional arguments to call functions of the diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 5f2843e56f46ad..d387fc7a23da26 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -3349,6 +3349,186 @@ def testfunc(n): self.assertIn("_CALL_BUILTIN_FAST_WITH_KEYWORDS", uops) self.assertNotIn("_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", uops) + def test_call_builtin_o_extra_flags(self): + # Extra method flags must not prevent callable guard elimination. + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + namespace = { + "METH_CLASS_O": _testcapi.MethClass.meth_o, + "METH_STATIC_O": _testcapi.MethStatic.meth_o, + "METH_COEXIST_O": {}.__contains__, + } + + @reset_code + def testfunc(n): + for _ in range(n): + class_result = METH_CLASS_O(1) + static_result = METH_STATIC_O(1) + coexist_result = METH_COEXIST_O(1) + return class_result, static_result, coexist_result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, ((_testcapi.MethClass, 1), (None, 1), False)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_BUILTIN_O"), 3) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_O", uops) + + def test_call_builtin_fast_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + obj = _testcapi.MethInstance() + namespace = { + "METH_CLASS_FASTCALL": _testcapi.MethClass.meth_fastcall, + "METH_STATIC_FASTCALL": _testcapi.MethStatic.meth_fastcall, + "METH_COEXIST_FASTCALL": obj.meth_fastcall_coexist, + } + + @reset_code + def testfunc(n): + for _ in range(n): + class_result = METH_CLASS_FASTCALL(1, 2) + static_result = METH_STATIC_FASTCALL(1, 2) + coexist_result = METH_COEXIST_FASTCALL(1, 2) + return class_result, static_result, coexist_result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, ( + (_testcapi.MethClass, (1, 2)), + (None, (1, 2)), + (obj, (1, 2)), + )) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_BUILTIN_FAST"), 3) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_FAST", uops) + + def test_call_builtin_fast_with_keywords_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + obj = _testcapi.MethInstance() + namespace = { + "METH_CLASS_FASTCALL_KEYWORDS": ( + _testcapi.MethClass.meth_fastcall_keywords), + "METH_STATIC_FASTCALL_KEYWORDS": ( + _testcapi.MethStatic.meth_fastcall_keywords), + "METH_COEXIST_FASTCALL_KEYWORDS": ( + obj.meth_fastcall_keywords_coexist), + } + + @reset_code + def testfunc(n): + # Use positional arguments to exercise CALL, not CALL_KW. + for _ in range(n): + class_result = METH_CLASS_FASTCALL_KEYWORDS(1, 2) + static_result = METH_STATIC_FASTCALL_KEYWORDS(1, 2) + coexist_result = METH_COEXIST_FASTCALL_KEYWORDS(1, 2) + return class_result, static_result, coexist_result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, ( + (_testcapi.MethClass, (1, 2), {}), + (None, (1, 2), {}), + (obj, (1, 2), {}), + )) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_BUILTIN_FAST_WITH_KEYWORDS"), 3) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", uops) + + def test_call_method_descriptor_o_extra_flags(self): + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + @reset_code + def testfunc(n): + d = {1: None} + for _ in range(n): + result = d.__contains__(1) + return result + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIs(res, True) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_METHOD_DESCRIPTOR_O_INLINE"), 1) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_O", uops) + + def test_call_method_descriptor_noargs_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + namespace = { + "METH_COEXIST_NOARGS_OBJECT": _testcapi.DocStringNoSignatureTest(), + } + + @reset_code + def testfunc(n): + for _ in range(n): + result = METH_COEXIST_NOARGS_OBJECT.meth_noargs_coexist() + return result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNone(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual( + uops.count("_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE"), 1) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS", uops) + + def test_call_method_descriptor_fast_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + obj = _testcapi.MethInstance() + namespace = {"METH_COEXIST_FAST_OBJECT": obj} + + @reset_code + def testfunc(n): + for _ in range(n): + result = METH_COEXIST_FAST_OBJECT.meth_fastcall_coexist(1, 2) + return result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, (obj, (1, 2))) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_METHOD_DESCRIPTOR_FAST_INLINE"), 1) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST", uops) + + def test_call_method_descriptor_fast_with_keywords_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + obj = _testcapi.MethInstance() + namespace = {"METH_COEXIST_FAST_OBJECT": obj} + + @reset_code + def testfunc(n): + # Use positional arguments to exercise CALL, not CALL_KW. + for _ in range(n): + result = ( + METH_COEXIST_FAST_OBJECT.meth_fastcall_keywords_coexist( + 1, 2)) + return result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, (obj, (1, 2), {})) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual( + uops.count("_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE"), 1) + self.assertNotIn( + "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", uops) + def test_call_method_descriptor_o(self): def testfunc(n): x = 0 diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 7946550ec0db63..60879e2774e707 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -2171,6 +2171,89 @@ class MyList(list): pass self.assert_no_opcode(my_list_append, "CALL_LIST_APPEND") self.assert_no_opcode(my_list_append, "CALL") + @cpython_only + @requires_specialization + def test_call_c_function_extra_flags(self): + # METH_CLASS, METH_STATIC and METH_COEXIST do not change the C + # calling convention, so the specialized instructions must not + # miss because of them. + _testcapi = import_module("_testcapi") + + def call_1(func, arg): + return func(arg) + + def call_2(func, arg1, arg2): + return func(arg1, arg2) + + def call_3(func, arg1, arg2, arg3): + return func(arg1, arg2, arg3) + + def call_method(obj, arg): + return obj.__contains__(arg) + + def call_method_noargs(obj): + return obj.meth_noargs_coexist() + + def call_method_fast(obj, arg1, arg2): + return obj.meth_fastcall_coexist(arg1, arg2) + + def call_method_fast_with_keywords(obj, arg1, arg2): + return obj.meth_fastcall_keywords_coexist(arg1, arg2) + + def call_site(f): + [call] = [instr for instr in dis.get_instructions(f, adaptive=True) + if instr.baseopname == "CALL"] + cache = {name: data for name, _, data in call.cache_info} + return call.opname, cache["counter"] + + def label(obj): + return getattr(obj, "__qualname__", type(obj).__name__) + + coexist = _testcapi.MethInstance() + cases = [ + # dict.__contains__ has METH_O | METH_COEXIST + (call_1, {}.__contains__, ("key",), "CALL_BUILTIN_O"), + (call_method, {}, ("key",), "CALL_METHOD_DESCRIPTOR_O"), + # meth_noargs_coexist has METH_NOARGS | METH_COEXIST + (call_method_noargs, _testcapi.DocStringNoSignatureTest(), (), + "CALL_METHOD_DESCRIPTOR_NOARGS"), + # METH_FASTCALL, with or without METH_KEYWORDS, and METH_COEXIST + (call_method_fast, coexist, (1, 2), + "CALL_METHOD_DESCRIPTOR_FAST"), + (call_method_fast_with_keywords, coexist, (1, 2), + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS"), + (call_3, _testcapi.MethInstance.meth_fastcall_coexist, + (coexist, 1, 2), "CALL_METHOD_DESCRIPTOR_FAST"), + (call_3, _testcapi.MethInstance.meth_fastcall_keywords_coexist, + (coexist, 1, 2), "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS"), + (call_2, coexist.meth_fastcall_coexist, (1, 2), + "CALL_BUILTIN_FAST"), + (call_2, coexist.meth_fastcall_keywords_coexist, (1, 2), + "CALL_BUILTIN_FAST_WITH_KEYWORDS"), + ] + for owner in (_testcapi.MethClass, _testcapi.MethStatic): + cases += [ + (call_1, owner.meth_o, (1,), "CALL_BUILTIN_O"), + (call_2, owner.meth_fastcall, (1, 2), "CALL_BUILTIN_FAST"), + (call_2, owner.meth_fastcall_keywords, (1, 2), + "CALL_BUILTIN_FAST_WITH_KEYWORDS"), + ] + + for f, func, args, opname in cases: + with self.subTest(call=f.__name__, func=label(func)): + reset_code(f) + expected = f(func, *args) + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f(func, *args) + self.assertEqual(call_site(f)[0], opname) + + # A hit leaves the counter of the call site unchanged. + # A miss decrements it. + before = call_site(f) + for _ in range(10): + self.assertEqual(f(func, *args), expected) + self.assertEqual(call_site(f), before) + @cpython_only @requires_specialization def test_load_attr_module_with_getattr(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-20-08-59-48.gh-issue-157833.3b0bac.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-20-08-59-48.gh-issue-157833.3b0bac.rst new file mode 100644 index 00000000000000..cf38270dc447a8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-20-08-59-48.gh-issue-157833.3b0bac.rst @@ -0,0 +1,2 @@ +Fix call specialization for C functions and methods with ``METH_CLASS``, +``METH_STATIC``, or ``METH_COEXIST`` flags, improving their call performance. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 577bb14df13405..19c02e6774e3d4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3360,6 +3360,11 @@ static PyMethodDef meth_instance_methods[] = { {"meth_noargs", meth_noargs, METH_NOARGS}, {"meth_fastcall", _PyCFunction_CAST(meth_fastcall), METH_FASTCALL}, {"meth_fastcall_keywords", _PyCFunction_CAST(meth_fastcall_keywords), METH_FASTCALL|METH_KEYWORDS}, + {"meth_fastcall_coexist", _PyCFunction_CAST(meth_fastcall), + METH_FASTCALL|METH_COEXIST}, + {"meth_fastcall_keywords_coexist", + _PyCFunction_CAST(meth_fastcall_keywords), + METH_FASTCALL|METH_KEYWORDS|METH_COEXIST}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index a04770bd83b176..19c6cb7517d6ff 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -2488,7 +2488,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2581,7 +2582,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2674,7 +2676,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4041,7 +4044,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_FASTCALL) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4160,7 +4164,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4283,7 +4288,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_NOARGS) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4406,7 +4412,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_O) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 0b2ab25050de49..63023887667a50 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -938,8 +938,7 @@ PyDescr_NewMethod(PyTypeObject *type, PyMethodDef *method) { /* Figure out correct vectorcall function to use */ vectorcallfunc vectorcall; - switch (method->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | - METH_O | METH_KEYWORDS | METH_METHOD)) + switch (method->ml_flags & _Py_METH_CALL_FLAGS) { case METH_VARARGS: vectorcall = method_vectorcall_VARARGS; diff --git a/Objects/methodobject.c b/Objects/methodobject.c index e6e469ca270ac9..9b37b1da95bb50 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -48,8 +48,7 @@ PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *c { /* Figure out correct vectorcall function to use */ vectorcallfunc vectorcall; - switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | - METH_O | METH_KEYWORDS | METH_METHOD)) + switch (ml->ml_flags & _Py_METH_CALL_FLAGS) { case METH_VARARGS: case METH_VARARGS | METH_KEYWORDS: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2ca52766c110b9..8799c48b110976 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -10,6 +10,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" +#include "pycore_call.h" // _Py_METH_CALL_FLAGS #include "pycore_cell.h" // PyCell_GetRef() #include "pycore_ceval.h" // _PyEval_LazyImportName(), _PyEval_LazyImportFrom() #include "pycore_code.h" @@ -4872,7 +4873,8 @@ dummy_func( op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O); + EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_O); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; @@ -4916,7 +4918,8 @@ dummy_func( op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); + EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_FASTCALL); } op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -4954,7 +4957,8 @@ dummy_func( op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); + EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)); } op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -5088,7 +5092,8 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - EXIT_IF(method->d_method->ml_flags != METH_O); + EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_O); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; @@ -5163,7 +5168,8 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - EXIT_IF(method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); + EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + (METH_FASTCALL | METH_KEYWORDS)); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { @@ -5237,7 +5243,8 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - EXIT_IF(method->d_method->ml_flags != METH_NOARGS); + EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_NOARGS); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; @@ -5306,7 +5313,8 @@ dummy_func( PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - EXIT_IF(method->d_method->ml_flags != METH_FASTCALL); + EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_FASTCALL); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0045296f1bc868..0f1acd7f6af160 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -18366,7 +18366,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_O) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -18440,7 +18441,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -18506,7 +18508,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -19153,7 +19156,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (method->d_method->ml_flags != METH_O) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_O) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -19358,7 +19362,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + (METH_FASTCALL | METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -19502,7 +19507,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (method->d_method->ml_flags != METH_NOARGS) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_NOARGS) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -19631,7 +19637,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (method->d_method->ml_flags != METH_FASTCALL) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 48f76ccf59f8bc..d9fb82d713154b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2488,7 +2488,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2581,7 +2582,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2674,7 +2676,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4041,7 +4044,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_FASTCALL) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4160,7 +4164,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4283,7 +4288,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_NOARGS) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4406,7 +4412,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_O) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e726dc0e6fd111..9b70874b35a009 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -13,6 +13,7 @@ * */ #include "Python.h" #include "opcode.h" +#include "pycore_call.h" // _Py_METH_CALL_FLAGS #include "pycore_dict.h" #include "pycore_interp.h" #include "pycore_opcode_metadata.h" diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 54b01e1ffbd322..ca4ce4f733ea34 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_call.h" // _Py_METH_CALL_FLAGS #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" @@ -1711,7 +1712,9 @@ dummy_func(void) { if (sym_is_not_null(self_or_null)) { total_args++; } - if (total_args == 1 && PyCFunction_GET_FLAGS(callable_o) == METH_O) { + if (total_args == 1 && + (PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) == METH_O) { ADD_OP(_NOP, 0, 0); } } @@ -1723,7 +1726,8 @@ dummy_func(void) { op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == + METH_FASTCALL) { ADD_OP(_NOP, 0, 0); } } @@ -1735,7 +1739,8 @@ dummy_func(void) { op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == + (METH_FASTCALL | METH_KEYWORDS)) { ADD_OP(_NOP, 0, 0); } } @@ -1786,9 +1791,10 @@ dummy_func(void) { else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args == 2 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_O && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == METH_O && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -1813,9 +1819,11 @@ dummy_func(void) { else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == (METH_FASTCALL|METH_KEYWORDS) && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + (METH_FASTCALL | METH_KEYWORDS) && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -1840,9 +1848,11 @@ dummy_func(void) { else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args == 1 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_NOARGS && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + METH_NOARGS && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -1918,9 +1928,11 @@ dummy_func(void) { else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_FASTCALL && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + METH_FASTCALL && self_type == d_type) { ADD_OP(_NOP, 0, 0); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 73a86bfe86dacd..57a9539016bbb2 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -4500,7 +4500,9 @@ if (sym_is_not_null(self_or_null)) { total_args++; } - if (total_args == 1 && PyCFunction_GET_FLAGS(callable_o) == METH_O) { + if (total_args == 1 && + (PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) == METH_O) { ADD_OP(_NOP, 0, 0); } } @@ -4546,7 +4548,8 @@ callable = stack_pointer[-2 - oparg]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == + METH_FASTCALL) { ADD_OP(_NOP, 0, 0); } } @@ -4569,7 +4572,8 @@ callable = stack_pointer[-2 - oparg]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == + (METH_FASTCALL | METH_KEYWORDS)) { ADD_OP(_NOP, 0, 0); } } @@ -4755,9 +4759,10 @@ else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args == 2 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_O && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == METH_O && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -4855,9 +4860,11 @@ else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == (METH_FASTCALL|METH_KEYWORDS) && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + (METH_FASTCALL | METH_KEYWORDS) && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -4910,9 +4917,11 @@ else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args == 1 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_NOARGS && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + METH_NOARGS && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -4998,9 +5007,11 @@ else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_FASTCALL && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + METH_FASTCALL && self_type == d_type) { ADD_OP(_NOP, 0, 0); } diff --git a/Python/specialize.c b/Python/specialize.c index 05cb76ff015ff4..f658df3bc5d222 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3,6 +3,7 @@ #include "opcode.h" #include "pycore_bytesobject.h" // _PyBytes_Concat +#include "pycore_call.h" // _Py_METH_CALL_FLAGS #include "pycore_code.h" #include "pycore_critical_section.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type @@ -1710,9 +1711,7 @@ static int specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, _Py_CODEUNIT *instr, int nargs) { - switch (descr->d_method->ml_flags & - (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { + switch (descr->d_method->ml_flags & _Py_METH_CALL_FLAGS) { case METH_NOARGS: { if (nargs != 1) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); @@ -1832,9 +1831,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER); return 1; } - switch (PyCFunction_GET_FLAGS(callable) & - (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { + switch (PyCFunction_GET_FLAGS(callable) & _Py_METH_CALL_FLAGS) { case METH_O: { if (nargs != 1) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); From c57c8772b0b133c5580e94006e8b084f78753f49 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Wed, 30 Sep 2026 15:45:37 +0100 Subject: [PATCH 021/240] Avoid Clang bug in UBSan CI (#157958) Co-authored-by: Victor Stinner --- .github/workflows/reusable-san.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 6b4a43b259638e..ad3232743874d6 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -90,10 +90,14 @@ jobs: --openssl="${OPENSSL_VER}" --system=Linux --tsan + # gh-157958: -O2 instead of the pydebug default -Og to avoid a clang 21 + # compile-time blowup on some interpreter files. + # (https://github.com/llvm/llvm-project/issues/179695) - name: Configure CPython run: >- ./configure --config-cache + OPT="-O2 -g" ${{ inputs.sanitizer == 'TSan' && '--with-thread-sanitizer' From a14a4a6bc9a6e1b930b1a4ba3678cb9ab551c006 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2026 17:19:51 +0200 Subject: [PATCH 022/240] gh-158445: Detect invalid UCS4 characters in debug mode (#158502) In debug mode, PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND) and PyUnicodeWriter_WriteUCS4() now raise a SystemError if a character is not in range [U+0000; U+10ffff], instead of creating an invalid str object. In release mode, PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND) raises a SystemError if the size is 1. * In debug mode, unicode_char() no longer fails with an assertion error if the character is invalid. Instead, raise SystemError. * Add tests on invalid UCS-4 characters. * Add _testinternalcapi._Py_MAX_UNICODE. * Add unicode_invalid_character() helper function. --- Doc/c-api/unicode.rst | 9 +++- Lib/test/test_capi/test_unicode.py | 80 ++++++++++++++++++++++++------ Modules/_testinternalcapi.c | 4 ++ Objects/unicodeobject.c | 73 +++++++++++++++++++++------ 4 files changed, 135 insertions(+), 31 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 5f918b447950d9..fcdcd26ff70cb0 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -425,6 +425,10 @@ APIs: the UCS1 range, it will be transformed into UCS1 (:c:macro:`PyUnicode_1BYTE_KIND`). + All characters must be in range [U+0000; U+10ffff]. If *kind* is + :c:macro:`PyUnicode_4BYTE_KIND` and the string contains invalid characters, + the behavior is undefined. + .. versionadded:: 3.3 @@ -1896,10 +1900,13 @@ object. .. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, const Py_UCS4 *str, Py_ssize_t size) - Writer the UCS4 string *str* into *writer*. + Write the UCS4 string *str* into *writer*. *size* is a number of UCS4 characters. + All characters must be in range [U+0000; U+10ffff]. If the string contains + invalid characters, the behavior is undefined. + On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 9c91d6eb512d2a..c5926dce397409 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -19,6 +19,11 @@ from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, SIZEOF_WCHAR_T +MAX_UNICODE = _testinternalcapi._Py_MAX_UNICODE +# The first invalid character after MAX_UNICODE +INVALID_CHAR = MAX_UNICODE + 1 +# Maximum invalid character which fits into 32-bit Py_UCS4 +MAX_INVALID_CHAR = 0xFFFF_FFFF NULL = None class Str(str): @@ -41,6 +46,12 @@ def __str__(self): SSTATE_INTERNED_IMMORTAL_STATIC = 3 +def assert_invalid_string(testcase, text): + # Check that a Unicode string contains invalid characters: + # not in range [U+0000; U+10ffff] + testcase.assertRaises(SystemError, list, text) + + class CAPITest(unittest.TestCase): def _test_check(self, check, *, exact): @@ -73,14 +84,14 @@ def test_new(self): self.assertEqual(new(0, maxchar), '') self.assertEqual(new(5, maxchar), chr(maxchar)*5) self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX, maxchar) - self.assertEqual(new(0, 0x110000), '') + self.assertEqual(new(0, INVALID_CHAR), '') self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x4f60) self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x4f60) self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x1f600) self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x1f600) self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4, 0x1f600) self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4+1, 0x1f600) - self.assertRaises(SystemError, new, 5, 0x110000) + self.assertRaises(SystemError, new, 5, INVALID_CHAR) self.assertRaises(SystemError, new, -1, 0) self.assertRaises(SystemError, new, PY_SSIZE_T_MIN, 0) @@ -115,7 +126,7 @@ def test_fill(self): s = strings[0] self.assertRaises(IndexError, fill, s, -1, 0, 0x78) self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78) - self.assertRaises(ValueError, fill, s, 0, 0, 0x110000) + self.assertRaises(ValueError, fill, s, 0, 0, INVALID_CHAR) self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78) self.assertRaises(SystemError, fill, [], 0, 0, 0x78) # CRASHES fill(s, 0, NULL, 0, 0) @@ -129,7 +140,7 @@ def _test_writechar(self, writechar, *, check): '\U0001f600\U0001f601\U0001f602' ] # one character for every kind + out of range code - chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000] + chars = [0x78, 0xa9, 0x20ac, 0x1f638, INVALID_CHAR] for i, s in enumerate(strings): for j, c in enumerate(chars): if j <= i: @@ -301,7 +312,22 @@ def test_fromkindanddata(self): self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1) self.assertRaises(ValueError, fromkindanddata, 1, NULL, PY_SSIZE_T_MIN) # CRASHES fromkindanddata(1, NULL, 1) - # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') + + # Test invalid UCS-4 string. Create an invalid string in release mode, + # or raise SystemError in debug mode. + for invalid_char in (INVALID_CHAR, MAX_INVALID_CHAR): + with self.subTest(invalid_char=invalid_char): + # Test single character + ucs4_char = invalid_char.to_bytes(4, byteorder=sys.byteorder) + self.assertRaises(SystemError, fromkindanddata, 4, ucs4_char) + + # Test multiple characters + s = 'valid'.encode(enc4) + ucs4_char + if support.Py_DEBUG: + self.assertRaises(SystemError, fromkindanddata, 4, s) + else: + result = fromkindanddata(4, s) + assert_invalid_string(self, result) def test_substring(self): """Test PyUnicode_Substring()""" @@ -446,7 +472,7 @@ def check_format(expected, format, *args): check_format('\U0010ffff', b'%c', c_int(0x10ffff)) with self.assertRaises(OverflowError): - PyUnicode_FromFormat(b'%c', c_int(0x110000)) + PyUnicode_FromFormat(b'%c', c_int(INVALID_CHAR)) # Issue #18183 check_format('\U00010000\U00100000', b'%c%c', c_int(0x10000), c_int(0x100000)) @@ -1023,7 +1049,7 @@ def test_fromordinal(self): self.assertEqual(fromordinal(0x20ac), '\u20ac') self.assertEqual(fromordinal(0x1f600), '\U0001f600') - self.assertRaises(ValueError, fromordinal, 0x110000) + self.assertRaises(ValueError, fromordinal, INVALID_CHAR) self.assertRaises(ValueError, fromordinal, -1) def test_asutf8(self): @@ -1375,8 +1401,8 @@ def test_findchar(self): self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i) str = "!>_ *maxchar) { *maxchar = ch; if (*maxchar > MAX_UNICODE) { - PyErr_Format(PyExc_ValueError, - "character U+%x is not in range [U+0000; U+%x]", - ch, MAX_UNICODE); + unicode_invalid_character(PyExc_ValueError, ch); return -1; } } @@ -1834,17 +1842,15 @@ get_latin1_char(Py_UCS1 ch) static PyObject* unicode_char(Py_UCS4 ch) { - PyObject *unicode; - - assert(ch <= MAX_UNICODE); - if (ch < 256) { return get_latin1_char(ch); } - unicode = PyUnicode_New(1, ch); - if (unicode == NULL) + // Raise SystemError if the character is not in range [U+0000; MAX_UNICODE] + PyObject *unicode = PyUnicode_New(1, ch); + if (unicode == NULL) { return NULL; + } assert(PyUnicode_KIND(unicode) != PyUnicode_1BYTE_KIND); if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { @@ -2224,15 +2230,36 @@ static PyObject* _PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size) { PyObject *res; - Py_UCS4 max_char; if (size == 0) _Py_RETURN_UNICODE_EMPTY(); assert(size > 0); - if (size == 1) + + if (size == 1) { + // Raise SystemError if the character is invalid return unicode_char(u[0]); + } + +#ifdef Py_DEBUG + // Check for invalid characters in debug mode + Py_UCS4 max_char = 127; + for (Py_ssize_t i = 0; i < size; i++) { + Py_UCS4 ch = u[i]; + if (ch > max_char) { + if (ch > MAX_UNICODE) { + unicode_invalid_character(PyExc_SystemError, ch); + return NULL; + } + max_char = ch; + } + } +#else + // gh-158445: Return MAX_UNICODE even if the string contains invalid + // characters. Checking for invalid characters would make the function + // slower whereas it's unlikely in practice. + Py_UCS4 max_char = ucs4lib_find_max_char(u, u + size); +#endif - max_char = ucs4lib_find_max_char(u, u + size); res = PyUnicode_New(size, max_char); if (!res) return NULL; @@ -2266,9 +2293,27 @@ PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer, return 0; } - Py_UCS4 max_char = ucs4lib_find_max_char(str, str + size); +#ifdef Py_DEBUG + // Check for invalid characters in debug mode + Py_UCS4 maxchar = 127; + for (Py_ssize_t i = 0; i < size; i++) { + Py_UCS4 ch = str[i]; + if (ch > maxchar) { + if (ch > MAX_UNICODE) { + unicode_invalid_character(PyExc_SystemError, ch); + return -1; + } + maxchar = ch; + } + } +#else + // gh-158445: Return MAX_UNICODE even if the string contains invalid + // characters. Checking for invalid characters would make the function + // slower whereas it's unlikely in practice. + Py_UCS4 maxchar = ucs4lib_find_max_char(str, str + size); +#endif - if (_PyUnicodeWriter_Prepare(writer, size, max_char) < 0) { + if (_PyUnicodeWriter_Prepare(writer, size, maxchar) < 0) { return -1; } assert(_PyUnicodeWriter_CanWrite(writer)); From 1697ea386c707142555d98a1263176bbbc014a96 Mon Sep 17 00:00:00 2001 From: Seth Larson Date: Wed, 30 Sep 2026 10:31:18 -0500 Subject: [PATCH 023/240] gh-156793: Validate SSLContext.wrap_bio() parameters like wrap_socket() (#158503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/asyncio-eventloop.rst | 4 + Doc/library/asyncio-stream.rst | 4 + Doc/library/ssl.rst | 13 ++- Lib/ssl.py | 27 ++++-- Lib/test/test_asyncio/test_sslproto.py | 44 +++++++++ Lib/test/test_ssl.py | 91 +++++++++++++++++++ ...-09-16-14-05-19.gh-issue-156793.Qa1T5Z.rst | 9 ++ ...-09-23-11-34-30.gh-issue-156793.zC_AjF.rst | 4 + 8 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-09-16-14-05-19.gh-issue-156793.Qa1T5Z.rst create mode 100644 Misc/NEWS.d/next/Security/2026-09-23-11-34-30.gh-issue-156793.zC_AjF.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 41abb2d7d0a53e..393f7a5fab4af5 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -581,6 +581,10 @@ Opening network connections .. versionchanged:: 3.12 *all_errors* was added. + .. versionchanged:: next + Raises a ``ValueError`` if ``ssl.check_hostname`` is ``True`` + and ``server_hostname`` is not supplied. + .. seealso:: The :func:`open_connection` function is a high-level alternative diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index d75846163edfdd..7d3275ade61094 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -425,6 +425,10 @@ StreamWriter .. versionchanged:: 3.12 Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: next + Raises a ``ValueError`` if ``sslcontext.check_hostname`` is ``True`` + and ``server_hostname`` is not supplied. + .. method:: is_closing() diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index c38763037d67c5..4fe76232536f59 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2004,7 +2004,11 @@ to speed up repeated connections from the same clients. outgoing BIO. The *server_side*, *server_hostname* and *session* parameters have the - same meaning as in :meth:`SSLContext.wrap_socket`. + same meaning as in :meth:`SSLContext.wrap_socket`, and are validated in + the same way: in particular a :exc:`ValueError` is raised when + :attr:`~SSLContext.check_hostname` is enabled but no *server_hostname* is + given, since there would be no name to match the peer's certificate + against. .. versionchanged:: 3.6 *session* argument was added. @@ -2013,6 +2017,13 @@ to speed up repeated connections from the same clients. The method returns an instance of :attr:`SSLContext.sslobject_class` instead of hard-coded :class:`SSLObject`. + .. versionchanged:: next + The *server_side*, *server_hostname* and *session* parameters are now + validated as :meth:`SSLContext.wrap_socket` validates them. Previously + a context with :attr:`~SSLContext.check_hostname` enabled and no + *server_hostname* was accepted, and verified the certificate chain but + never the peer's identity. + .. attribute:: SSLContext.sslobject_class The return type of :meth:`SSLContext.wrap_bio`, defaults to diff --git a/Lib/ssl.py b/Lib/ssl.py index 44dc0b046f4518..59593cfe7d5e70 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -374,6 +374,20 @@ def _ipaddress_match(cert_ipaddress, host_ip): return ip == host_ip +def _check_sslobject_params(server_side, context=None, server_hostname=None, session=None): + """Raises a ValueError if SSLObject._create() parameters aren't valid. + """ + if server_side: + if server_hostname: + raise ValueError("server_hostname can only be specified " + "in client mode") + if session is not None: + raise ValueError("session can only be specified in " + "client mode") + if context.check_hostname and not server_hostname: + raise ValueError("check_hostname requires server_hostname") + + DefaultVerifyPaths = namedtuple("DefaultVerifyPaths", "cafile capath openssl_cafile_env openssl_cafile openssl_capath_env " "openssl_capath") @@ -812,6 +826,8 @@ def __init__(self, *args, **kwargs): @classmethod def _create(cls, incoming, outgoing, server_side=False, server_hostname=None, session=None, context=None): + _check_sslobject_params(server_side=server_side, context=context, + server_hostname=server_hostname, session=session) self = cls.__new__(cls) sslobj = context._wrap_bio( incoming, outgoing, server_side=server_side, @@ -1008,15 +1024,8 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, context=None, session=None): if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM: raise NotImplementedError("only stream sockets are supported") - if server_side: - if server_hostname: - raise ValueError("server_hostname can only be specified " - "in client mode") - if session is not None: - raise ValueError("session can only be specified in " - "client mode") - if context.check_hostname and not server_hostname: - raise ValueError("check_hostname requires server_hostname") + _check_sslobject_params(server_side=server_side, context=context, + server_hostname=server_hostname, session=session) sock_timeout = sock.gettimeout() kwargs = dict( diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 656cdf570fad7b..253a470680ff05 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -70,6 +70,50 @@ def test_handshake_timeout_negative(self): sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, ssl_handshake_timeout=-10) + def test_check_hostname_accepts_server_hostname(self): + # Supplying a server_hostname succeeds with check_hostname enabled. + sslcontext = test_utils.simple_client_sslcontext(disable_verify=False) + sslcontext.check_hostname = True + app_proto = mock.Mock() + waiter = mock.Mock() + + # No ValueError is raised from SSLProtocol with 'server_hostname'. + ssl_proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, + server_hostname='example.org') + self.addCleanup(ssl_proto._app_transport.close) + + @support.subTests("server_hostname", [None, '']) + def test_check_hostname_requires_server_hostname(self, server_hostname): + # A caller-supplied context asking for hostname checking used to be + # taken through wrap_bio() with no name to check against, verifying + # the certificate chain but never the peer's identity. + # loop.start_tls() defaults server_hostname to None, and + # loop.create_connection() turns server_hostname='' into None here, + # so both reached that state. + sslcontext = test_utils.simple_client_sslcontext(disable_verify=False) + sslcontext.check_hostname = True + app_proto = mock.Mock() + waiter = mock.Mock() + + # Supplying an empty server_hostname fails with check_hostname enabled. + with self.assertRaisesRegex( + ValueError, + 'check_hostname requires server_hostname'): + sslproto.SSLProtocol(self.loop, app_proto, sslcontext, + waiter, + server_hostname=server_hostname) + + # Disabling check_hostname allows for an empty or unset server_hostname. + sslcontext.check_hostname = False + + ssl_proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter) + self.addCleanup(ssl_proto._app_transport.close) + + ssl_proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, + waiter, + server_hostname=server_hostname) + self.addCleanup(ssl_proto._app_transport.close) + def test_eof_received_waiter(self): waiter = self.loop.create_future() ssl_proto = self.ssl_protocol(waiter=waiter) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 9a6118d94fbbb2..2008069609ab60 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -399,6 +399,34 @@ def do_ssl_object_handshake(sslobject, outgoing, max_retry=25): return data +def connected_bio_pair(client_context, server_context, hostname, max_retry=5): + """Handshake a client and a server SSLObject against each other. + + Everything happens in memory, so this needs no socket and no thread. + Returns the two objects followed by their four BIOs, in the order + client, server, c_in, c_out, s_in, s_out. + """ + c_in, c_out = ssl.MemoryBIO(), ssl.MemoryBIO() + s_in, s_out = ssl.MemoryBIO(), ssl.MemoryBIO() + client = client_context.wrap_bio(c_in, c_out, server_hostname=hostname) + server = server_context.wrap_bio(s_in, s_out, server_side=True) + + # Loop on the handshake for a bit to get it settled + for _ in range(max_retry): + with contextlib.suppress(ssl.SSLWantReadError): + client.do_handshake() + if c_out.pending: + s_in.write(c_out.read()) + with contextlib.suppress(ssl.SSLWantReadError): + server.do_handshake() + if s_out.pending: + c_in.write(s_out.read()) + # Now the handshakes should be complete (don't raise WantReadError) + client.do_handshake() + server.do_handshake() + return client, server, c_in, c_out, s_in, s_out + + class BasicSocketTests(unittest.TestCase): def test_constants(self): @@ -1859,6 +1887,7 @@ def test__create_stdlib_context_check_hostname(self): def test_delete_sslobject_attributes(self): # None of the attributes of _ssl._SSLSocket can be deleted. ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False sslobj = ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO())._sslobj for name in 'context', 'owner', 'session', 'session_reused': with self.subTest(name=name): @@ -2053,6 +2082,10 @@ def test_subclass(self): def test_bad_server_hostname(self): ctx = ssl.create_default_context() + # Omitting the name entirely is bad too: this context checks it. + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname=None) with self.assertRaises(ValueError): ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), server_hostname="") @@ -2137,6 +2170,64 @@ def test_private_init(self): with self.assertRaisesRegex(TypeError, "public constructor"): ssl.SSLObject(bio, bio) + def test_check_hostname_requires_server_hostname(self): + # wrap_bio() used to accept a context asking for hostname checking + # without a name to check against, and then verify the certificate + # chain but never the peer's identity, with check_hostname still + # reporting True and nothing reporting the check had been skipped. + # It must refuse that call, as wrap_socket() already did. + client_context, _, hostname = testing_context() + self.assertTrue(client_context.check_hostname) + + for server_hostname in (None, ""): + with self.subTest(server_hostname=server_hostname): + with self.assertRaisesRegex( + ValueError, + "check_hostname requires server_hostname"): + client_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname=server_hostname) + # The sibling constructor refuses the very same call. + with socket.socket() as sock: + with self.assertRaisesRegex( + ValueError, + "check_hostname requires server_hostname"): + client_context.wrap_socket( + sock, server_hostname=server_hostname) + + # A name was all that was missing. + client_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname=hostname) + + # Asking for no hostname check remains a way to say so explicitly. + context = make_test_context() + self.assertFalse(context.check_hostname) + context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO()) + + def test_server_side_bad_params(self): + # A server neither sends a hostname nor resumes a client's session, + # so wrap_bio() rejects both in server mode like wrap_socket() + client_context, server_context, hostname = testing_context() + + with self.assertRaisesRegex( + ValueError, + "server_hostname can only be specified in client mode"): + server_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_side=True, + server_hostname=hostname) + + client, server, *_ = connected_bio_pair( + client_context, server_context, hostname) + session = client.session + self.assertIsNotNone(session) + with self.assertRaisesRegex( + ValueError, "session can only be specified in client mode"): + server_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_side=True, session=session) + + # Neither argument is what a server passes, so this still works. + server_context.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_side=True) + def test_unwrap(self): client_ctx, server_ctx, hostname = testing_context() c_in = ssl.MemoryBIO() diff --git a/Misc/NEWS.d/next/Security/2026-09-16-14-05-19.gh-issue-156793.Qa1T5Z.rst b/Misc/NEWS.d/next/Security/2026-09-16-14-05-19.gh-issue-156793.Qa1T5Z.rst new file mode 100644 index 00000000000000..2a521dbc9dc4e6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-09-16-14-05-19.gh-issue-156793.Qa1T5Z.rst @@ -0,0 +1,9 @@ +:meth:`ssl.SSLContext.wrap_bio` now validates its *server_side*, +*server_hostname* and *session* arguments similar to +:meth:`ssl.SSLContext.wrap_socket`. + +In particular, a context with :attr:`~ssl.SSLContext.check_hostname` enabled +and no *server_hostname* passed to :meth:`!wrap_bio` now raises :exc:`ValueError` +instead of completing a handshake that verified the certificate chain +without verifying the peer's identity, with no indication that the +check had been skipped. diff --git a/Misc/NEWS.d/next/Security/2026-09-23-11-34-30.gh-issue-156793.zC_AjF.rst b/Misc/NEWS.d/next/Security/2026-09-23-11-34-30.gh-issue-156793.zC_AjF.rst new file mode 100644 index 00000000000000..42a31b3c28f5ce --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-09-23-11-34-30.gh-issue-156793.zC_AjF.rst @@ -0,0 +1,4 @@ +:mod:`asyncio`: :meth:`loop.start_tls() ` and +:meth:`loop.create_connection() ` now +validate the *server_hostname* argument if an :class:`ssl.SSLContext` is +passed with *check_hostname* set to ``True``. From 34a53dce8174da2fceb12fe084a4def02a10053d Mon Sep 17 00:00:00 2001 From: Seth Larson Date: Wed, 30 Sep 2026 10:48:00 -0500 Subject: [PATCH 024/240] gh-156293: Use-after-free for server-side SSLContext with sni_callback (#158504) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Doc/library/ssl.rst | 11 +++ Lib/test/test_ssl.py | 80 +++++++++++++++++++ ...-08-10-12-00-00.gh-issue-156293.sNIcbk.rst | 7 ++ Modules/_ssl.c | 28 +++++-- 4 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-08-10-12-00-00.gh-issue-156293.sNIcbk.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 4fe76232536f59..3723f79778cef0 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1859,6 +1859,12 @@ to speed up repeated connections from the same clients. :class:`SSLContext` representing a certificate chain that matches the server name. + If the callback assigns a new context to :attr:`SSLSocket.context`, any + further ClientHello message on the same connection (for example after a + TLS 1.3 HelloRetryRequest) is dispatched to the new context's + *sni_callback*, if it has one; the original callback is not called again + for that connection. + Due to the early negotiation phase of the TLS connection, only limited methods and attributes are usable like :meth:`SSLSocket.selected_alpn_protocol` and :attr:`SSLSocket.context`. @@ -1883,6 +1889,11 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 + .. versionchanged:: next + After the callback assigns a new :attr:`SSLSocket.context`, later + ClientHello messages on the connection are dispatched to the new + context's *sni_callback*. + .. method:: SSLContext.set_servername_callback(server_name_callback) This is a legacy API retained for backwards compatibility. When possible, diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 2008069609ab60..d8eebc33deebfa 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2270,6 +2270,86 @@ def test_unwrap(self): c_in.write(s_out.read()) client.unwrap() + def test_sni_callback_context_released_and_callback_raises(self): + # Variant of the test below without a HelloRetryRequest: the callback + # switches the connection to another context, drops the last + # references to the context that carries it, and raises. The C + # callback must not touch that context after the Python callback + # returned. + client_ctx, server_ctx, hostname = testing_context() + leaf_ctx = server_ctx + + def sni_cb(sslobj, server_name, ctx): + sslobj.context = leaf_ctx + del ctx + raise LookupError("no certificate for " + repr(server_name)) + + def make_server(): + dispatch_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + dispatch_ctx.load_cert_chain(SIGNED_CERTFILE) + dispatch_ctx.sni_callback = sni_cb + s_in, s_out = ssl.MemoryBIO(), ssl.MemoryBIO() + server = dispatch_ctx.wrap_bio(s_in, s_out, server_side=True) + return server, s_in, s_out + + server, s_in, s_out = make_server() + c_in, c_out = ssl.MemoryBIO(), ssl.MemoryBIO() + client = client_ctx.wrap_bio(c_in, c_out, server_hostname=hostname) + with self.assertRaises(ssl.SSLWantReadError): + client.do_handshake() + s_in.write(c_out.read()) + with support.catch_unraisable_exception() as cm: + with self.assertRaises(ssl.SSLError): + server.do_handshake() + self.assertIsInstance(cm.unraisable.exc_value, LookupError) + self.assertIs(server.context, leaf_ctx) + + def test_sni_callback_context_released_before_second_client_hello(self): + # The SSLContext carrying sni_callback may be released by the + # application once the callback has switched the connection over to + # another context. If the server then sends a HelloRetryRequest, the + # second ClientHello makes OpenSSL consult the original SSL_CTX's + # servername callback again; that must not use the deallocated + # SSLContext object. + client_ctx, leaf_ctx, hostname = testing_context() + calls = [] + + def make_server(): + dispatch_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + dispatch_ctx.load_cert_chain(SIGNED_CERTFILE) + # Force a HelloRetryRequest: the client offers an X25519 key + # share first, the server only accepts P-384. + dispatch_ctx.set_ecdh_curve("secp384r1") + def sni_cb(sslobj, server_name, ctx): + calls.append(server_name) + sslobj.context = leaf_ctx + dispatch_ctx.sni_callback = sni_cb + s_in, s_out = ssl.MemoryBIO(), ssl.MemoryBIO() + server = dispatch_ctx.wrap_bio(s_in, s_out, server_side=True) + return server, s_in, s_out, weakref.ref(dispatch_ctx) + + # After this only the C-level SSL object references dispatch_ctx. + server, s_in, s_out, dispatch_ref = make_server() + c_in, c_out = ssl.MemoryBIO(), ssl.MemoryBIO() + client = client_ctx.wrap_bio(c_in, c_out, server_hostname=hostname) + for _ in range(10): + for obj, out, peer_in in ((client, c_out, s_in), + (server, s_out, c_in)): + try: + obj.do_handshake() + except ssl.SSLWantReadError: + pass + if out.pending: + peer_in.write(out.read()) + client.do_handshake() + server.do_handshake() + support.gc_collect() + self.assertIsNone(dispatch_ref()) + self.assertGreaterEqual(len(calls), 1) + self.assertEqual(calls[0], hostname) + self.assertIs(server.context, leaf_ctx) + self.assertIsNotNone(client.cipher()) + class SimpleBackgroundTests(unittest.TestCase): """Tests that connect to a simple server running in the background""" diff --git a/Misc/NEWS.d/next/Security/2026-08-10-12-00-00.gh-issue-156293.sNIcbk.rst b/Misc/NEWS.d/next/Security/2026-08-10-12-00-00.gh-issue-156293.sNIcbk.rst new file mode 100644 index 00000000000000..0cc620b3fdca20 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-08-10-12-00-00.gh-issue-156293.sNIcbk.rst @@ -0,0 +1,7 @@ +Fix a crash in :mod:`ssl` when an :attr:`~ssl.SSLContext.sni_callback` +switches a connection to another :class:`~ssl.SSLContext` and the context +that carries the callback is no longer referenced by the application. +Servers that keep their ``sni_callback`` context alive (the usual case when +it wraps the listening socket or is stored on the server object) were not +affected. +This addresses :cve:`2026-19445`. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 360aa3e2454f8b..5fc029af24821a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -3647,6 +3647,9 @@ context_dealloc(PyObject *op) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); (void)context_clear(op); + /* The SSL_CTX may outlive this object as the session_ctx of sockets that + were switched to another context; leave no Python callback behind. */ + SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); SSL_CTX_free(self->ctx); PyMem_FREE(self->alpn_protocols); tp->tp_free(self); @@ -5119,10 +5122,10 @@ _ssl__SSLContext_set_ecdh_curve_impl(PySSLContext *self, PyObject *name) } static int -_servername_callback(SSL *s, int *al, void *args) +_servername_callback(SSL *s, int *al, void *Py_UNUSED(args)) { int ret; - PySSLContext *sslctx = (PySSLContext *) args; + PySSLContext *sslctx; PySSLSocket *ssl; PyObject *result; /* The high-level ssl.SSLSocket object */ @@ -5131,18 +5134,30 @@ _servername_callback(SSL *s, int *al, void *args) const char *servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name); PyGILState_STATE gstate = PyGILState_Ensure(); + /* Do not use the SSL_CTX's servername arg to find the context: it is a + borrowed pointer to whichever _SSLContext installed the callback, and + that object may already be gone while OpenSSL still reaches this + callback through the connection's session_ctx (e.g. on the second + ClientHello after a HelloRetryRequest, once sni_callback has switched + the socket to another context). The socket's current context is + always alive. */ + ssl = SSL_get_app_data(s); + assert(ssl != NULL); + Py_BEGIN_CRITICAL_SECTION(ssl); + sslctx = (PySSLContext *)Py_NewRef(ssl->ctx); + Py_END_CRITICAL_SECTION(); + assert(Py_IS_TYPE(ssl, get_state_ctx(sslctx)->PySSLSocket_Type)); + Py_BEGIN_CRITICAL_SECTION(sslctx); sni_cb = Py_XNewRef(sslctx->set_sni_cb); Py_END_CRITICAL_SECTION(); if (sni_cb == NULL) { + Py_DECREF(sslctx); PyGILState_Release(gstate); return SSL_TLSEXT_ERR_OK; } - ssl = SSL_get_app_data(s); - assert(Py_IS_TYPE(ssl, get_state_ctx(sslctx)->PySSLSocket_Type)); - /* The servername callback expects an argument that represents the current * SSL connection and that has a .context attribute that can be changed to * identify the requested hostname. Since the official API is the Python @@ -5225,12 +5240,14 @@ _servername_callback(SSL *s, int *al, void *args) } Py_DECREF(sni_cb); + Py_DECREF(sslctx); PyGILState_Release(gstate); return ret; error: Py_XDECREF(ssl_socket); Py_XDECREF(sni_cb); + Py_DECREF(sslctx); *al = SSL_AD_INTERNAL_ERROR; ret = SSL_TLSEXT_ERR_ALERT_FATAL; PyGILState_Release(gstate); @@ -5289,7 +5306,6 @@ _ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value) } else { Py_XSETREF(self->set_sni_cb, Py_NewRef(value)); - SSL_CTX_set_tlsext_servername_arg(self->ctx, self); SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); } return 0; From e580c888746411d3b576b5c3c2ce8b34293ddaf2 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Wed, 30 Sep 2026 17:31:05 +0100 Subject: [PATCH 025/240] Update `Doc/pylock.toml` (#158519) --- Doc/pylock.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/pylock.toml b/Doc/pylock.toml index 62f144e511fe08..e616fa78c756fa 100644 --- a/Doc/pylock.toml +++ b/Doc/pylock.toml @@ -220,9 +220,9 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf [[packages]] name = "python-docs-theme" -version = "2026.9" -sdist = { url = "https://files.pythonhosted.org/packages/5a/5e/fee7dbfd6a5ebf27f4a91c8cdf204f35f0744fd1fbde7f50cd479d1d890c/python_docs_theme-2026.9.tar.gz", upload-time = 2026-09-19T08:41:45Z, size = 40572, hashes = { sha256 = "1a400cb5e6ac6c8b4788eb0a98067a8c35e8aea15e38941e90a7f764582702fc" } } -wheels = [{ url = "https://files.pythonhosted.org/packages/68/e1/0d603158c9d9a59cd05dfa98a1682c2fffbc387f1e3a724794480f8acaab/python_docs_theme-2026.9-py3-none-any.whl", upload-time = 2026-09-19T08:41:43Z, size = 49214, hashes = { sha256 = "1a4bdc4f9c664c35a58714af0a98efee4ecb7c3bc4ee0752782c953164ac8fc3" } }] +version = "2026.9.1" +sdist = { url = "https://files.pythonhosted.org/packages/10/84/74b9dc8e57aad8e77e20e56b42db5ca101ef250272b178734c3e638e6c9c/python_docs_theme-2026.9.1.tar.gz", upload-time = 2026-09-30T16:17:50Z, size = 92406, hashes = { sha256 = "575743d4c38f39c8615c564269bb3eac5856c1e0191e5798168656cff828ec59" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/85/cd/1b313c9440012be2bca5aebac85f86990aa7deafb37a3cac816c21fd6c04/python_docs_theme-2026.9.1-py3-none-any.whl", upload-time = 2026-09-30T16:17:49Z, size = 99993, hashes = { sha256 = "3e4743ec90bdaf5d1144741b3b9c38cf9bf82c605d3c9a23ae178d9cea0f4429" } }] [[packages]] name = "requests" @@ -316,6 +316,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/87/55/ab40a0d1378ee5c [[packages]] name = "urllib3" -version = "2.7.0" -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", upload-time = 2026-05-07T16:13:18Z, size = 433602, hashes = { sha256 = "231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c" } } -wheels = [{ url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", upload-time = 2026-05-07T16:13:17Z, size = 131087, hashes = { sha256 = "9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897" } }] +version = "2.8.0" +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", upload-time = 2026-09-15T19:29:36Z, size = 458972, hashes = { sha256 = "63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", upload-time = 2026-09-15T19:29:34Z, size = 135717, hashes = { sha256 = "0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3" } }] From 763b6edb0ec959bdfb78f098b76829365de36ca5 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Sep 2026 23:48:34 +0100 Subject: [PATCH 026/240] gh-158536: Skip some Android logging tests on OS versions with aggressive "chatty" filtering (#158531) Address a test suite failure when run on Android 26-30 (inclusive) due to the adb logger dropping lines with a "chatty ... Identical 3 lines" marker. --- Lib/test/test_android.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 201d701aff11eb..45c2911afa8bba 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -1,4 +1,5 @@ import io +import platform import queue import re import subprocess @@ -16,6 +17,11 @@ if sys.platform != "android": raise unittest.SkipTest("Android-specific") +# Some API levels filter out consecutive identical lines and replace them with a +# "chatty" marker. +api_level = platform.android_ver().api_level +chatty_issue = (26 <= api_level <= 30) + # (name, level, fileno) STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] @@ -179,15 +185,13 @@ def write(s, lines=None, *, write_len=None): write("\u0000b", [r"\xc0\x80b"]) write("a\u0000b", [r"a\xc0\x80b"]) - # Multi-line messages. Avoid identical consecutive lines, as - # they may activate "chatty" filtering and break the tests. - # - # Additional spaces will appear in the output where necessary to - # protect leading newlines. + # Multi-line messages. Additional spaces will appear in the output where + # necessary to protect leading newlines. write("\nx", [" "]) write("\na\n", ["x", "a"]) write("\n", [" "]) - write("\n\n", [" ", " "]) + if not chatty_issue: + write("\n\n", [" ", " "]) write("b\n", ["b"]) write("c\n\n", ["c", " "]) write("d\ne", ["d"]) @@ -206,7 +210,8 @@ def write(s, lines=None, *, write_len=None): write("\nx", [" ", "x"]) write("\na\n", [" ", "a"]) write("\n", [" "]) - write("\n\n", [" ", " "]) + if not chatty_issue: + write("\n\n", [" ", " "]) write("b\n", ["b"]) write("c\n\n", ["c", " "]) write("d\ne", ["d", "e"]) From 1071f7400ec6f3ffb63b48310440b2ae73c632b4 Mon Sep 17 00:00:00 2001 From: An Long Date: Thu, 1 Oct 2026 20:08:48 +0900 Subject: [PATCH 027/240] gh-155823: Fix stale stack on _SEND_VIRTUAL_TIER_TWO exhausted exit (GH-155844) --- Lib/test/test_capi/test_opt.py | 14 ++++++++++++++ .../2026-08-15-20-31-20.gh-issue-155823.OhNmzi.rst | 4 ++++ Python/bytecodes.c | 2 -- Python/executor_cases.c.h | 10 +++------- 4 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-15-20-31-20.gh-issue-155823.OhNmzi.rst diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d387fc7a23da26..64b6e82936d2f1 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5821,6 +5821,20 @@ def testfunc(n): self.assertIn("_FOR_ITER_GEN_FRAME", uops) self.assertIn("_SEND_VIRTUAL_TIER_TWO", uops) + def test_send_virtual_exhausted(self): + # gh-155823: warm up on a non-empty list, then take the exhausted exit. + def gen(x): + yield from x + def testfunc(n, x): + total = 0 + for _ in range(n): + for v in gen(x): + total += v + return total + + testfunc(TIER2_THRESHOLD * 10, [1]) + self.assertEqual(testfunc(1000, []), 0) + def test_binary_op_subscr_init_frame(self): class B: def __getitem__(self, other): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-15-20-31-20.gh-issue-155823.OhNmzi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-15-20-31-20.gh-issue-155823.OhNmzi.rst new file mode 100644 index 00000000000000..7dfbaf02575fd6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-15-20-31-20.gh-issue-155823.OhNmzi.rst @@ -0,0 +1,4 @@ +Fix a JIT bug where the side exit taken when a virtual iterator is exhausted +left a stale value on the stack, so the tier one interpreter resumed with +corrupted state and ``yield from`` raised :exc:`AttributeError` instead of +finishing normally. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8799c48b110976..8dfb76ef7884d1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1786,8 +1786,6 @@ dummy_func( if (index < 0) { ERROR_NO_POP(); } - next = none; - DEAD(none); EXIT_IF(true); } DEAD(none); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0f1acd7f6af160..b60e9b2b52daa8 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9175,7 +9175,6 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - next = none; if (true) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -9219,10 +9218,9 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - next = none; if (true) { UOP_STAT_INC(uopcode, miss); - _tos_cache0 = stack_pointer[0]; + _tos_cache0 = none; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } @@ -9266,10 +9264,9 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - next = none; if (true) { UOP_STAT_INC(uopcode, miss); - _tos_cache1 = stack_pointer[1]; + _tos_cache1 = none; _tos_cache0 = null_or_index; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); @@ -9316,10 +9313,9 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - next = none; if (true) { UOP_STAT_INC(uopcode, miss); - _tos_cache2 = stack_pointer[2]; + _tos_cache2 = none; _tos_cache1 = null_or_index; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(3); From 0906d2ab93fc9f4c7e0db031c965e7d8a0db7d37 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 1 Oct 2026 12:15:51 +0100 Subject: [PATCH 028/240] gh-157468: Validate correct builtins used under JIT (GH-157766) * Expose two possible builtin JIT gaps * Add trace and runtime guard for builtin dict * Check _GUARD_BUILTINS_IS_CANONICAL once per frame * Add check and fix for globals being folded --- Include/internal/pycore_optimizer_types.h | 1 + Include/internal/pycore_uop_ids.h | 9 +- Include/internal/pycore_uop_metadata.h | 21 ++++ Lib/test/test_capi/test_opt.py | 114 ++++++++++++++++++ ...-09-18-20-24-43.gh-issue-157468.zVIAva.rst | 7 ++ Objects/dictobject.c | 4 + Python/bytecodes.c | 4 + Python/executor_cases.c.h | 70 +++++++++++ Python/optimizer_bytecodes.c | 13 +- Python/optimizer_cases.c.h | 16 ++- Python/optimizer_symbols.c | 1 + 11 files changed, 256 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-18-20-24-43.gh-issue-157468.zVIAva.rst diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index a722652cc8163a..c1928ec9ebdac3 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -140,6 +140,7 @@ typedef union { typedef struct _Py_UOpsAbstractFrame { bool globals_watched; + bool builtins_checked; // The version number of the globals dicts, once checked. 0 if unchecked. uint32_t globals_checked_version; // Max stacklen diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 840a32f0e4481f..ba6e0252024a6d 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -142,6 +142,7 @@ enum { _GUARD_BIT_IS_UNSET_POP_5, _GUARD_BIT_IS_UNSET_POP_6, _GUARD_BIT_IS_UNSET_POP_7, + _GUARD_BUILTINS_IS_CANONICAL, _GUARD_CALLABLE_BUILTIN_CLASS, _GUARD_CALLABLE_BUILTIN_FAST, _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, @@ -365,7 +366,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 653 +#define MAX_UOP_ID 654 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -750,6 +751,10 @@ enum { _GUARD_BIT_IS_UNSET_POP_7_r10, _GUARD_BIT_IS_UNSET_POP_7_r21, _GUARD_BIT_IS_UNSET_POP_7_r32, + _GUARD_BUILTINS_IS_CANONICAL_r00, + _GUARD_BUILTINS_IS_CANONICAL_r11, + _GUARD_BUILTINS_IS_CANONICAL_r22, + _GUARD_BUILTINS_IS_CANONICAL_r33, _GUARD_CALLABLE_BUILTIN_CLASS_r00, _GUARD_CALLABLE_BUILTIN_FAST_r00, _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00, @@ -1439,7 +1444,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1650 +#define MAX_UOP_REGS_ID 1655 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1c440fe9f1e978..177564a7545941 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -196,6 +196,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, [_LOAD_GLOBAL_MODULE] = HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS] = HAS_DEOPT_FLAG, + [_GUARD_BUILTINS_IS_CANONICAL] = HAS_DEOPT_FLAG, [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -1894,6 +1895,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_BUILTINS_IS_CANONICAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BUILTINS_IS_CANONICAL_r00 }, + { 1, 1, _GUARD_BUILTINS_IS_CANONICAL_r11 }, + { 2, 2, _GUARD_BUILTINS_IS_CANONICAL_r22 }, + { 3, 3, _GUARD_BUILTINS_IS_CANONICAL_r33 }, + }, + }, [_DELETE_FAST] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -4315,6 +4325,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_GLOBALS_VERSION_r33] = _GUARD_GLOBALS_VERSION, [_LOAD_GLOBAL_MODULE_r01] = _LOAD_GLOBAL_MODULE, [_LOAD_GLOBAL_BUILTINS_r01] = _LOAD_GLOBAL_BUILTINS, + [_GUARD_BUILTINS_IS_CANONICAL_r00] = _GUARD_BUILTINS_IS_CANONICAL, + [_GUARD_BUILTINS_IS_CANONICAL_r11] = _GUARD_BUILTINS_IS_CANONICAL, + [_GUARD_BUILTINS_IS_CANONICAL_r22] = _GUARD_BUILTINS_IS_CANONICAL, + [_GUARD_BUILTINS_IS_CANONICAL_r33] = _GUARD_BUILTINS_IS_CANONICAL, [_DELETE_FAST_r00] = _DELETE_FAST, [_MAKE_CELL_r00] = _MAKE_CELL, [_DELETE_DEREF_r00] = _DELETE_DEREF, @@ -5292,6 +5306,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_BIT_IS_UNSET_POP_7_r10] = "_GUARD_BIT_IS_UNSET_POP_7_r10", [_GUARD_BIT_IS_UNSET_POP_7_r21] = "_GUARD_BIT_IS_UNSET_POP_7_r21", [_GUARD_BIT_IS_UNSET_POP_7_r32] = "_GUARD_BIT_IS_UNSET_POP_7_r32", + [_GUARD_BUILTINS_IS_CANONICAL] = "_GUARD_BUILTINS_IS_CANONICAL", + [_GUARD_BUILTINS_IS_CANONICAL_r00] = "_GUARD_BUILTINS_IS_CANONICAL_r00", + [_GUARD_BUILTINS_IS_CANONICAL_r11] = "_GUARD_BUILTINS_IS_CANONICAL_r11", + [_GUARD_BUILTINS_IS_CANONICAL_r22] = "_GUARD_BUILTINS_IS_CANONICAL_r22", + [_GUARD_BUILTINS_IS_CANONICAL_r33] = "_GUARD_BUILTINS_IS_CANONICAL_r33", [_GUARD_CALLABLE_BUILTIN_CLASS] = "_GUARD_CALLABLE_BUILTIN_CLASS", [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = "_GUARD_CALLABLE_BUILTIN_CLASS_r00", [_GUARD_CALLABLE_BUILTIN_FAST] = "_GUARD_CALLABLE_BUILTIN_FAST", @@ -6488,6 +6507,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_GLOBAL_BUILTINS: return 0; + case _GUARD_BUILTINS_IS_CANONICAL: + return 0; case _DELETE_FAST: return 0; case _MAKE_CELL: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 64b6e82936d2f1..88077aacecbf5b 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1,3 +1,4 @@ +import builtins import contextlib import dis import itertools @@ -5276,6 +5277,119 @@ def jitted(funcs): with self.assertRaises(NameError): jitted([f, f_with_bad_globals]) + def test_jitted_code_sees_changed_copied_builtins(self): + # Trace-time check. The traced function's builtins is a copy of the + # canonical dict with the same keys version, so a version check + # cannot tell them apart. The optimizer must see that func_builtins + # is not interp->builtins and keep _LOAD_GLOBAL_BUILTINS, which reads + # the frame's own dict, rather than fold a constant from the + # canonical one. No runtime guard is involved. + + def f(n): + return [len("hello") for _ in range(n)] + + copied_builtins = vars(builtins).copy() + f = types.FunctionType(f.__code__, {"__builtins__": copied_builtins}) + + f(TIER2_THRESHOLD) + ex = get_first_executor(f) + self.assertIsNotNone(ex) + # Not folded: the load must still consult the frame's builtins. + self.assertIn("_LOAD_GLOBAL_BUILTINS", get_opnames(ex)) + + # Replacing an existing value does not change the keys version. + copied_builtins["len"] = lambda s: 42 + self.assertEqual(f(8), [42] * 8) + + def test_jitted_code_sees_changed_copied_globals(self): + # Copying a dict must not carry over the keys version of the source. + # The optimizer folds a global to a constant guarded only by the + # globals keys version plus a watcher on the traced dict. A copy is + # not watched, and replacing an existing value does not change the + # keys version, so a function whose globals are a copy of the traced + # dict would otherwise pass _GUARD_GLOBALS_VERSION and see the stale + # constant. + def f(n): + for _ in range(n): + x = COPIED_GLOBAL + return x + + for copy in (dict.copy, dict): + with self.subTest(copy=copy): + original = {"COPIED_GLOBAL": 1} + # A fresh code object, so that each subtest traces anew. + f_original = types.FunctionType(f.__code__.replace(), original) + self.assertEqual(f_original(TIER2_THRESHOLD), 1) + ex = get_first_executor(f_original) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_GLOBALS_VERSION", uops) + # The global was folded to a constant. + self.assertNotIn("_LOAD_GLOBAL_MODULE", uops) + + copied = copy(original) + copied["COPIED_GLOBAL"] = 2 + # Share the code object, so that the same executor is entered. + f_copied = types.FunctionType(f_original.__code__, copied) + self.assertEqual(f_copied(TIER2_THRESHOLD), 2) + self.assertEqual(f_original(TIER2_THRESHOLD), 1) + + def test_jitted_code_sees_different_builtins(self): + # Runtime check. The traced function's builtins IS the canonical + # dict, so folding len to a constant is correct at trace time. + # A second function sharing the code object then enters the same + # executor with other builtins, so only the runtime guard on the + # executing frame's builtins can catch it. + def f(n): + return [len("hello") for _ in range(n)] + + namespace = {"__builtins__": builtins} + f_canonical = types.FunctionType(f.__code__, namespace) + copied_builtins = vars(builtins).copy() + namespace["__builtins__"] = copied_builtins + f_copied = types.FunctionType(f.__code__, namespace) + + + f_canonical(TIER2_THRESHOLD) + ex = get_first_executor(f_canonical) + self.assertIsNotNone(ex) + self.assertIn("_GUARD_BUILTINS_IS_CANONICAL", get_opnames(ex)) + + copied_builtins["len"] = lambda s: 42 + # The executor's owner still sees the canonical len. + self.assertEqual(f_canonical(8), [5] * 8) + # A different function enters the same executor with other builtins. + self.assertEqual(f_copied(8), [42] * 8) + + def test_builtins_guard_emitted_once_per_frame(self): + # A frame's builtins cannot change once the frame is pushed, so + # repeated builtin loads in one frame share a single guard, just as + # they already share a single _GUARD_GLOBALS_VERSION. + + def warmup(n): + x = 0 + for _ in range(n): + x += len("ab") + return x + + def one_frame(n): + x = 0 + for _ in range(n): + x += len("ab") + abs(-1) + ord("c") + return x + + # The optimizer context is reused for every compilation, so compile an + # unrelated trace first: state that is not reset per frame leaks here. + warmup(TIER2_THRESHOLD) + self.assertIsNotNone(get_first_executor(warmup)) + + _, ex = self._run_with_optimizer(one_frame, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uop_names = get_opnames(ex) + self.assertNotIn("_LOAD_GLOBAL_BUILTINS", uop_names) # all folded + self.assertEqual(uop_names.count("_GUARD_BUILTINS_IS_CANONICAL"), 1) + self.assertEqual(uop_names.count("_GUARD_GLOBALS_VERSION"), 1) + def test_reference_tracking_across_call_doesnt_crash(self): def f1(): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-18-20-24-43.gh-issue-157468.zVIAva.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-18-20-24-43.gh-issue-157468.zVIAva.rst new file mode 100644 index 00000000000000..bec475ae711423 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-18-20-24-43.gh-issue-157468.zVIAva.rst @@ -0,0 +1,7 @@ +Fix the JIT optimizer folding a builtin name to a constant taken from the +interpreter's builtins dictionary even when the running function uses a +different ``__builtins__`` mapping, such as one created with +``vars(builtins).copy()``. The optimizer now only folds when the function's +builtins is the interpreter's, and the folded constant is guarded at runtime so +that another function sharing the same code object but a different builtins +mapping does not use it. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 15377ac083c2b2..49eb8481e128d0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1041,6 +1041,10 @@ clone_combined_dict_keys(PyDictObject *orig) memcpy(keys, orig->ma_keys, keys_size); + /* The keys version must be unique per keys object: the specializer + and the JIT optimizer rely on it to identify a dict's keys. */ + keys->dk_version = 0; + /* After copying key/value pairs, we need to incref all keys and values and they are about to be co-owned by a new dict object. */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8dfb76ef7884d1..3f329274f06b11 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2356,6 +2356,10 @@ dummy_func( STAT_INC(LOAD_GLOBAL, hit); } + tier2 op(_GUARD_BUILTINS_IS_CANONICAL, (--)) { + DEOPT_IF(BUILTINS() != tstate->interp->builtins); + } + macro(LOAD_GLOBAL_MODULE) = unused/1 + // Skip over the counter NOP + // For guard insertion in the JIT optimizer diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b60e9b2b52daa8..400f07b58e7fa5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10482,6 +10482,76 @@ break; } + case _GUARD_BUILTINS_IS_CANONICAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + if (BUILTINS() != tstate->interp->builtins) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_BUILTINS_IS_CANONICAL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + if (BUILTINS() != tstate->interp->builtins) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_BUILTINS_IS_CANONICAL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (BUILTINS() != tstate->interp->builtins) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_BUILTINS_IS_CANONICAL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (BUILTINS() != tstate->interp->builtins) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _DELETE_FAST_r00: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index ca4ce4f733ea34..6284885c428832 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2531,13 +2531,24 @@ dummy_func(void) { else if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { /* Do nothing */ } + else if (ctx->frame->func == NULL || + ctx->frame->func->func_builtins != builtins) { + /* Do nothing */ + } else { if (!ctx->builtins_watched) { PyDict_Watch(BUILTINS_WATCHER_ID, builtins); ctx->builtins_watched = true; } - if (ctx->frame->globals_checked_version != 0 && ctx->frame->globals_watched) { + if (ctx->frame->globals_checked_version != 0 && + ctx->frame->globals_watched) + { cnst = convert_global_to_const(this_instr, builtins); + if (cnst != NULL && !ctx->frame->builtins_checked) { + ctx->frame->builtins_checked = true; + ADD_OP(_GUARD_BUILTINS_IS_CANONICAL, 0, 0); + ADD_OP(this_instr->opcode, 0, (uintptr_t)cnst); + } } } if (cnst == NULL) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 57a9539016bbb2..bd3ed9ce99a555 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2301,13 +2301,23 @@ } else if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { } + else if (ctx->frame->func == NULL || + ctx->frame->func->func_builtins != builtins) { + } else { if (!ctx->builtins_watched) { PyDict_Watch(BUILTINS_WATCHER_ID, builtins); ctx->builtins_watched = true; } - if (ctx->frame->globals_checked_version != 0 && ctx->frame->globals_watched) { + if (ctx->frame->globals_checked_version != 0 && + ctx->frame->globals_watched) + { cnst = convert_global_to_const(this_instr, builtins); + if (cnst != NULL && !ctx->frame->builtins_checked) { + ctx->frame->builtins_checked = true; + ADD_OP(_GUARD_BUILTINS_IS_CANONICAL, 0, 0); + ADD_OP(this_instr->opcode, 0, (uintptr_t)cnst); + } } } if (cnst == NULL) { @@ -2328,6 +2338,10 @@ break; } + case _GUARD_BUILTINS_IS_CANONICAL: { + break; + } + case _DELETE_FAST: { break; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 3147e9c5065edd..eaa4c8dbbdc5de 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -1378,6 +1378,7 @@ _Py_uop_frame_new( frame->stack_pointer = frame->stack; frame->globals_checked_version = 0; frame->globals_watched = false; + frame->builtins_checked = false; frame->func = NULL; frame->caller = false; frame->is_c_recursion_checked = false; From eb30e3d9d4b523e3f8de2cd1485c9ae9a996ab03 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2026 14:25:38 +0200 Subject: [PATCH 029/240] Optimize PyFloat_Pack*() functions (#158483) Use _Py_bswap16/32/64() functions to swap bytes. PyFloat_Pack4() and PyFloat_Pack8() avoid temporary buffer copy in the native byte order. --- Objects/floatobject.c | 69 +++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 46 deletions(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 1bb5311ac678a5..d3ae337236a693 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -5,6 +5,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_bitutils.h" // _Py_bswap16() #include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() #include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() @@ -1894,7 +1895,6 @@ _PyFloat_DebugMallocStats(FILE *out) int PyFloat_Pack2(double x, char *data, int le) { - unsigned char *p = (unsigned char *)data; #if _Py_HAVE_FLOAT16 /* Conversion can change NaNs type or alter payload. Here we just fallback to the generic code, instead of providing @@ -1906,16 +1906,14 @@ PyFloat_Pack2(double x, char *data, int le) goto Overflow; } - unsigned char s[sizeof(_Float16)]; - - memcpy(s, &y, sizeof(_Float16)); if ((_PY_FLOAT_LITTLE_ENDIAN && !le) || (_PY_FLOAT_BIG_ENDIAN && le)) { - p[1] = s[0]; - p[0] = s[1]; + uint16_t word; + memcpy(&word, &y, 2); + word = _Py_bswap16(word); + memcpy(data, &word, 2); } else { - p[0] = s[0]; - p[1] = s[1]; + memcpy(data, &y, sizeof(_Float16)); } return 0; } @@ -1923,8 +1921,7 @@ PyFloat_Pack2(double x, char *data, int le) unsigned char sign; int e; double f; - unsigned short bits; - int incr = 1; + uint16_t bits; if (x == 0.0) { sign = (copysign(1.0, x) == -1.0); @@ -2004,18 +2001,10 @@ PyFloat_Pack2(double x, char *data, int le) bits |= (e << 10) | (sign << 15); /* Write out result. */ - if (le) { - p += 1; - incr = -1; + if ((_PY_FLOAT_LITTLE_ENDIAN && !le) || (_PY_FLOAT_BIG_ENDIAN && le)) { + bits = _Py_bswap16(bits); } - - /* First byte */ - *p = (unsigned char)((bits >> 8) & 0xFF); - p += incr; - - /* Second byte */ - *p = (unsigned char)(bits & 0xFF); - + memcpy(data, &bits, 2); return 0; Overflow: @@ -2027,10 +2016,7 @@ PyFloat_Pack2(double x, char *data, int le) int PyFloat_Pack4(double x, char *data, int le) { - unsigned char *p = (unsigned char *)data; float y = (float)x; - int i, incr = 1; - if (isinf(y) && !isinf(x)) { PyErr_SetString(PyExc_OverflowError, "float too large to pack with f format"); @@ -2054,8 +2040,8 @@ PyFloat_Pack4(double x, char *data, int le) } #else uint32_t u32; - memcpy(&u32, &y, 4); + /* Workaround RISC-V: "If a NaN value is converted to a * different floating-point type, the result is the * canonical NaN of the new type". The canonical NaN here @@ -2075,17 +2061,14 @@ PyFloat_Pack4(double x, char *data, int le) #endif } - unsigned char s[sizeof(float)]; - memcpy(s, &y, sizeof(float)); - if ((_PY_FLOAT_LITTLE_ENDIAN && !le) || (_PY_FLOAT_BIG_ENDIAN && le)) { - p += 3; - incr = -1; + uint32_t word; + memcpy(&word, &y, 4); + word = _Py_bswap32(word); + memcpy(data, &word, 4); } - - for (i = 0; i < 4; i++) { - *p = s[i]; - p += incr; + else { + memcpy(data, &y, sizeof(float)); } return 0; } @@ -2093,20 +2076,14 @@ PyFloat_Pack4(double x, char *data, int le) int PyFloat_Pack8(double x, char *data, int le) { - unsigned char *p = (unsigned char *)data; - unsigned char as_bytes[8]; - memcpy(as_bytes, &x, 8); - const unsigned char *s = as_bytes; - int i, incr = 1; - if ((_PY_FLOAT_LITTLE_ENDIAN && !le) || (_PY_FLOAT_BIG_ENDIAN && le)) { - p += 7; - incr = -1; + uint64_t word; + memcpy(&word, &x, 8); + word = _Py_bswap64(word); + memcpy(data, &word, 8); } - - for (i = 0; i < 8; i++) { - *p = *s++; - p += incr; + else { + memcpy(data, &x, 8); } return 0; } From 82fcaf881cddc8c735ed036ba5c1db81c5864276 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2026 16:58:55 +0200 Subject: [PATCH 030/240] Cleanup Python/getopt.c: add _PyOS_GetOpt structure (#158362) Replace global variables with a new _PyOS_GetOpt structure: * _PyOS_opterr => getopt->error * _PyOS_optind => getopt->index * _PyOS_optarg => getopt->arg Changes: * Add tests to test_cmd_line. * Rename _PyOS_ResetGetOpt() to _PyOS_GetOpt_Init(). Add argc and argv parameters. * Don't expose longindex in the structure, it's only used internally. * Move _PyOS_LongOption structure from the internal C API to getopt.c. --- Include/internal/pycore_getopt.h | 25 +++--- Lib/test/test_cmd_line.py | 125 +++++++++++++++++++++++++++--- Python/getopt.c | 128 ++++++++++++++++++------------- Python/initconfig.c | 47 ++++++------ Python/preconfig.c | 13 ++-- 5 files changed, 234 insertions(+), 104 deletions(-) diff --git a/Include/internal/pycore_getopt.h b/Include/internal/pycore_getopt.h index 7f0dd13ae577f7..e767cb1049d079 100644 --- a/Include/internal/pycore_getopt.h +++ b/Include/internal/pycore_getopt.h @@ -5,18 +5,21 @@ # error "this header requires Py_BUILD_CORE define" #endif -extern int _PyOS_opterr; -extern Py_ssize_t _PyOS_optind; -extern const wchar_t *_PyOS_optarg; +struct _PyOS_GetOpt { + int error; // generate error messages + Py_ssize_t index; // index into argv array + const wchar_t *arg; // optional argument + const wchar_t *ptr; + Py_ssize_t argc; + wchar_t * const *argv; +}; -extern void _PyOS_ResetGetOpt(void); +extern void _PyOS_GetOpt_Init( + struct _PyOS_GetOpt *getopt, + Py_ssize_t argc, + wchar_t * const *argv); -typedef struct { - const wchar_t *name; - int has_arg; - int val; -} _PyOS_LongOption; - -extern int _PyOS_GetOpt(Py_ssize_t argc, wchar_t * const *argv, int *longindex); +extern int _PyOS_GetOpt( + struct _PyOS_GetOpt *getopt); #endif /* !Py_INTERNAL_PYGETOPT_H */ diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 476a481a0ba54b..6d07238a9d08ab 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -14,6 +14,7 @@ from test import support from test.support import os_helper from test.support import force_not_colorized +from test.support import import_helper from test.support import threading_helper from test.support.script_helper import ( spawn_python, kill_python, assert_python_ok, assert_python_failure, @@ -60,14 +61,17 @@ def verify_valid_flag(self, cmd_line): @support.cpython_only @support.force_not_colorized def test_help(self): - self.verify_valid_flag('-h') - self.verify_valid_flag('-?') - out = self.verify_valid_flag('--help') - lines = out.splitlines() - self.assertIn(b'usage', lines[0]) - self.assertNotIn(b'PYTHONHOME', out) - self.assertNotIn(b'-X dev', out) - self.assertLess(len(lines), 50) + options = ['-h', '-?', '--help'] + if support.MS_WINDOWS: + options.append('/?') + for opt in options: + with self.subTest(opt=opt): + out = self.verify_valid_flag(opt) + lines = out.splitlines() + self.assertIn(b'usage', lines[0]) + self.assertNotIn(b'PYTHONHOME', out) + self.assertNotIn(b'-X dev', out) + self.assertLess(len(lines), 50) @support.cpython_only @support.force_not_colorized @@ -680,16 +684,22 @@ def test_del___main__(self): @support.cpython_only def test_unknown_options(self): - rc, out, err = assert_python_failure('-E', '-z') - self.assertIn(b'Unknown option: -z', err) - self.assertEqual(err.splitlines().count(b'Unknown option: -z'), 1) - self.assertEqual(b'', out) + # Test unknown option -a + for option in ('-z', '--long-option', '---'): + with self.subTest(option=option): + rc, out, err = assert_python_failure('-E', option) + errmsg = f'Unknown option: {option}'.encode() + self.assertIn(errmsg, err) + self.assertEqual(err.splitlines().count(errmsg), 1) + self.assertEqual(b'', out) + # Add "without='-E'" to prevent _assert_python to append -E # to env_vars and change the output of stderr rc, out, err = assert_python_failure('-z', without='-E') self.assertIn(b'Unknown option: -z', err) self.assertEqual(err.splitlines().count(b'Unknown option: -z'), 1) self.assertEqual(b'', out) + rc, out, err = assert_python_failure('-a', '-z', without='-E') self.assertIn(b'Unknown option: -a', err) # only the first unknown option is reported @@ -1403,6 +1413,97 @@ def test_dump_path_config(self): self.assertIn(b'Python path configuration:', proc.err) self.assertIn(f"PYTHONHOME = '{nonexistent}'".encode(), proc.err) + def test_short_options(self): + # Skip the test if _testcapi cannot be imported: + # the test uses _testcapi.config_get(). + import_helper.import_module('_testcapi') + + # Test short command line options + def check(options, config_names, expected): + if isinstance(config_names, str): + config_names = (config_names,) + if isinstance(options, str): + options = (options,) + expr = ', '.join(f'config_get({name!a})' for name in config_names) + code = f'from _testcapi import config_get; print({expr})' + args = options + ("-c", code) + proc = assert_python_ok(*args) + self.assertEqual(proc.out.rstrip(), expected.encode()) + + def check_ignored(option): + # Just test that passing the option doesn't fail + assert_python_ok(option, "-c", "pass") + + check('-b', 'bytes_warning', '1') + check('-bb', 'bytes_warning', '2') + check('-B', 'write_bytecode', 'False') + check('-d', 'parser_debug', 'True') + check('-E', 'use_environment', 'False') + check('-i', ('inspect', 'interactive'), 'True True') + check('-I', 'isolated', 'True') + check('-O', 'optimization_level', '1') + check('-OO', 'optimization_level', '2') + check('-P', 'safe_path', 'True') + check('-q', 'quiet', 'True') + check('-R', 'use_hash_seed', 'False') + check('-s', 'user_site_directory', 'False') + check('-S', 'site_import', 'False') + check_ignored('-t') + check('-u', 'buffered_stdio', 'False') + check('-v', 'verbose', '1') + check('-Wignore', 'warnoptions', "['ignore']") + check('-x', 'skip_source_first_line', 'True') + check(('-X', 'xoption=value'), 'xoptions', + # assert_python_ok() adds -X faulthandler + "{'faulthandler': True, 'xoption': 'value'}") + + # Short options can be combined + check('-bIs', ('bytes_warning', 'isolated', 'user_site_directory'), + '1 True False') + + # -c, -h, -m, -V and -? are tested elsewhere + + def test_missing_argument(self): + def check_missing_arg(option): + proc = assert_python_failure(option) + self.assertEqual(proc.rc, 2) + errmsg = f"Argument expected for the {option} option" + self.assertStartsWith(proc.err.rstrip(), errmsg.encode()) + + check_missing_arg('-c') + check_missing_arg('-m') + check_missing_arg('-W') + check_missing_arg('-X') + check_missing_arg('--check-hash-based-pycs') + + def test_long_options(self): + # Test long command line options + + # Test --check-hash-based-pycs option + code = 'import _imp; print(_imp.check_hash_based_pycs)' + opt = f"--check-hash-based-pycs" + for value in ('always', 'never', 'default'): + with self.subTest(value=value): + proc = assert_python_ok(opt, value, "-c", code) + self.assertEqual(proc.out.rstrip(), value.encode()) + + # "Expected long option" error + proc = assert_python_ok("-b-") + self.assertEqual(proc.out, b'') + self.assertEqual(proc.err.rstrip(), b"Expected long option") + + # Other long options --help-all, --help-env, --help-xoptions + # and --version are tested elsewhere + + def test_dash_option(self): + # Test -- in the command line + code = ( + 'import sys; ' + 'print(sys.flags.isolated, sys.flags.optimize, sys.argv)' + ) + proc = assert_python_ok('-I', '-c', code, '--', '-O') + self.assertEqual(proc.out.rstrip(), b"1 0 ['-c', '--', '-O']") + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') diff --git a/Python/getopt.c b/Python/getopt.c index 7e918189c716a9..45454d39fc6779 100644 --- a/Python/getopt.c +++ b/Python/getopt.c @@ -27,18 +27,19 @@ #include #include #include -#include "pycore_getopt.h" +#include "pycore_getopt.h" // struct _PyOS_GetOpt -int _PyOS_opterr = 1; /* generate error messages */ -Py_ssize_t _PyOS_optind = 1; /* index into argv array */ -const wchar_t *_PyOS_optarg = NULL; /* optional argument */ - -static const wchar_t *opt_ptr = L""; /* Python command line short and long options */ #define SHORT_OPTS L"bBc:dEhiIm:OPqRsStuvVW:xX:?" +typedef struct { + const wchar_t *name; + int has_arg; + int val; +} _PyOS_LongOption; + static const _PyOS_LongOption longopts[] = { /* name, has_arg, val (used in switch in initconfig.c) */ {L"check-hash-based-pycs", 1, 1}, @@ -49,113 +50,136 @@ static const _PyOS_LongOption longopts[] = { }; -void _PyOS_ResetGetOpt(void) +void +_PyOS_GetOpt_Init(struct _PyOS_GetOpt *getopt, + Py_ssize_t argc, wchar_t * const *argv) { - _PyOS_opterr = 1; - _PyOS_optind = 1; - _PyOS_optarg = NULL; - opt_ptr = L""; + getopt->error = 1; + getopt->index = 1; + getopt->arg = NULL; + getopt->ptr = L""; + getopt->argc = argc; + getopt->argv = argv; } -int _PyOS_GetOpt(Py_ssize_t argc, wchar_t * const *argv, int *longindex) +// Parse a command line option. +// +// Return a character for short option (ex: return 'h' for -h). +// Return a number for long options (see 'longopts' array). +// Return '_' on unknown option or missing argument. +// Return -1 when done. +// +// Return 'h' for --help and return 'V' for --version. +// +// If an option has an argument, set getopt->arg to the argument. +// If getopt->error is non-error, write error messages to stderr. +int +_PyOS_GetOpt(struct _PyOS_GetOpt *getopt) { - wchar_t *ptr; - wchar_t option; + // Local copy of read-only members to omit "getopt->" + const Py_ssize_t argc = getopt->argc; + wchar_t * const *argv = getopt->argv; + const int error = getopt->error; - if (*opt_ptr == '\0') { - - if (_PyOS_optind >= argc) + if (*getopt->ptr == '\0') { + if (getopt->index >= argc) { return -1; + } + + const wchar_t *arg = argv[getopt->index]; #ifdef MS_WINDOWS - else if (wcscmp(argv[_PyOS_optind], L"/?") == 0) { - ++_PyOS_optind; + if (wcscmp(arg, L"/?") == 0) { + ++getopt->index; return 'h'; } #endif - else if (argv[_PyOS_optind][0] != L'-' || - argv[_PyOS_optind][1] == L'\0' /* lone dash */ ) + if (arg[0] != L'-' || arg[1] == L'\0' /* lone dash */ ) { return -1; + } - else if (wcscmp(argv[_PyOS_optind], L"--") == 0) { - ++_PyOS_optind; + if (wcscmp(arg, L"--") == 0) { + ++getopt->index; return -1; } - - else if (wcscmp(argv[_PyOS_optind], L"--help") == 0) { - ++_PyOS_optind; + if (wcscmp(arg, L"--help") == 0) { + ++getopt->index; return 'h'; } - - else if (wcscmp(argv[_PyOS_optind], L"--version") == 0) { - ++_PyOS_optind; + if (wcscmp(arg, L"--version") == 0) { + ++getopt->index; return 'V'; } - opt_ptr = &argv[_PyOS_optind++][1]; + getopt->ptr = &argv[getopt->index++][1]; } - if ((option = *opt_ptr++) == L'\0') + wchar_t option = *getopt->ptr++; + if (option == L'\0') { return -1; + } if (option == L'-') { // Parse long option. - if (*opt_ptr == L'\0') { - if (_PyOS_opterr) { + if (*getopt->ptr == L'\0') { + if (error) { fprintf(stderr, "Expected long option\n"); } return -1; } - *longindex = 0; + int longindex = 0; const _PyOS_LongOption *opt; - for (opt = &longopts[*longindex]; opt->name; opt = &longopts[++(*longindex)]) { - if (!wcscmp(opt->name, opt_ptr)) + for (opt = &longopts[longindex]; opt->name; opt = &longopts[++longindex]) { + if (wcscmp(opt->name, getopt->ptr) == 0) { break; + } } + if (!opt->name) { - if (_PyOS_opterr) { - fprintf(stderr, "Unknown option: %ls\n", argv[_PyOS_optind - 1]); + if (error) { + fprintf(stderr, "Unknown option: %ls\n", argv[getopt->index - 1]); } return '_'; } - opt_ptr = L""; + + getopt->ptr = L""; if (!opt->has_arg) { return opt->val; } - if (_PyOS_optind >= argc) { - if (_PyOS_opterr) { + if (getopt->index >= argc) { + if (error) { fprintf(stderr, "Argument expected for the %ls options\n", - argv[_PyOS_optind - 1]); + argv[getopt->index - 1]); } return '_'; } - _PyOS_optarg = argv[_PyOS_optind++]; + getopt->arg = argv[getopt->index++]; return opt->val; } - if ((ptr = wcschr(SHORT_OPTS, option)) == NULL) { - if (_PyOS_opterr) { + wchar_t *ptr = wcschr(SHORT_OPTS, option); + if (ptr == NULL) { + if (error) { fprintf(stderr, "Unknown option: -%c\n", (char)option); } return '_'; } if (*(ptr + 1) == L':') { - if (*opt_ptr != L'\0') { - _PyOS_optarg = opt_ptr; - opt_ptr = L""; + if (*getopt->ptr != L'\0') { + getopt->arg = getopt->ptr; + getopt->ptr = L""; } - else { - if (_PyOS_optind >= argc) { - if (_PyOS_opterr) { + if (getopt->index >= argc) { + if (error) { fprintf(stderr, "Argument expected for the -%c option\n", (char)option); } return '_'; } - _PyOS_optarg = argv[_PyOS_optind++]; + getopt->arg = argv[getopt->index++]; } } diff --git a/Python/initconfig.c b/Python/initconfig.c index 6de12db9d600ec..178e6f992fa11f 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -3009,11 +3009,11 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, const PyWideStringList *argv = &config->argv; int print_version = 0; - _PyOS_ResetGetOpt(); + struct _PyOS_GetOpt getopt; + _PyOS_GetOpt_Init(&getopt, argv->length, argv->items); do { - int longindex = -1; - int c = _PyOS_GetOpt(argv->length, argv->items, &longindex); - if (c == EOF) { + int c = _PyOS_GetOpt(&getopt); + if (c == -1) { break; } @@ -3022,12 +3022,12 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, /* -c is the last option; following arguments that look like options are left for the command to interpret. */ - size_t len = wcslen(_PyOS_optarg) + 1 + 1; + size_t len = wcslen(getopt.arg) + 1 + 1; wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); if (command == NULL) { return _PyStatus_NO_MEMORY(); } - memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); + memcpy(command, getopt.arg, (len - 2) * sizeof(wchar_t)); command[len - 2] = '\n'; command[len - 1] = 0; config->run_command = command; @@ -3040,7 +3040,7 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, that look like options are left for the module to interpret. */ if (config->run_module == NULL) { - config->run_module = _PyMem_RawWcsdup(_PyOS_optarg); + config->run_module = _PyMem_RawWcsdup(getopt.arg); if (config->run_module == NULL) { return _PyStatus_NO_MEMORY(); } @@ -3051,13 +3051,13 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, switch (c) { // Integers represent long options, see Python/getopt.c case 1: - // check-hash-based-pycs - if (wcscmp(_PyOS_optarg, L"always") == 0 - || wcscmp(_PyOS_optarg, L"never") == 0 - || wcscmp(_PyOS_optarg, L"default") == 0) + // --check-hash-based-pycs option + if (wcscmp(getopt.arg, L"always") == 0 + || wcscmp(getopt.arg, L"never") == 0 + || wcscmp(getopt.arg, L"default") == 0) { status = PyConfig_SetString(config, &config->check_hash_pycs_mode, - _PyOS_optarg); + getopt.arg); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -3067,17 +3067,17 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, break; case 2: - // help-all + // --help-all option DEFER_OPTION(c); break; case 3: - // help-env + // --help-env option DEFER_OPTION(c); break; case 4: - // help-xoptions + // --help-xoptions option DEFER_OPTION(c); break; @@ -3146,7 +3146,7 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, break; case 'W': - status = PyWideStringList_Append(warnoptions, _PyOS_optarg); + status = PyWideStringList_Append(warnoptions, getopt.arg); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -3177,22 +3177,22 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, } if (config->run_command == NULL && config->run_module == NULL - && _PyOS_optind < argv->length - && wcscmp(argv->items[_PyOS_optind], L"-") != 0 + && getopt.index < argv->length + && wcscmp(argv->items[getopt.index], L"-") != 0 && config->run_filename == NULL) { - config->run_filename = _PyMem_RawWcsdup(argv->items[_PyOS_optind]); + config->run_filename = _PyMem_RawWcsdup(argv->items[getopt.index]); if (config->run_filename == NULL) { return _PyStatus_NO_MEMORY(); } } if (config->run_command != NULL || config->run_module != NULL) { - /* Backup _PyOS_optind */ - _PyOS_optind--; + /* Backup getopt.index */ + getopt.index--; } - *opt_index = _PyOS_optind; + *opt_index = getopt.index; return _PyStatus_OK(); @@ -3257,6 +3257,9 @@ _PyConfig_ProcessDeferredCmdlineOption(PyConfig *config) config_xoptions_usage(); return 0; + case '_': + // Unknown option or missing argument + _Py_FALLTHROUGH; default: config_usage(1, program); return 2; diff --git a/Python/preconfig.c b/Python/preconfig.c index 844ac8e6372fc5..c8005ccdce1a73 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -187,15 +187,14 @@ precmdline_parse_cmdline(_PyPreCmdline *cmdline) { const PyWideStringList *argv = &cmdline->argv; - _PyOS_ResetGetOpt(); + struct _PyOS_GetOpt getopt; + _PyOS_GetOpt_Init(&getopt, argv->length, argv->items); /* Don't log parsing errors into stderr here: PyConfig_Read() is responsible for that */ - _PyOS_opterr = 0; + getopt.error = 0; do { - int longindex = -1; - int c = _PyOS_GetOpt(argv->length, argv->items, &longindex); - - if (c == EOF || c == 'c' || c == 'm') { + int c = _PyOS_GetOpt(&getopt); + if (c == -1 || c == 'c' || c == 'm') { break; } @@ -211,7 +210,7 @@ precmdline_parse_cmdline(_PyPreCmdline *cmdline) case 'X': { PyStatus status = PyWideStringList_Append(&cmdline->xoptions, - _PyOS_optarg); + getopt.arg); if (_PyStatus_EXCEPTION(status)) { return status; } From c7a79bd2f4d3dd3bcaebd77c47f07d1feabe320a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2026 18:19:26 +0200 Subject: [PATCH 031/240] gh-158567: Workaround GCC crash in PyFloat_Pack2() (#158568) Avoid __builtin_bswap16() with _Float16 on s390x. --- Objects/floatobject.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index d3ae337236a693..5095da01e45dd2 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1907,10 +1907,20 @@ PyFloat_Pack2(double x, char *data, int le) } if ((_PY_FLOAT_LITTLE_ENDIAN && !le) || (_PY_FLOAT_BIG_ENDIAN && le)) { +#if defined(__s390x__) && defined(__GNUC__) + // gh-158567: Workaround GCC crash on s390x. + // Avoid __builtin_bswap16() with _Float16. + // https://bugzilla.redhat.com/show_bug.cgi?id=2544649 + char buffer[2]; + memcpy(buffer, &y, 2); + data[0] = buffer[1]; + data[1] = buffer[0]; +#else uint16_t word; memcpy(&word, &y, 2); word = _Py_bswap16(word); memcpy(data, &word, 2); +#endif } else { memcpy(data, &y, sizeof(_Float16)); From a6a63b6f5b5e4b921dd98ca8eeea90bca3519814 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 1 Oct 2026 17:20:45 +0100 Subject: [PATCH 032/240] gh-142349: Simplify lazy import resolution (#158282) Co-authored-by: Petr Viktorin Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Include/internal/pycore_dict.h | 9 +- Include/internal/pycore_import.h | 14 +- Include/internal/pycore_interp_structs.h | 5 +- Include/internal/pycore_lazyimportobject.h | 24 +- Include/internal/pycore_opcode_metadata.h | 3 +- Include/internal/pycore_tstate.h | 3 + Include/internal/pycore_uop_metadata.h | 15 + Lib/test/test_lazy_import/__init__.py | 32 +- .../data/broken_module_chained_cause.py | 3 + .../data/broken_module_chained_context.py | 5 + .../data/broken_module_chained_suppressed.py | 2 + Modules/_testinternalcapi/test_cases.c.h | 136 ++--- Objects/dictobject.c | 122 ++--- Objects/lazyimportobject.c | 463 +++++++++++++++-- Objects/moduleobject.c | 133 ++--- Python/bytecodes.c | 83 +-- Python/ceval.c | 91 +--- Python/clinic/import.c.h | 4 +- Python/executor_cases.c.h | 97 ++-- Python/generated_cases.c.h | 136 ++--- Python/import.c | 484 ++++-------------- Python/optimizer_cases.c.h | 7 +- Python/pystate.c | 4 + 23 files changed, 935 insertions(+), 940 deletions(-) create mode 100644 Lib/test/test_lazy_import/data/broken_module_chained_cause.py create mode 100644 Lib/test/test_lazy_import/data/broken_module_chained_context.py create mode 100644 Lib/test/test_lazy_import/data/broken_module_chained_suppressed.py diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 7032b61d7654b8..81f1de3a5be865 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -23,6 +23,11 @@ PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, int (*predicate)(PyObject *value, void *arg), void *arg); +// Atomically replace an existing value if it is expected (by identity). +// Return 1 if replaced, 0 if absent or different, or -1 on error. +extern int _PyDict_ReplaceItemIf(PyObject *dict, PyObject *key, + PyObject *expected, PyObject *replacement); + // "KnownHash" variants // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, @@ -147,8 +152,8 @@ PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyOb */ PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); -PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); -PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); +// Return the borrowed source dictionary, or NULL if absent or on error. +PyAPI_FUNC(PyObject *) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); // Loads the __builtins__ object from the globals dict. Returns a new reference. extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 669e328c266d00..81448e23787fb3 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -32,20 +32,16 @@ extern int _PyImport_FixupBuiltin( PyObject *modules ); -extern PyObject * _PyImport_ResolveName( - PyThreadState *tstate, PyObject *name, PyObject *globals, int level); extern PyObject * _PyImport_GetAbsName( PyThreadState *tstate, PyObject *name, PyObject *globals, int level); // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); -typedef enum { - _Py_LAZY_SUBMODULE_ERROR = -1, - _Py_LAZY_SUBMODULE_NOT_FOUND = 0, - _Py_LAZY_SUBMODULE_LOADED = 1, -} _PyLazySubmoduleImportResult; -extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule( - PyObject *mod_name, PyObject *attr_name, PyObject **result); +// Bind a pending child on module before removing its registration. Return a +// new reference, NULL without an exception if absent, or NULL on error. +// Set recheck_dict if a concurrent lookup may have already bound the child. +extern PyObject * _PyImport_TryLoadLazySubmodule( + PyObject *module, PyObject *attr_name, int *recheck_dict); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 6c907e0cf79894..a9c61ea6db3a8c 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -352,15 +352,12 @@ struct _import_state { PyObject *lazy_import_func; int lazy_imports_mode; PyObject *lazy_imports_filter; - PyObject *lazy_importing_modules; // The set stored in sys.lazy_modules if values that have been // lazily imported. This value is only for debugging/introspection // purposes and is not used by the runtime. PyObject *lazy_modules; // A dict mapping package names to a set of submodule names that - // have been imported lazily from packages which have been imported - // lazily. When the package is reified we need to add a - // LazyImportObject which refers to the submodule on the module. + // may need to be imported when an attribute is missing on the package. PyObject *lazy_pending_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index b81e4211b08ff3..c3f8417c089316 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -14,20 +14,18 @@ extern "C" { PyAPI_DATA(PyTypeObject) PyLazyImport_Type; #define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type) -typedef struct { - PyObject_HEAD - PyObject *lz_builtins; - PyObject *lz_from; - PyObject *lz_attr; - // Frame information for the original import location. - PyCodeObject *lz_code; // Code object where the lazy import was created. - int lz_instr_offset; // Instruction offset where the lazy import was created. -} PyLazyImportObject; - - -PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); PyAPI_FUNC(PyObject *) _PyLazyImport_New( - struct _PyInterpreterFrame *frame, PyObject *import_func, PyObject *from, PyObject *attr); + struct _PyInterpreterFrame *frame, PyObject *builtins, + PyObject *name, PyObject *fromlist); + +extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); + +// Resolve a placeholder and replace its binding if it is unchanged or holds +// the child module published by the normal importer during resolution. +// namespace is the source captured during lookup, before resolution runs. +PyAPI_FUNC(PyObject *) _PyLazyImport_Reify( + PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *ns); #ifdef __cplusplus } diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 38bccdd48c0f86..d3546119bc9ee0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1253,7 +1253,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1476,6 +1476,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, OPARG_SIMPLE, 0 } } }, [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, OPARG_SIMPLE, 0 } } }, + [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, OPARG_SIMPLE, 0 } } }, [LOAD_GLOBAL] = { .nuops = 2, .uops = { { _LOAD_GLOBAL, OPARG_SIMPLE, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 3 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 4, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_BUILTINS, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 4 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 4, .uops = { { _NOP, OPARG_SIMPLE, 1 }, { _LOAD_GLOBAL_MODULE, 1, 1 }, { _LOAD_GLOBAL_MODULE, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 4 } } }, diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index eb2b0c84acdc7c..58dd66b69efc9d 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -51,6 +51,9 @@ typedef struct _PyThreadStateImpl { PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference + // Reused for the thread's lifetime; owns placeholders while they resolve. + PyObject *lazy_imports; + // Distinguishes between yield and return from PyEval_EvalFrame(). // See gen_send_ex2() in Objects/genobject.c enum { diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 177564a7545941..86b449158f00d0 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -190,6 +190,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG, + [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL_CONDITIONAL] = HAS_ARG_FLAG, @@ -1841,6 +1842,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_LOAD_FROM_DICT_OR_GLOBALS] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_FROM_DICT_OR_GLOBALS_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_LOAD_NAME] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -4316,6 +4326,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_LOCALS_r01] = _LOAD_LOCALS, [_LOAD_LOCALS_r12] = _LOAD_LOCALS, [_LOAD_LOCALS_r23] = _LOAD_LOCALS, + [_LOAD_FROM_DICT_OR_GLOBALS_r11] = _LOAD_FROM_DICT_OR_GLOBALS, [_LOAD_NAME_r01] = _LOAD_NAME, [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, @@ -5829,6 +5840,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_FAST_CHECK_r23] = "_LOAD_FAST_CHECK_r23", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", [_LOAD_FROM_DICT_OR_DEREF_r11] = "_LOAD_FROM_DICT_OR_DEREF_r11", + [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", + [_LOAD_FROM_DICT_OR_GLOBALS_r11] = "_LOAD_FROM_DICT_OR_GLOBALS_r11", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_LOAD_GLOBAL_r00] = "_LOAD_GLOBAL_r00", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", @@ -6495,6 +6508,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_LOCALS: return 0; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; case _LOAD_NAME: return 0; case _LOAD_GLOBAL: diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 5026c9670d81d2..f9ef8ce2cfef8c 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -13,7 +13,7 @@ import contextlib from test import support -from test.support.script_helper import assert_python_ok +from test.support.script_helper import assert_python_ok, assert_python_failure try: import _testcapi @@ -617,6 +617,14 @@ def test_dunder_lazy_import_invalid_arguments(self): with self.assertRaises(TypeError): __lazy_import__("sys", globals=1) + code = textwrap.dedent(""" + __lazy_import__("sys", fromlist=(1, 2, 3)) + """) + result = assert_python_failure("-c", code, NO_COLOR='y') + self.assertIn( + b"TypeError: Item in ``from list'' must be str, not int", + result.err) + def test_dunder_lazy_import_builtins(self): """__lazy_import__ should use module's __builtins__ for __import__.""" from test.test_lazy_import.data import dunder_lazy_import_builtins @@ -766,6 +774,28 @@ def test_missing_lazy_from_import_shows_chained_traceback(self): """) assert_python_ok("-c", code) + @support.subTests('name', ( + 'test.test_lazy_import.data.broken_module_chained_cause', + 'test.test_lazy_import.data.broken_module_chained_context', + 'test.test_lazy_import.data.broken_module_chained_suppressed', + )) + def test_chained_exception_import_shows_notes(self, name): + """Accessing missing attribute from lazy from-import should chain errors.""" + code = textwrap.dedent(f""" + lazy import {name} + + try: + _ = test + except ValueError as e: + assert any( + note.startswith("lazy import of '{name}' declared in ") + for note in e.__notes__ + ), e.__notes__ + else: + raise AssertionError("ImportError was not raised") + """) + assert_python_ok("-c", code) + def test_reification_retries_on_failure(self): """Failed reification should allow retry on subsequent access. diff --git a/Lib/test/test_lazy_import/data/broken_module_chained_cause.py b/Lib/test/test_lazy_import/data/broken_module_chained_cause.py new file mode 100644 index 00000000000000..786607b7bb557e --- /dev/null +++ b/Lib/test/test_lazy_import/data/broken_module_chained_cause.py @@ -0,0 +1,3 @@ +# Module that raises an exception with explicit cause during import +cause = ValueError("Cause of failure") +raise ValueError("This module always fails to import") from cause diff --git a/Lib/test/test_lazy_import/data/broken_module_chained_context.py b/Lib/test/test_lazy_import/data/broken_module_chained_context.py new file mode 100644 index 00000000000000..82640856e3e2d8 --- /dev/null +++ b/Lib/test/test_lazy_import/data/broken_module_chained_context.py @@ -0,0 +1,5 @@ +# Module that raises an exception with context during import +try: + raise ValueError("Cause of failure") +except: + raise ValueError("This module always fails to import") diff --git a/Lib/test/test_lazy_import/data/broken_module_chained_suppressed.py b/Lib/test/test_lazy_import/data/broken_module_chained_suppressed.py new file mode 100644 index 00000000000000..5c34654afc13bf --- /dev/null +++ b/Lib/test/test_lazy_import/data/broken_module_chained_suppressed.py @@ -0,0 +1,2 @@ +# Module that raises an exception with suppressed context during import +raise ValueError("This module always fails to import") from None diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 19c6cb7517d6ff..3bdc16437e2bc6 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -9914,6 +9914,16 @@ JUMP_TO_LABEL(error); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + JUMP_TO_LABEL(error); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9939,11 +9949,21 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + err = -1; + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9954,81 +9974,17 @@ JUMP_TO_LABEL(error); } if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - } - JUMP_TO_LABEL(error); - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } - } - else { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { + JUMP_TO_LABEL(error); } } - v = PyStackRef_FromPyObjectSteal(v_o); + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10282,38 +10238,6 @@ if (v_o == NULL) { JUMP_TO_LABEL(error); } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 49eb8481e128d0..c6d055a412a515 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2013,6 +2013,35 @@ _PyDict_InsertSplitValue(PyDictObject *mp, PyObject *key, PyObject *value, Py_ss ASSERT_CONSISTENT(mp); } +// Replace a value at an existing entry. Steals the new value reference. +static void +replace_value(PyDictObject *mp, PyObject *key, Py_ssize_t ix, + PyObject *old_value, PyObject *value) +{ + assert(can_modify_dict(mp)); + assert(old_value != NULL); + + if (old_value != value) { + _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); + if (DK_IS_UNICODE(mp->ma_keys)) { + if (_PyDict_HasSplitTable(mp)) { + STORE_SPLIT_VALUE(mp, ix, value); + } + else { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } + } + else { + PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } + } + Py_DECREF(old_value); /* which **CAN** re-enter (see gh-66843) */ + + ASSERT_CONSISTENT(mp); +} + /* Internal routine to insert a new item into the table. Used both by the internal resize routine and by the public insert routine. @@ -2060,25 +2089,7 @@ insertdict(PyDictObject *mp, return 0; } - if (old_value != value) { - _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); - assert(old_value != NULL); - if (DK_IS_UNICODE(mp->ma_keys)) { - if (_PyDict_HasSplitTable(mp)) { - STORE_SPLIT_VALUE(mp, ix, value); - } - else { - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); - } - } - else { - PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); - } - } - Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ - ASSERT_CONSISTENT(mp); + replace_value(mp, key, ix, old_value, value); Py_DECREF(key); return 0; @@ -2704,44 +2715,7 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) return rv; } -/* Fast version of global value lookup (LOAD_GLOBAL). - * Lookup in globals, then builtins. - * - * - * - * - * Raise an exception and return NULL if an error occurred (ex: computing the - * key hash failed, key comparison failed, ...). Return NULL if the key doesn't - * exist. Return the value if the key exists. - * - * Returns a new reference. - */ PyObject * -_PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) -{ - Py_ssize_t ix; - Py_hash_t hash; - PyObject *value; - - hash = _PyObject_HashDictKey(key); - if (hash == -1) { - return NULL; - } - - /* namespace 1: globals */ - ix = _Py_dict_lookup_threadsafe(globals, key, hash, &value); - if (ix == DKIX_ERROR) - return NULL; - if (ix != DKIX_EMPTY && value != NULL) - return value; - - /* namespace 2: builtins */ - ix = _Py_dict_lookup_threadsafe(builtins, key, hash, &value); - assert(ix >= 0 || value == NULL); - return value; -} - -void _PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObject *key, _PyStackRef *res) { Py_ssize_t ix; @@ -2750,21 +2724,22 @@ _PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObje hash = _PyObject_HashDictKey(key); if (hash == -1) { *res = PyStackRef_NULL; - return; + return NULL; } /* namespace 1: globals */ ix = _Py_dict_lookup_threadsafe_stackref(globals, key, hash, res); if (ix == DKIX_ERROR) { - return; + return NULL; } if (ix != DKIX_EMPTY && !PyStackRef_IsNull(*res)) { - return; + return (PyObject *)globals; } /* namespace 2: builtins */ ix = _Py_dict_lookup_threadsafe_stackref(builtins, key, hash, res); assert(ix >= 0 || PyStackRef_IsNull(*res)); + return PyStackRef_IsNull(*res) ? NULL : (PyObject *)builtins; } PyObject * @@ -3109,6 +3084,35 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return res; } +int +_PyDict_ReplaceItemIf(PyObject *op, PyObject *key, + PyObject *expected, PyObject *replacement) +{ + assert(PyDict_Check(op)); + assert(expected != NULL); + assert(replacement != NULL); + + Py_hash_t hash = PyObject_Hash(key); + if (hash == -1) { + return -1; + } + int result = 0; + Py_BEGIN_CRITICAL_SECTION(op); + PyDictObject *mp = (PyDictObject *)op; + PyObject *current; + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, ¤t); + if (ix == DKIX_ERROR) { + result = -1; + } + else if (current == expected) { + // Do not look up the key again: equality can execute Python code. + replace_value(mp, key, ix, current, Py_NewRef(replacement)); + result = 1; + } + Py_END_CRITICAL_SECTION(); + return result; +} + static void clear_embedded_values(PyDictValues *values, Py_ssize_t nentries) { diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index e008c076352ebb..e4a949a5456cfd 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -2,18 +2,39 @@ #include "Python.h" #include "pycore_ceval.h" -#include "pycore_frame.h" +#include "pycore_dict.h" +#include "pycore_gc.h" #include "pycore_import.h" #include "pycore_interpframe.h" #include "pycore_lazyimportobject.h" -#include "pycore_modsupport.h" +#include "pycore_long.h" +#include "pycore_moduleobject.h" +#include "pycore_pyerrors.h" +#include "pycore_traceback.h" +#include "pycore_tstate.h" + +typedef struct { + PyObject_HEAD + PyObject *lz_builtins; // Roots own the mapping; projections retain the root. + // A root stores its absolute name (PyUnicode) in lz_from, and original + // fromlist in lz_attr. + // A projection stores its source placeholder (a PyLazyImportObject) + // in lz_from, and the attribute to import from (PyUnicode) it in lz_attr. + PyObject *lz_from; + PyObject *lz_attr; + // Declaration location. + PyCodeObject *lz_code; + int lz_instr_offset; +} PyLazyImportObject; #define PyLazyImportObject_CAST(op) ((PyLazyImportObject *)(op)) +static PyObject *lazy_import_name(PyLazyImportObject *m); + PyObject * -_PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name, PyObject *fromlist) +_PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, + PyObject *name, PyObject *fromlist) { - PyLazyImportObject *m; if (!name || !(PyUnicode_Check(name) || PyLazyImport_CheckExact(name))) { PyErr_SetString(PyExc_TypeError, "expected str or lazy_import for name"); return NULL; @@ -26,7 +47,20 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name "lazy_import: fromlist must be None, a string, or a tuple"); return NULL; } - m = PyObject_GC_New(PyLazyImportObject, &PyLazyImport_Type); +#ifndef NDEBUG + if (PyLazyImport_CheckExact(name)) { + // projection + assert(builtins == NULL); + assert(fromlist != NULL); + assert(PyUnicode_Check(fromlist)); + } + else { + // root + assert(builtins != NULL); + } +#endif + PyLazyImportObject *m = PyObject_GC_New( + PyLazyImportObject, &PyLazyImport_Type); if (m == NULL) { return NULL; } @@ -34,23 +68,83 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name m->lz_from = Py_NewRef(name); m->lz_attr = Py_XNewRef(fromlist); - // Capture frame information for the original import location. m->lz_code = NULL; m->lz_instr_offset = -1; if (frame != NULL) { - PyCodeObject *code = _PyFrame_GetCode(frame); - if (code != NULL) { - m->lz_code = (PyCodeObject *)Py_NewRef(code); - // Calculate the instruction offset from the current frame. - m->lz_instr_offset = _PyInterpreterFrame_LASTI(frame); - } + m->lz_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + m->lz_instr_offset = _PyInterpreterFrame_LASTI(frame); } _PyObject_GC_TRACK(m); return (PyObject *)m; } +// Reuse concrete attributes of initialized modules without waiting for imports +// or resolving lazy attributes. Failed cache lookups are retried at resolution. +// May return NULL with or without an exception set. +static PyObject * +lazy_import_get_loaded_attr(PyThreadState *tstate, PyObject *name, + PyObject *attr_name) +{ + PyObject *mod = NULL, *spec = NULL, *current = NULL, *attr = NULL; + PyObject *modules = Py_XNewRef(_PyImport_GetModules(tstate->interp)); + if (modules == NULL) { + return NULL; + } + int rc = PyMapping_GetOptionalItem(modules, name, &mod); + if (rc <= 0 || !PyModule_Check(mod)) { + goto done; + } + PyObject *dict = _PyModule_GetDict(mod); + if (PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec) < 0 || + _PyModuleSpec_IsInitializing(spec) != 0) { + goto done; + } + // An initialization check can replace the module in sys.modules. + if (modules != _PyImport_GetModules(tstate->interp) || + PyMapping_GetOptionalItem(modules, name, ¤t) <= 0 || + current != mod) { + goto done; + } + if (PyDict_GetItemRef(dict, attr_name, &attr) < 0) { + goto done; + } + if (attr != NULL && PyLazyImport_CheckExact(attr)) { + Py_CLEAR(attr); + } + +done: + Py_XDECREF(current); + Py_XDECREF(spec); + Py_XDECREF(mod); + Py_DECREF(modules); + if (PyErr_ExceptionMatches(PyExc_Exception)) { + PyErr_Clear(); + } + return attr; +} + +PyObject * +_PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, + PyObject *v, PyObject *name) +{ + assert(PyLazyImport_CheckExact(v)); + assert(name); + assert(PyUnicode_Check(name)); + PyLazyImportObject *lz = PyLazyImportObject_CAST(v); + // Only `from a import b` can take b off an already imported a; + // `import a.b as c` has to import a.b first. + if (lz->lz_attr != NULL && PyTuple_Check(lz->lz_attr) && + PyTuple_GET_SIZE(lz->lz_attr) > 0) { + PyObject *attr = lazy_import_get_loaded_attr(tstate, lz->lz_from, name); + if (attr != NULL || PyErr_Occurred()) { + return attr; + } + } + return _PyLazyImport_New(frame, NULL, v, name); +} + static int lazy_import_traverse(PyObject *op, visitproc visit, void *arg) { @@ -85,23 +179,20 @@ lazy_import_dealloc(PyObject *op) static PyObject * lazy_import_getattro(PyObject *op, PyObject *name) { - PyObject *value = _PyObject_GenericGetAttrWithDict(op, name, NULL, /* suppress */1); - if (value == NULL) { - if (PyErr_Occurred()) { - // pass up non-AttributeError exception - return NULL; - } - PyObject *lz_name = _PyLazyImport_GetName(op); - if (lz_name == NULL) { - return NULL; - } - PyErr_Format(PyExc_AttributeError, - "cannot access attribute %R on unresolved lazy import %R", - name, lz_name); - Py_DECREF(lz_name); + PyObject *value = _PyObject_GenericGetAttrWithDict( + op, name, NULL, /* suppress */ 1); + if (value != NULL || PyErr_Occurred()) { + return value; + } + PyObject *lz_name = lazy_import_name(PyLazyImportObject_CAST(op)); + if (lz_name == NULL) { return NULL; } - return value; + PyErr_Format(PyExc_AttributeError, + "cannot access attribute %R on unresolved lazy import %R", + name, lz_name); + Py_DECREF(lz_name); + return NULL; } // The dotted name of the object that resolving the placeholder returns. @@ -160,12 +251,320 @@ lazy_import_repr(PyObject *op) return res; } -PyObject * -_PyLazyImport_GetName(PyObject *op) +// Consume a result, resolving a placeholder returned by an import hook or +// an attribute lookup under the same cycle and recursion checks. +static PyObject * +lazy_import_resolve_result(PyThreadState *tstate, PyObject *obj) +{ + if (obj == NULL || !PyLazyImport_CheckExact(obj)) { + return obj; + } + PyObject *result = _PyImport_LoadLazyImportTstate(tstate, obj); + Py_DECREF(obj); + return result; +} + +// Look up, in order, the attributes recorded from the root placeholder to lz +// on the module the root's import returned. +static PyObject * +lazy_import_replay_from(PyThreadState *tstate, PyObject *mod, + PyLazyImportObject *lz) +{ + if (!PyLazyImport_CheckExact(lz->lz_from)) { + return Py_NewRef(mod); + } + PyObject *from = lazy_import_replay_from( + tstate, mod, (PyLazyImportObject *)lz->lz_from); + if (from == NULL) { + return NULL; + } + PyObject *obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr); + Py_DECREF(from); + return lazy_import_resolve_result(tstate, obj); +} + +// Preserve the resolution error and attach the import's declaration location. +static void +lazy_import_add_exception_cause(PyThreadState *tstate, PyLazyImportObject *lz) { - PyLazyImportObject *lazy_import = PyLazyImportObject_CAST(op); + if (!PyErr_Occurred() || lz->lz_code == NULL || lz->lz_instr_offset < 0) { + return; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyObject *name = lazy_import_name(lz); + if (name == NULL) { + goto done; + } + int lineno = PyCode_Addr2Line(lz->lz_code, lz->lz_instr_offset * 2); + const char *filename = PyUnicode_AsUTF8(lz->lz_code->co_filename); + if (filename == NULL) { + goto done; + } + const char *funcname = PyUnicode_AsUTF8(lz->lz_code->co_name); + if (funcname == NULL) { + goto done; + } + PyBaseExceptionObject *base_exc = (PyBaseExceptionObject *)exc; + if (base_exc->cause != NULL || base_exc->context != NULL || + base_exc->suppress_context) { + // Preserve the original chain, including `raise ... from None`. + PyObject *note = PyUnicode_FromFormat( + "lazy import of '%U' declared in %s at %s:%d", + name, funcname, filename, lineno); + if (note != NULL) { + PyObject *notes; + if (PyObject_GetOptionalAttr(exc, &_Py_ID(__notes__), ¬es) >= 0) { + if (notes == NULL || PySequence_Contains(notes, note) == 0) { + (void)_PyException_AddNote(exc, note); + } + Py_XDECREF(notes); + } + Py_DECREF(note); + } + goto done; + } + PyObject *msg = PyUnicode_FromFormat( + "lazy import of '%U' raised an exception during resolution", name); + if (msg == NULL) { + goto done; + } + PyObject *cause = PyObject_CallOneArg(PyExc_ImportError, msg); + Py_DECREF(msg); + if (cause == NULL) { + goto done; + } + _PyErr_SetRaisedException(tstate, cause); + _PyTraceback_Add(funcname, filename, lineno); + PyException_SetCause(exc, _PyErr_GetRaisedException(tstate)); + +done: + Py_XDECREF(name); + _PyErr_SetRaisedException(tstate, exc); +} + +int +_PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op) +{ + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(PyLazyImport_CheckExact(op)); + int active = ts->lazy_imports == NULL ? 0 : PySet_Contains(ts->lazy_imports, op); + assert(active >= 0); // Exact placeholders use identity hashing and equality. + return active; +} + +static PyObject * +lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, + PyObject **imported_module) +{ + PyObject *obj = NULL; + PyObject *fromlist = NULL; + PyObject *import_func = NULL; + PyObject *resolving = NULL; + assert(lazy_import != NULL); assert(PyLazyImport_CheckExact(lazy_import)); - return lazy_import_name(lazy_import); + + PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import; + + // Walk back to the placeholder IMPORT_NAME left, and the first lookup on it. + PyLazyImportObject *root = lz, *first = NULL; + while (PyLazyImport_CheckExact(root->lz_from)) { + first = root; + root = (PyLazyImportObject *)root->lz_from; + } + + if (_PyLazyImport_IsResolving(tstate, lazy_import)) { + PyObject *name = lazy_import_name(lz); + if (name == NULL) { + return NULL; + } + PyObject *errmsg = PyUnicode_FromFormat( + "cannot import name %R (most likely due to a circular import)", + name); + Py_DECREF(name); + if (errmsg != NULL) { + PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, + root->lz_from, NULL); + Py_DECREF(errmsg); + } + return NULL; + } + if (_Py_EnterRecursiveCallTstate(tstate, " while resolving a lazy import")) { + return NULL; + } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + if (ts->lazy_imports == NULL) { + ts->lazy_imports = PySet_New(NULL); + if (ts->lazy_imports == NULL) { + goto done; + } + } + resolving = ts->lazy_imports; + if (PySet_Add(resolving, lazy_import) < 0) { + goto done; + } + + // `from a import b, c`: import only the name being resolved. + // Keep an empty tuple intact for custom __import__ hooks. + if (first != NULL && root->lz_attr != NULL && + PyTuple_Check(root->lz_attr) && PyTuple_GET_SIZE(root->lz_attr) > 0) { + fromlist = PyTuple_Pack(1, first->lz_attr); + } + else { + fromlist = Py_NewRef(root->lz_attr != NULL ? root->lz_attr : Py_None); + } + if (fromlist == NULL) { + goto done; + } + + PyObject *globals = PyEval_GetGlobals(); + if (globals == NULL) { + globals = Py_None; + } + + if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), + &import_func) < 0) { + goto done; + } + if (import_func == NULL) { + PyErr_SetString(PyExc_ImportError, "__import__ not found"); + goto done; + } + obj = _PyEval_ImportNameWithImport( + tstate, import_func, globals, globals, + root->lz_from, fromlist, _PyLong_GetZero() + ); + obj = lazy_import_resolve_result(tstate, obj); + // The normal importer may publish this module on its parent. Custom + // hooks retain control of their own assignments to the parent. + if (imported_module != NULL && obj != NULL && PyModule_Check(obj) && + _PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { + *imported_module = Py_NewRef(obj); + } + if (obj != NULL && first != NULL) { + // Keep the hook and root result alive until all attribute lookups finish. + PyObject *from = obj; + obj = lazy_import_replay_from(tstate, from, lz); + Py_DECREF(from); + } + +done: + if (obj == NULL) { + lazy_import_add_exception_cause(tstate, lz); + } + assert(obj == NULL || !PyLazyImport_CheckExact(obj)); + if (resolving != NULL) { + // A failed set resize can leave the placeholder inserted. Removing by + // identity also permits greenlets to finish in a different order. + if (PySet_Discard(resolving, lazy_import) < 0) { + Py_CLEAR(obj); + } + if (PySet_GET_SIZE(resolving) == 0) { + // Keep the set, but release the capacity used by deep resolutions. + (void)PySet_Clear(resolving); + } + } + + Py_XDECREF(fromlist); + Py_XDECREF(import_func); + _Py_LeaveRecursiveCallTstate(tstate); + return obj; +} + +PyObject * +_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) +{ + return lazy_import_resolve_impl(tstate, lazy_import, NULL); +} + +// Loading pkg.child can replace a placeholder in pkg.child with the module +// before a from-import retrieves the value that belongs in that binding. +// This is an optimization that can be safely skipped. +static int +lazy_import_replace_child(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace, + PyObject *child, PyObject *value) +{ + PyLazyImportObject *root = (PyLazyImportObject *)placeholder; + if (!PyLazyImport_CheckExact(root->lz_from)) { + return 0; + } + while (PyLazyImport_CheckExact(root->lz_from)) { + root = (PyLazyImportObject *)root->lz_from; + } + Py_ssize_t end = PyUnicode_GET_LENGTH(root->lz_from); + Py_ssize_t dot = PyUnicode_FindChar(root->lz_from, '.', 0, end, -1); + if (dot < 0) { + return dot == -1 ? 0 : -1; + } + if (end - dot - 1 != PyUnicode_GET_LENGTH(name)) { + return 0; + } + Py_ssize_t matches = PyUnicode_Tailmatch(root->lz_from, name, dot + 1, end, 1); + if (matches <= 0) { + return matches ? -1 : 0; + } + PyObject *parent_name = PyUnicode_Substring(root->lz_from, 0, dot); + if (parent_name == NULL) { + return -1; + } + PyObject *modules = Py_XNewRef(_PyImport_GetModules(tstate->interp)); + PyObject *parent = NULL; + int rc = 0; + if (modules != NULL) { + rc = PyMapping_GetOptionalItem(modules, parent_name, &parent); + if (rc > 0 && PyModule_Check(parent) && + _PyModule_GetDict(parent) == namespace) { + rc = _PyDict_ReplaceItemIf(namespace, name, child, value); + } + } + Py_XDECREF(parent); + Py_XDECREF(modules); + Py_DECREF(parent_name); + return rc; +} + +PyObject * +_PyLazyImport_Reify(PyThreadState *tstate, PyObject *placeholder, + PyObject *name, PyObject *namespace) +{ + PyObject *imported_module = NULL; + PyObject *value = lazy_import_resolve_impl( + tstate, placeholder, &imported_module); + if (value == NULL) { + Py_XDECREF(imported_module); + return NULL; + } + int rc; + if (PyDict_CheckExact(namespace)) { + rc = _PyDict_ReplaceItemIf(namespace, name, placeholder, value); + if (rc == 0 && imported_module != NULL) { + rc = lazy_import_replace_child( + tstate, placeholder, name, namespace, imported_module, value); + } + } + else if (Py_TYPE(namespace)->tp_as_mapping == NULL || + Py_TYPE(namespace)->tp_as_mapping->mp_ass_subscript == NULL) { + // Read-only namespaces can resolve a value without caching it. + Py_XDECREF(imported_module); + return value; + } + else { + // Custom namespaces retain their mapping protocol. Atomic replacement + // is only available for exact dictionaries. + PyObject *current; + rc = PyMapping_GetOptionalItem(namespace, name, ¤t); + if (rc > 0) { + if (current == placeholder) { + rc = PyObject_SetItem(namespace, name, value); + } + Py_DECREF(current); + } + } + if (rc < 0) { + Py_CLEAR(value); + } + Py_XDECREF(imported_module); + return value; } static PyObject * @@ -177,7 +576,7 @@ lazy_import_resolve(PyObject *self, PyObject *args) static PyMethodDef lazy_import_methods[] = { { "resolve", lazy_import_resolve, METH_NOARGS, - PyDoc_STR("resolves the lazy import and returns the actual object") + PyDoc_STR("Resolve the lazy import and return the imported object.") }, {NULL, NULL} }; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b8cd6025c20ba5..57df195b7f3201 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -7,7 +7,7 @@ #include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_import.h" // _PyImport_GetNextModuleIndex() #include "pycore_interp.h" // PyInterpreterState.importlib -#include "pycore_lazyimportobject.h" // _PyLazyImportObject_Check() +#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() @@ -1299,32 +1299,21 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } -static PyObject * -try_load_lazy_submodule(PyModuleObject *m, PyObject *name) +static int +module_get_resolved_dict_item(PyObject *dict, PyObject *name, PyObject **result) { - PyObject *mod_name; - int rc = PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name); - if (rc <= 0) { - return NULL; - } - if (!PyUnicode_Check(mod_name)) { - Py_DECREF(mod_name); - return NULL; - } - PyObject *result = NULL; - _PyLazySubmoduleImportResult status = - _PyImport_TryLoadLazySubmodule(mod_name, name, &result); - Py_DECREF(mod_name); - if (status != _Py_LAZY_SUBMODULE_LOADED) { - assert(status == _Py_LAZY_SUBMODULE_ERROR || - status == _Py_LAZY_SUBMODULE_NOT_FOUND); - return NULL; + int rc = PyDict_GetItemRef(dict, name, result); + if (rc <= 0 || !PyLazyImport_CheckExact(*result)) { + return rc; } - if (PyDict_SetItem(m->md_dict, name, result) < 0) { - Py_DECREF(result); - return NULL; + PyThreadState *tstate = PyThreadState_GET(); + if (_PyLazyImport_IsResolving(tstate, *result)) { + Py_CLEAR(*result); + return 0; } - return result; + PyObject *value = _PyLazyImport_Reify(tstate, *result, name, dict); + Py_SETREF(*result, value); + return value == NULL ? -1 : 1; } PyObject* @@ -1333,11 +1322,52 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) // When suppress=1, this function suppresses AttributeError. PyObject *attr, *mod_name, *getattr; attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); + if (attr == NULL) { + if (suppress == 1) { + if (PyErr_Occurred()) { + // pass up non-AttributeError exception + return NULL; + } + } + else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + // pass up non-AttributeError exception + return NULL; + } + PyErr_Clear(); + } + assert(m->md_dict != NULL); + int recheck_dict; + attr = _PyImport_TryLoadLazySubmodule((PyObject *)m, name, &recheck_dict); + if (attr != NULL || PyErr_Occurred()) { + return attr; + } + // A concurrent load may have bound the child and removed its pending + // registration after our initial dictionary lookup. + if (recheck_dict) { + if (Py_TYPE(m) != &PyModule_Type) { + PyObject *descr = _PyType_LookupRef(Py_TYPE(m), name); + if (descr != NULL) { + // Preserve the fallback after a descriptor raised. + recheck_dict = Py_TYPE(descr)->tp_descr_get == NULL; + Py_DECREF(descr); + } + } + if (recheck_dict && + PyDict_GetItemRef(m->md_dict, name, &attr) < 0) { + return NULL; + } + } + } if (attr) { if (PyLazyImport_CheckExact(attr)) { // gh-144957: Module __getattr__ should get a chance to provide // the attribute before resolving a lazy import placeholder. - if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + // Resolving __getattr__ itself must not invoke the hook. + getattr = NULL; + if (!_PyUnicode_EqualToASCIIString(name, "__getattr__") && + module_get_resolved_dict_item( + m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { Py_DECREF(attr); return NULL; } @@ -1354,51 +1384,21 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) } PyErr_Clear(); } - PyObject *new_value = _PyImport_LoadLazyImportTstate( - PyThreadState_GET(), attr); - if (new_value == NULL) { - if (suppress && - PyErr_ExceptionMatches(PyExc_ImportCycleError)) { - // ImportCycleError is raised when a lazy object tries - // to import itself. In this case, the error should not - // propagate to the caller and instead treated as if the - // attribute doesn't exist. - PyErr_Clear(); - } + PyThreadState *tstate = PyThreadState_GET(); + if (suppress && _PyLazyImport_IsResolving(tstate, attr)) { + // Only direct reentry means this attribute is unavailable. Py_DECREF(attr); return NULL; } - - if (PyDict_SetItem(m->md_dict, name, new_value) < 0) { - Py_CLEAR(new_value); - } + PyObject *new_value = _PyLazyImport_Reify( + tstate, attr, name, m->md_dict); Py_DECREF(attr); return new_value; } return attr; } - if (suppress == 1) { - if (PyErr_Occurred()) { - // pass up non-AttributeError exception - return NULL; - } - } - else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - // pass up non-AttributeError exception - return NULL; - } - PyErr_Clear(); - } - assert(m->md_dict != NULL); - attr = try_load_lazy_submodule(m, name); - if (attr != NULL) { - return attr; - } - if (PyErr_Occurred()) { - return NULL; - } - if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + if (module_get_resolved_dict_item( + m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { return NULL; } if (getattr) { @@ -1589,11 +1589,14 @@ module_dir(PyObject *self, PyObject *args) if (dict != NULL) { if (PyDict_Check(dict)) { - PyObject *dirfunc = PyDict_GetItemWithError(dict, &_Py_ID(__dir__)); - if (dirfunc) { + PyObject *dirfunc; + int rc = module_get_resolved_dict_item( + dict, &_Py_ID(__dir__), &dirfunc); + if (rc > 0) { result = _PyObject_CallNoArgs(dirfunc); + Py_DECREF(dirfunc); } - else if (!PyErr_Occurred()) { + else if (rc == 0) { result = PyDict_Keys(dict); } } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3f329274f06b11..31eaeab0d67841 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2192,80 +2192,30 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); - + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + if (v_o == NULL) { + err = -1; + } + } PyStackRef_CLOSE(mod_or_class_dict); ERROR_IF(err < 0); if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - ERROR_NO_POP(); - } - - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - Py_SETREF(v_o, l_v); - ERROR_IF(v_o == NULL); - } - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - ERROR_IF(err < 0); - if (v_o == NULL) { - /* namespace 2: builtins */ - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - ERROR_IF(err < 0); - if (v_o == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - ERROR_IF(true); - } - } - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - Py_SETREF(v_o, l_v); - ERROR_IF(v_o == NULL); - } - } + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + ERROR_IF(PyStackRef_IsNull(v)); + } + else { + v = PyStackRef_FromPyObjectSteal(v_o); } - v = PyStackRef_FromPyObjectSteal(v_o); } inst(LOAD_NAME, (-- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); ERROR_IF(v_o == NULL); - if (PyLazyImport_CheckExact(v_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - // cannot early-decref v_o as it may cause a side-effect on l_v - if (l_v == NULL) { - Py_DECREF(v_o); - ERROR_IF(true); - } - int err = PyDict_SetItem(GLOBALS(), name, l_v); - if (err < 0) { - Py_DECREF(v_o); - Py_DECREF(l_v); - ERROR_IF(true); - } - Py_SETREF(v_o, l_v); - } - v = PyStackRef_FromPyObjectSteal(v_o); } @@ -2431,6 +2381,13 @@ dummy_func( ERROR_NO_POP(); } } + else if (PyLazyImport_CheckExact(value_o)) { + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + if (value_o == NULL) { + ERROR_NO_POP(); + } + } PyStackRef_CLOSE(class_dict_st); value = PyStackRef_FromPyObjectSteal(value_o); } diff --git a/Python/ceval.c b/Python/ceval.c index c1fbcb0611384c..b90de789accd44 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3150,8 +3150,8 @@ _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, goto error; } - PyObject *args[6] = {name, globals, locals, fromlist, level, builtins}; - res = PyObject_Vectorcall(lazy_import_func, args, 6, NULL); + PyObject *args[5] = {name, globals, locals, fromlist, level}; + res = PyObject_Vectorcall(lazy_import_func, args, 5, NULL); error: Py_XDECREF(lazy_import_func); return res; @@ -3325,43 +3325,6 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) return NULL; } -PyObject * -_PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name) -{ - assert(PyLazyImport_CheckExact(v)); - assert(name); - assert(PyUnicode_Check(name)); - PyObject *ret; - PyLazyImportObject *d = (PyLazyImportObject *)v; - PyObject *mod = NULL; - // Only `from a import b` can take b off an already imported a; - // `import a.b as c` has to import a.b first. - if (d->lz_attr != NULL && PyTuple_Check(d->lz_attr) && - PyTuple_GET_SIZE(d->lz_attr) > 0) { - mod = PyImport_GetModule(d->lz_from); - } - if (mod != NULL) { - // Check if the module already has the attribute, if so, resolve it - // eagerly. - if (PyModule_Check(mod)) { - PyObject *mod_dict = PyModule_GetDict(mod); - if (mod_dict != NULL) { - if (PyDict_GetItemRef(mod_dict, name, &ret) < 0) { - Py_DECREF(mod); - return NULL; - } - if (ret != NULL) { - Py_DECREF(mod); - return ret; - } - } - } - Py_DECREF(mod); - } - - return _PyLazyImport_New(frame, d->lz_builtins, v, name); -} - #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" @@ -3627,13 +3590,13 @@ _PyEval_GetANext(PyObject *aiter) void _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name, _PyStackRef *writeto) { + PyObject *namespace = globals; if (PyAnyDict_CheckExact(globals) && PyAnyDict_CheckExact(builtins)) { - _PyDict_LoadGlobalStackRef((PyDictObject *)globals, - (PyDictObject *)builtins, - name, writeto); + namespace = _PyDict_LoadGlobalStackRef((PyDictObject *)globals, + (PyDictObject *)builtins, + name, writeto); if (PyStackRef_IsNull(*writeto) && !PyErr_Occurred()) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ + // A missing key does not set an exception in the dictionary helper. _PyEval_FormatExcCheckArg(PyThreadState_GET(), PyExc_NameError, NAME_ERROR_MSG, name); } @@ -3648,6 +3611,7 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name } if (res == NULL) { /* namespace 2: builtins */ + namespace = builtins; if (PyMapping_GetOptionalItem(builtins, name, &res) < 0) { *writeto = PyStackRef_NULL; return; @@ -3665,20 +3629,10 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name PyObject *res_o = PyStackRef_AsPyObjectBorrow(*writeto); if (res_o != NULL && PyLazyImport_CheckExact(res_o)) { - PyObject *l_v = _PyImport_LoadLazyImportTstate(PyThreadState_GET(), res_o); + PyObject *l_v = _PyLazyImport_Reify( + PyThreadState_GET(), res_o, name, namespace); PyStackRef_CLOSE(writeto[0]); - if (l_v == NULL) { - assert(PyErr_Occurred()); - *writeto = PyStackRef_NULL; - return; - } - int err = PyDict_SetItem(globals, name, l_v); - if (err < 0) { - Py_DECREF(l_v); - *writeto = PyStackRef_NULL; - return; - } - *writeto = PyStackRef_FromPyObjectSteal(l_v); + *writeto = l_v == NULL ? PyStackRef_NULL : PyStackRef_FromPyObjectSteal(l_v); } } @@ -3709,32 +3663,39 @@ _PyEval_GetAwaitable(PyObject *iterable, int oparg) PyObject * _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name) { - PyObject *value; - if (frame->f_locals == NULL) { + PyObject *namespace = frame->f_locals; + if (namespace == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); return NULL; } - if (PyMapping_GetOptionalItem(frame->f_locals, name, &value) < 0) { + if (PyMapping_GetOptionalItem(namespace, name, &value) < 0) { return NULL; } if (value != NULL) { - return value; + goto found; } - if (PyDict_GetItemRef(frame->f_globals, name, &value) < 0) { + namespace = frame->f_globals; + if (PyDict_GetItemRef(namespace, name, &value) < 0) { return NULL; } if (value != NULL) { - return value; + goto found; } - if (PyMapping_GetOptionalItem(frame->f_builtins, name, &value) < 0) { + namespace = frame->f_builtins; + if (PyMapping_GetOptionalItem(namespace, name, &value) < 0) { return NULL; } if (value == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); + return NULL; + } +found: + if (PyLazyImport_CheckExact(value)) { + Py_SETREF(value, _PyLazyImport_Reify(tstate, value, name, namespace)); } return value; } diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 2e4e178b308406..749eaebca19dc5 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -627,7 +627,7 @@ PyDoc_STRVAR(_imp__set_lazy_attributes__doc__, "_set_lazy_attributes($module, modobj, name, /)\n" "--\n" "\n" -"Sets attributes to lazy submodules on the module, as side effects."); +"Remove the resolved module name from sys.lazy_modules."); #define _IMP__SET_LAZY_ATTRIBUTES_METHODDEF \ {"_set_lazy_attributes", _PyCFunction_CAST(_imp__set_lazy_attributes), METH_FASTCALL, _imp__set_lazy_attributes__doc__}, @@ -665,4 +665,4 @@ _imp__set_lazy_attributes(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=0974db098d601372 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ae9cf67e39955555 input=a9049054013a1b77]*/ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 400f07b58e7fa5..c5b2dfcf5f618f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10179,55 +10179,77 @@ break; } - /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - - case _LOAD_NAME_r01: { - CHECK_CURRENT_CACHED_VALUES(0); + case _LOAD_FROM_DICT_OR_GLOBALS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef mod_or_class_dict; _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); + mod_or_class_dict = _stack_item_0; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); + int err; + stack_pointer[0] = mod_or_class_dict; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (PyLazyImport_CheckExact(v_o)) { + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + if (v_o == NULL) { + err = -1; } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(mod_or_class_dict); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (v_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); + } + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } + _tos_cache0 = v; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _LOAD_NAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } v = PyStackRef_FromPyObjectSteal(v_o); _tos_cache0 = v; @@ -10669,6 +10691,17 @@ JUMP_TO_ERROR(); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d9fb82d713154b..dd0ce41e4b06b4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9912,6 +9912,16 @@ JUMP_TO_LABEL(error); } } + else if (PyLazyImport_CheckExact(value_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(value_o, _PyLazyImport_Reify( + tstate, value_o, name, class_dict)); + _PyFrame_StackPointerInvalidate(frame); + if (value_o == NULL) { + JUMP_TO_LABEL(error); + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9937,11 +9947,21 @@ _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *namespace = PyStackRef_AsPyObjectBorrow(mod_or_class_dict); int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); + PyObject *v_o = _PyMapping_GetOptionalItem2(namespace, name, &err); _PyFrame_StackPointerInvalidate(frame); + if (v_o != NULL && PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, _PyLazyImport_Reify(tstate, v_o, name, namespace)); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + err = -1; + } + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9952,81 +9972,17 @@ JUMP_TO_LABEL(error); } if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - } - JUMP_TO_LABEL(error); - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } - } - else { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (v_o == NULL) { - JUMP_TO_LABEL(error); - } - } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, &v); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { + JUMP_TO_LABEL(error); } } - v = PyStackRef_FromPyObjectSteal(v_o); + else { + v = PyStackRef_FromPyObjectSteal(v_o); + } stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10280,38 +10236,6 @@ if (v_o == NULL) { JUMP_TO_LABEL(error); } - if (PyLazyImport_CheckExact(v_o)) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - _PyFrame_StackPointerInvalidate(frame); - if (l_v == NULL) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); - if (err < 0) { - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(v_o); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_DECREF(l_v); - _PyFrame_StackPointerInvalidate(frame); - JUMP_TO_LABEL(error); - } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - Py_SETREF(v_o, l_v); - _PyFrame_StackPointerInvalidate(frame); - } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; diff --git a/Python/import.c b/Python/import.c index 2fa63dd01833a6..576b8d47639fa4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -11,7 +11,6 @@ #include "pycore_interp.h" // struct _import_runtime_state #include "pycore_interpframe.h" #include "pycore_lazyimportobject.h" -#include "pycore_long.h" // _PyLong_GetZero #include "pycore_magic_number.h" // PYC_MAGIC_NUMBER_TOKEN #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_namespace.h" // _PyNamespace_Type @@ -25,7 +24,6 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_time.h" // _PyTime_AsMicroseconds() -#include "pycore_traceback.h" #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -3891,227 +3889,6 @@ resolve_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level return NULL; } -PyObject * -_PyImport_ResolveName(PyThreadState *tstate, PyObject *name, - PyObject *globals, int level) -{ - return resolve_name(tstate, name, globals, level); -} - -// Look up, in order, the attributes recorded from the root placeholder to lz -// on the module the root's import returned. -static PyObject * -lazy_import_replay_from(PyThreadState *tstate, PyObject *mod, - PyLazyImportObject *lz) -{ - if (!PyLazyImport_CheckExact(lz->lz_from)) { - return Py_NewRef(mod); - } - PyObject *from = lazy_import_replay_from( - tstate, mod, (PyLazyImportObject *)lz->lz_from); - if (from == NULL) { - return NULL; - } - PyObject *obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr); - Py_DECREF(from); - return obj; -} - -PyObject * -_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) -{ - PyObject *obj = NULL; - PyObject *fromlist = Py_None; - PyObject *import_func = NULL; - assert(lazy_import != NULL); - assert(PyLazyImport_CheckExact(lazy_import)); - - PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import; - PyInterpreterState *interp = tstate->interp; - - // Walk back to the placeholder IMPORT_NAME left, and the first lookup on it. - PyLazyImportObject *root = lz, *first = NULL; - while (PyLazyImport_CheckExact(root->lz_from)) { - first = root; - root = (PyLazyImportObject *)root->lz_from; - } - - // Acquire the global import lock to serialize reification - _PyImport_AcquireLock(interp); - - // Check if we are already importing this module, if so, then we want to - // return an error that indicates we've hit a cycle which will indicate - // the value isn't yet available. - PyObject *importing = interp->imports.lazy_importing_modules; - if (importing == NULL) { - importing = interp->imports.lazy_importing_modules = PySet_New(NULL); - if (importing == NULL) { - _PyImport_ReleaseLock(interp); - return NULL; - } - } - - assert(PyAnySet_CheckExact(importing)); - int is_loading = _PySet_Contains((PySetObject *)importing, lazy_import); - if (is_loading < 0) { - _PyImport_ReleaseLock(interp); - return NULL; - } - else if (is_loading == 1) { - PyObject *name = _PyLazyImport_GetName(lazy_import); - if (name == NULL) { - _PyImport_ReleaseLock(interp); - return NULL; - } - PyObject *errmsg = PyUnicode_FromFormat( - "cannot import name %R (most likely due to a circular import)", - name); - if (errmsg == NULL) { - Py_DECREF(name); - _PyImport_ReleaseLock(interp); - return NULL; - } - PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg, - root->lz_from, NULL); - Py_DECREF(errmsg); - Py_DECREF(name); - _PyImport_ReleaseLock(interp); - return NULL; - } - else if (PySet_Add(importing, lazy_import) < 0) { - goto error; - } - - if (root->lz_attr != NULL) { - // `from a import b, c`: import only the name being resolved. - // Keep an empty tuple intact for custom __import__ hooks. - fromlist = first && PyTuple_GET_SIZE(root->lz_attr) > 0 - ? PyTuple_Pack(1, first->lz_attr) - : Py_NewRef(root->lz_attr); - if (fromlist == NULL) { - goto error; - } - } - - PyObject *globals = PyEval_GetGlobals(); - - if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__), - &import_func) < 0) { - goto error; - } - if (import_func == NULL) { - PyErr_SetString(PyExc_ImportError, "__import__ not found"); - goto error; - } - obj = _PyEval_ImportNameWithImport( - tstate, import_func, globals, globals, - root->lz_from, fromlist, _PyLong_GetZero() - ); - if (obj == NULL) { - goto error; - } - - PyObject *from = obj; - obj = lazy_import_replay_from(tstate, from, lz); - Py_DECREF(from); - if (obj == NULL) { - goto error; - } - - assert(!PyLazyImport_CheckExact(obj)); - - goto ok; - -error: - Py_CLEAR(obj); - - // If an error occurred and we have frame information, add it to the - // exception. - if (PyErr_Occurred() && lz->lz_code != NULL && lz->lz_instr_offset >= 0) { - // Get the current exception - this already has the full traceback - // from the access point. - PyObject *exc = _PyErr_GetRaisedException(tstate); - - // Get import name - this can fail and set an exception. - PyObject *import_name = _PyLazyImport_GetName(lazy_import); - if (!import_name) { - // Failed to get import name, just restore original exception. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Resolve line number from instruction offset on demand. - int lineno = PyCode_Addr2Line((PyCodeObject *)lz->lz_code, - lz->lz_instr_offset*2); - - // Get strings - these can return NULL on encoding errors. - const char *filename_str = PyUnicode_AsUTF8(lz->lz_code->co_filename); - if (!filename_str) { - // Unicode conversion failed - clear error and restore original - // exception. - PyErr_Clear(); - Py_DECREF(import_name); - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - const char *funcname_str = PyUnicode_AsUTF8(lz->lz_code->co_name); - if (!funcname_str) { - // Unicode conversion failed - clear error and restore original - // exception. - PyErr_Clear(); - Py_DECREF(import_name); - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Create a cause exception showing where the lazy import was declared. - PyObject *msg = PyUnicode_FromFormat( - "lazy import of '%U' raised an exception during resolution", - import_name - ); - Py_DECREF(import_name); // Done with import_name. - - if (!msg) { - // Failed to create message - restore original exception. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - PyObject *cause_exc = PyObject_CallOneArg(PyExc_ImportError, msg); - Py_DECREF(msg); // Done with msg. - - if (!cause_exc) { - // Failed to create exception - restore original. - _PyErr_SetRaisedException(tstate, exc); - goto ok; - } - - // Add traceback entry for the lazy import declaration. - _PyErr_SetRaisedException(tstate, cause_exc); - _PyTraceback_Add(funcname_str, filename_str, lineno); - PyObject *cause_with_tb = _PyErr_GetRaisedException(tstate); - - // Set the cause on the original exception. - PyException_SetCause(exc, cause_with_tb); // Steals ref to cause_with_tb. - - // Restore the original exception with its full traceback. - _PyErr_SetRaisedException(tstate, exc); - } - -ok: - if (PySet_Discard(importing, lazy_import) < 0) { - Py_CLEAR(obj); - } - - // Release the global import lock. - _PyImport_ReleaseLock(interp); - - Py_XDECREF(fromlist); - Py_XDECREF(import_func); - return obj; -} - static PyObject * import_find_and_load_with_name(PyThreadState *tstate, PyObject *abs_name, PyObject *find_and_load, @@ -4180,9 +3957,9 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) tstate, abs_name, &_Py_ID(_find_and_load), NULL); } -static PyObject * -get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, - int level) +PyObject * +_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, PyObject *globals, + int level) { if (level > 0) { return resolve_name(tstate, name, globals, level); @@ -4194,13 +3971,6 @@ get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, return Py_NewRef(name); } -PyObject * -_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, - PyObject *globals, int level) -{ - return get_abs_name(tstate, name, globals, level); -} - PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, @@ -4232,7 +4002,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - abs_name = get_abs_name(tstate, name, globals, level); + abs_name = _PyImport_GetAbsName(tstate, name, globals, level); if (abs_name == NULL) { goto error; } @@ -4358,7 +4128,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } -// ensure we have the set for the parent module name in sys.lazy_modules. +// Ensure a set of pending submodule names exists for the parent. // Returns a new reference. static PyObject * ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) @@ -4387,59 +4157,35 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) static int register_lazy_on_parent(PyThreadState *tstate, PyObject *name) { - int ret = -1; - PyObject *parent = NULL; - PyObject *child = NULL; - - PyInterpreterState *interp = tstate->interp; - PyObject *lazy_pending_submodules = LAZY_PENDING_SUBMODULES(interp); - assert(lazy_pending_submodules != NULL); - - Py_INCREF(name); + PyDictObject *pending = + (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp); + assert(pending != NULL); + Py_ssize_t end = PyUnicode_GET_LENGTH(name); while (true) { - Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, - PyUnicode_GET_LENGTH(name), -1); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); if (dot < 0) { - PyObject *lazy_submodules = ensure_lazy_pending_submodules( - (PyDictObject *)lazy_pending_submodules, name); - if (lazy_submodules == NULL) { - goto done; - } - Py_DECREF(lazy_submodules); - ret = 0; - goto done; + return dot == -1 ? 0 : -1; } - parent = PyUnicode_Substring(name, 0, dot); + PyObject *parent = PyUnicode_Substring(name, 0, dot); if (parent == NULL) { - goto done; + return -1; } - Py_XDECREF(child); - child = PyUnicode_Substring(name, dot + 1, PyUnicode_GET_LENGTH(name)); + PyObject *child = PyUnicode_Substring(name, dot + 1, end); if (child == NULL) { - goto done; - } - - PyObject *lazy_submodules = ensure_lazy_pending_submodules( - (PyDictObject *)lazy_pending_submodules, parent); - if (lazy_submodules == NULL) { - goto done; + Py_DECREF(parent); + return -1; } - if (PySet_Add(lazy_submodules, child) < 0) { - Py_DECREF(lazy_submodules); - goto done; + PyObject *submodules = ensure_lazy_pending_submodules(pending, parent); + Py_DECREF(parent); + int err = submodules == NULL ? -1 : PySet_Add(submodules, child); + Py_DECREF(child); + Py_XDECREF(submodules); + if (err < 0) { + return -1; } - Py_DECREF(lazy_submodules); - - Py_SETREF(name, parent); - parent = NULL; + end = dot; } - -done: - Py_XDECREF(child); - Py_XDECREF(parent); - Py_XDECREF(name); - return ret; } static int @@ -4453,80 +4199,65 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, // Add the module name to sys.lazy_modules set (PEP 810). PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - if (PySet_Add(lazy_modules, fromname) < 0) { - Py_DECREF(fromname); - return -1; + int res = PySet_Add(lazy_modules, fromname); + if (res == 0) { + res = register_lazy_on_parent(tstate, fromname); } - - int res = register_lazy_on_parent(tstate, fromname); Py_DECREF(fromname); return res; } -_PyLazySubmoduleImportResult -_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name, - PyObject **result) +PyObject * +_PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, + int *recheck_dict) { - assert(result != NULL); - *result = NULL; - - PyThreadState *tstate = _PyThreadState_GET(); - PyInterpreterState *interp = tstate->interp; - PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); - if (lazy_pending == NULL) { - return _Py_LAZY_SUBMODULE_NOT_FOUND; + *recheck_dict = 0; + PyObject *mod_dict = _PyModule_GetDict(module); + PyObject *mod_name; + if (PyDict_GetItemRef(mod_dict, &_Py_ID(__name__), &mod_name) <= 0) { + return NULL; } - - PyObject *pending_set; - int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); - if (rc < 0) { - return _Py_LAZY_SUBMODULE_ERROR; + PyObject *mod = NULL, *pending_set = NULL; + if (!PyUnicode_Check(mod_name)) { + goto done; } - if (rc == 0) { - return _Py_LAZY_SUBMODULE_NOT_FOUND; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(tstate->interp); + if (lazy_pending == NULL || + PyDict_GetItemRef(lazy_pending, mod_name, &pending_set) <= 0) { + goto done; } - int contains = PySet_Contains(pending_set, attr_name); - if (contains < 0) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_ERROR; - } - if (contains == 0) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_NOT_FOUND; + *recheck_dict = 1; + if (PySet_Contains(pending_set, attr_name) <= 0) { + goto done; } - PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { - Py_DECREF(pending_set); - return _Py_LAZY_SUBMODULE_ERROR; + goto done; } - - PyObject *mod = import_find_and_load_with_name( - tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); + // Match eager from-imports: accept a partial module in an import cycle. + mod = PyImport_GetModule(full_name); + if ((mod == NULL && !PyErr_Occurred()) || mod == Py_None) { + Py_XDECREF(mod); + mod = import_find_and_load_with_name( + tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); + } + Py_DECREF(full_name); if (mod == NULL) { - Py_DECREF(pending_set); - Py_DECREF(full_name); remove_importlib_frames(tstate); - return _Py_LAZY_SUBMODULE_ERROR; } - if (mod == Py_None) { - Py_DECREF(mod); - Py_DECREF(pending_set); - Py_DECREF(full_name); - return _Py_LAZY_SUBMODULE_NOT_FOUND; + /* Keep missing children pending: a finder may provide them later. */ + else if (mod == Py_None || + PyDict_SetItem(mod_dict, attr_name, mod) < 0 || + PySet_Discard(pending_set, attr_name) < 0) { + Py_CLEAR(mod); } - if (PySet_Discard(pending_set, attr_name) < 0) { - Py_DECREF(mod); - Py_DECREF(pending_set); - Py_DECREF(full_name); - return _Py_LAZY_SUBMODULE_ERROR; - } - Py_DECREF(pending_set); - Py_DECREF(full_name); - *result = mod; - return _Py_LAZY_SUBMODULE_LOADED; +done: + Py_XDECREF(pending_set); + Py_DECREF(mod_name); + return mod; } PyObject * @@ -4546,12 +4277,11 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, return NULL; } - PyObject *abs_name = get_abs_name(tstate, name, globals, level); + PyObject *abs_name = _PyImport_GetAbsName(tstate, name, globals, level); if (abs_name == NULL) { return NULL; } - PyInterpreterState *interp = tstate->interp; _PyInterpreterFrame *frame = _PyEval_GetFrame(); if (frame == NULL || frame->f_globals != frame->f_locals) { Py_DECREF(abs_name); @@ -4560,71 +4290,69 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, return NULL; } + PyObject *res = NULL; + if (fromlist != NULL && PyUnicode_Check(fromlist)) { + fromlist = PyTuple_Pack(1, fromlist); + if (fromlist == NULL) { + goto done; + } + } + else { + fromlist = Py_NewRef(fromlist != NULL ? fromlist : Py_None); + } + // Check if the filter disables the lazy import. - // We must hold a reference to the filter while calling it to prevent - // use-after-free if another thread replaces it via - // PyImport_SetLazyImportsFilter. - LAZY_IMPORTS_LOCK(interp); - PyObject *filter = Py_XNewRef(LAZY_IMPORTS_FILTER(interp)); - LAZY_IMPORTS_UNLOCK(interp); + PyObject *filter = PyImport_GetLazyImportsFilter(); if (filter != NULL) { PyObject *modname; if (PyDict_GetItemRef(globals, &_Py_ID(__name__), &modname) < 0) { Py_DECREF(filter); - Py_DECREF(abs_name); - return NULL; + goto done; } if (modname == NULL) { assert(!PyErr_Occurred()); modname = Py_NewRef(Py_None); } - if (fromlist == NULL) { - assert(!PyErr_Occurred()); - fromlist = Py_None; - } PyObject *args[] = {modname, abs_name, fromlist}; - PyObject *res = PyObject_Vectorcall(filter, args, 3, NULL); + PyObject *decision = PyObject_Vectorcall(filter, args, 3, NULL); Py_DECREF(modname); Py_DECREF(filter); - if (res == NULL) { - Py_DECREF(abs_name); - return NULL; + if (decision == NULL) { + goto done; } - int is_true = PyObject_IsTrue(res); - Py_DECREF(res); + int is_true = PyObject_IsTrue(decision); + Py_DECREF(decision); if (is_true < 0) { - Py_DECREF(abs_name); - return NULL; + goto done; } if (!is_true) { - Py_DECREF(abs_name); - return PyImport_ImportModuleLevelObject( + res = PyImport_ImportModuleLevelObject( name, globals, locals, fromlist, level ); + goto done; } } - // here, 'filter' is either NULL or is equivalent to a borrowed reference - if (fromlist && PyUnicode_Check(fromlist)) { - fromlist = PyTuple_Pack(1, fromlist); - if (fromlist == NULL) { - Py_DECREF(abs_name); - return NULL; + // Validate the entire fromlist before registering any pending imports. + if (PyTuple_Check(fromlist)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { + PyObject *item = PyTuple_GET_ITEM(fromlist, i); + if (!PyUnicode_Check(item)) { + _PyErr_Format(tstate, PyExc_TypeError, + "Item in ``from list'' must be str, not %.200s", + Py_TYPE(item)->tp_name); + goto done; + } } } - else { - Py_XINCREF(fromlist); - } - PyObject *res = _PyLazyImport_New(frame, builtins, abs_name, fromlist); + res = _PyLazyImport_New(frame, builtins, abs_name, fromlist); if (res == NULL) { - Py_XDECREF(fromlist); - Py_DECREF(abs_name); - return NULL; + goto done; } // Add the module name to sys.lazy_modules set (PEP 810). @@ -4633,7 +4361,7 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, goto error; } - if (fromlist && PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { + if (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { if (register_from_lazy_on_parent(tstate, abs_name, PyTuple_GET_ITEM(fromlist, i)) < 0) @@ -4646,14 +4374,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, goto error; } - Py_XDECREF(fromlist); - Py_DECREF(abs_name); - return res; + goto done; error: + Py_CLEAR(res); +done: Py_XDECREF(fromlist); Py_DECREF(abs_name); - Py_DECREF(res); - return NULL; + return res; } PyObject * @@ -4865,7 +4592,6 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(LAZY_IMPORT_FUNC(interp)); Py_CLEAR(interp->imports.lazy_pending_submodules); Py_CLEAR(interp->imports.lazy_modules); - Py_CLEAR(interp->imports.lazy_importing_modules); Py_CLEAR(interp->imports.lazy_imports_filter); } @@ -5660,13 +5386,13 @@ _imp._set_lazy_attributes modobj: object name: unicode / -Sets attributes to lazy submodules on the module, as side effects. +Remove the resolved module name from sys.lazy_modules. [clinic start generated code]*/ static PyObject * _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, PyObject *name) -/*[clinic end generated code: output=3369bb3242b1f043 input=38ea6f30956dd7d6]*/ +/*[clinic end generated code: output=3369bb3242b1f043 input=900339e013ab2b82]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index bd3ed9ce99a555..9cd20c495db441 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2176,7 +2176,12 @@ break; } - /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ + case _LOAD_FROM_DICT_OR_GLOBALS: { + JitOptRef v; + v = sym_new_not_null(ctx); + stack_pointer[-1] = v; + break; + } case _LOAD_NAME: { JitOptRef v; diff --git a/Python/pystate.c b/Python/pystate.c index 76ad958478a3ca..97323ea2ba2bce 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1630,6 +1630,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; + _tstate->lazy_imports = NULL; #ifdef _Py_TIER2 _tstate->jit_tracer_state = NULL; @@ -1873,6 +1874,9 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->context); + // Finalizers above may resolve imports and create this set. + Py_CLEAR(((_PyThreadStateImpl *)tstate)->lazy_imports); + #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. struct _Py_freelists *freelists = _Py_freelists_GET(); From ab01d184746b3ea383f69317c16fc726d885c968 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev <144015565+v0ropaev@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:36:43 +0300 Subject: [PATCH 033/240] gh-142083: Document the 'w' type code in the array.array docstring (#158040) Add a test asserting that the table lists exactly the supported type codes and that the minimum size it gives for each of them is one the implementation really meets. --- Lib/test/test_array.py | 20 +++++++++++++++++++ ...-09-24-01-20-36.gh-issue-142083.WOlkvr.rst | 3 +++ Modules/arraymodule.c | 1 + 3 files changed, 24 insertions(+) create mode 100644 Misc/NEWS.d/next/Documentation/2026-09-24-01-20-36.gh-issue-142083.WOlkvr.rst diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index ba9c25c835bc39..580966589fdd6b 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -12,6 +12,7 @@ import weakref import pickle import operator +import re import struct import sys @@ -110,6 +111,25 @@ def test_typecodes(self): self.assertIsInstance(typecode, str) self.assertGreaterEqual(len(typecode), 1) + @support.cpython_only + @support.requires_docstrings + def test_typecodes_documented(self): + # The type code table in the array.array docstring must list every + # supported type code, and nothing else, and the minimum size it + # gives for each of them must be one the implementation meets. + row = re.compile(r"^ {4}'(\w+)' +\S.*? +(\d+)(?: \(see note\))?$") + documented = {} + for line in array.array.__doc__.splitlines(): + match = row.match(line) + if match is not None: + documented[match.group(1)] = int(match.group(2)) + + self.assertEqual(sorted(documented), sorted(array.typecodes)) + for typecode, minimum_size in documented.items(): + with self.subTest(typecode=typecode): + self.assertGreaterEqual(array.array(typecode).itemsize, + minimum_size) + # Machine format codes. # diff --git a/Misc/NEWS.d/next/Documentation/2026-09-24-01-20-36.gh-issue-142083.WOlkvr.rst b/Misc/NEWS.d/next/Documentation/2026-09-24-01-20-36.gh-issue-142083.WOlkvr.rst new file mode 100644 index 00000000000000..d8f20396b4556a --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-09-24-01-20-36.gh-issue-142083.WOlkvr.rst @@ -0,0 +1,3 @@ +Document the ``'w'`` type code in the :class:`array.array` docstring, so +that it lists the same type codes as :data:`array.typecodes` and the +:mod:`array` module documentation. diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index ef492d143d5644..64dacd1b8f1f61 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3109,6 +3109,7 @@ The following type codes are defined:\n\ Type code C Type Minimum size in bytes\n\ 'b' signed integer 1\n\ 'B' unsigned integer 1\n\ + 'w' Py_UCS4 4\n\ 'h' signed integer 2\n\ 'H' unsigned integer 2\n\ 'i' signed integer 2\n\ From a4f28a52b4b54c34100ee0891b9a15640ed3a7b2 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 1 Oct 2026 22:39:34 +0100 Subject: [PATCH 034/240] gh-158316: Avoid second buffer fetch on bytearray_decode() (#158485) Co-authored-by: Victor Stinner --- Lib/test/test_bytes.py | 25 +++++++++++++++++++ ...-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst | 2 ++ Objects/bytearrayobject.c | 14 ++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index b55863256cc37c..ea0a54dd5e1d35 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -18,6 +18,7 @@ import textwrap import threading import unittest +from _codecs import _unregister_error as _codecs_unregister_error import test.support from test import support @@ -2195,6 +2196,30 @@ def delslice(): self.assertRaises(BufferError, delslice) self.assertEqual(b, orig) + def test_decode_resize_forbidden(self): + # The storage is pinned while it is decoded, so an error handler + # cannot resize the bytearray. + b = bytearray(b'ab\xffcd') + errors = 'test.bytearray_decode_resize' + def handler(exc): + self.assertRaises(BufferError, b.clear) + self.assertRaises(BufferError, b.append, 0) + return ('?', exc.end) + self.addCleanup(_codecs_unregister_error, errors) + codecs.register_error(errors, handler) + for encoding in 'utf-8', 'utf-8-sig': + with self.subTest(encoding=encoding): + self.assertEqual(b.decode(encoding, errors), 'ab?cd') + self.assertEqual(b, b'ab\xffcd') + + def test_decode_subclass_buffer(self): + # decode() decodes the buffer that the object exports. + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b'other') + self.assertEqual(B(b'mine').decode(), 'other') + self.assertEqual(B(b'mine').decode('latin-1'), 'other') + @test.support.cpython_only def test_obsolete_write_lock(self): _testcapi = import_helper.import_module('_testcapi') diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst new file mode 100644 index 00000000000000..c95bab19af393a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst @@ -0,0 +1,2 @@ +On Free Threading, speed up :meth:`bytearray.decode` by decoding the underlying +storage directly instead of going through the buffer protocol. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index caf8bad88a280d..496d04a1704d46 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2599,7 +2599,19 @@ bytearray_decode_impl(PyByteArrayObject *self, const char *encoding, { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); - return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); + if (Py_TYPE(self)->tp_as_buffer->bf_getbuffer != bytearray_getbuffer) { + /* A subclass may export a different buffer. */ + return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); + } + + /* Decode the storage directly instead of exporting a buffer, which + would re-acquire the critical section we already hold. Increase + exports to prevent the storage from changing during the decode. */ + self->ob_exports++; + PyObject *res = PyUnicode_Decode(PyByteArray_AS_STRING(self), + Py_SIZE(self), encoding, errors); + self->ob_exports--; + return res; } PyDoc_STRVAR(alloc_doc, From ee782e6143b3fda2e7feb1216ff23860ad39b234 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2026 05:06:58 +0200 Subject: [PATCH 035/240] gh-158585: Optimize PyBytesWriter_FinishWithSize() (#158586) Add specialized bytes_resize_inplace() and inline PyBytesWriter_Discard() in PyBytesWriter_FinishWithSize(). No longer copy the buffer if _PyObject_IsUniquelyReferenced() is false. Add also bytes_get_char() function. --- Objects/bytesobject.c | 145 +++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 60 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 7e677108983f0f..cb2c2f7bfae634 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -42,7 +42,8 @@ static Py_ssize_t _PyBytesWriter_ResizeToAllocated(PyBytesWriter *writer); // Return a reference to the immortal empty bytes string singleton. -static inline PyObject* bytes_get_empty(void) +static inline PyObject* +bytes_get_empty(void) { PyObject *empty = &EMPTY->ob_base.ob_base; assert(_Py_IsImmortal(empty)); @@ -50,6 +51,16 @@ static inline PyObject* bytes_get_empty(void) } +// The function cannot fail +static inline PyObject* +bytes_get_char(uint8_t ch) +{ + PyObject *obj = (PyObject*)CHARACTER(ch); + assert(_Py_IsImmortal(obj)); + return obj; +} + + static inline void set_ob_shash(PyBytesObject *a, Py_hash_t hash) { @@ -130,9 +141,7 @@ PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) return NULL; } if (size == 1 && str != NULL) { - op = CHARACTER(*str & 255); - assert(_Py_IsImmortal(op)); - return (PyObject *)op; + return bytes_get_char((uint8_t)*str); } if (size == 0) { return bytes_get_empty(); @@ -166,9 +175,7 @@ PyBytes_FromString(const char *str) return bytes_get_empty(); } else if (size == 1) { - op = CHARACTER(*str & 255); - assert(_Py_IsImmortal(op)); - return (PyObject *)op; + return bytes_get_char((uint8_t)*str); } /* Inline PyObject_NewVar */ @@ -3347,6 +3354,53 @@ _PyBytes_IsMutable(PyObject *self) #endif +static inline int +bytes_resize_inplace(PyObject **pv, Py_ssize_t newsize) +{ + PyObject *v = *pv; + // Do not test _PyObject_IsUniquelyReferenced(). The function is used by + // PyBytesWriter_FinishWithSize() and its caller can have its own lock. + assert(Py_REFCNT(v) == 1); + assert(PyBytes_GET_SIZE(v) >= 1); + assert(newsize >= 1); + + // Only mutable bytes can be resized in-place + assert(_PyBytes_IsMutable(v)); + + if ((size_t)newsize > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { + PyErr_SetString(PyExc_OverflowError, + "byte string is too large"); + return -1; + } + +#ifdef Py_TRACE_REFS + _Py_ForgetReference(v); +#endif + _PyReftracerTrack(v, PyRefTracer_DESTROY); + + PyObject *result = PyObject_Realloc(v, PyBytesObject_SIZE + newsize); + if (result == NULL) { +#ifdef Py_TRACE_REFS + _Py_AddToAllObjects(v); +#endif + _PyReftracerTrack(v, PyRefTracer_CREATE); + + PyErr_NoMemory(); + return -1; + } + + *pv = result; + v = result; + _Py_NewReferenceNoTotal(v); + PyBytesObject *sv = (PyBytesObject *)v; + Py_SET_SIZE(sv, newsize); + sv->ob_sval[newsize] = '\0'; + set_ob_shash(sv, -1); /* invalidate cached hash value */ + assert(_PyBytes_IsMutable(*pv)); + return 0; +} + + /* The following function breaks the notion that bytes are immutable: it changes the size of a bytes object. You can think of it as creating a new bytes object and destroying the old one, only @@ -3410,40 +3464,7 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) return 0; } - // Only mutable bytes can be resized in-place - assert(_PyBytes_IsMutable(v)); - - if ((size_t)newsize > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { - PyErr_SetString(PyExc_OverflowError, - "byte string is too large"); - return -1; - } - -#ifdef Py_TRACE_REFS - _Py_ForgetReference(v); -#endif - _PyReftracerTrack(v, PyRefTracer_DESTROY); - - result = (PyObject *)PyObject_Realloc(v, PyBytesObject_SIZE + newsize); - if (result == NULL) { -#ifdef Py_TRACE_REFS - _Py_AddToAllObjects(v); -#endif - _PyReftracerTrack(v, PyRefTracer_CREATE); - - PyErr_NoMemory(); - return -1; - } - - *pv = result; - v = result; - _Py_NewReferenceNoTotal(v); - PyBytesObject *sv = (PyBytesObject *)v; - Py_SET_SIZE(sv, newsize); - sv->ob_sval[newsize] = '\0'; - set_ob_shash(sv, -1); /* invalidate cached hash value */ - assert(_PyBytes_IsMutable(*pv)); - return 0; + return bytes_resize_inplace(pv, newsize); } @@ -3903,9 +3924,6 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) { assert(byteswriter_check_consistency(writer)); - // Check for negative size here to raise ValueError in all cases, rather - // than having a different exception depending on the code path. For - // example, _PyBytes_Resize() raises SystemError on negative size. if (size < 0) { PyErr_Format(PyExc_ValueError, "size must be positive"); goto error; @@ -3919,6 +3937,13 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) PyObject *result; if (size == 0 && !writer->use_bytearray) { result = bytes_get_empty(); + if (writer->obj != NULL) { +#ifdef Py_DEBUG + byteswriter_reset_trailing_byte(writer); +#endif + Py_DECREF(writer->obj); + writer->obj = NULL; + } } else if (writer->obj != NULL) { // Truncate the bytes/bytearray object if needed @@ -3935,14 +3960,11 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) } else { if (size == 1) { - // Get the single byte singleton unsigned char ch = PyBytes_AS_STRING(writer->obj)[0]; - PyObject *op = (PyObject*)CHARACTER(ch); - assert(_Py_IsImmortal(op)); - Py_SETREF(writer->obj, op); + Py_SETREF(writer->obj, bytes_get_char(ch)); } else if (size != PyBytes_GET_SIZE(writer->obj)) { - if (_PyBytes_Resize(&writer->obj, size)) { + if (bytes_resize_inplace(&writer->obj, size)) { goto error; } } @@ -3953,23 +3975,26 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) } else { // Create an object from the small buffer + const char *buffer = (const char *)writer->small_buffer; if (writer->use_bytearray) { - result = PyByteArray_FromStringAndSize(writer->small_buffer, size); + result = PyByteArray_FromStringAndSize(buffer, size); } else { - // The function returns single byte singleton if size equals 1 - result = PyBytes_FromStringAndSize(writer->small_buffer, size); + if (size == 1) { + result = bytes_get_char((uint8_t)buffer[0]); + } + else { + result = _PyBytes_FromSize(size, 0); + if (result == NULL) { + goto error; + } + memcpy(PyBytes_AS_STRING(result), buffer, size); + } } } -#ifdef Py_DEBUG - // Reset the writer, so byteswriter_check_consistency() doesn't fail - // in PyBytesWriter_Discard(). - writer->size = 0; - byteswriter_write_canary_byte(writer); -#endif - - PyBytesWriter_Discard(writer); + assert(writer->obj == NULL); + _Py_FREELIST_FREE(bytes_writers, writer, PyMem_Free); return result; error: From 5c7445be431698c6764af1246b2f5d7ca5c5c930 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 2 Oct 2026 14:05:24 +0900 Subject: [PATCH 036/240] gh-158165: Fix _Py_get_machine_stack_pointer() compilation with MinGW (#158167) --- Include/internal/pycore_pystate.h | 8 +++++--- .../2026-09-25-22-34-50.gh-issue-158165.GVPxKy.rst | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-25-22-34-50.gh-issue-158165.GVPxKy.rst diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 8b8f0a98bedf76..16f876140020bb 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -335,14 +335,16 @@ static uintptr_t return_pointer_as_int(char* p) { static inline uintptr_t _Py_get_machine_stack_pointer(void) { uintptr_t result; -#if defined(_M_ARM64) +#if _Py__has_builtin(__builtin_stack_address) + result = (uintptr_t)__builtin_stack_address(); +#elif defined(_MSC_VER) && defined(_M_ARM64) result = __getReg(31); -#elif defined(_M_X64) || defined(_M_IX86) +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) result = (uintptr_t)_AddressOfReturnAddress(); #elif defined(__aarch64__) __asm__ ("mov %0, sp" : "=r" (result)); #elif defined(__x86_64__) - __asm__("{movq %%rsp, %0" : "=r" (result)); + __asm__ ("{movq %%rsp, %0|mov %0, rsp}" : "=r" (result)); #else char here; result = (uintptr_t)&here; diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-25-22-34-50.gh-issue-158165.GVPxKy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-25-22-34-50.gh-issue-158165.GVPxKy.rst new file mode 100644 index 00000000000000..055bfba4688809 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-25-22-34-50.gh-issue-158165.GVPxKy.rst @@ -0,0 +1,2 @@ +Fix compilation of ``_Py_get_machine_stack_pointer()`` with MinGW on Windows. +Patch by Donghee Na. From 2095eab5f5005da2e773bd909c83e2a900b00ae2 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 2 Oct 2026 18:26:49 +0530 Subject: [PATCH 037/240] gh-156321: Don't eagerly log exceptions from a cancelled asyncio.shield() (#158590) When the outer future returned by shield() was cancelled, a callback was added to the inner future that reported its exception through the loop exception handler as soon as it completed, even if the exception was retrieved afterwards by awaiting the inner future or calling `exception()`. Remove the callback; an exception nobody retrieves is still reported by the inner future itself when it is garbage collected, like any other future. --- Lib/asyncio/tasks.py | 22 ------------ Lib/test/test_asyncio/test_tasks.py | 34 ++++++++++++++++--- ...-10-01-16-45-00.gh-issue-156321.Kq3xVt.rst | 5 +++ 3 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-01-16-45-00.gh-issue-156321.Kq3xVt.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index cf4787db173059..979f8d2f83eb1e 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -938,25 +938,6 @@ def _done_callback(fut, cur_task=cur_task): return outer -def _log_on_exception(fut): - if fut.cancelled(): - return - - exc = fut.exception() - if exc is None: - return - - context = { - 'message': - f'{exc.__class__.__name__} exception in shielded future', - 'exception': exc, - 'future': fut, - } - if fut._source_traceback: - context['source_traceback'] = fut._source_traceback - fut._loop.call_exception_handler(context) - - def shield(arg): """Wait for a future, shielding it from cancellation. @@ -1021,9 +1002,6 @@ def _inner_done_callback(inner): def _outer_done_callback(outer): if not inner.done(): inner.remove_done_callback(_inner_done_callback) - # Keep only one callback to log on cancel - inner.remove_done_callback(_log_on_exception) - inner.add_done_callback(_log_on_exception) if cur_task is not None: inner.remove_done_callback(_clear_awaited_by_callback) futures.future_discard_from_awaited_by(inner, cur_task) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 570810a231b48d..c380dcfec21395 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2153,7 +2153,7 @@ def test_shield_cancel_outer(self): self.assertTrue(outer.cancelled()) self.assertEqual(0, 0 if outer._callbacks is None else len(outer._callbacks)) self.assertFalse(inner._asyncio_awaited_by) - self.assertTrue({f for f, _ctx in inner._callbacks or []} <= {asyncio.tasks._log_on_exception}) + self.assertFalse(inner._callbacks) def test_shield_cancel_outer_result(self): mock_handler = mock.Mock() @@ -2168,6 +2168,8 @@ def test_shield_cancel_outer_result(self): mock_handler.assert_not_called() def test_shield_cancel_outer_exception(self): + # gh-156321: an exception in the inner future must not be reported + # eagerly, as it may still be retrieved later. mock_handler = mock.Mock() self.loop.set_exception_handler(mock_handler) inner = self.new_future(self.loop) @@ -2177,7 +2179,29 @@ def test_shield_cancel_outer_exception(self): test_utils.run_briefly(self.loop) inner.set_exception(Exception('foo')) test_utils.run_briefly(self.loop) + mock_handler.assert_not_called() + self.assertIsInstance(inner.exception(), Exception) + + def test_shield_cancel_outer_exception_never_retrieved(self): + # gh-156321: an exception nobody retrieves is reported by the inner + # future itself when it is garbage collected, like any other future. + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_not_called() + inner = None + outer = None + support.gc_collect() mock_handler.assert_called_once() + context = mock_handler.call_args[0][1] + self.assertEndsWith(context['message'], 'exception was never retrieved') + self.assertIsInstance(context['exception'], Exception) def test_shield_cancel_outer_in_task(self): inner = self.new_future(self.loop) @@ -2192,9 +2216,9 @@ async def coro(): task = self.new_task(self.loop, coro()) self.loop.run_until_complete(task) self.assertFalse(inner._asyncio_awaited_by) - self.assertTrue({f for f, _ctx in inner._callbacks or []} <= {asyncio.tasks._log_on_exception}) + self.assertFalse(inner._callbacks) - def test_shield_duplicate_log_once(self): + def test_shield_cancel_outer_twice_exception(self): mock_handler = mock.Mock() self.loop.set_exception_handler(mock_handler) inner = self.new_future(self.loop) @@ -2206,9 +2230,11 @@ def test_shield_duplicate_log_once(self): test_utils.run_briefly(self.loop) outer.cancel() test_utils.run_briefly(self.loop) + self.assertFalse(inner._callbacks) inner.set_exception(Exception('foo')) test_utils.run_briefly(self.loop) - mock_handler.assert_called_once() + mock_handler.assert_not_called() + self.assertIsInstance(inner.exception(), Exception) def test_shield_shortcut(self): fut = self.new_future(self.loop) diff --git a/Misc/NEWS.d/next/Library/2026-10-01-16-45-00.gh-issue-156321.Kq3xVt.rst b/Misc/NEWS.d/next/Library/2026-10-01-16-45-00.gh-issue-156321.Kq3xVt.rst new file mode 100644 index 00000000000000..2101e9a899e507 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-01-16-45-00.gh-issue-156321.Kq3xVt.rst @@ -0,0 +1,5 @@ +Fix :func:`asyncio.shield` reporting an exception from the inner future +through the loop exception handler as soon as it completes after the shield +was cancelled, even when the exception is retrieved afterwards. The +exception is now reported only if it is never retrieved, as with any other +future. From f7586aa88dbaf3f4bd515e972b82f530820795a9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 2 Oct 2026 15:11:09 +0200 Subject: [PATCH 038/240] gh-157675: Add *limit* argument to encodings.idna.nameprep (GH-157682) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a *limit* argument to nameprep to allow ToASCII and ToUnicode to reject extremely large input early. This replaces the check in #99092, while allowing any number of harmless "characters mapped to nothing" (RFC 3454 §3.1). The default stays unlimited, to not change behaviour for users that call nameprep manually (and don't necessarily follow up with punycode). Co-authored-by: Stan Ulbrych --- Doc/library/codecs.rst | 36 +++++++++------ Doc/whatsnew/3.16.rst | 4 ++ Lib/encodings/idna.py | 46 +++++++++++++------ Lib/test/test_codecs.py | 32 ++++++++++++- ...-09-17-15-14-08.gh-issue-157675.C4-ze9.rst | 5 ++ 5 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-17-15-14-08.gh-issue-157675.C4-ze9.rst diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 311437a67f6e81..ed20a435faf434 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1395,15 +1395,6 @@ encodings. | | | :mod:`encodings.idna`. | | | | Only ``errors='strict'`` | | | | is supported. | -| | | | -| | | .. warning:: | -| | | | -| | | This codec builds on | -| | | ``punycode``, whose | -| | | algorithms scale | -| | | poorly, so limit the | -| | | length of untrusted | -| | | input. | +--------------------+---------+---------------------------+ | mbcs | ansi, | Windows only: Encode the | | | dbcs | operand according to the | @@ -1655,11 +1646,6 @@ Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. -.. warning:: - - This module builds on ``punycode``, whose algorithms scale poorly, so limit - the length of untrusted input. - If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the third-party :pypi:`idna` module. @@ -1697,11 +1683,31 @@ international domain names, and to unify similar characters. The nameprep functions can be used directly if desired. -.. function:: nameprep(label) +.. function:: nameprep(label, *, limit=None) Return the nameprepped version of *label*. The implementation currently assumes query strings, so ``AllowUnassigned`` is true. + Raise :exc:`UnicodeEncodeError` if the nameprep algorithm emits an error. + + If the *limit* argument is given, it should be set to the maximum size + of an encoded A-label (that is, 63 for IDNA). + :func:`!nameprep` will raise :exc:`UnicodeEncodeError` if the label is + **much** larger than *limit*. + Note that this is only a rough check meant to skip expensive processing + of extremely large input; the caller should check any exact + limits separately. + + .. warning:: + + For backwards compatibility, label size is unlimited by default. + This may cause issues when processing the result with the + ``punycode`` encoding, whose algorithms scale poorly. + + .. versionchanged:: next + + Added the *limit* parameter. + .. function:: ToASCII(label) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 29bd7dcd749ebc..16e03e9ef06532 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -397,6 +397,10 @@ encodings used for international IMAP4 mailbox names (:rfc:`3501`). (Contributed by Serhiy Storchaka in :gh:`66788`.) +* :func:`encodings.idna.nameprep` now takes a *limit* argument that allows + rejecting extremely large input early. + (Contributed by Petr Viktorin in :gh:`157675`.) + gzip ---- diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index c896ffdeadfef7..98d7d891594c64 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -1,5 +1,6 @@ # This module implements the RFCs 3490 (IDNA) and 3491 (Nameprep) +import sys import stringprep, re, codecs from unicodedata import ucd_3_2_0 as unicodedata @@ -11,7 +12,24 @@ sace_prefix = "xn--" # This assumes query strings, so AllowUnassigned is true -def nameprep(label): # type: (str) -> str +def nameprep(label, *, limit=None): # type: (str) -> str + if limit is None: + limit = sys.maxsize + else: + # Protection from gh-98433 and gh-157675 (passing unbounded input to + # the quadratic-complexity punycode algorithm). + # While the "map" step can remove characters, later steps (in ToASCII + # and FromASCII) will not shorten the result *drastically*. + # (NFKC normalization can compress e.g. '\u03c9\u0314\u0300\u0345' + # to '\u1fa3' -- a 4-fold reduction. Non-ASCII labels then get + # longer via prefixing & punycode). + # We bail if the number of non-ignored input characters exceeds 8 times + # the limit, which gives ample room for future Unicode versions to + # include long normalizations, while still preventing us from wasting + # time decoding a big thing that'll just hit the actual <= 63 limit in + # ToASCII. + limit *= 8 + # Map newlabel = [] for c in label: @@ -19,6 +37,11 @@ def nameprep(label): # type: (str) -> str # Map to nothing continue newlabel.append(stringprep.map_table_b2(c)) + + if len(newlabel) > limit: + raise UnicodeEncodeError("idna", label, 0, len(label), + "label way too long") + label = "".join(newlabel) # Normalize @@ -80,7 +103,7 @@ def ToASCII(label): # type: (str) -> bytes raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 2: nameprep - label = nameprep(label) + label = nameprep(label, limit=63) # Step 3: UseSTD3ASCIIRules is false # Step 4: try ASCII @@ -115,18 +138,6 @@ def ToASCII(label): # type: (str) -> bytes raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") def ToUnicode(label): - if len(label) > 1024: - # Protection from https://github.com/python/cpython/issues/98433. - # https://datatracker.ietf.org/doc/html/rfc5894#section-6 - # doesn't specify a label size limit prior to NAMEPREP. But having - # one makes practical sense. - # This leaves ample room for nameprep() to remove Nothing characters - # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still - # preventing us from wasting time decoding a big thing that'll just - # hit the actual <= 63 length limit in Step 6. - if isinstance(label, str): - label = label.encode("utf-8", errors="backslashreplace") - raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long") # Step 1: Check for ASCII if isinstance(label, bytes): pure_ascii = True @@ -139,7 +150,7 @@ def ToUnicode(label): if not pure_ascii: assert isinstance(label, str) # Step 2: Perform nameprep - label = nameprep(label) + label = nameprep(label, limit=63) # It doesn't say this, but apparently, it should be ASCII now try: label = label.encode("ascii") @@ -151,6 +162,11 @@ def ToUnicode(label): if not label.lower().startswith(ace_prefix): return str(label, "ascii") + # Below in steps 6-7, `label` must match the result of `ToASCII`, so it's + # limited to 63 chars. Check before the expensive punycode decode. + if len(label) >= 64: + raise UnicodeDecodeError("idna", label, 0, len(label), "label too long") + # Step 4: Remove ACE prefix label1 = label[len(ace_prefix):] diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index c4a9ffa5ff1c77..1916ee5507e972 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1646,6 +1646,14 @@ def test_nameprep(self): except Exception as e: raise support.TestFailed("Test 3.%d: %s" % (pos+1, str(e))) + def test_long_input(self): + from encodings.idna import nameprep + self.assertEqual(nameprep("x" + "\N{ZWJ}" * 10_000 + "y"), 'xy') + self.assertEqual(nameprep("x" + "\N{ZWJ}" * 10_000 + "y", limit=2), + 'xy') + with self.assertRaises(UnicodeEncodeError): + nameprep("x" + "\N{SNAKE}" * 10_000 + "y", limit=10) + class IDNACodecTest(unittest.TestCase): @@ -1716,11 +1724,33 @@ def test_builtin_encode_invalid(self): self.assertEqual(exc.end, expected.end) def test_builtin_decode_length_limit(self): - with self.assertRaisesRegex(UnicodeDecodeError, "way too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "too long"): (b"xn--016c"+b"a"*1100).decode("idna") with self.assertRaisesRegex(UnicodeDecodeError, "too long"): (b"xn--016c"+b"a"*70).decode("idna") + def test_builtin_encode_length_limit(self): + with self.assertRaisesRegex(UnicodeEncodeError, "too long"): + ("x" * 64).encode("idna") + with self.assertRaisesRegex(UnicodeEncodeError, "too long"): + ("short." + "x" * 64).encode("idna") + + # Test at both sides of the limit (<64 bytes) + self.assertEqual(len(("\N{SNAKE}" * 56).encode("idna")), 63) + with self.assertRaisesRegex(UnicodeEncodeError, "too long"): + ("\N{SNAKE}" * 57).encode("idna") + with self.assertRaisesRegex(UnicodeEncodeError, "too long"): + ("short." + "\N{SNAKE}" * 57).encode("idna") + + # Very long names are handled + with self.assertRaisesRegex(UnicodeEncodeError, "way too long"): + ("\N{SNAKE}"*50_000).encode("idna") + + # The limit doesn't apply to ignored characters + self.assertEqual(('a' + "\N{ZWSP}"*50_000 + 'b').encode('idna'), b'ab') + self.assertEqual(('a' + "\N{ZWSP}"*50_000 + '\N{SNAKE}').encode('idna'), + b'xn--a-012s') + def test_stream(self): r = codecs.getreader("idna")(io.BytesIO(b"abc")) r.read(3) diff --git a/Misc/NEWS.d/next/Library/2026-09-17-15-14-08.gh-issue-157675.C4-ze9.rst b/Misc/NEWS.d/next/Library/2026-09-17-15-14-08.gh-issue-157675.C4-ze9.rst new file mode 100644 index 00000000000000..37f8e3e50c31ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-17-15-14-08.gh-issue-157675.C4-ze9.rst @@ -0,0 +1,5 @@ +:func:`encodings.idna.nameprep` now takes a *limit* argument that allows +rejecting extremely large input early. The ``idna`` encoding and the +:func:`!encodings.idna.ToASCII` and :func:`!encodings.idna.ToUnicode` +functions use this to avoid passing unbounded input to quadratic-time +algorithms. From f610e8cc7f0c9687d01592a08c78326351c6264d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 2 Oct 2026 15:34:44 +0200 Subject: [PATCH 039/240] gh-158098: Add Setuptools workaround to abi3t migration guide (#158405) --- Doc/howto/abi3t-migration.rst | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/Doc/howto/abi3t-migration.rst b/Doc/howto/abi3t-migration.rst index ec78a33ed25820..501106679b02bd 100644 --- a/Doc/howto/abi3t-migration.rst +++ b/Doc/howto/abi3t-migration.rst @@ -217,9 +217,9 @@ Module export hook Unless you've done this step already, your extension module defines a :ref:`module initialization function ` -named :samp:`PyInit_{}`. +named :samp:`PyInit_{}` (where ``modname`` is the name of your module). You will need to port it to a :ref:`module export hook `, -:samp:`PyModExport_{}`, a feature added in CPython 3.15 in +:samp:`PyModExport_{}`, a feature added in CPython 3.15 in :pep:`793`. Your existing init function should look like this (with your own names @@ -297,6 +297,34 @@ pointer to static data. If you cannot avoid additional code, refer to the :ref:`caveats in PyModExport documentation `. +.. note:: + + When building for Windows using the Setuptools_ build tool, + removing the :samp:`PyInit_{}` function may result in the linker error + :samp:`LINK : error LNK2001: unresolved external symbol PyInit_{}`. + This is caused by Setuptools passing an ``/EXPORT`` linker flag, which + is redundant since Python 3.15 (see :gh:`141671`). + A workaround is to add a dummy :samp:`PyInit_{}` function + to your code. + Python 3.15+ will never call this function if + :samp:`PyModExport_{}` is present, so it can always fail: + + .. code-block:: c + + // Workaround for https://github.com/pypa/distutils/issues/387 + PyMODINIT_FUNC + PyInit_(void) + { + PyErr_SetString(PyExc_SystemError, + "PyInit_* called for module with PyModExport_*"); + return NULL; + } + + (This issue is present in Setuptools 84.0.0; it might be fixed in newer + versions.) + +.. _Setuptools: https://setuptools.pypa.io/ + Existing slots -------------- From a831abba141ceeb0ef8acfada37602aa578cc670 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2026 16:43:44 +0200 Subject: [PATCH 040/240] gh-158445: Add C API tests on build information functions (#158603) Add tests on functions: * Py_GetBuildInfo() * Py_GetCompiler() * Py_GetCopyright() * Py_GetPlatform() * Py_GetVersion() Changes: * Add Modules/_testlimitedcapi.c and Lib/test/test_capi/test_build.py files. * Add internal _Py_GetBuiltWithAssert() function. * Reimplemenet support.built_with_c_assertions() using _Py_GetBuiltWithAssert() to cover more cases like buildbots using CFLAGS=-UNDEBUG in release mode. --- Lib/test/support/__init__.py | 15 ++--- Lib/test/test_capi/test_build.py | 39 ++++++++++++ Lib/test/test_support.py | 4 ++ Modules/Setup.stdlib.in | 2 +- Modules/_testlimitedcapi.c | 3 + Modules/_testlimitedcapi/build.c | 78 ++++++++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 1 + Modules/getbuildinfo.c | 16 +++++ PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + 10 files changed, 149 insertions(+), 11 deletions(-) create mode 100644 Lib/test/test_capi/test_build.py create mode 100644 Modules/_testlimitedcapi/build.c diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f348c9a5bbb31c..e9fc4cca229a03 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3525,17 +3525,12 @@ def check_immutable_type(testcase, type): def built_with_c_assertions(): """Check if Python was built with C assertions (assert()).""" - - if MS_WINDOWS: - # On Windows, rely on the Py_DEBUG macro to check for assertions + try: + import _testlimitedcapi + except ImportError: return Py_DEBUG - - # Check if the NDEBUG macro is defined in C compiler flags - PY_CFLAGS = (sysconfig.get_config_var('PY_CFLAGS') or '') - if '-DNDEBUG' in PY_CFLAGS: - return False - - return True + else: + return bool(_testlimitedcapi._py_getbuiltwithassert()) def inject_memory_error(start=0, stop=0): diff --git a/Lib/test/test_capi/test_build.py b/Lib/test/test_capi/test_build.py new file mode 100644 index 00000000000000..8319ad1ec77da8 --- /dev/null +++ b/Lib/test/test_capi/test_build.py @@ -0,0 +1,39 @@ +import sys +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +class CAPITest(unittest.TestCase): + # Test Py_GetVersion() + def test_getversion(self): + self.assertEqual(_testlimitedcapi.py_getversion(), sys.version) + + # Test Py_GetPlatform() + def test_getplatform(self): + self.assertEqual(_testlimitedcapi.py_getplatform(), sys.platform) + + # Test Py_GetCopyright() + def test_getcopyright(self): + self.assertEqual(_testlimitedcapi.py_getcopyright(), sys.copyright) + + # Test Py_GetCompiler() + def test_getcompiler(self): + compiler = _testlimitedcapi.py_getcompiler() + self.assertIsInstance(compiler, str) + self.assertNotEqual(compiler, '') + + # Test Py_GetBuildInfo() + def test_getbuildinfo(self): + buildinfo = _testlimitedcapi.py_getbuildinfo() + self.assertIsInstance(buildinfo, str) + self.assertNotEqual(buildinfo, '') + + # Test internal _Py_GetBuiltWithAssert() + def test__getbuiltwithassert(self): + self.assertIn(_testlimitedcapi._py_getbuiltwithassert(), (0, 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index a8c7e883bd4dd8..b83e0c371db33d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -838,6 +838,10 @@ def test_linked_to_musl(self): for v in linked: self.assertIsInstance(v, int) + def test_built_with_c_assertions(self): + # Check that calling the function doesn't raise an exception + result = support.built_with_c_assertions() + self.assertIsInstance(result, bool) # XXX -follows a list of untested API # make_legacy_pyc diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 0ca1d90ac4f30b..dd9f38e59f7a3f 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -174,7 +174,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tokenizer.c _testinternalcapi/tuple.c _testinternalcapi/typecache.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c _testcapi/marshal.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slice.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c _testlimitedcapi/type.c _testlimitedcapi/hash.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slice.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c _testlimitedcapi/type.c _testlimitedcapi/hash.c _testlimitedcapi/build.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 0d290eb5ef5b9e..1aee3bc9bf3c1d 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -97,6 +97,9 @@ module_exec(PyObject *mod) if (_PyTestLimitedCAPI_Init_Hash(mod) < 0) { return -1; } + if (_PyTestLimitedCAPI_Init_Build(mod) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testlimitedcapi/build.c b/Modules/_testlimitedcapi/build.c new file mode 100644 index 00000000000000..ae26fdc07fe908 --- /dev/null +++ b/Modules/_testlimitedcapi/build.c @@ -0,0 +1,78 @@ +#include "parts.h" +#include "util.h" + + +static PyObject* +return_string(const char *str) +{ + assert(str != NULL); + return PyUnicode_FromString(str); +} + + +/* Test Py_GetVersion() */ +static PyObject* +py_getversion(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return return_string(Py_GetVersion()); +} + + +/* Test Py_GetPlatform() */ +static PyObject* +py_getplatform(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return return_string(Py_GetPlatform()); +} + + +/* Test Py_GetCopyright() */ +static PyObject* +py_getcopyright(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return return_string(Py_GetCopyright()); +} + + +/* Test Py_GetCompiler() */ +static PyObject* +py_getcompiler(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return return_string(Py_GetCompiler()); +} + + +/* Test Py_GetBuildInfo() */ +static PyObject* +py_getbuildinfo(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return return_string(Py_GetBuildInfo()); +} + + +/* Test _Py_GetBuiltWithAssert() */ +static PyObject* +_py_getbuiltwithassert(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // Function only exported for _testlimitedcapi + PyAPI_FUNC(int) _Py_GetBuiltWithAssert(void); + + return PyLong_FromLong(_Py_GetBuiltWithAssert()); +} + + +static PyMethodDef test_methods[] = { + {"py_getversion", py_getversion, METH_NOARGS}, + {"py_getplatform", py_getplatform, METH_NOARGS}, + {"py_getcopyright", py_getcopyright, METH_NOARGS}, + {"py_getcompiler", py_getcompiler, METH_NOARGS}, + {"py_getbuildinfo", py_getbuildinfo, METH_NOARGS}, + {"_py_getbuiltwithassert", _py_getbuiltwithassert, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Build(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index ab6f76f3d06699..1e9d140899cecf 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -53,5 +53,6 @@ int _PyTestLimitedCAPI_Init_Weakref(PyObject *module); int _PyTestLimitedCAPI_Init_Run(PyObject *module); int _PyTestLimitedCAPI_Init_Type(PyObject *module); int _PyTestLimitedCAPI_Init_Hash(PyObject *module); +int _PyTestLimitedCAPI_Init_Build(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index b68f0f5cc56a44..006043d7dc3ba6 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -77,3 +77,19 @@ _Py_gitidentifier(void) gitid = GITBRANCH; return gitid; } + + +// Check if Python was built with NDEBUG macro defined or not. Implement the +// function in Modules/getbuildinfo.c so it's built with the same compiler +// flags than the Python core C code. +// +// Export the function for '_testlimitedcapi' shared extension. +PyAPI_FUNC(int) +_Py_GetBuiltWithAssert(void) +{ +#ifdef NDEBUG + return 0; +#else + return 1; +#endif +} diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 2995dbb469ed3d..1a40cd0d2271f5 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -123,6 +123,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 92f1c5dedbd62e..1b13da0ebca7ba 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -39,6 +39,7 @@ + From 35bdbcad63d629e7e4209117411241d8ac0bd6cf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2026 17:22:13 +0200 Subject: [PATCH 041/240] gh-158445: Skip invalid UCS-4 tests if Python is built with assertions (#158604) If Python is built in release mode with assertions, PyUnicode_FromKindAndData() and PyUnicodeWriter_Finish() fail with an assertion error on invalid characters. --- Lib/test/test_capi/test_unicode.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index c5926dce397409..6e2fd13567f5fb 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -315,6 +315,7 @@ def test_fromkindanddata(self): # Test invalid UCS-4 string. Create an invalid string in release mode, # or raise SystemError in debug mode. + built_with_assert = support.built_with_c_assertions() for invalid_char in (INVALID_CHAR, MAX_INVALID_CHAR): with self.subTest(invalid_char=invalid_char): # Test single character @@ -325,9 +326,12 @@ def test_fromkindanddata(self): s = 'valid'.encode(enc4) + ucs4_char if support.Py_DEBUG: self.assertRaises(SystemError, fromkindanddata, 4, s) - else: + elif not built_with_assert : result = fromkindanddata(4, s) assert_invalid_string(self, result) + else: + # PyUnicode_FromKindAndData() fails with an assertion error + pass def test_substring(self): """Test PyUnicode_Substring()""" @@ -2170,6 +2174,7 @@ def test_ucs4(self): # Invalid UCS-4 characters. Create an invalid string in release mode, # or raise SystemError in debug mode. + built_with_assert = support.built_with_c_assertions() writer = self.create_writer(0) for invalid_char in (INVALID_CHAR, MAX_INVALID_CHAR): with self.subTest(invalid_char=invalid_char): @@ -2189,9 +2194,12 @@ def test_ucs4(self): if support.Py_DEBUG: self.assertEqual(writer.finish(), '') - else: + elif not built_with_assert: result = writer.finish() assert_invalid_string(self, result) + else: + # PyUnicodeWriter_Finish() fails with an assertion error + pass # Invalid size writer = self.create_writer(0) From b5aa1954277494d80e83b7642f633b53e12e3579 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2026 17:51:47 +0200 Subject: [PATCH 042/240] gh-155742: Use PyMem_Malloc() in memoryview.hex() (#158582) Replace PyBytesWriter with a simple PyMem_Malloc(), since no bytes object is needed. PyMem_Malloc() should be more efficient in this case. --- Objects/memoryobject.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 5a8ae6ad149a75..e6d49b6c29a3e9 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2468,22 +2468,19 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, return ret; } - PyBytesWriter *writer = PyBytesWriter_Create(src->len); - if (writer == NULL) { + char *buffer = PyMem_Malloc(src->len); + if (buffer == NULL) { + PyErr_NoMemory(); return NULL; } - if (PyBuffer_ToContiguous(PyBytesWriter_GetData(writer), - src, src->len, 'C') < 0) { - PyBytesWriter_Discard(writer); + if (PyBuffer_ToContiguous(buffer, src, src->len, 'C') < 0) { + PyMem_Free(buffer); return NULL; } - PyObject *ret = _Py_strhex_with_sep( - PyBytesWriter_GetData(writer), - PyBytesWriter_GetSize(writer), - sep, bytes_per_sep); - PyBytesWriter_Discard(writer); + PyObject *ret = _Py_strhex_with_sep(buffer, src->len, sep, bytes_per_sep); + PyMem_Free(buffer); return ret; } From be87a85e2fb0cc22e6c4292ecec1bf7e3d0d3ace Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Fri, 2 Oct 2026 18:17:48 +0200 Subject: [PATCH 043/240] gh-155695: Remove resolved names from sys.lazy_modules more consistently (#157714) Co-authored-by: Pablo Galindo Salgado Co-authored-by: Petr Viktorin --- Include/internal/pycore_import.h | 2 + Lib/test/test_lazy_import/__init__.py | 259 +++++++++++++++++- Lib/test/test_lazy_import/data/init_fails.py | 4 + .../data/lazy_on_init_fails.py | 1 + ...-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst | 3 + Objects/lazyimportobject.c | 8 + Python/import.c | 82 +++++- 7 files changed, 345 insertions(+), 14 deletions(-) create mode 100644 Lib/test/test_lazy_import/data/init_fails.py create mode 100644 Lib/test/test_lazy_import/data/lazy_on_init_fails.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 81448e23787fb3..983ee0ad22789b 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -87,6 +87,8 @@ extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp); extern PyObject * _PyImport_InitLazyModules( PyInterpreterState *interp); extern void _PyImport_ClearLazyModules(PyInterpreterState *interp); +extern int _PyImport_DiscardLazyModule( + PyInterpreterState *interp, PyObject *name); extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp); extern int _PyImport_IsDefaultImportFunc( diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index f9ef8ce2cfef8c..69b1f775a29c12 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -53,7 +53,8 @@ def test_sys_lazy_modules(self): self.fail('lazy import failed') self.assertFalse("test.test_lazy_import.data.basic2" in sys.modules) - self.assertIn("test.test_lazy_import.data", sys.lazy_modules) + # The package is already loaded, so it is not a pending import. + self.assertNotIn("test.test_lazy_import.data", sys.lazy_modules) self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) test.test_lazy_import.data.basic_from_unused.basic2 self.assertNotIn("test.test_import.data", sys.lazy_modules) @@ -1305,6 +1306,262 @@ def test_lazy_module_without_children_is_tracked(self): """) assert_python_ok("-c", code) + def test_already_loaded_module_is_not_tracked(self): + """A lazy import of an already loaded module should not be tracked.""" + code = textwrap.dedent(""" + import sys + + # Loaded by a regular import. + import json + lazy import json as lazy_json + assert "json" not in sys.lazy_modules, ( + f"expected 'json' not in sys.lazy_modules, got {sys.lazy_modules}" + ) + + # Loaded by reifying an earlier lazy import. + lazy import base64 + _ = base64.b64encode + lazy import base64 as lazy_base64 + assert "base64" not in sys.lazy_modules, ( + f"expected 'base64' not in sys.lazy_modules, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_already_loaded_submodule_is_not_tracked(self): + """`lazy from` a loaded submodule should not be tracked either.""" + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg.b + lazy from test.test_lazy_import.data.pkg import b + assert "test.test_lazy_import.data.pkg.b" not in sys.lazy_modules, ( + f"expected 'pkg.b' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_attribute_entry_removed_on_reification(self): + """`lazy from x import attr` should untrack "x.attr" once resolved.""" + code = textwrap.dedent(""" + import sys + lazy from test.test_lazy_import.data.basic2 import x + assert "test.test_lazy_import.data.basic2.x" in sys.lazy_modules, ( + f"expected 'basic2.x' tracked, got {sys.lazy_modules}" + ) + _ = x + assert "test.test_lazy_import.data.basic2.x" not in sys.lazy_modules, ( + f"expected 'basic2.x' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_already_loaded_attribute_is_not_tracked(self): + code = textwrap.dedent(""" + import sys + import math + + lazy from math import pi + assert "math.pi" not in sys.lazy_modules, sys.lazy_modules + assert pi == math.pi + """) + assert_python_ok("-c", code) + + def test_cached_lazy_attribute_tracks_its_target(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.basic_from_unused + + holder = "test.test_lazy_import.data.basic_from_unused" + target = "test.test_lazy_import.data.basic2" + + namespace = {"__name__": "cached_import_test"} + exec(f"lazy from {holder} import basic2", namespace) + assert holder + ".basic2" not in sys.lazy_modules, sys.lazy_modules + assert target in sys.lazy_modules, sys.lazy_modules + exec("assert basic2.x == 42", namespace) + assert target not in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_cached_attribute_keeps_pending_module_tracked(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg as pkg + pkg.b = 42 + + lazy import test.test_lazy_import.data.pkg.b as pending + lazy from test.test_lazy_import.data.pkg import b + name = "test.test_lazy_import.data.pkg.b" + assert b == 42, b + assert name not in sys.modules, sys.modules + assert name in sys.lazy_modules, sys.lazy_modules + assert pending.foo() == "foo" + assert name not in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_failed_reification_stays_tracked(self): + """A lazy import that fails to resolve must stay tracked.""" + code = textwrap.dedent(""" + import sys + lazy import test.test_lazy_import.data.broken_module + try: + _ = test.test_lazy_import.data.broken_module + except ValueError: + pass + else: + raise AssertionError("ValueError was not raised") + assert "test.test_lazy_import.data.broken_module" in sys.lazy_modules, ( + f"failed reification must stay tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_blocked_module_is_still_tracked(self): + """A ``None`` entry in sys.modules must not count as loaded.""" + code = textwrap.dedent(""" + import sys + sys.modules['test.test_lazy_import.data.basic2'] = None + lazy import test.test_lazy_import.data.basic2 + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules, ( + f"blocked module must stay tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_initializing_module_is_still_tracked(self): + """A module that is still executing must not count as loaded.""" + code = textwrap.dedent(""" + import sys + name = "test.test_lazy_import.data.init_fails" + try: + import test.test_lazy_import.data.init_fails + except ValueError: + pass + else: + raise AssertionError("ValueError was not raised") + assert name not in sys.modules, "failed import left a module behind" + assert name in sys.lazy_modules, ( + f"expected {name!r} tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_module_spec_descriptor_is_not_run(self): + """Checking whether a module is loaded must not run its descriptors.""" + code = textwrap.dedent(""" + import sys + import types + + class RaisingSpec(types.ModuleType): + @property + def __spec__(self): + raise RuntimeError("__spec__ descriptor was run") + + sys.modules["raising_spec"] = RaisingSpec("raising_spec") + lazy import raising_spec + assert "raising_spec" not in sys.lazy_modules, ( + f"expected 'raising_spec' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_spec_initializing_descriptor_is_not_run(self): + code = textwrap.dedent(""" + import sys + import types + + class Spec: + @property + def _initializing(self): + raise RuntimeError("_initializing descriptor was run") + + @property + def __dict__(self): + raise RuntimeError("__dict__ descriptor was run") + + class SlottedSpec: + __slots__ = () + + @property + def _initializing(self): + raise RuntimeError("_initializing descriptor was run") + + for spec in (Spec(), SlottedSpec()): + module = types.ModuleType("custom_spec") + module.__spec__ = spec + sys.modules["custom_spec"] = module + lazy import custom_spec + """) + assert_python_ok("-c", code) + + def test_cached_attribute_does_not_check_spec_twice(self): + code = textwrap.dedent(""" + import sys + import types + + class Spec: + def __init__(self): + self.calls = 0 + + @property + def _initializing(self): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("_initializing was read twice") + return False + + module = types.ModuleType("cached_spec") + module.__spec__ = Spec() + module.attr = 1 + sys.modules["cached_spec"] = module + lazy from cached_spec import attr + assert attr == 1, attr + """) + assert_python_ok("-c", code) + + def test_spec_initializing_truth_conversion_is_not_run(self): + code = textwrap.dedent(""" + import sys + import types + from importlib.machinery import ModuleSpec + + class Flag: + def __bool__(self): + raise RuntimeError("_initializing truth conversion was run") + + released = memoryview(b"") + released.release() + for flag in (Flag(), released): + sys.lazy_modules.discard("custom_spec") + module = types.ModuleType("custom_spec") + module.__spec__ = ModuleSpec("custom_spec", None) + module.__spec__._initializing = flag + sys.modules["custom_spec"] = module + lazy import custom_spec + assert "custom_spec" in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_pending_submodule_is_still_tracked(self): + """`lazy from` a submodule that is not loaded must stay tracked.""" + code = textwrap.dedent(""" + import sys + lazy from test.test_lazy_import.data.pkg import b + assert "test.test_lazy_import.data.pkg.b" in sys.lazy_modules, ( + f"expected 'pkg.b' tracked, got {sys.lazy_modules}" + ) + import test.test_lazy_import.data.pkg + assert "test.test_lazy_import.data.pkg.b" not in sys.modules, ( + "loading the package must not load the submodule" + ) + assert "test.test_lazy_import.data.pkg.b" in sys.lazy_modules, ( + f"loading the package must not untrack the submodule, " + f"got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + @support.requires_subprocess() class CommandLineAndEnvVarTests(unittest.TestCase): diff --git a/Lib/test/test_lazy_import/data/init_fails.py b/Lib/test/test_lazy_import/data/init_fails.py new file mode 100644 index 00000000000000..16e7ad363b0eba --- /dev/null +++ b/Lib/test/test_lazy_import/data/init_fails.py @@ -0,0 +1,4 @@ +# Imported by test_initializing_module_is_still_tracked. The module it imports +# lazily imports this one back while this one is still initializing. +import test.test_lazy_import.data.lazy_on_init_fails +raise ValueError("initialization failed") diff --git a/Lib/test/test_lazy_import/data/lazy_on_init_fails.py b/Lib/test/test_lazy_import/data/lazy_on_init_fails.py new file mode 100644 index 00000000000000..7203fcbb4aaa45 --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazy_on_init_fails.py @@ -0,0 +1 @@ +lazy import test.test_lazy_import.data.init_fails diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst new file mode 100644 index 00000000000000..f0e8c7c849be22 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst @@ -0,0 +1,3 @@ +Resolved names are now removed from :data:`sys.lazy_modules` more +consistently: a lazy import of an already loaded module is no longer recorded, +and reifying ``lazy from pkg import attr`` now discards the ``pkg.attr`` entry. diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index e4a949a5456cfd..69b3e1663d0786 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -452,6 +452,14 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, lazy_import_add_exception_cause(tstate, lz); } assert(obj == NULL || !PyLazyImport_CheckExact(obj)); + if (obj != NULL) { + PyObject *name = lazy_import_name(lz); + if (name == NULL || + _PyImport_DiscardLazyModule(tstate->interp, name) < 0) { + Py_CLEAR(obj); + } + Py_XDECREF(name); + } if (resolving != NULL) { // A failed set resize can leave the placeholder inserted. Removing by // identity also permits greenlets to finish in a different order. diff --git a/Python/import.c b/Python/import.c index 576b8d47639fa4..823201d38b64ec 100644 --- a/Python/import.c +++ b/Python/import.c @@ -284,6 +284,12 @@ _PyImport_ClearLazyModules(PyInterpreterState *interp) Py_CLEAR(LAZY_PENDING_SUBMODULES(interp)); } +int +_PyImport_DiscardLazyModule(PyInterpreterState *interp, PyObject *name) +{ + return PySet_Discard(LAZY_MODULES(interp), name); +} + static PyObject * get_importtime_name(PyObject *name) { @@ -4128,6 +4134,47 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } +// Add *name* to `sys.lazy_modules`. +// Skip the registration if *name* is already imported, and fully initialized. +// If *existing_mod_p* is not NULL, set it to the already imported object (or +// NULL if not found), for use by register_lazy_on_parent. +static int +lazy_modules_add(PyThreadState *tstate, PyObject *name, + PyObject **existing_mod_p) +{ + PyObject *mod = import_get_module(tstate, name); + if (mod == NULL && PyErr_Occurred()) { + return -1; + } + int loaded = mod != NULL && mod != Py_None; + if (loaded && PyModule_Check(mod)) { + PyObject *spec; + if (PyDict_GetItemRef(_PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) < 0) { + Py_DECREF(mod); + return -1; + } + loaded = spec == NULL || spec == Py_None; + if (!loaded) { + // Read the usual flag without callbacks or dictionary allocation. + // Unsupported spec representations stay conservatively tracked. + PyObject *initializing = NULL; + if ((Py_TYPE(spec)->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(spec, &_Py_ID(_initializing), &initializing)) { + loaded = initializing == NULL || initializing == Py_False; + } + Py_XDECREF(initializing); + } + Py_XDECREF(spec); + } + if (existing_mod_p != NULL) { + *existing_mod_p = mod; + } + else { + Py_XDECREF(mod); + } + return loaded ? 0 : PySet_Add(LAZY_MODULES(tstate->interp), name); +} + // Ensure a set of pending submodule names exists for the parent. // Returns a new reference. static PyObject * @@ -4189,22 +4236,30 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name) } static int -register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, - PyObject *from) -{ +register_from_lazy_on_parent(PyThreadState *tstate, PyObject *existing_module, + PyObject *abs_name, PyObject *from) +{ + // IMPORT_FROM returns stored attributes directly. Their imports are + // already resolved or tracked by their own placeholders, so skip the alias. + int rc = 0; + if (existing_module != NULL && PyModule_Check(existing_module)) { + rc = PyDict_Contains(_PyModule_GetDict(existing_module), from); + if (rc != 0) { + return rc < 0 ? -1 : 0; + } + } + PyObject *fromname = PyUnicode_FromFormat("%U.%U", abs_name, from); if (fromname == NULL) { return -1; } - // Add the module name to sys.lazy_modules set (PEP 810). - PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - int res = PySet_Add(lazy_modules, fromname); - if (res == 0) { - res = register_lazy_on_parent(tstate, fromname); + rc = lazy_modules_add(tstate, fromname, NULL); + if (rc == 0) { + rc = register_lazy_on_parent(tstate, fromname); } Py_DECREF(fromname); - return res; + return rc; } PyObject * @@ -4291,6 +4346,8 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, } PyObject *res = NULL; + PyObject *existing_module = NULL; + if (fromlist != NULL && PyUnicode_Check(fromlist)) { fromlist = PyTuple_Pack(1, fromlist); if (fromlist == NULL) { @@ -4355,15 +4412,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, goto done; } - // Add the module name to sys.lazy_modules set (PEP 810). - PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - if (PySet_Add(lazy_modules, abs_name) < 0) { + if (lazy_modules_add(tstate, abs_name, &existing_module) < 0) { goto error; } if (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { - if (register_from_lazy_on_parent(tstate, abs_name, + if (register_from_lazy_on_parent(tstate, existing_module, abs_name, PyTuple_GET_ITEM(fromlist, i)) < 0) { goto error; @@ -4380,6 +4435,7 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, done: Py_XDECREF(fromlist); Py_DECREF(abs_name); + Py_XDECREF(existing_module); return res; } From 9d3b0b889b72df25ec5c894437a32e8867ef4e4e Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Fri, 2 Oct 2026 17:45:11 +0100 Subject: [PATCH 044/240] gh-158563: Fix various bugs in `turtle`'s undo handling (#158564) --- Lib/test/test_turtle.py | 28 ++++ Lib/turtle.py | 135 ++++++++++-------- ...-10-01-13-59-02.gh-issue-158563.kR7wQe.rst | 4 + 3 files changed, 105 insertions(+), 62 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-01-13-59-02.gh-issue-158563.kR7wQe.rst diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 7e24f1d526a1bc..88e18e7daa5c5a 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -694,6 +694,34 @@ def test_dot_signature(self): self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, (0, 257, 0)) self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, 0, 257, 0) + def test_circle_undo(self): + self.turtle.circle(50, 90) + self.turtle.undo() + self.assertEqual(self.turtle.pos(), (0, 0)) + self.assertEqual(self.turtle.undobufferentries(), 0) + + def test_undo_sequence_resets_after_exception(self): + with unittest.mock.patch.object(self.turtle, "_write", + side_effect=ValueError): + self.assertRaises(ValueError, self.turtle.write, "spam") + self.assertFalse(self.turtle.undobuffer.cumulate) + + def test_nested_undo_sequence(self): + with self.turtle._undo_sequence(): + self.turtle.teleport(10, 20) + self.turtle.forward(10) + self.assertEqual(self.turtle.undobufferentries(), 1) + self.turtle.undo() + self.assertEqual(self.turtle.pos(), (0, 0)) + + def test_stamp_without_undobuffer(self): + shape = turtle.Shape("polygon", ((0, 0), (5, 9), (-5, 9))) + self.turtle.screen._shapes = {self.turtle.shape(): shape} + self.turtle.setundobuffer(None) + stamp = self.turtle.stamp() + self.turtle.clearstamp(stamp) + self.assertEqual(self.turtle.stampItems, []) + class TestModuleLevel(unittest.TestCase): def test_all_signatures(self): import inspect diff --git a/Lib/turtle.py b/Lib/turtle.py index 6e77d257ba8c7f..a731bd69664a4a 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -897,6 +897,15 @@ def pop(self): self.ptr = (self.ptr - 1) % self.bufsize return (item) + def remove(self, item): + if item not in self.buffer: + return + index = self.buffer.index(item) + self.buffer.remove(item) + if index <= self.ptr: + self.ptr = (self.ptr - 1) % self.bufsize + self.buffer.insert((self.ptr+1) % self.bufsize, [None]) + def nr_of_items(self): return self.bufsize - self.buffer.count([None]) @@ -1643,6 +1652,20 @@ def _goto(self, end): """Move the turtle to the end position.""" self._position = end + @contextmanager + def _undo_sequence(self): + """Record the enclosed actions as a single undo step.""" + undobuffer = self.undobuffer + if not undobuffer or undobuffer.cumulate: + yield + return + undobuffer.push(["seq"]) + undobuffer.cumulate = True + try: + yield + finally: + undobuffer.cumulate = False + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: """To be overwritten by child class RawTurtle. Includes no TPen references.""" @@ -1985,15 +2008,12 @@ def circle(self, radius, extent = None, steps = None): >>> turtle.circle(50) >>> turtle.circle(120, 180) # draw a semicircle """ - if self.undobuffer: - self.undobuffer.push(["seq"]) - self.undobuffer.cumulate = True speed = self.speed() if extent is None: extent = self._fullcircle if steps is None: frac = abs(extent)/self._fullcircle - steps = 1+int(min(11+abs(radius)/6.0, 59.0)*frac) + steps = 1 + int(min(11 + abs(radius) / 6.0, 59.0) * frac) w = 1.0 * extent / steps w2 = 0.5 * w l = 2.0 * radius * math.sin(math.radians(w2)*self._degreesPerAU) @@ -2001,22 +2021,21 @@ def circle(self, radius, extent = None, steps = None): l, w, w2 = -l, -w, -w2 tr = self._tracer() dl = self._delay() - if speed == 0: - self._tracer(0, 0) - else: - self.speed(0) - self._rotate(w2) - for i in range(steps): + with self._undo_sequence(): + if speed == 0: + self._tracer(0, 0) + else: + self.speed(0) + self._rotate(w2) + for i in range(steps): + self.speed(speed) + self._go(l) + self.speed(0) + self._rotate(w) + self._rotate(-w2) + if speed == 0: + self._tracer(tr, dl) self.speed(speed) - self._go(l) - self.speed(0) - self._rotate(w) - self._rotate(-w2) - if speed == 0: - self._tracer(tr, dl) - self.speed(speed) - if self.undobuffer: - self.undobuffer.cumulate = False # Three dummy methods to be implemented by the child class: @@ -2787,16 +2806,19 @@ def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: """ pendown = self.isdown() was_filling = self.filling() - if pendown: - self.pen(pendown=False) - if was_filling and not fill_gap: - self.end_fill() - new_x = x if x is not None else self._position[0] - new_y = y if y is not None else self._position[1] - self._position = Vec2D(new_x, new_y) - self.pen(pendown=pendown) - if was_filling and not fill_gap: - self.begin_fill() + with self._undo_sequence(): + if pendown: + self.pen(pendown=False) + if was_filling and not fill_gap: + self.end_fill() + new_x = x if x is not None else self._position[0] + new_y = y if y is not None else self._position[1] + if self.undobuffer: + self.undobuffer.push(("teleport", self._position)) + self._position = Vec2D(new_x, new_y) + self.pen(pendown=pendown) + if was_filling and not fill_gap: + self.begin_fill() def clone(self): """Create and return a clone of the turtle. @@ -3147,7 +3169,8 @@ def stamp(self): screen._drawpoly(item, poly, fill=self._cc(fc), outline=self._cc(oc), width=self._outlinewidth, top=True) self.stampItems.append(stitem) - self.undobuffer.push(("stamp", stitem)) + if self.undobuffer: + self.undobuffer.push(("stamp", stitem)) return stitem def _clearstamp(self, stampid): @@ -3162,15 +3185,8 @@ def _clearstamp(self, stampid): self.stampItems.remove(stampid) # Delete stampitem from undobuffer if necessary # if clearstamp is called directly. - item = ("stamp", stampid) - buf = self.undobuffer - if item not in buf.buffer: - return - index = buf.buffer.index(item) - buf.buffer.remove(item) - if index <= buf.ptr: - buf.ptr = (buf.ptr - 1) % buf.bufsize - buf.buffer.insert((buf.ptr+1)%buf.bufsize, [None]) + if self.undobuffer: + self.undobuffer.remove(("stamp", stampid)) def clearstamp(self, stampid): """Delete stamp with given stampid @@ -3468,20 +3484,16 @@ def dot(self, size=None, *color): color = self._colorstr(color) # If screen were to gain a dot function, see GH #104218. pen = self.pen() - if self.undobuffer: - self.undobuffer.push(["seq"]) - self.undobuffer.cumulate = True - try: - if self.resizemode() == 'auto': - self.ht() - self.pendown() - self.pensize(size) - self.pencolor(color) - self.forward(0) - finally: - self.pen(pen) - if self.undobuffer: - self.undobuffer.cumulate = False + with self._undo_sequence(): + try: + if self.resizemode() == 'auto': + self.ht() + self.pendown() + self.pensize(size) + self.pencolor(color) + self.forward(0) + finally: + self.pen(pen) def _write(self, txt, align, font): """Performs the writing for write() @@ -3513,15 +3525,11 @@ def write(self, arg, move=False, align="left", font=("Arial", 8, "normal")): >>> turtle.write('Home = ', True, align="center") >>> turtle.write((0,0), True) """ - if self.undobuffer: - self.undobuffer.push(["seq"]) - self.undobuffer.cumulate = True - end = self._write(str(arg), align.lower(), font) - if move: - x, y = self.pos() - self.setpos(end, y) - if self.undobuffer: - self.undobuffer.cumulate = False + with self._undo_sequence(): + end = self._write(str(arg), align.lower(), font) + if move: + x, y = self.pos() + self.setpos(end, y) @contextmanager def poly(self): @@ -3709,6 +3717,9 @@ def _undo(self, action, data): self.clearstamp(stitem) elif action == "go": self._undogoto(data) + elif action == "teleport": + self._position = data[0] + self._update() elif action in ["wri", "dot"]: item = data[0] self.screen._delete(item) diff --git a/Misc/NEWS.d/next/Library/2026-10-01-13-59-02.gh-issue-158563.kR7wQe.rst b/Misc/NEWS.d/next/Library/2026-10-01-13-59-02.gh-issue-158563.kR7wQe.rst new file mode 100644 index 00000000000000..938a1b69fd8077 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-01-13-59-02.gh-issue-158563.kR7wQe.rst @@ -0,0 +1,4 @@ +Fix :func:`turtle.undo` after :func:`turtle.teleport` and after an exception +in :func:`turtle.circle`, :func:`turtle.dot` or :func:`turtle.write`. Fix +:func:`turtle.stamp`, :func:`turtle.clearstamp`, :func:`turtle.clear` and +:func:`turtle.reset` when the undo buffer is disabled. From 00a1c3a0c94401095510a7022002f12bad118fbb Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 2 Oct 2026 18:20:04 +0100 Subject: [PATCH 045/240] gh-158140: Resolve lazily imported sibling submodules independently (#158521) * gh-158140: Resolve lazy siblings through the pending-child registry Co-authored-by: Petr Viktorin Co-authored-by: T. Wouters --- Include/internal/pycore_import.h | 4 +- Include/internal/pycore_interp_structs.h | 6 +- Include/internal/pycore_lazyimportobject.h | 4 + Lib/test/test_lazy_import/__init__.py | 117 +++++++++++- ...6-10-01-20-00-00.gh-issue-158140.small.rst | 2 + Objects/lazyimportobject.c | 131 +++++++++++++- Objects/moduleobject.c | 3 +- Python/import.c | 169 +++++++++++++++--- 8 files changed, 408 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-20-00-00.gh-issue-158140.small.rst diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 983ee0ad22789b..f64b65e0f890df 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -41,7 +41,9 @@ PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( // new reference, NULL without an exception if absent, or NULL on error. // Set recheck_dict if a concurrent lookup may have already bound the child. extern PyObject * _PyImport_TryLoadLazySubmodule( - PyObject *module, PyObject *attr_name, int *recheck_dict); + PyObject *module, PyObject *attr_name, int suppress, int *recheck_dict); +extern int _PyImport_ClearLazySubmodule( + PyThreadState *tstate, PyObject *name, int bind); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index a9c61ea6db3a8c..fe501be1afc59b 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -356,9 +356,11 @@ struct _import_state { // lazily imported. This value is only for debugging/introspection // purposes and is not used by the runtime. PyObject *lazy_modules; - // A dict mapping package names to a set of submodule names that - // may need to be imported when an attribute is missing on the package. + // Package names map to pending children: declarations for plain imports, + // or None for from-import names that may be ordinary attributes. PyObject *lazy_pending_submodules; + // Avoid pending-child work for ordinary cached imports. + int has_lazy_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; #endif diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index c3f8417c089316..2da037b0b6fb84 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -18,6 +18,10 @@ PyAPI_FUNC(PyObject *) _PyLazyImport_New( struct _PyInterpreterFrame *frame, PyObject *builtins, PyObject *name, PyObject *fromlist); +extern PyObject *_PyLazyImport_LoadChild( + PyThreadState *tstate, PyObject *declaration, PyObject *name); +extern int _PyLazyImport_IsActive(PyObject *declaration); + extern int _PyLazyImport_IsResolving(PyThreadState *tstate, PyObject *op); // Resolve a placeholder and replace its binding if it is unchanged or holds diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 69b1f775a29c12..e0c5552fe11c53 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -449,6 +449,121 @@ def test_lazy_with_from(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) +@support.requires_subprocess() +class IndependentSubmoduleTests(LazyImportTestCase): + def check(self, code): + assert_python_ok('-c', textwrap.dedent(code)) + + def test_siblings(self): + for package, children in ( + ('test.test_lazy_import.data.pkg', ('b', 'bar')), + ('test.test_lazy_import.data.metasyntactic.foo', ('ack', 'bar')), + ): + for first, second in (children, children[::-1]): + with self.subTest(package=package, first=first): + self.check(f""" + import sys + lazy import {package}.{first} + lazy import {package}.{second} + assert {package}.{first} is sys.modules['{package}.{first}'] + assert '{package}.{second}' not in sys.modules + assert {package}.{second} is sys.modules['{package}.{second}'] + """) + + def test_star_import(self): + self.check(""" + lazy import urllib.nonexistent + lazy import urllib + assert urllib.__name__ == 'urllib' + from urllib import * + """) + + def test_deleted_child(self): + self.check(""" + lazy from xml import dom + assert dom.__name__ == 'xml.dom' + import xml + del xml.dom + assert not hasattr(xml, 'dom') + lazy from xml import dom + assert hasattr(xml, 'dom') + """) + + def test_cached_import_releases_builtins(self): + for statement in ('lazy from xml.dom import Node', + 'lazy import xml.dom.minidom'): + with self.subTest(statement=statement): + self.check(f""" + import builtins, gc, weakref + import xml.dom.minidom + class Payload: + pass + payload = Payload() + reference = weakref.ref(payload) + namespace = {{'__builtins__': dict(vars(builtins), payload=payload)}} + exec({statement!r}, namespace) + exec('Node' if 'from' in {statement!r} else 'xml.dom.minidom', namespace) + del namespace, payload + gc.collect() + assert reference() is None + """) + + def test_invalid_descendant_retries(self): + self.check(""" + lazy import test.test_lazy_import.data.pkg.b.foo + for _ in range(2): + try: + test.test_lazy_import.data.pkg.b.foo + except ModuleNotFoundError: + pass + else: + raise AssertionError('imported an attribute as a module') + import test.test_lazy_import.data.pkg.b + assert callable(test.test_lazy_import.data.pkg.b.foo) + """) + + def test_namespace_import_hooks(self): + self.check(""" + import builtins + first = {'__builtins__': vars(builtins).copy()} + second = {'__builtins__': vars(builtins).copy()} + exec('lazy import xml.dom', first) + exec('lazy import xml.dom', second) + def denied(*args): + raise AssertionError('used another namespace import hook') + second['__builtins__']['__import__'] = denied + exec('assert xml.dom.__name__ == "xml.dom"', first) + """) + + def test_retry_after_recursive_access(self): + self.check(""" + import sys, xml + from importlib.machinery import ModuleSpec + class Loader: + calls = 0 + def find_spec(self, name, path=None, target=None): + if name == 'xml.broken': + return ModuleSpec(name, self) + def create_module(self, spec): + return None + def exec_module(self, module): + self.calls += 1 + xml.broken + raise RuntimeError('failed initialization') + loader = Loader() + sys.meta_path.insert(0, loader) + lazy import xml.broken + for _ in range(2): + try: + xml.broken + except RuntimeError as exc: + assert str(exc) == 'failed initialization' + else: + raise AssertionError('returned a failed partial module') + assert loader.calls == 2 + """) + + class PackageTests(LazyImportTestCase): """Tests for lazy imports with packages.""" @@ -786,7 +901,7 @@ def test_chained_exception_import_shows_notes(self, name): lazy import {name} try: - _ = test + _ = {name} except ValueError as e: assert any( note.startswith("lazy import of '{name}' declared in ") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-20-00-00.gh-issue-158140.small.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-20-00-00.gh-issue-158140.small.rst new file mode 100644 index 00000000000000..b9a0fc5e7a80c7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-20-00-00.gh-issue-158140.small.rst @@ -0,0 +1,2 @@ +Resolve lazily imported sibling submodules independently, so accessing one +does not import another merely because its declaration came last. diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 69b3e1663d0786..bb35ddcdae1755 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -9,6 +9,7 @@ #include "pycore_lazyimportobject.h" #include "pycore_long.h" #include "pycore_moduleobject.h" +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pyerrors.h" #include "pycore_traceback.h" #include "pycore_tstate.h" @@ -25,6 +26,8 @@ typedef struct { // Declaration location. PyCodeObject *lz_code; int lz_instr_offset; + // The root was accessed, so its declared path requires import semantics. + int lz_active; } PyLazyImportObject; #define PyLazyImportObject_CAST(op) ((PyLazyImportObject *)(op)) @@ -70,6 +73,7 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, m->lz_code = NULL; m->lz_instr_offset = -1; + m->lz_active = 0; if (frame != NULL) { m->lz_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); @@ -429,10 +433,81 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, PyErr_SetString(PyExc_ImportError, "__import__ not found"); goto done; } + PyObject *name = Py_NewRef(root->lz_from); + if (first == NULL && + (fromlist == Py_None || + (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist) == 0)) && + _PyImport_IsDefaultImportFunc(tstate->interp, import_func) && + root->lz_builtins == tstate->interp->builtins) { + // A plain import binds the root package. Its pending children are + // imported separately when their attributes are accessed. + Py_ssize_t dot = PyUnicode_FindChar( + name, '.', 0, PyUnicode_GET_LENGTH(name), 1); + if (dot >= 0) { + // Preserve eager traversal through aliased or non-package parents. + int regular = 1; + int complete = 1; + Py_ssize_t end = PyUnicode_GET_LENGTH(name); + for (Py_ssize_t i = dot; i >= 0 && regular; + i = PyUnicode_FindChar(name, '.', i + 1, end, 1)) { + PyObject *prefix = PyUnicode_Substring(name, 0, i); + PyObject *cached = prefix == NULL ? NULL : + lazy_import_get_loaded_attr(tstate, prefix, &_Py_ID(__name__)); + if (cached == NULL) { + complete = 0; + Py_XDECREF(prefix); + break; + } + PyObject *path = lazy_import_get_loaded_attr( + tstate, prefix, &_Py_ID(__path__)); + regular = path != NULL && PyUnicode_Check(cached) && + PyUnicode_Compare(cached, prefix) == 0; + Py_XDECREF(path); + Py_DECREF(cached); + Py_DECREF(prefix); + } + PyObject *loaded = lazy_import_get_loaded_attr( + tstate, name, &_Py_ID(__name__)); + if (regular && (!complete || loaded == NULL) && !PyErr_Occurred()) { + Py_SETREF(name, PyUnicode_Substring(name, 0, dot)); + } + Py_XDECREF(loaded); + if (PyErr_Occurred()) { + Py_CLEAR(name); + } + } + else if (dot == -2) { + Py_CLEAR(name); + } + if (name == NULL) { + goto done; + } + } obj = _PyEval_ImportNameWithImport( tstate, import_func, globals, globals, - root->lz_from, fromlist, _PyLong_GetZero() + name, fromlist, _PyLong_GetZero() ); + if (obj != NULL && name != root->lz_from) { + FT_ATOMIC_STORE_INT_RELAXED(root->lz_active, 1); + FT_ATOMIC_STORE_INT_RELAXED(tstate->interp->imports.has_lazy_submodules, 1); + int package = PyModule_CheckExact(obj) ? + PyDict_Contains(_PyModule_GetDict(obj), &_Py_ID(__path__)) : 0; + if (package < 0) { + Py_CLEAR(obj); + } + else if (!package) { + Py_DECREF(obj); + obj = _PyEval_ImportNameWithImport( + tstate, import_func, globals, globals, + root->lz_from, fromlist, _PyLong_GetZero()); + } + } + if (obj != NULL && name == root->lz_from && + _PyImport_IsDefaultImportFunc(tstate->interp, import_func) && + _PyImport_ClearLazySubmodule(tstate, name, 0) < 0) { + Py_CLEAR(obj); + } + Py_DECREF(name); obj = lazy_import_resolve_result(tstate, obj); // The normal importer may publish this module on its parent. Custom // hooks retain control of their own assignments to the parent. @@ -453,7 +528,7 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, } assert(obj == NULL || !PyLazyImport_CheckExact(obj)); if (obj != NULL) { - PyObject *name = lazy_import_name(lz); + PyObject *name = lazy_import_path(lz); if (name == NULL || _PyImport_DiscardLazyModule(tstate->interp, name) < 0) { Py_CLEAR(obj); @@ -484,6 +559,58 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) return lazy_import_resolve_impl(tstate, lazy_import, NULL); } +// Resolve a pending child as an aliased import, preserving its declaration. +PyObject * +_PyLazyImport_LoadChild(PyThreadState *tstate, PyObject *declaration, + PyObject *name) +{ + assert(PyLazyImport_CheckExact(declaration)); + PyLazyImportObject *source = PyLazyImportObject_CAST(declaration); + PyObject *lz = _PyLazyImport_New(NULL, source->lz_builtins, name, NULL); + Py_ssize_t end = PyUnicode_GET_LENGTH(name); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, 1); + if (dot == -2) { + Py_CLEAR(lz); + } + while (lz != NULL && dot >= 0) { + Py_ssize_t start = dot + 1; + dot = PyUnicode_FindChar(name, '.', start, end, 1); + if (dot == -2) { + Py_CLEAR(lz); + break; + } + PyObject *attr = PyUnicode_Substring(name, start, dot < 0 ? end : dot); + PyObject *next = attr == NULL ? NULL : + _PyLazyImport_New(NULL, NULL, lz, attr); + Py_XDECREF(attr); + Py_SETREF(lz, next); + } + if (lz == NULL) { + return NULL; + } + PyLazyImportObject *child = PyLazyImportObject_CAST(lz); + child->lz_code = (PyCodeObject *)Py_XNewRef(source->lz_code); + child->lz_instr_offset = source->lz_instr_offset; + PyObject *result = _PyImport_LoadLazyImportTstate(tstate, lz); + Py_DECREF(lz); + if (result != NULL && PyModule_Check(result) && + PyUnicode_Compare(source->lz_from, name) != 0 && + !PyDict_Contains(_PyModule_GetDict(result), &_Py_ID(__path__))) { + // Reaching a non-package must still reject an invalid dotted import. + Py_DECREF(result); + return _PyLazyImport_LoadChild(tstate, declaration, source->lz_from); + } + return result; +} + +int +_PyLazyImport_IsActive(PyObject *declaration) +{ + assert(PyLazyImport_CheckExact(declaration)); + return FT_ATOMIC_LOAD_INT_RELAXED( + PyLazyImportObject_CAST(declaration)->lz_active); +} + // Loading pkg.child can replace a placeholder in pkg.child with the module // before a from-import retrieves the value that belongs in that binding. // This is an optimization that can be safely skipped. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 57df195b7f3201..d13b513ab468e4 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1338,7 +1338,8 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) } assert(m->md_dict != NULL); int recheck_dict; - attr = _PyImport_TryLoadLazySubmodule((PyObject *)m, name, &recheck_dict); + attr = _PyImport_TryLoadLazySubmodule((PyObject *)m, name, suppress, + &recheck_dict); if (attr != NULL || PyErr_Occurred()) { return attr; } diff --git a/Python/import.c b/Python/import.c index 823201d38b64ec..4d3e74f45e5e36 100644 --- a/Python/import.c +++ b/Python/import.c @@ -303,6 +303,7 @@ get_importtime_name(PyObject *name) return encoded; } +// Return whether the cached module was still initializing. static int import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name) { @@ -348,7 +349,7 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n #undef import_level } - return 0; + return rc; } static void remove_importlib_frames(PyThreadState *tstate); @@ -4019,7 +4020,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } if (mod != NULL && mod != Py_None) { - if (import_ensure_initialized(tstate->interp, mod, abs_name) < 0) { + int initializing = import_ensure_initialized(tstate->interp, mod, abs_name); + if (initializing < 0) { goto error; } /* Verify the module is still in sys.modules. Another thread may have @@ -4042,6 +4044,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } else { Py_DECREF(mod_check); + if (!initializing && + FT_ATOMIC_LOAD_INT_RELAXED(interp->imports.has_lazy_submodules) && + _PyImport_ClearLazySubmodule(tstate, abs_name, 1) < 0) { + goto error; + } } } else { @@ -4175,7 +4182,7 @@ lazy_modules_add(PyThreadState *tstate, PyObject *name, return loaded ? 0 : PySet_Add(LAZY_MODULES(tstate->interp), name); } -// Ensure a set of pending submodule names exists for the parent. +// Ensure a dict of pending submodule names exists for the parent. // Returns a new reference. static PyObject * ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) @@ -4186,7 +4193,7 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) &lazy_submodules); if (err == 0) { // value isn't present - lazy_submodules = PySet_New(NULL); + lazy_submodules = PyDict_New(); if (lazy_submodules != NULL && _PyDict_SetItem_LockHeld(lazy_modules, parent, lazy_submodules) < 0) { @@ -4202,7 +4209,7 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) // is accessed, _Py_module_getattro_impl will check lazy_pending_submodules // and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *source) { PyDictObject *pending = (PyDictObject *)LAZY_PENDING_SUBMODULES(tstate->interp); @@ -4225,7 +4232,18 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name) PyObject *submodules = ensure_lazy_pending_submodules(pending, parent); Py_DECREF(parent); - int err = submodules == NULL ? -1 : PySet_Add(submodules, child); + PyObject *fullname = PyUnicode_Substring(name, 0, end); + PyObject *cached = fullname == NULL ? NULL : + import_get_module(tstate, fullname); + Py_XDECREF(fullname); + PyObject *value = cached != NULL && cached != Py_None ? Py_None : source; + int err = -1; + if (submodules != NULL && !PyErr_Occurred()) { + err = source == Py_None ? + PyDict_SetDefaultRef(submodules, child, value, NULL) : + PyDict_SetItem(submodules, child, value); + } + Py_XDECREF(cached); Py_DECREF(child); Py_XDECREF(submodules); if (err < 0) { @@ -4256,15 +4274,92 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *existing_module, rc = lazy_modules_add(tstate, fromname, NULL); if (rc == 0) { - rc = register_lazy_on_parent(tstate, fromname); + rc = register_lazy_on_parent(tstate, fromname, Py_None); } Py_DECREF(fromname); return rc; } +// Release completed declarations, retaining names while their parent may fail. +// Cached eager imports also restore a child removed after failed validation. +int +_PyImport_ClearLazySubmodule(PyThreadState *tstate, PyObject *name, int bind) +{ + if (LAZY_PENDING_SUBMODULES(tstate->interp) == NULL || + PyDict_GET_SIZE(LAZY_PENDING_SUBMODULES(tstate->interp)) == 0) { + return 0; + } + Py_ssize_t end = PyUnicode_GET_LENGTH(name); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, end, -1); + if (dot < 0) { + return dot == -2 ? -1 : 0; + } + PyObject *parent = PyUnicode_Substring(name, 0, dot); + PyObject *child = PyUnicode_Substring(name, dot + 1, end); + PyObject *pending = NULL; + int rc = -1; + if (parent != NULL && child != NULL) { + rc = PyDict_GetItemRef(LAZY_PENDING_SUBMODULES(tstate->interp), + parent, &pending); + if (rc > 0) { + rc = PyDict_Contains(pending, child); + } + if (rc > 0) { + PyObject *loaded = import_get_module(tstate, name), *loaded_spec = NULL; + int loading = loaded == NULL ? (PyErr_Occurred() ? -1 : 0) : + PyObject_GetOptionalAttr(loaded, &_Py_ID(__spec__), &loaded_spec); + if (loading > 0) { + loading = _PyModuleSpec_IsInitializing(loaded_spec); + } + Py_XDECREF(loaded_spec); + Py_XDECREF(loaded); + if (loading != 0) { + if (PyErr_ExceptionMatches(PyExc_Exception)) { + PyErr_Clear(); + loading = 1; + } + rc = loading < 0 ? -1 : 0; + goto done; + } + PyObject *module = import_get_module(tstate, parent); + PyObject *spec = NULL; + int initializing = module == NULL ? (PyErr_Occurred() ? -1 : 0) : + PyObject_GetOptionalAttr(module, &_Py_ID(__spec__), &spec); + if (initializing > 0) { + initializing = _PyModuleSpec_IsInitializing(spec); + } + Py_XDECREF(spec); + if (bind && initializing == 0 && module != NULL && + PyModule_CheckExact(module) && PyDict_Contains(pending, child) > 0) { + PyObject *value = import_get_module(tstate, name); + if (value != NULL && value != Py_None) { + initializing = PyDict_SetDefaultRef( + _PyModule_GetDict(module), child, value, NULL) < 0 ? -1 : 0; + } + Py_XDECREF(value); + } + Py_XDECREF(module); + rc = initializing < 0 ? -1 : + (initializing ? PyDict_SetItem(pending, child, Py_None) : + PyDict_Pop(pending, child, NULL)); + } + } +done: + Py_XDECREF(pending); + Py_XDECREF(parent); + Py_XDECREF(child); + return rc < 0 ? -1 : 0; +} + +static int +lazy_submodule_matches(PyObject *value, void *expected) +{ + return value == expected; +} + PyObject * _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, - int *recheck_dict) + int suppress, int *recheck_dict) { *recheck_dict = 0; PyObject *mod_dict = _PyModule_GetDict(module); @@ -4272,7 +4367,7 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, if (PyDict_GetItemRef(mod_dict, &_Py_ID(__name__), &mod_name) <= 0) { return NULL; } - PyObject *mod = NULL, *pending_set = NULL; + PyObject *mod = NULL, *pending_set = NULL, *source = NULL; if (!PyUnicode_Check(mod_name)) { goto done; } @@ -4284,7 +4379,7 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, } *recheck_dict = 1; - if (PySet_Contains(pending_set, attr_name) <= 0) { + if (PyDict_GetItemRef(pending_set, attr_name, &source) <= 0) { goto done; } PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); @@ -4292,24 +4387,53 @@ _PyImport_TryLoadLazySubmodule(PyObject *module, PyObject *attr_name, goto done; } // Match eager from-imports: accept a partial module in an import cycle. - mod = PyImport_GetModule(full_name); - if ((mod == NULL && !PyErr_Occurred()) || mod == Py_None) { - Py_XDECREF(mod); - mod = import_find_and_load_with_name( - tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); + if (source != Py_None && !suppress && + (_PyLazyImport_IsActive(source) || + !PyDict_Contains(mod_dict, &_Py_ID(__getattr__)))) { + mod = _PyLazyImport_LoadChild(tstate, source, full_name); + } + else { + mod = PyImport_GetModule(full_name); + if ((mod == NULL && !PyErr_Occurred()) || mod == Py_None) { + Py_XDECREF(mod); + mod = import_find_and_load_with_name( + tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); + } } - Py_DECREF(full_name); if (mod == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + PyObject *loaded = import_get_module(tstate, full_name); + if (loaded != NULL && loaded != Py_None) { + (void)_PyDict_DelItemIf(mod_dict, attr_name, + lazy_submodule_matches, loaded); + (void)PyDict_SetDefaultRef(pending_set, attr_name, source, NULL); + } + Py_XDECREF(loaded); + PyErr_SetRaisedException(exc); remove_importlib_frames(tstate); } /* Keep missing children pending: a finder may provide them later. */ - else if (mod == Py_None || - PyDict_SetItem(mod_dict, attr_name, mod) < 0 || - PySet_Discard(pending_set, attr_name) < 0) { - Py_CLEAR(mod); + else { + PyObject *spec = NULL; + int rc = mod == Py_None ? 0 : + PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec); + int initializing = rc > 0 ? _PyModuleSpec_IsInitializing(spec) : rc; + Py_XDECREF(spec); + if (initializing < 0 && PyErr_ExceptionMatches(PyExc_Exception)) { + PyErr_Clear(); + initializing = 0; + } + if (mod == Py_None || initializing < 0 || + (initializing == 0 && + (PyDict_SetItem(mod_dict, attr_name, mod) < 0 || + PyDict_Pop(pending_set, attr_name, NULL) < 0))) { + Py_CLEAR(mod); + } } + Py_DECREF(full_name); done: + Py_XDECREF(source); Py_XDECREF(pending_set); Py_DECREF(mod_name); return mod; @@ -4425,7 +4549,7 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, } } } - else if (register_lazy_on_parent(tstate, abs_name) < 0) { + else if (register_lazy_on_parent(tstate, abs_name, res) < 0) { goto error; } @@ -5454,6 +5578,9 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { return NULL; } + if (_PyImport_ClearLazySubmodule(_PyThreadState_GET(), name, 0) < 0) { + return NULL; + } Py_RETURN_NONE; } From 20d5e67fc51eee8cbf7e38d75830e5edd140d37e Mon Sep 17 00:00:00 2001 From: Bhuvansh Date: Sat, 3 Oct 2026 01:52:05 +0530 Subject: [PATCH 046/240] gh-156204: Guard recursion in `PyErr_GivenExceptionMatches` (GH-156205) --- Lib/test/test_exceptions.py | 26 +++++++++++++++ ...-08-21-18-49-12.gh-issue-156204.ZusA7e.rst | 3 ++ Modules/_testcapi/clinic/exceptions.c.h | 33 ++++++++++++++++++- Modules/_testcapi/exceptions.c | 21 ++++++++++++ Python/errors.c | 29 +++++++++++----- 5 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-18-49-12.gh-issue-156204.ZusA7e.rst diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 4cdd269c7761ff..963da1b0eae27c 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2806,6 +2806,32 @@ def test_except_star_invalid_exception_type(self): except (ValueError, 42): pass + @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_given_exception_matches_nested_tuple(self): + # Nested tuples are searched recursively. + self.assertTrue( + _testcapi.err_givenexceptionmatches(ValueError(), ((ValueError,),))) + self.assertFalse( + _testcapi.err_givenexceptionmatches(TypeError(), ((ValueError,),))) + + @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + @support.run_with_limited_c_stack(depth=500_000) + def test_given_exception_matches_deeply_nested_tuple(self): + # gh-156204: PyErr_GivenExceptionMatches() used to exhaust the C stack + # and crash the interpreter on deeply nested tuples of exception types. + tup = (ValueError,) + for _ in range(500_000): + tup = (tup,) + + with support.catch_unraisable_exception() as cm: + self.assertFalse( + _testcapi.err_givenexceptionmatches(ValueError(), tup)) + self.assertIsInstance(cm.unraisable.exc_value, RecursionError) + class PEP626Tests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-18-49-12.gh-issue-156204.ZusA7e.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-18-49-12.gh-issue-156204.ZusA7e.rst new file mode 100644 index 00000000000000..fade87c2d09ae1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-18-49-12.gh-issue-156204.ZusA7e.rst @@ -0,0 +1,3 @@ +Fix a crash in :c:func:`PyErr_GivenExceptionMatches` when evaluating deeply +nested exception tuples. The recursion is now bounded, and exceeding the +limit is reported as an unraisable exception. diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index cac5d288f437b7..89b78923bdb2d0 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -16,6 +16,37 @@ PyDoc_STRVAR(_testcapi_err_set_raised__doc__, #define _TESTCAPI_ERR_SET_RAISED_METHODDEF \ {"err_set_raised", (PyCFunction)_testcapi_err_set_raised, METH_O, _testcapi_err_set_raised__doc__}, +PyDoc_STRVAR(_testcapi_err_givenexceptionmatches__doc__, +"err_givenexceptionmatches($module, err, exc, /)\n" +"--\n" +"\n" +"Test PyErr_GivenExceptionMatches()."); + +#define _TESTCAPI_ERR_GIVENEXCEPTIONMATCHES_METHODDEF \ + {"err_givenexceptionmatches", _PyCFunction_CAST(_testcapi_err_givenexceptionmatches), METH_FASTCALL, _testcapi_err_givenexceptionmatches__doc__}, + +static PyObject * +_testcapi_err_givenexceptionmatches_impl(PyObject *module, PyObject *err, + PyObject *exc); + +static PyObject * +_testcapi_err_givenexceptionmatches(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *err; + PyObject *exc; + + if (!_PyArg_CheckPositional("err_givenexceptionmatches", nargs, 2, 2)) { + goto exit; + } + err = args[0]; + exc = args[1]; + return_value = _testcapi_err_givenexceptionmatches_impl(module, err, exc); + +exit: + return return_value; +} + PyDoc_STRVAR(_testcapi_exception_print__doc__, "exception_print($module, exception, legacy=False, /)\n" "--\n" @@ -459,4 +490,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=357caea020348789 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a21ce5554900dba1 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index c0254e044bc2d5..cc62b47c99414a 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -54,6 +54,26 @@ err_restore(PyObject *self, PyObject *args) { return NULL; } +/*[clinic input] +_testcapi.err_givenexceptionmatches + err: object + exc: object + / + +Test PyErr_GivenExceptionMatches(). +[clinic start generated code]*/ + +static PyObject * +_testcapi_err_givenexceptionmatches_impl(PyObject *module, PyObject *err, + PyObject *exc) +/*[clinic end generated code: output=e40994ab6dd75001 input=7b8ef542df07575b]*/ +{ + assert(!PyErr_Occurred()); + int res = PyErr_GivenExceptionMatches(err, exc); + assert(!PyErr_Occurred()); + return PyBool_FromLong(res); +} + /*[clinic input] _testcapi.exception_print exception as exc: object @@ -552,6 +572,7 @@ static PyMethodDef test_methods[] = { _TESTCAPI_MAKE_EXCEPTION_WITH_DOC_METHODDEF _TESTCAPI_EXC_SET_OBJECT_METHODDEF _TESTCAPI_EXC_SET_OBJECT_FETCH_METHODDEF + _TESTCAPI_ERR_GIVENEXCEPTIONMATCHES_METHODDEF _TESTCAPI_ERR_SETSTRING_METHODDEF _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF _TESTCAPI_RAISE_EXCEPTION_METHODDEF diff --git a/Python/errors.c b/Python/errors.c index eb148998fc4652..edb1557e23f63b 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_audit.h" // _PySys_Audit() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _Py_ReachedRecursionLimitWithMargin() #include "pycore_fileutils.h" // _PyFile_Flush #include "pycore_initconfig.h" // _PyStatus_ERR() #include "pycore_pyerrors.h" // _PyErr_Format() @@ -337,17 +338,27 @@ PyErr_GivenExceptionMatches(PyObject *err, PyObject *exc) return 0; } if (PyTuple_Check(exc)) { - Py_ssize_t i, n; - n = PyTuple_Size(exc); - for (i = 0; i < n; i++) { + PyThreadState *tstate = _PyThreadState_GET(); + if (_Py_ReachedRecursionLimitWithMargin(tstate, 2)) { + PyObject *exc_value = _PyErr_GetRaisedException(tstate); + _PyErr_SetString(tstate, PyExc_RecursionError, + "maximum recursion depth exceeded while " + "checking exception tuple"); + PyErr_FormatUnraisable("Exception ignored while " + "checking exception tuple"); + _PyErr_SetRaisedException(tstate, exc_value); + return 0; + } + int res = 0; + Py_ssize_t n = PyTuple_GET_SIZE(exc); + for (Py_ssize_t i = 0; i < n; i++) { /* Test recursively */ - if (PyErr_GivenExceptionMatches( - err, PyTuple_GET_ITEM(exc, i))) - { - return 1; - } + if (PyErr_GivenExceptionMatches(err, PyTuple_GET_ITEM(exc, i))) { + res = 1; + break; + } } - return 0; + return res; } /* err might be an instance, so check its class. */ if (PyExceptionInstance_Check(err)) From 8ba1f31def0d79f230728fbf0fe400755c0856c2 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 2 Oct 2026 22:57:12 +0200 Subject: [PATCH 047/240] gh-150816: Speed up inspect.signature() for Python functions (#157463) * Speed up inspect.signature() for Python functions For real Python functions the parameter kinds and defaults come straight from the code object, so the validation done by Parameter.__init__ is redundant. Construct the Parameter objects directly, falling back to the regular constructor for names that are not plain identifiers (e.g. the ".0" implicit argument of comprehensions). Parameter subclasses and duck-typed functions keep the full validation. inspect.signature() on a function with 7 parameters: 23.4 us -> 14.5 us. Co-Authored-By: Claude Opus 5 (1M context) * Make the fast Parameter constructor a classmethod and add tests Move the helper to Parameter._from_code(). Add tests that a Signature subclass overriding _parameter_cls still gets its own Parameter class (the concern raised on gh-150823), and that parameter names of function-like objects are still validated. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- Lib/inspect.py | 14 +++++ Lib/test/test_inspect/test_inspect.py | 56 +++++++++++++++++++ ...-09-13-21-45-00.gh-issue-150816.pQ3kZs.rst | 2 + 3 files changed, 72 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-09-13-21-45-00.gh-issue-150816.pQ3kZs.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 3683f8c3fd5330..bceed35a53abba 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2348,6 +2348,8 @@ def _signature_from_function(cls, func, skip_bound_arg=True, return _signature_fromstr(cls, func, s, skip_bound_arg) Parameter = cls._parameter_cls + if Parameter is Signature._parameter_cls and not is_duck_function: + Parameter = Parameter._from_code # Parameter information. func_code = func.__code__ @@ -2746,6 +2748,18 @@ def __init__(self, name, kind, *, default=_empty, annotation=_empty): self._name = name + @classmethod + def _from_code(cls, name, kind, *, default=_empty, annotation=_empty): + # Fast path for Python functions: only the name needs validation. + if iskeyword(name) or not name.isidentifier(): + return cls(name, kind, default=default, annotation=annotation) + self = object.__new__(cls) + self._name = name + self._kind = kind + self._default = default + self._annotation = annotation + return self + def __reduce__(self): return (type(self), (self._name, self._kind), diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 38c7a9bde823d9..3f14a9e66226ea 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3556,6 +3556,48 @@ def __init__(self, marker): self.assertEqual(str(inspect.signature(funclike)), '(marker)') + @cpython_only + def test_signature_functionlike_invalid_names(self): + # The code object of a function-like object is not guaranteed + # to have valid parameter names, so they must be validated. + def func(a, b): + pass + + class funclike: + __name__ = func.__name__ + __code__ = func.__code__.replace(co_varnames=('a', '$b')) + __annotations__ = {} + __defaults__ = None + __kwdefaults__ = None + + def __call__(self, *args): + pass + + with self.assertRaisesRegex(ValueError, + 'is not a valid parameter name'): + inspect.signature(funclike()) + + def test_signature_parameter_cls_subclass(self): + # A Signature subclass can override _parameter_cls with a + # Parameter subclass that has its own constructor. + class MyParameter(inspect.Parameter): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.extra = 'spam' + + class MySignature(inspect.Signature): + _parameter_cls = MyParameter + + def f(a, /, b=1, *args, c, d=2, **kwargs): + pass + + sig = MySignature.from_callable(f) + self.assertEqual(len(sig.parameters), 6) + for param in sig.parameters.values(): + self.assertIs(type(param), MyParameter) + self.assertEqual(param.extra, 'spam') + self.assertEqual(sig, inspect.signature(f)) + def test_signature_on_method(self): class Test: def __init__(*args): @@ -5656,6 +5698,20 @@ def test_signature_parameter_implicit(self): self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_ONLY) self.assertEqual(param.name, 'implicit0') + @cpython_only + def test_signature_from_code_unusual_names(self): + def f(a, b): pass + f.__code__ = f.__code__.replace(co_varnames=('.0', 'b')) + sig = inspect.signature(f) + self.assertEqual(list(sig.parameters), ['implicit0', 'b']) + self.assertEqual(sig.parameters['implicit0'].kind, + inspect.Parameter.POSITIONAL_ONLY) + + f.__code__ = f.__code__.replace(co_varnames=('if', 'b')) + with self.assertRaisesRegex(ValueError, + 'is not a valid parameter name'): + inspect.signature(f) + def test_signature_parameter_immutability(self): p = inspect.Parameter('spam', kind=inspect.Parameter.KEYWORD_ONLY) diff --git a/Misc/NEWS.d/next/Library/2026-09-13-21-45-00.gh-issue-150816.pQ3kZs.rst b/Misc/NEWS.d/next/Library/2026-09-13-21-45-00.gh-issue-150816.pQ3kZs.rst new file mode 100644 index 00000000000000..4f67195f45fc6d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-13-21-45-00.gh-issue-150816.pQ3kZs.rst @@ -0,0 +1,2 @@ +Speed up :func:`inspect.signature` for Python functions by skipping +redundant validation when creating :class:`inspect.Parameter` objects. From b4197da019f4a77a649a17e0b9c4ddf7c137229c Mon Sep 17 00:00:00 2001 From: Taeknology <20297177+Taeknology@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:01:21 +0900 Subject: [PATCH 048/240] gh-150942: Speed up BytesIO.readlines() using reference-stealing append (#155171) * gh-150942: Use reference-stealing append in BytesIO.readlines * gh-150942: Add NEWS entry for BytesIO.readlines optimization * gh-150942: Fix NEWS cross-reference * gh-150942: Simplify BytesIO.readlines NEWS entry --- .../Library/2026-08-04-19-50-52.gh-issue-150942.B8ytes.rst | 2 ++ Modules/_io/bytesio.c | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-04-19-50-52.gh-issue-150942.B8ytes.rst diff --git a/Misc/NEWS.d/next/Library/2026-08-04-19-50-52.gh-issue-150942.B8ytes.rst b/Misc/NEWS.d/next/Library/2026-08-04-19-50-52.gh-issue-150942.B8ytes.rst new file mode 100644 index 00000000000000..0ffe54b9cd2cbd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-04-19-50-52.gh-issue-150942.B8ytes.rst @@ -0,0 +1,2 @@ +Improve the performance of :meth:`~io.IOBase.readlines` for +:class:`io.BytesIO`. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index ff9cc0f2166bd7..1f97450f53beec 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytes_ResizeKeepOnError() #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_list.h" // _PyList_AppendTakeRef() #include "pycore_object.h" #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_sysmodule.h" // _PySys_GetSizeOf() @@ -664,11 +665,9 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) line = PyBytes_FromStringAndSize(output, n); if (!line) goto on_error; - if (PyList_Append(result, line) == -1) { - Py_DECREF(line); + if (_PyList_AppendTakeRef((PyListObject *)result, line) < 0) { goto on_error; } - Py_DECREF(line); size += n; if (maxsize > 0 && size >= maxsize) break; From 1a85213940896f180cbcf76921cd2d35a8aa50de Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2026 00:34:38 +0200 Subject: [PATCH 049/240] gh-158585: Adjust allocation in bytes_fromformat() (#158621) Take in account preallocated bytes when calling PyBytesWriter_GrowAndUpdatePointer(). Use sprintf() return value instead of calling strlen(). --- Lib/test/test_capi/test_bytes.py | 32 ++++++++- Objects/bytesobject.c | 120 +++++++++++++++++-------------- 2 files changed, 96 insertions(+), 56 deletions(-) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index f8487a2b8fb085..910f2bedac0bcb 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -607,15 +607,45 @@ def test_format_i(self): writer = self.create_writer() writer.format_i(b'x=%i, ', 123) - writer.format_i(b'y=%i', 456) + writer.format_i(b'y=%d', 456) self.bytes_equal(writer.finish(), b'x=123, y=456') + # Make sure that %i allocates the right number of bytes + n_str = '1234567' + for i in range(1, len(n_str) + 1): + n = int(n_str[:i]) + writer = self.create_writer(0) + writer.format_i(b'n=%i', n) + self.bytes_equal(writer.finish(), f'n={n}'.encode()) + + # Test invalid format: "%t" is invalid, stop the parser and copy + # remaining bytes + writer = self.create_writer() + writer.write_bytes(b'prefix ', -1) + writer.format_i(b'y=%._ t %i', 0) + writer.write_bytes(b'.', 1) + self.bytes_equal(writer.finish(), b'prefix y=%._ t %i.') + def test_format_s(self): # Test PyBytesWriter_Format() writer = self.create_writer() writer.format_s(b's=%s', b'Hello World') self.bytes_equal(writer.finish(), b's=Hello World') + # Test precision + writer = self.create_writer() + writer.format_s(b'%.6s', b'Monty Python') + writer.format_s(b'%.20s', b'3.0') + self.bytes_equal(writer.finish(), b'Monty 3.0') + + # Make sure that %s allocates the right number of bytes + text = b'1234567' + for i in range(1, len(text) + 1): + s = text[:i] + writer = self.create_writer(0) + writer.format_s(b's=%s', s) + self.bytes_equal(writer.finish(), b's=' + s) + @support.nomemtest def test_format_s_memory_error(self): writer = self.create_writer() diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index cb2c2f7bfae634..2339e8bc1d2819 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -194,73 +194,77 @@ static char* bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, const char *format, va_list vargs) { - const char *f; - const char *p; - Py_ssize_t prec; - int longflag; - int size_tflag; /* Longest 64-bit formatted numbers: - "18446744073709551615\0" (21 bytes) - "-9223372036854775808\0" (21 bytes) Decimal takes the most space (it isn't enough for octal.) Longest 64-bit pointer representation: - "0xffffffffffffffff\0" (19 bytes). */ + "0xffffffffffffffff\0" (19 bytes). + + Longest 64-bit "%p" with "0x" prefix: len(hex(2**64-1)+'\0') = 19. + */ char buffer[21]; char *s = (char*)PyBytesWriter_GetData(writer) + writer_pos; -#define WRITE_BYTES_LEN(str, len_expr) \ +#define WRITE_BYTES(str, len_expr) \ do { \ - size_t len = (len_expr); \ - s = PyBytesWriter_GrowAndUpdatePointer(writer, len, s); \ - if (s == NULL) { \ - return NULL; \ + size_t _len = (len_expr); \ + size_t _prealloc = (f - p + 1); \ + if (_len > _prealloc) { \ + s = PyBytesWriter_GrowAndUpdatePointer(writer, _len - _prealloc, s); \ + if (s == NULL) { \ + return NULL; \ + } \ } \ - memcpy(s, (str), len); \ - s += len; \ + memcpy(s, (str), _len); \ + s += _len; \ } while (0) -#define WRITE_BYTES(str) WRITE_BYTES_LEN(str, strlen(str)) - for (f = format; *f; f++) { + for (const char *f = format; *f; f++) { if (*f != '%') { *s++ = *f; continue; } - p = f++; + const char *p = f++; /* ignore the width (ex: 10 in "%10s") */ while (Py_ISDIGIT(*f)) f++; /* parse the precision (ex: 10 in "%.10s") */ - prec = 0; + Py_ssize_t prec = 0; if (*f == '.') { f++; for (; Py_ISDIGIT(*f); f++) { prec = (prec * 10) + (*f - '0'); } } + assert(prec >= 0); - while (*f && *f != '%' && !Py_ISALPHA(*f)) + while (*f && *f != '%' && !Py_ISALPHA(*f)) { f++; + } /* handle the long flag ('l'), but only for %ld and %lu. others can be added when necessary. */ - longflag = 0; + int longflag = 0; if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { longflag = 1; ++f; } /* handle the size_t flag ('z'). */ - size_tflag = 0; + int size_tflag = 0; if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { size_tflag = 1; ++f; } + Py_ssize_t len; + switch (*f) { case 'c': { @@ -277,74 +281,77 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, case 'd': if (longflag) { - sprintf(buffer, "%ld", va_arg(vargs, long)); + len = sprintf(buffer, "%ld", va_arg(vargs, long)); } else if (size_tflag) { - sprintf(buffer, "%zd", va_arg(vargs, Py_ssize_t)); + len = sprintf(buffer, "%zd", va_arg(vargs, Py_ssize_t)); } else { - sprintf(buffer, "%d", va_arg(vargs, int)); + len = sprintf(buffer, "%d", va_arg(vargs, int)); } - assert(strlen(buffer) < sizeof(buffer)); - WRITE_BYTES(buffer); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); + WRITE_BYTES(buffer, len); break; case 'u': if (longflag) { - sprintf(buffer, "%lu", va_arg(vargs, unsigned long)); + len = sprintf(buffer, "%lu", va_arg(vargs, unsigned long)); } else if (size_tflag) { - sprintf(buffer, "%zu", va_arg(vargs, size_t)); + len = sprintf(buffer, "%zu", va_arg(vargs, size_t)); } else { - sprintf(buffer, "%u", va_arg(vargs, unsigned int)); + len = sprintf(buffer, "%u", va_arg(vargs, unsigned int)); } - assert(strlen(buffer) < sizeof(buffer)); - WRITE_BYTES(buffer); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); + WRITE_BYTES(buffer, len); break; case 'i': - sprintf(buffer, "%i", va_arg(vargs, int)); - assert(strlen(buffer) < sizeof(buffer)); - WRITE_BYTES(buffer); + len = sprintf(buffer, "%i", va_arg(vargs, int)); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); + WRITE_BYTES(buffer, len); break; case 'x': - sprintf(buffer, "%x", va_arg(vargs, int)); - assert(strlen(buffer) < sizeof(buffer)); - WRITE_BYTES(buffer); + len = sprintf(buffer, "%x", va_arg(vargs, int)); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); + WRITE_BYTES(buffer, len); break; case 's': { - Py_ssize_t i; - - p = va_arg(vargs, const char*); - if (prec <= 0) { - i = strlen(p); + const char *str = va_arg(vargs, const char*); + if (prec == 0) { + len = strlen(str); } else { - i = 0; - while (i < prec && p[i]) { - i++; + const char *end = memchr(str, 0, prec); + if (end != NULL) { + len = (end - str); + } + else { + len = (size_t)prec; } } - WRITE_BYTES_LEN(p, i); + WRITE_BYTES(str, len); break; } case 'p': - sprintf(buffer, "%p", va_arg(vargs, void*)); - assert(strlen(buffer) < sizeof(buffer)); + len = sprintf(buffer, "%p", va_arg(vargs, void*)); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); /* %p is ill-defined: ensure leading 0x. */ - if (buffer[1] == 'X') + if (buffer[1] == 'X') { buffer[1] = 'x'; + } else if (buffer[1] != 'x') { - memmove(buffer+2, buffer, strlen(buffer)+1); + memmove(buffer + 2, buffer, len + 1); buffer[0] = '0'; buffer[1] = 'x'; + len += 2; } - WRITE_BYTES(buffer); + WRITE_BYTES(buffer, len); break; case '%': @@ -352,16 +359,19 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, break; default: - /* invalid format string: copy unformatted string and exit */ - WRITE_BYTES(p); + // Invalid format string: copy unformatted string and exit. + // No need to grow the writer buffer, we already preallocated + // enough bytes. + len = strlen(p); + memcpy(s, p, len); + s += len; return s; } } -#undef WRITE_BYTES -#undef WRITE_BYTES_LEN - return s; + +#undef WRITE_BYTES } From f5e913c7f710794421e48437f45a65f190830cbf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2026 15:15:08 +0200 Subject: [PATCH 050/240] gh-158445: Allocate memory in the heap in Py_GetVersion() (#158608) Py_GetVersion() now allocates memory on the heap, instead of using a static buffer, to no longer truncate the version if it's longer than 299 bytes. Update Py_GetCompiler() and Py_GetBuildInfo() tests: they are now always a part of sys.version. --- Doc/c-api/interp-lifecycle.rst | 2 +- Include/internal/pycore_pylifecycle.h | 3 +- Lib/test/test_capi/test_build.py | 8 +--- Modules/main.c | 1 + Python/getcompiler.c | 9 ++-- Python/getversion.c | 67 ++++++++++++++++++++++----- Python/pylifecycle.c | 6 ++- Tools/c-analyzer/cpython/ignored.tsv | 3 +- 8 files changed, 74 insertions(+), 25 deletions(-) diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index efd208263bdf33..226fb8fb4b14b3 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -514,7 +514,7 @@ Process-wide parameters Return the version of this Python interpreter. This is a string that looks something like :: - "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" + "3.15.0rc2 (3.15.0~rc2-1.fc44.x86_64, Sep 3 2026, 00:00:00) [GCC 16.2.1 20260819 (Red Hat 16.2.1-2)]" .. index:: single: version (in module sys) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index bfc94e3e8529b7..99d09c5e0d4e11 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -23,7 +23,7 @@ extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc); /* Various one-time initializers */ -extern void _Py_InitVersion(void); +extern PyStatus _Py_GetVersion_Init(void); extern PyStatus _PyFaulthandler_Init(int enable); extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); extern int _PyBuiltin_InitPythonFunctions(PyObject *dict); @@ -43,6 +43,7 @@ extern PyStatus _PyDateTime_InitTypes(PyInterpreterState *interp); /* Various internal finalizers */ +extern void _Py_GetVersion_Fini(void); extern int _PySignal_Init(int install_signal_handlers); extern void _PySignal_Fini(void); diff --git a/Lib/test/test_capi/test_build.py b/Lib/test/test_capi/test_build.py index 8319ad1ec77da8..d45c9dbc1c5882 100644 --- a/Lib/test/test_capi/test_build.py +++ b/Lib/test/test_capi/test_build.py @@ -20,15 +20,11 @@ def test_getcopyright(self): # Test Py_GetCompiler() def test_getcompiler(self): - compiler = _testlimitedcapi.py_getcompiler() - self.assertIsInstance(compiler, str) - self.assertNotEqual(compiler, '') + self.assertIn(_testlimitedcapi.py_getcompiler(), sys.version) # Test Py_GetBuildInfo() def test_getbuildinfo(self): - buildinfo = _testlimitedcapi.py_getbuildinfo() - self.assertIsInstance(buildinfo, str) - self.assertNotEqual(buildinfo, '') + self.assertIn(_testlimitedcapi.py_getbuildinfo(), sys.version) # Test internal _Py_GetBuiltWithAssert() def test__getbuiltwithassert(self): diff --git a/Modules/main.c b/Modules/main.c index 44a025d500e35e..097c10504d69a3 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -847,6 +847,7 @@ pymain_free(void) _PyPathConfig_ClearGlobal(); _Py_ClearArgcArgv(); _PyRuntime_Finalize(); + _Py_GetVersion_Fini(); } diff --git a/Python/getcompiler.c b/Python/getcompiler.c index cc56ad8c895551..baa03679d79fb8 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -3,6 +3,7 @@ #include "Python.h" +// PC/pyconfig.h defines _Py_COMPILER #ifdef _Py_COMPILER # define COMPILER _Py_COMPILER #endif @@ -12,14 +13,14 @@ // Note the __clang__ conditional has to come before the __GNUC__ one because // clang pretends to be GCC. #if defined(__clang__) -#define COMPILER "[Clang " __clang_version__ "]" +# define COMPILER "[Clang " __clang_version__ "]" #elif defined(__GNUC__) -#define COMPILER "[GCC " __VERSION__ "]" +# define COMPILER "[GCC " __VERSION__ "]" // Generic fallbacks. #elif defined(__cplusplus) -#define COMPILER "[C++]" +# define COMPILER "[C++]" #else -#define COMPILER "[C]" +# define COMPILER "[C]" #endif #endif /* !COMPILER */ diff --git a/Python/getversion.c b/Python/getversion.c index 8d8bc6ea70048c..5c05b841de6fc8 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -2,32 +2,77 @@ /* Return the full version string. */ #include "Python.h" +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() -#include "patchlevel.h" static int initialized = 0; -static char version[300]; +// Use the smallest possible buffer: it's only needed if malloc() fails *and* +// Py_GetVersion() is called before Py_Initialize(). The buffer should be big +// enough to store "3.16.0a0 " string. +static char static_version[20]; +static char *heap_version = NULL; -void _Py_InitVersion(void) + +PyStatus +_Py_GetVersion_Init(void) { if (initialized) { - return; + return _PyStatus_OK(); } - initialized = 1; + #ifdef Py_GIL_DISABLED - const char *buildinfo_format = "%.80s free-threading build (%.80s) %.80s"; + const char *format = "%s free-threading build (%s) %s"; + size_t format_len = strlen(" free-threading build () "); #else - const char *buildinfo_format = "%.80s (%.80s) %.80s"; + const char *format = "%s (%s) %s"; + size_t format_len = strlen(" () "); #endif - PyOS_snprintf(version, sizeof(version), buildinfo_format, - PY_VERSION, Py_GetBuildInfo(), Py_GetCompiler()); + const char *version_str = PY_VERSION; + const char *buildinfo = Py_GetBuildInfo(); + const char *compiler = Py_GetCompiler(); + // +1 for the trailing NUL byte + size_t len = (format_len + strlen(version_str) + strlen(buildinfo) + + strlen(compiler) + 1); + + // Always format the static version + PyOS_snprintf(static_version, sizeof(static_version), format, + version_str, buildinfo, compiler); + + heap_version = malloc(len); + if (heap_version == NULL) { + // If malloc() failed, don't set initialized to 1, so next + // Py_GetVersion() will try again to allocate memory. + return _PyStatus_NO_MEMORY(); + } + + PyOS_snprintf(heap_version, len, format, + version_str, buildinfo, compiler); + initialized = 1; + return _PyStatus_OK(); +} + +void +_Py_GetVersion_Fini(void) +{ + if (heap_version) { + free(heap_version); + heap_version = NULL; + } } const char * Py_GetVersion(void) { - _Py_InitVersion(); - return version; + PyStatus status = _Py_GetVersion_Init(); + // Ignore error: Py_GetVersion() API cannot report error + (void)status; + + if (heap_version) { + return heap_version; + } + else { + return static_version; + } } // Export the Python hex version as a constant. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8888f523559123..ba1b94c47984ab 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -559,7 +559,11 @@ pycore_init_runtime(_PyRuntimeState *runtime, */ _PyRuntimeState_SetFinalizing(runtime, NULL); - _Py_InitVersion(); + status = _Py_GetVersion_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _Py_DumpTraceback_Init(); status = _Py_HashRandomization_Init(config); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 2ad801c671855c..220e84ef1f60c2 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -48,7 +48,8 @@ Objects/unicodeobject.c _init_global_state initialized - Modules/getbuildinfo.c - buildinfo - Modules/getbuildinfo.c - initialized - Python/getversion.c - initialized - -Python/getversion.c - version - +Python/getversion.c - static_version - +Python/getversion.c - heap_version - ## public C-API - set during first init Python/bootstrap_hash.c - _Py_HashSecret_Initialized - From b42fdf6b44b6701dc643c733983fff4500e077c4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2026 15:37:58 +0200 Subject: [PATCH 051/240] Fix _PyOS_GetOpt() error message for non-ASCII characters (#158626) Replace %c with %lc. --- Lib/test/test_cmd_line.py | 2 +- Python/getopt.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 6d07238a9d08ab..1e303dc3bf6770 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -684,7 +684,7 @@ def test_del___main__(self): @support.cpython_only def test_unknown_options(self): - # Test unknown option -a + # Test unknown option for option in ('-z', '--long-option', '---'): with self.subTest(option=option): rc, out, err = assert_python_failure('-E', option) diff --git a/Python/getopt.c b/Python/getopt.c index 45454d39fc6779..2dcabc9962d25f 100644 --- a/Python/getopt.c +++ b/Python/getopt.c @@ -160,7 +160,7 @@ _PyOS_GetOpt(struct _PyOS_GetOpt *getopt) wchar_t *ptr = wcschr(SHORT_OPTS, option); if (ptr == NULL) { if (error) { - fprintf(stderr, "Unknown option: -%c\n", (char)option); + fprintf(stderr, "Unknown option: -%lc\n", option); } return '_'; } @@ -174,7 +174,7 @@ _PyOS_GetOpt(struct _PyOS_GetOpt *getopt) if (getopt->index >= argc) { if (error) { fprintf(stderr, - "Argument expected for the -%c option\n", (char)option); + "Argument expected for the -%lc option\n", option); } return '_'; } From 9a1544073ff63d57bc723b9913788fccc91f4cc0 Mon Sep 17 00:00:00 2001 From: Sacul <183588943+Sacul0457@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:06:44 +0800 Subject: [PATCH 052/240] gh-142349: Update lazy import comments in whatsnew (#158651) * Update lazy import comments in whatsnew * Correct wording --- Doc/whatsnew/3.15.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ffd0d64b229741..ca0f67e7c76ace 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -188,10 +188,10 @@ name: lazy import json lazy from pathlib import Path - print("Starting up...") # json and pathlib not loaded yet + print("Starting up...") # json and pathlib are not fully imported yet - data = json.loads('{"key": "value"}') # json loads here - p = Path(".") # pathlib loads here + data = json.loads('{"key": "value"}') # json gets fully imported here + p = Path(".") # pathlib gets fully imported here This mechanism is particularly useful for applications that import many modules at the top level but may only use a subset of them in any given run. From d6d1ebcdc9f758cff7564dd3aa894c004fc55762 Mon Sep 17 00:00:00 2001 From: Bhuvansh Date: Sat, 3 Oct 2026 21:05:03 +0530 Subject: [PATCH 053/240] gh-158643: Document that `Py_EnterRecursiveCall` uses the C stack, not a counter (GH-158644) Co-authored-by: An Long --- Doc/c-api/exceptions.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 40522f8c7b1375..0c8d14dc134ebc 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1034,6 +1034,10 @@ because the :ref:`call protocol ` takes care of recursion handling. case, a :exc:`RecursionError` is set and a nonzero value is returned. Otherwise, zero is returned. + The check is based on the remaining C stack space of the current thread, + not on a count of calls, so it is unaffected by + :c:func:`Py_SetRecursionLimit` and :func:`sys.setrecursionlimit`. + *where* should be a UTF-8 encoded string such as ``" in instance check"`` to be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. @@ -1044,6 +1048,10 @@ because the :ref:`call protocol ` takes care of recursion handling. .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. + .. versionchanged:: 3.14 + The check is based on the remaining C stack space. Previously, a + separate counter of C-level calls was used. + .. c:function:: void Py_LeaveRecursiveCall(void) Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each From b06dd2233e89715dad546f7b5d4950ad5cd93658 Mon Sep 17 00:00:00 2001 From: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:35:39 +0300 Subject: [PATCH 054/240] gh-158239: Fix `asyncio.gather()` performance regression (#158240) --- Lib/asyncio/tasks.py | 9 ++++----- Lib/test/test_asyncio/test_tasks.py | 31 +++++++++++++++++++++++++++++ Misc/ACKS | 1 + 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 979f8d2f83eb1e..33284288a7cca7 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -786,7 +786,7 @@ def cancel(self, msg=None): return ret -def _discard_awaited_by(children, waiter, outer): +def _discard_awaited_by(children, waiter): for fut in children: futures.future_discard_from_awaited_by(fut, waiter) @@ -852,11 +852,14 @@ def _done_callback(fut, cur_task=cur_task): # 'fut.exception()' will *raise* a CancelledError # instead of returning it. exc = fut._make_cancelled_error() + # gh-157213: children outliving gather() must lose the edge + _discard_awaited_by(children, cur_task) outer.set_exception(exc) return else: exc = fut.exception() if exc is not None: + _discard_awaited_by(children, cur_task) outer.set_exception(exc) return @@ -924,10 +927,6 @@ def _done_callback(fut, cur_task=cur_task): children.append(fut) outer = _GatheringFuture(children, loop=loop) - if cur_task is not None: - # gh-157213: a child outliving gather() must lose the awaited-by edge - outer.add_done_callback( - functools.partial(_discard_awaited_by, children, cur_task)) # Run done callbacks after GatheringFuture created so any post-processing # can be performed at this point # optimization: in the special case that *all* futures finished eagerly, diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index c380dcfec21395..f460029aff839d 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1250,6 +1250,37 @@ async def coro(): self.loop.run_until_complete(self.new_task(self.loop, coro())) + def test_gather_discards_awaited_by_for_cancelled_sibling(self): + # gh-157213: same, when gather() is ended by a cancelled child + async def survivor(): + await asyncio.Future() + + async def coro(): + t = self.new_task(self.loop, survivor()) + victim = self.new_task(self.loop, asyncio.sleep(10)) + victim.cancel() + with self.assertRaises(asyncio.CancelledError): + await asyncio.gather(t, victim) + self.assertFalse(t._asyncio_awaited_by) + t.cancel() + with self.assertRaises(asyncio.CancelledError): + await t + + self.loop.run_until_complete(self.new_task(self.loop, coro())) + + def test_gather_does_not_add_callback_to_outer(self): + # gh-158239: gather() must not add an internal done callback to + # the outer future just to maintain the await graph. + async def child(): + await asyncio.sleep(0) + + async def coro(): + outer = asyncio.gather(child(), child()) + self.assertFalse(outer._callbacks) + await outer + + self.loop.run_until_complete(self.new_task(self.loop, coro())) + def test_wait_really_done(self): # there is possibility that some tasks in the pending list # became done but their callbacks haven't all been called yet diff --git a/Misc/ACKS b/Misc/ACKS index 4a3f6482294e08..ad1b1119aa243a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -884,6 +884,7 @@ Dean Inwood Bob Ippolito Roger Irwin Atsuo Ishimoto +Timofei Ivankov Alexey Izbyshev Kasia Jachim Adam Jackson From 83b40d06f6380a09e93233bfd14d980ea9c8da63 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:05:42 +0300 Subject: [PATCH 055/240] gh-158618: Split `test_free_threading` into per-module tests to avoid timeouts (#158620) --- Lib/test/libregrtest/findtests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index e7692c5156812e..88a105488d95d9 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -21,6 +21,7 @@ "test_asyncio", "test_concurrent_futures", "test_doctests", + "test_free_threading", "test_future_stmt", "test_gdb", "test_inspect", From 1b015e66f058f29de4d97031b465587ec15cace9 Mon Sep 17 00:00:00 2001 From: Pengyu Lee Date: Sun, 4 Oct 2026 02:36:01 +0800 Subject: [PATCH 056/240] gh-157906: fix an incorrect architecture macro check in free-threaded GC (gh-157907) Fix typo in free-threaded GC prefetch check for 32-bit x86 MSVC. --- .../next/Windows/2026-09-21-20-54-37.gh-issue-157906.RlyLkY.rst | 2 ++ Python/gc_free_threading.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-09-21-20-54-37.gh-issue-157906.RlyLkY.rst diff --git a/Misc/NEWS.d/next/Windows/2026-09-21-20-54-37.gh-issue-157906.RlyLkY.rst b/Misc/NEWS.d/next/Windows/2026-09-21-20-54-37.gh-issue-157906.RlyLkY.rst new file mode 100644 index 00000000000000..3e0ca08a58abec --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-09-21-20-54-37.gh-issue-157906.RlyLkY.rst @@ -0,0 +1,2 @@ +Fix an incorrect architecture macro check that prevented CPU prefetching in +the free-threaded garbage collector on 32-bit Windows builds. diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index f865ac2f7db085..004b5871d196e4 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -548,7 +548,7 @@ static_assert(BUFFER_HI < BUFFER_SIZE && #define PREFETCH_T1(ptr) __builtin_prefetch(ptr, 0, 2) #define PREFETCH_T2(ptr) __builtin_prefetch(ptr, 0, 1) #define PREFETCH_NTA(ptr) __builtin_prefetch(ptr, 0, 0) -#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) && !defined(_M_ARM64EC) #include #define PREFETCH_T0(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) #define PREFETCH_T1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) From 6e7738bb5e6a294d888f8af6f409f52508194249 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2026 23:35:46 +0200 Subject: [PATCH 057/240] gh-158585: Fix _PyBytes_Resize() if hash value is already computed (#158589) _PyBytes_Resize() now creates a new bytes object if the hash value was already computed. bytes_resize_inplace() no longer sets the hash value to -1, since this function must not be called if the hash value was already computed. Add tests on PyUnicode_Resize(). --- Lib/test/test_capi/test_bytes.py | 77 +++++++++++++++++++----------- Lib/test/test_capi/test_unicode.py | 67 ++++++++++++++++++++++---- Modules/_testcapi/bytes.c | 24 ++++++++-- Modules/_testlimitedcapi/unicode.c | 52 ++++++++++++++++---- Objects/bytesobject.c | 10 ++-- 5 files changed, 177 insertions(+), 53 deletions(-) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 910f2bedac0bcb..ea5d3a3d59b6fd 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -239,39 +239,60 @@ def test_resize(self): """Test _PyBytes_Resize()""" _resize = _testcapi.bytes_resize - def resize(obj, size, new): - result = _resize(obj, size, new) - if 1 <= len(result): - if new or size != len(obj): - # gh-156995: Make sure that the result is a fresh object. - # Previously, _PyBytes_Resize(&obj, 1) returned a singleton - # if _PyObject_IsUniquelyReferenced() is false. - self.assertEqual(sys.getrefcount(result), 1) - self.assertFalse(sys._is_immortal(result)) - else: + def assert_is_fresh_copy(result, refcnt, is_new_obj): + self.assertEqual(refcnt, 1) + self.assertTrue(is_new_obj) + self.assertFalse(sys._is_immortal(result)) + + def resize(obj, size, new, compute_hash=False): + old_size = len(obj) + result, refcnt, is_new_obj = _resize(obj, size, new, + compute_hash=compute_hash) + + if size == old_size: + # Return the same object unchanged + self.assertFalse(is_new_obj) + elif old_size == 0: + assert_is_fresh_copy(result, refcnt, is_new_obj) + elif size == 0: # check that the result is the empty bytes string singleton + self.assertEqual(result, b'') self.assertTrue(sys._is_immortal(result)) + self.assertTrue(is_new_obj) + elif (not new) or compute_hash: + # gh-156995: Make sure that the result is a fresh object. + # Previously, _PyBytes_Resize(&obj, 1) returned a singleton + # if _PyObject_IsUniquelyReferenced() is false. + assert_is_fresh_copy(result, refcnt, is_new_obj) + else: + # An in-place resize can return the same memory address, or + # not. 'is_new_obj' cannot be tested. + self.assertEqual(refcnt, 1) + self.assertFalse(sys._is_immortal(result)) + return result for new in True, False: - with self.subTest(new=new): - self.assertEqual(resize(b'abc', 0, new), b'') - self.assertEqual(resize(b'abc', 1, new), b'a') - self.assertEqual(resize(b'abc', 2, new), b'ab') - self.assertEqual(resize(b'abc', 3, new), b'abc') - b = resize(b'abc', 4, new) - self.assertEqual(len(b), 4) - self.assertEqual(b[:3], b'abc') - - self.assertEqual(resize(b'a', 0, new), b'') - self.assertEqual(resize(b'a', 1, new), b'a') - b = resize(b'a', 2, new) - self.assertEqual(len(b), 2) - self.assertEqual(b[:1], b'a') - - self.assertEqual(resize(b'', 0, new), b'') - self.assertEqual(len(resize(b'', 1, new)), 1) - self.assertEqual(len(resize(b'', 2, new)), 2) + for compute_hash in True, False: + with self.subTest(new=new, compute_hash=compute_hash): + self.assertEqual(resize(b'abc', 0, new, compute_hash), b'') + self.assertEqual(resize(b'abc', 1, new, compute_hash), b'a') + self.assertEqual(resize(b'abc', 2, new, compute_hash), b'ab') + self.assertEqual(resize(b'abc', 3, new, compute_hash), b'abc') + + b = resize(b'abc', 4, new, compute_hash) + self.assertEqual(len(b), 4) + self.assertEqual(b[:3], b'abc') + + self.assertEqual(resize(b'a', 0, new, compute_hash), b'') + self.assertEqual(resize(b'a', 1, new, compute_hash), b'a') + b = resize(b'a', 2, new, compute_hash) + self.assertEqual(len(b), 2) + self.assertEqual(b[:1], b'a') + + self.assertEqual(resize(b'', 0, new, compute_hash), b'') + self.assertEqual(len(resize(b'', 1, new, compute_hash)), 1) + self.assertEqual(len(resize(b'', 2, new, compute_hash)), 2) self.assertRaises(SystemError, resize, b'abc', -1, False) self.assertRaises(SystemError, resize, bytearray(b'abc'), 3, False) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 6e2fd13567f5fb..d43318770a4e62 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -170,28 +170,75 @@ def test_write_macro(self): def test_resize(self): """Test PyUnicode_Resize()""" - resize = _testlimitedcapi.unicode_resize + _resize = _testlimitedcapi.unicode_resize + resize_null = _testlimitedcapi.unicode_resize_null + + def resize(s, length, new=True, compute_hash=False): + if s is not NULL and isinstance(s, str): + old_length = len(s) + else: + old_length = 0 + result, int_result, refcnt, is_new_obj = _resize(s, length, + new, compute_hash) + self.assertEqual(int_result, 0) + + if length == old_length: + # Return the same object unchanged + self.assertFalse(is_new_obj) + elif length == 0: + # Get the empty Unicode string + self.assertEqual(result, '') + self.assertTrue(sys._is_immortal(result)) + self.assertTrue(is_new_obj) + elif (not new) or compute_hash: + # Get a fresh copy + self.assertEqual(refcnt, 1) + self.assertTrue(is_new_obj) + self.assertFalse(sys._is_immortal(result)) + else: + # In-size replace can return the same address, or not. + # So 'is_new_obj' cannot be tested. + self.assertFalse(sys._is_immortal(result)) + + return result strings = [ # all strings have exactly 3 characters 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', '\U0001f600\U0001f601\U0001f602' ] - for s in strings: - self.assertEqual(resize(s, 3), (s, 0)) - self.assertEqual(resize(s, 2), (s[:2], 0)) - self.assertEqual(resize(s, 4), (s + '\0', 0)) - self.assertEqual(resize(s, 10), (s + '\0'*7, 0)) - self.assertEqual(resize(s, 0), ('', 0)) - self.assertRaises(MemoryError, resize, s, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, resize, s, -1) - self.assertRaises(SystemError, resize, s, PY_SSIZE_T_MIN) + for new in (True, False): + for compute_hash in (True, False): + for s in strings: + with self.subTest(new=new, compute_hash=compute_hash, s=s): + self.assertEqual(resize(s, 3, new, compute_hash), + s) + self.assertEqual(resize(s, 2, new, compute_hash), + s[:2]) + self.assertEqual(resize(s, 4, new, compute_hash), + s + '\0') + self.assertEqual(resize(s, 10, new, compute_hash), + s + '\0'*7) + self.assertEqual(resize(s, 0, new, compute_hash), + '') + + with self.assertRaises(MemoryError): + resize(s, PY_SSIZE_T_MAX, new, compute_hash) + with self.assertRaises(SystemError): + resize(s, -1, new, compute_hash) + with self.assertRaises(SystemError): + resize(s, PY_SSIZE_T_MIN, new, compute_hash) + self.assertRaises(SystemError, resize, b'abc', 0) self.assertRaises(SystemError, resize, [], 0) self.assertRaises(SystemError, resize, NULL, 0) # TODO: Test PyUnicode_Resize() with non-modifiable and legacy unicode # and with NULL as the address. + # Test PyUnicode_Resize(NULL, length) + self.assertRaises(SystemError, resize_null, 0) + self.assertRaises(SystemError, resize_null, 123) + def test_append(self): """Test PyUnicode_Append()""" append = _testlimitedcapi.unicode_append diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index f5a5fb0481d263..e9ed765850a907 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -11,14 +11,19 @@ /* Test _PyBytes_Resize() */ static PyObject * -bytes_resize(PyObject *Py_UNUSED(module), PyObject *args) +bytes_resize(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) { + static char *kwlist[] = {"obj", "newsize", "new", "compute_hash", NULL}; PyObject *obj; Py_ssize_t newsize; int new; + int compute_hash = 0; - if (!PyArg_ParseTuple(args, "Onp", &obj, &newsize, &new)) + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "Onp|p", kwlist, + &obj, &newsize, &new, &compute_hash)) { return NULL; + } NULLABLE(obj); if (new) { @@ -34,13 +39,24 @@ bytes_resize(PyObject *Py_UNUSED(module), PyObject *args) else { Py_XINCREF(obj); } + + if (compute_hash) { + if (PyObject_Hash(obj) == -1) { + return NULL; + } + } + + PyObject *old_obj = obj; if (_PyBytes_Resize(&obj, newsize) < 0) { assert(obj == NULL); + return NULL; } else { assert(obj != NULL); } - return obj; + + Py_ssize_t refcnt = Py_REFCNT(obj); + return Py_BuildValue("Onp", obj, refcnt, obj != old_obj); } @@ -604,7 +620,7 @@ corrupt_bytes(PyObject *Py_UNUSED(module), PyObject *args) static PyMethodDef test_methods[] = { - {"bytes_resize", bytes_resize, METH_VARARGS}, + {"bytes_resize", _PyCFunction_CAST(bytes_resize), METH_VARARGS | METH_KEYWORDS}, {"bytes_join", bytes_join, METH_VARARGS}, {"byteswriter_abc", byteswriter_abc, METH_NOARGS}, {"byteswriter_resize", byteswriter_resize, METH_NOARGS}, diff --git a/Modules/_testlimitedcapi/unicode.c b/Modules/_testlimitedcapi/unicode.c index a8511a8242fe7a..53fff1916ac1ab 100644 --- a/Modules/_testlimitedcapi/unicode.c +++ b/Modules/_testlimitedcapi/unicode.c @@ -183,24 +183,59 @@ unicode_resize(PyObject *self, PyObject *args) PyObject *obj, *copy; Py_ssize_t length; int result; + int new = 1; + int compute_hash = 0; - if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + if (!PyArg_ParseTuple(args, "On|pp", &obj, &length, &new, &compute_hash)) { return NULL; } NULLABLE(obj); - if (!(copy = unicode_copy(obj)) && obj) { - return NULL; + Py_ssize_t old_len = obj ? PyUnicode_GetLength(obj) : 0; + if (obj != NULL && new) { + copy = unicode_copy(obj); + if (!copy) { + return NULL; + } + obj = copy; } - result = PyUnicode_Resize(©, length); + else { + Py_XINCREF(obj); + } + + if (compute_hash && obj != NULL) { + if (PyObject_Hash(obj) == -1) { + Py_DECREF(obj); + return NULL; + } + } + + PyObject *old_obj = obj; + result = PyUnicode_Resize(&obj, length); if (result == -1 && PyErr_Occurred()) { - Py_XDECREF(copy); + Py_XDECREF(obj); return NULL; } - if (obj && PyUnicode_Check(obj) && length > PyUnicode_GetLength(obj)) { - unicode_fill(copy, PyUnicode_GetLength(obj), length, 0U); + if (obj && PyUnicode_Check(obj) && length > old_len) { + unicode_fill(obj, old_len, length, 0U); } - return Py_BuildValue("(Ni)", copy, result); + Py_ssize_t refcnt = Py_REFCNT(obj); + return Py_BuildValue("(Ninp)", obj, result, refcnt, obj != old_obj); +} + +// Test PyUnicode_Resize(NULL, length) +static PyObject * +unicode_resize_null(PyObject *self, PyObject *args) +{ + Py_ssize_t length; + if (!PyArg_ParseTuple(args, "n", &length)) { + return NULL; + } + + if (PyUnicode_Resize(NULL, length)) { + return NULL; + } + Py_RETURN_NONE; } /* Test PyUnicode_Append() */ @@ -1882,6 +1917,7 @@ static PyMethodDef TestMethods[] = { {"test_widechar", test_widechar, METH_NOARGS}, {"unicode_writechar", unicode_writechar, METH_VARARGS}, {"unicode_resize", unicode_resize, METH_VARARGS}, + {"unicode_resize_null", unicode_resize_null, METH_VARARGS}, {"unicode_append", unicode_append, METH_VARARGS}, {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2339e8bc1d2819..ac3a3340ec3201 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3405,7 +3405,6 @@ bytes_resize_inplace(PyObject **pv, Py_ssize_t newsize) PyBytesObject *sv = (PyBytesObject *)v; Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; - set_ob_shash(sv, -1); /* invalidate cached hash value */ assert(_PyBytes_IsMutable(*pv)); return 0; } @@ -3437,7 +3436,9 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) Py_ssize_t oldsize = PyBytes_GET_SIZE(v); if (oldsize == newsize) { - /* return early if newsize equals to v->ob_size */ + // Leave the object unchanged if the new size is the same as the old + // size, even if the object is not uniquely referenced or if the hash + // value was already computed. return 0; } @@ -3458,7 +3459,10 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) return 0; } - if (!_PyObject_IsUniquelyReferenced(v)) { + if (!_PyObject_IsUniquelyReferenced(v) + // Return a copy if the hash value was already computed + || get_ob_shash((PyBytesObject *)v) != -1) + { // Allocate and then copy so we don't get a shared immortal // one-character singleton! result = _PyBytes_FromSize(newsize, 0); From 880696af1aff6f4170fbacdfb8aa73a84be73da5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 4 Oct 2026 00:15:45 +0200 Subject: [PATCH 058/240] gh-158585: Use PyBytesWriter in io.BufferedReader.readline() (#158615) Replace a list of bytes object with PyBytesWriter. --- Modules/_io/bufferedio.c | 126 ++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 2cc3774e75c9da..d6318af1263d9c 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1205,109 +1205,115 @@ _io__Buffered_readinto1_impl(buffered *self, Py_buffer *buffer) static PyObject * _buffered_readline(buffered *self, Py_ssize_t limit) { - PyObject *res = NULL; - PyObject *chunks = NULL; - Py_ssize_t n; - const char *start, *s, *end; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); CHECK_CLOSED(self, "readline of closed file") /* First, try to find a line in the buffer. This can run unlocked because the calls to the C API are simple enough that they can't trigger any thread switch. */ - n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); - if (limit >= 0 && n > limit) + Py_ssize_t n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + if (limit >= 0 && n > limit) { n = limit; - start = self->buffer + self->pos; - s = memchr(start, '\n', n); + } + const char *start = self->buffer + self->pos; + const char *s = memchr(start, '\n', n); if (s != NULL) { - res = PyBytes_FromStringAndSize(start, s - start + 1); - if (res != NULL) - self->pos += s - start + 1; - goto end_unlocked; + n = s - start + 1; + PyObject *res = PyBytes_FromStringAndSize(start, n); + if (res == NULL) { + return NULL; + } + self->pos += n; + return res; } + if (n == limit) { - res = PyBytes_FromStringAndSize(start, n); - if (res != NULL) - self->pos += n; - goto end_unlocked; + PyObject *res = PyBytes_FromStringAndSize(start, n); + if (res == NULL) { + return NULL; + } + self->pos += n; + return res; } - if (!ENTER_BUFFERED(self)) - goto end_unlocked; + PyBytesWriter *writer = NULL; + int locked = 0; + if (!ENTER_BUFFERED(self)) { + goto error; + } + locked = 1; /* Now we try to get some more from the raw stream */ - chunks = PyList_New(0); - if (chunks == NULL) - goto end; + writer = PyBytesWriter_Create(0); + if (writer == NULL) { + goto error; + } + if (n > 0) { - res = PyBytes_FromStringAndSize(start, n); - if (res == NULL) - goto end; - if (PyList_Append(chunks, res) < 0) { - Py_CLEAR(res); - goto end; + if (PyBytesWriter_WriteBytes(writer, start, n) < 0) { + goto error; } - Py_CLEAR(res); self->pos += n; - if (limit >= 0) + if (limit >= 0) { limit -= n; + } } if (self->writable) { - PyObject *r = buffered_flush_and_rewind_unlocked(self); - if (r == NULL) - goto end; - Py_DECREF(r); + PyObject *res = buffered_flush_and_rewind_unlocked(self); + if (res == NULL) { + goto error; + } + Py_DECREF(res); } for (;;) { _bufferedreader_reset_buf(self); n = _bufferedreader_fill_buffer(self); - if (n == -1) - goto end; - if (n <= 0) + if (n == -1) { + goto error; + } + if (n <= 0) { break; - if (limit >= 0 && n > limit) + } + if (limit >= 0 && n > limit) { n = limit; + } start = self->buffer; - end = start + n; + const char *end = start + n; s = start; while (s < end) { if (*s++ == '\n') { - res = PyBytes_FromStringAndSize(start, s - start); - if (res == NULL) - goto end; + if (PyBytesWriter_WriteBytes(writer, start, s - start) < 0) { + goto error; + } self->pos = s - start; goto found; } } - res = PyBytes_FromStringAndSize(start, n); - if (res == NULL) - goto end; + + if (PyBytesWriter_WriteBytes(writer, start, n) < 0) { + goto error; + } if (n == limit) { self->pos = n; break; } - if (PyList_Append(chunks, res) < 0) { - Py_CLEAR(res); - goto end; - } - Py_CLEAR(res); - if (limit >= 0) + if (limit >= 0) { limit -= n; + } } -found: - if (res != NULL && PyList_Append(chunks, res) < 0) { - Py_CLEAR(res); - goto end; - } - Py_XSETREF(res, PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks)); -end: +found: LEAVE_BUFFERED(self) -end_unlocked: - Py_XDECREF(chunks); - return res; + return PyBytesWriter_Finish(writer); + +error: + PyBytesWriter_Discard(writer); + if (locked) { + LEAVE_BUFFERED(self) + } + return NULL; } /*[clinic input] From 9d22a5334bd5273962adceeb697a1337e9a0ca21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 4 Oct 2026 01:51:37 +0200 Subject: [PATCH 059/240] gh-158583: Fix uninitialized memory read in bytes.fromhex() (#158584) --- Lib/test/test_bytes.py | 9 +++++++++ ...6-10-03-16-14-26.gh-issue-158583.7EUMvS.rst | 2 ++ Objects/bytesobject.c | 18 ++++++++++-------- 3 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-16-14-26.gh-issue-158583.7EUMvS.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index ea0a54dd5e1d35..0f2eec38631213 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -518,6 +518,15 @@ def test_fromhex(self): self.type2test.fromhex(data) self.assertIn('at position %s' % pos, str(cm.exception)) + # gh-158583: Check for out of bounds reads (uninitialized bytes). + # Create an array from a list to not overallocate. + a = array.array('B', list(b'1234 ')) # Py_ISSPACE() loop + self.assertEqual(self.type2test.fromhex(a), b'\x12\x34') + + a = array.array('B', list(b'12345')) # Missing second digit + with self.assertRaises(ValueError): + self.type2test.fromhex(a) + def test_hex(self): self.assertRaises(TypeError, self.type2test.hex) self.assertRaises(TypeError, self.type2test.hex, 1) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-16-14-26.gh-issue-158583.7EUMvS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-16-14-26.gh-issue-158583.7EUMvS.rst new file mode 100644 index 00000000000000..c94fcc58add88c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-16-14-26.gh-issue-158583.7EUMvS.rst @@ -0,0 +1,2 @@ +:meth:`bytes.fromhex` and :meth:`bytearray.fromhex`: Fix uninitialized +memory read. Patch by Victor Stinner. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index ac3a3340ec3201..e7bc465a87f4a0 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2702,9 +2702,10 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) if (Py_ISSPACE(*str)) { do { str++; + if (str >= end) { + goto done; + } } while (Py_ISSPACE(*str)); - if (str >= end) - break; } top = _PyLong_DigitValue[*str]; @@ -2712,16 +2713,16 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) invalid_char = str - start; goto error; } + str++; + if (str >= end) { + invalid_char = -1; + goto error; + } bot = _PyLong_DigitValue[*str]; if (bot >= 16) { - /* Check if we had a second digit */ - if (str >= end){ - invalid_char = -1; - } else { - invalid_char = str - start; - } + invalid_char = str - start; goto error; } str++; @@ -2729,6 +2730,7 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) *buf++ = (unsigned char)((top << 4) + bot); } + done: if (view.obj != NULL) { PyBuffer_Release(&view); } From 8dba03970833d67b5b97dc98e1be8423f9f1d908 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 4 Oct 2026 23:01:31 +0200 Subject: [PATCH 060/240] gh-158585: Optimize bytes object creation (#158664) Add bytes_alloc() helper function. bytes_alloc() and _PyBytes_FromStringAndSize() have less conditional branches than previous code, and so should be a little bit faster. * Rename existing bytes_alloc() to bytes_type_alloc(). * Replace _PyBytes_FromSize(calloc=1) with _PyBytes_FromSizeZero(). * Add set_ob_shash_unsafe(): similar to set_ob_shash() but don't use an atomic operation on Free Threading. Use this new function on newly allocated bytes objects and in bytes_resize_inplace(). * Replace PyBytes_FromStringAndSize(NULL, size) with bytes_alloc(size). * Use bytes_alloc() in PyBytes_FromString() and _PyBytes_Repeat(). * Add _PyBytes_FromStringAndSize(): similar to PyBytes_FromStringAndSize(), but str must not be NULL. * Replace PyBytes_FromStringAndSize() with _PyBytes_FromStringAndSize(). * Test that PyBytes_FromString() and PyBytes_FromStringAndSize() return singletons for 0 or 1 bytes. * Fix test_tracemalloc.test_set_traceback_limit(). --- Lib/test/test_bytes.py | 41 ++++++ Lib/test/test_capi/test_bytes.py | 15 ++- Lib/test/test_tracemalloc.py | 4 +- Objects/bytesobject.c | 215 ++++++++++++++++++------------- 4 files changed, 183 insertions(+), 92 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 0f2eec38631213..f87a924c57016e 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -410,6 +410,8 @@ def test_concat(self): self.assertRaises(TypeError, lambda: b1 + "def") self.assertRaises(TypeError, lambda: "abc" + b2) + self.assertEqual(self.type2test() + memoryview(b''), b'') + def test_repeat(self): for b in b"abc", self.type2test(b"abc"): self.assertEqual(b * 3, b"abcabcabc") @@ -1447,6 +1449,45 @@ class SubBytes(bytes): self.assertNotEqual(id(s), id(1 * s)) self.assertNotEqual(id(s), id(s * 2)) + @support.cpython_only + def test_concat_cpython(self): + # Test optimizations + empty = b'' + abc = b"abc" + self.assertIs(abc + empty, abc) + self.assertIs(empty + abc, abc) + self.assertIs(empty + bytearray(b''), empty) + self.assertIs(empty + memoryview(b''), empty) + + class Subclass(bytes): + pass + + empty_subclass = Subclass(b"") + self.assertIs(empty + empty_subclass, empty) + self.assertIs(empty_subclass + empty, empty) + self.assertIs(abc + empty_subclass, abc) + self.assertIs(empty_subclass + abc, abc) + + def assert_copy(result, expected): + self.assertIsNot(result, expected) + self.assertEqual(type(result), bytes) + self.assertEqual(result, expected) + + # Copy the string if it's a subclass + def_subclass = Subclass(b"def") + assert_copy(def_subclass + empty, b'def') + assert_copy(empty + def_subclass, b'def') + assert_copy(abc + def_subclass, b'abcdef') + assert_copy(def_subclass + abc, b'defabc') + + @support.cpython_only + def test_repeat_cpython(self): + # Test optimizations + for b in b'', b'x', b"abc": + self.assertIs(b * 1, b) + self.assertIs(b * 0, b'') + self.assertIs(b * -23, b'') + class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index ea5d3a3d59b6fd..0f53d915c63dd3 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -61,10 +61,12 @@ def test_fromstringandsize(self): self.assertEqual(fromstringandsize(b'abc'), b'abc') self.assertEqual(fromstringandsize(b'abc', 2), b'ab') self.assertEqual(fromstringandsize(b'abc\0def'), b'abc\0def') - self.assertEqual(fromstringandsize(b'a'), b'a') - self.assertEqual(fromstringandsize(b'a', 1), b'a') - self.assertEqual(fromstringandsize(b'', 0), b'') - self.assertEqual(fromstringandsize(NULL, 0), b'') + + # Return singleton for 0 or 1 bytes + self.assertIs(fromstringandsize(b'', 0), b'') + self.assertIs(fromstringandsize(NULL, 0), b'') + self.assertIs(fromstringandsize(b'a', 1), b'a') + self.assertEqual(len(fromstringandsize(NULL, 3)), 3) self.assertRaises((MemoryError, OverflowError), fromstringandsize, NULL, PY_SSIZE_T_MAX) @@ -79,7 +81,10 @@ def test_fromstring(self): fromstring = _testlimitedcapi.bytes_fromstring self.assertEqual(fromstring(b'abc\0def'), b'abc') - self.assertEqual(fromstring(b''), b'') + + # Return singleton for 0 or 1 bytes + self.assertIs(fromstring(b''), b'') + self.assertIs(fromstring(b'a'), b'a') # CRASHES fromstring(NULL) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index ee02fbd16af0c4..32a869c706270d 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -41,6 +41,8 @@ def get_frames(nframe, lineno_delta): def allocate_bytes(size): nframe = tracemalloc.get_traceback_limit() bytes_len = (size - EMPTY_STRING_SIZE) + if bytes_len < 0: + raise ValueError(f"{size} bytes is too small") frames = get_frames(nframe, 1) data = b'x' * bytes_len return data, tracemalloc.Traceback(frames, min(len(frames), nframe)) @@ -158,7 +160,7 @@ def test_new_reference(self): self.assertEqual(traceback, obj_traceback) def test_set_traceback_limit(self): - obj_size = 10 + obj_size = 123 tracemalloc.stop() self.assertRaises(ValueError, tracemalloc.start, -1) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e7bc465a87f4a0..af5dce763e1f25 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -74,6 +74,21 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS _Py_COMP_DIAG_POP } +// Similar to set_ob_shash() but don't use an atomic operation on Free +// Threading. This function must only be used on newly allocated bytes objects. +static inline void +set_ob_shash_unsafe(PyBytesObject *a, Py_hash_t hash) +{ + // Don't test _PyObject_IsUniquelyReferenced(), so the function can be + // called by bytes_resize_inplace(). + assert(Py_REFCNT(a) == 1); + +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + a->ob_shash = hash; +_Py_COMP_DIAG_POP +} + #define get_ob_shash(op) _PyBytes_GET_CACHED_HASH(op) @@ -99,8 +114,36 @@ _Py_COMP_DIAG_POP PyBytes_FromStringAndSize()) or the length of the string in the 'str' parameter (for PyBytes_FromString()). */ + +// Allocate a new bytes object with contents left uninitialized. The caller is +// responsible to fill the contents. size must be greater than 0. static PyObject * -_PyBytes_FromSize(Py_ssize_t size, int use_calloc) +bytes_alloc(Py_ssize_t size) +{ + assert(size >= 1); + + if ((size_t)size > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { + PyErr_SetString(PyExc_OverflowError, + "byte string is too large"); + return NULL; + } + size_t alloc = PyBytesObject_SIZE + size; + + /* Inline PyObject_NewVar */ + PyBytesObject *op = (PyBytesObject *)PyObject_Malloc(alloc); + if (op == NULL) { + return PyErr_NoMemory(); + } + + _PyObject_InitVar(_PyVarObject_CAST(op), &PyBytes_Type, size); + set_ob_shash_unsafe(op, -1); + op->ob_sval[size] = '\0'; + return (PyObject *) op; +} + +// Create a new bytes object with contents initialized to zero. size can be 0. +static PyObject * +_PyBytes_FromSizeZero(Py_ssize_t size) { PyBytesObject *op; assert(size >= 0); @@ -114,77 +157,77 @@ _PyBytes_FromSize(Py_ssize_t size, int use_calloc) "byte string is too large"); return NULL; } + size_t alloc = PyBytesObject_SIZE + size; /* Inline PyObject_NewVar */ - if (use_calloc) - op = (PyBytesObject *)PyObject_Calloc(1, PyBytesObject_SIZE + size); - else - op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size); + op = (PyBytesObject *)PyObject_Calloc(1, alloc); if (op == NULL) { return PyErr_NoMemory(); } _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); - set_ob_shash(op, -1); - if (!use_calloc) { - op->ob_sval[size] = '\0'; + set_ob_shash_unsafe(op, -1); + // op->ob_sval[size] is equal to '\0' + return (PyObject *) op; +} + +// Similar to PyBytes_FromStringAndSize(), but str must not be NULL. +static PyObject * +_PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) +{ + assert(str != NULL); + assert(size >= 0); + + if (size == 0) { + return bytes_get_empty(); } + if (size == 1) { + return bytes_get_char((uint8_t)str[0]); + } + + PyBytesObject *op = (PyBytesObject *)bytes_alloc(size); + if (op == NULL) { + return NULL; + } + memcpy(op->ob_sval, str, size); return (PyObject *) op; } PyObject * PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) { - PyBytesObject *op; if (size < 0) { PyErr_SetString(PyExc_SystemError, "Negative size passed to PyBytes_FromStringAndSize"); return NULL; } - if (size == 1 && str != NULL) { - return bytes_get_char((uint8_t)*str); + + if (str != NULL) { + return _PyBytes_FromStringAndSize(str, size); } - if (size == 0) { - return bytes_get_empty(); + else { + if (size == 0) { + return bytes_get_empty(); + } + return bytes_alloc(size); } - - op = (PyBytesObject *)_PyBytes_FromSize(size, 0); - if (op == NULL) - return NULL; - if (str == NULL) - return (PyObject *) op; - - memcpy(op->ob_sval, str, size); - return (PyObject *) op; } PyObject * PyBytes_FromString(const char *str) { - size_t size; - PyBytesObject *op; - assert(str != NULL); - size = strlen(str); - if (size > PY_SSIZE_T_MAX - PyBytesObject_SIZE) { - PyErr_SetString(PyExc_OverflowError, - "byte string is too long"); - return NULL; - } - + size_t size = strlen(str); if (size == 0) { return bytes_get_empty(); } - else if (size == 1) { - return bytes_get_char((uint8_t)*str); + if (size == 1) { + return bytes_get_char((uint8_t)str[0]); } - /* Inline PyObject_NewVar */ - op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size); + PyBytesObject *op = (PyBytesObject *)bytes_alloc(size); if (op == NULL) { return PyErr_NoMemory(); } - _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); - set_ob_shash(op, -1); memcpy(op->ob_sval, str, size+1); return (PyObject *) op; } @@ -489,7 +532,7 @@ formatfloat(PyObject *v, Py_ssize_t argidx, PyObject *key, return str; } - result = PyBytes_FromStringAndSize(p, len); + result = _PyBytes_FromStringAndSize(p, len); PyMem_Free(p); *p_result = result; return result != NULL ? str : NULL; @@ -743,8 +786,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, (Py_ssize_t)(fmtstart - format - 1)); goto error; } - key = PyBytes_FromStringAndSize(keystart, - keylen); + key = _PyBytes_FromStringAndSize(keystart, keylen); if (key == NULL) goto error; if (args_owned) { @@ -1572,10 +1614,15 @@ _PyBytes_Concat(PyObject *a, PyObject *b) goto done; } - result = PyBytes_FromStringAndSize(NULL, va.len + vb.len); - if (result != NULL) { - memcpy(PyBytes_AS_STRING(result), va.buf, va.len); - memcpy(PyBytes_AS_STRING(result) + va.len, vb.buf, vb.len); + if ((va.len + vb.len) != 0) { + result = bytes_alloc(va.len + vb.len); + if (result != NULL) { + memcpy(PyBytes_AS_STRING(result), va.buf, va.len); + memcpy(PyBytes_AS_STRING(result) + va.len, vb.buf, vb.len); + } + } + else { + result = bytes_get_empty(); } done: @@ -1600,26 +1647,22 @@ _PyBytes_Repeat(PyObject *self, Py_ssize_t n) "repeated bytes are too long"); return NULL; } - Py_ssize_t size = Py_SIZE(a) * n; - if (size == Py_SIZE(a) && PyBytes_CheckExact(a)) { + Py_ssize_t old_size = Py_SIZE(a); + Py_ssize_t new_size = old_size * n; + if (new_size == old_size && PyBytes_CheckExact(a)) { return Py_NewRef(a); } - size_t nbytes = (size_t)size; - if (nbytes + PyBytesObject_SIZE <= nbytes) { - PyErr_SetString(PyExc_OverflowError, - "repeated bytes are too long"); - return NULL; + + if (new_size == 0) { + return bytes_get_empty(); } - PyBytesObject *op = PyObject_Malloc(PyBytesObject_SIZE + nbytes); + + PyBytesObject *op = (PyBytesObject *)bytes_alloc(new_size); if (op == NULL) { return PyErr_NoMemory(); } - _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); - set_ob_shash(op, -1); - op->ob_sval[size] = '\0'; - - _PyBytes_RepeatBuffer(op->ob_sval, size, a->ob_sval, Py_SIZE(a)); + _PyBytes_RepeatBuffer(op->ob_sval, new_size, a->ob_sval, old_size); return (PyObject *) op; } @@ -1755,7 +1798,6 @@ bytes_subscript(PyObject *op, PyObject* item) size_t cur; const char* source_buf; char* result_buf; - PyObject* result; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; @@ -1772,22 +1814,22 @@ bytes_subscript(PyObject *op, PyObject* item) return Py_NewRef(self); } else if (step == 1) { - return PyBytes_FromStringAndSize( + return _PyBytes_FromStringAndSize( PyBytes_AS_STRING(self) + start, slicelength); } else { source_buf = PyBytes_AS_STRING(self); - result = PyBytes_FromStringAndSize(NULL, slicelength); - if (result == NULL) + PyObject* result = bytes_alloc(slicelength); + if (result == NULL) { return NULL; + } result_buf = PyBytes_AS_STRING(result); for (cur = start, i = 0; i < slicelength; cur += step, i++) { result_buf[i] = source_buf[cur]; } - return result; } } @@ -1843,7 +1885,7 @@ bytes___bytes___impl(PyBytesObject *self) return Py_NewRef(self); } else { - return PyBytes_FromStringAndSize(self->ob_sval, Py_SIZE(self)); + return _PyBytes_FromStringAndSize(self->ob_sval, Py_SIZE(self)); } } @@ -2134,7 +2176,7 @@ do_xstrip(PyBytesObject *self, int striptype, PyObject *sepobj) return Py_NewRef(self); } else - return PyBytes_FromStringAndSize(s+i, j-i); + return _PyBytes_FromStringAndSize(s+i, j-i); } @@ -2163,7 +2205,7 @@ do_strip(PyBytesObject *self, int striptype) return Py_NewRef(self); } else - return PyBytes_FromStringAndSize(s+i, j-i); + return _PyBytes_FromStringAndSize(s+i, j-i); } @@ -2468,15 +2510,15 @@ bytes_removeprefix_impl(PyBytesObject *self, Py_buffer *prefix) && prefix_len > 0 && memcmp(self_start, prefix_start, prefix_len) == 0) { - return PyBytes_FromStringAndSize(self_start + prefix_len, - self_len - prefix_len); + return _PyBytes_FromStringAndSize(self_start + prefix_len, + self_len - prefix_len); } if (PyBytes_CheckExact(self)) { return Py_NewRef(self); } - return PyBytes_FromStringAndSize(self_start, self_len); + return _PyBytes_FromStringAndSize(self_start, self_len); } /*[clinic input] @@ -2507,15 +2549,15 @@ bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) && memcmp(self_start + self_len - suffix_len, suffix_start, suffix_len) == 0) { - return PyBytes_FromStringAndSize(self_start, - self_len - suffix_len); + return _PyBytes_FromStringAndSize(self_start, + self_len - suffix_len); } if (PyBytes_CheckExact(self)) { return Py_NewRef(self); } - return PyBytes_FromStringAndSize(self_start, self_len); + return _PyBytes_FromStringAndSize(self_start, self_len); } /*[clinic input] @@ -2955,7 +2997,7 @@ bytes_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, PyErr_SetString(PyExc_ValueError, "negative count"); return NULL; } - bytes = _PyBytes_FromSize(size, 1); + bytes = _PyBytes_FromSizeZero(size); } } else { @@ -3144,13 +3186,14 @@ PyBytes_FromObject(PyObject *x) * This allocator will be removed when ob_shash is removed. */ static PyObject * -bytes_alloc(PyTypeObject *self, Py_ssize_t nitems) +bytes_type_alloc(PyTypeObject *self, Py_ssize_t nitems) { + // Initialize memory to zero PyBytesObject *obj = (PyBytesObject*)PyType_GenericAlloc(self, nitems); if (obj == NULL) { return NULL; } - set_ob_shash(obj, -1); + set_ob_shash_unsafe(obj, -1); return (PyObject*)obj; } @@ -3165,10 +3208,9 @@ bytes_subtype_new(PyTypeObject *type, PyObject *tmp) n = PyBytes_GET_SIZE(tmp); pnew = type->tp_alloc(type, n); if (pnew != NULL) { - memcpy(PyBytes_AS_STRING(pnew), - PyBytes_AS_STRING(tmp), n+1); - set_ob_shash((PyBytesObject *)pnew, - get_ob_shash((PyBytesObject *)tmp)); + memcpy(PyBytes_AS_STRING(pnew), PyBytes_AS_STRING(tmp), n+1); + set_ob_shash_unsafe((PyBytesObject*)pnew, + get_ob_shash((PyBytesObject *)tmp)); } return pnew; } @@ -3269,7 +3311,7 @@ PyTypeObject PyBytes_Type = { 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ - bytes_alloc, /* tp_alloc */ + bytes_type_alloc, /* tp_alloc */ bytes_new, /* tp_new */ PyObject_Free, /* tp_free */ .tp_version_tag = _Py_TYPE_VERSION_BYTES, @@ -3375,6 +3417,7 @@ bytes_resize_inplace(PyObject **pv, Py_ssize_t newsize) assert(Py_REFCNT(v) == 1); assert(PyBytes_GET_SIZE(v) >= 1); assert(newsize >= 1); + assert(get_ob_shash((PyBytesObject *)v) == -1); // Only mutable bytes can be resized in-place assert(_PyBytes_IsMutable(v)); @@ -3429,7 +3472,6 @@ int _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) { PyObject *v = *pv; - PyObject *result; if (!PyBytes_Check(v) || newsize < 0) { PyErr_BadInternalCall(); @@ -3445,7 +3487,8 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) } if (oldsize == 0) { - result = _PyBytes_FromSize(newsize, 0); + assert(newsize >= 1); + PyObject *result = bytes_alloc(newsize); if (result == NULL) { return -1; } @@ -3467,7 +3510,7 @@ _PyBytes_ResizeKeepOnError(PyObject **pv, Py_ssize_t newsize) { // Allocate and then copy so we don't get a shared immortal // one-character singleton! - result = _PyBytes_FromSize(newsize, 0); + PyObject *result = bytes_alloc(newsize); if (!result) { return -1; } @@ -3783,7 +3826,7 @@ byteswriter_check_consistency(PyBytesWriter *writer) static inline int byteswriter_resize(PyBytesWriter *writer, Py_ssize_t new_size, int resize) { - assert(new_size >= 0); + assert(new_size >= 1); Py_ssize_t old_allocated = byteswriter_allocated(writer); if (new_size <= old_allocated) { @@ -3830,7 +3873,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t new_size, int resize) data = PyByteArray_AS_STRING(writer->obj); } else { - writer->obj = PyBytes_FromStringAndSize(NULL, alloc); + writer->obj = bytes_alloc(alloc); if (writer->obj == NULL) { return -1; } @@ -4000,7 +4043,7 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) result = bytes_get_char((uint8_t)buffer[0]); } else { - result = _PyBytes_FromSize(size, 0); + result = bytes_alloc(size); if (result == NULL) { goto error; } From a7b5fc50e98c1abf38f246b0802f6c9a44779e80 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 4 Oct 2026 17:09:32 -0400 Subject: [PATCH 061/240] Docs: Clarify that `ctypes.pythonapi` *may* be `None` under a statically linked interpreter (GH-158624) --- Doc/library/ctypes.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 35658aeba70505..c492451148341d 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1755,8 +1755,9 @@ These prefabricated library loaders are available: :c:expr:`int`, which is of course not always the truth, so you have to assign the correct :attr:`!restype` attribute to use these functions. - Note that if the Python interpreter is statically linked, this will be - ``None``, as ``dlopen`` is not possible in this case. + .. note:: + + If the Python interpreter is statically linked, this may be ``None``. .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader From d19febb00c5a5ff20fee4b251a9656d088cf6965 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 4 Oct 2026 23:22:18 +0200 Subject: [PATCH 062/240] Rework locale encoding C functions (#158680) Add output_length to _Py_EncodeLocale(), _Py_EncodeUTF8(), encode_current_locale() and encode_ascii(). So unicode_encode_locale() and wstr_to_utf8() can now use the output_length, instead of having to compute strlen(). * Rename functions: * _Py_DecodeUTF8Ex() => _Py_DecodeUTF8() * _Py_EncodeUTF8Ex() => _Py_EncodeUTF8() * _Py_EncodeLocaleEx() => _Py_EncodeLocale() * _Py_DecodeLocaleEx() => _Py_DecodeLocale() * Add tests on Py_DecodeLocale() and Py_EncodeLocale() in test_codecs. Clarify also their documentation. * _Py_EncodeLocale() and _Py_DecodeLocale() now sets all output parameters on error. Add more tests on these output parameters. * Add encode_current_locale_impl() to simplify encode_current_locale(). * Remove reason parameter of _Py_EncodeLocale() and _Py_DecodeLocale() to simplify the code. Only the UTF-8 decoder used specific reasons. * _Py_EncodeUTF8() now uses size_t instead of Py_ssize_t to iterate on the input string. * On error, encode_current_locale_impl() now returns ENCODE_ERROR (new constant) instead of DECODE_ERROR. * Mark _Py_EncodeLocaleRaw() as static (it's not used outside fileutils.c). --- Doc/c-api/sys.rst | 43 +- Include/internal/pycore_fileutils.h | 23 +- Lib/test/test_codecs.py | 101 ++++- Modules/_testinternalcapi.c | 93 ++-- Modules/_testlimitedcapi/codec.c | 89 +++- Objects/unicodeobject.c | 181 ++++---- Python/fileutils.c | 628 +++++++++++++++++----------- Python/initconfig.c | 11 +- 8 files changed, 734 insertions(+), 435 deletions(-) diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 0a446f86e22eaf..2f59e8498b88a2 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -152,28 +152,29 @@ Operating System Utilities ` and so that the LC_CTYPE locale is properly configured: see the :c:func:`Py_PreInitialize` function. - Decode a byte string from the :term:`filesystem encoding and error handler`. - If the error handler is :ref:`surrogateescape error handler - `, undecodable bytes are decoded as characters in range - U+DC80..U+DCFF; and if a byte sequence can be decoded as a surrogate - character, the bytes are escaped using the surrogateescape error handler - instead of decoding them. + Decode a byte string from the :term:`filesystem encoding ` with the :ref:`surrogateescape error handler + `. + + Undecodable bytes are decoded as characters in range U+DC80..U+DCFF. If a + byte sequence can be decoded as a surrogate character, escape the bytes + using the surrogateescape error handler instead of decoding them. Return a pointer to a newly allocated wide character string, use :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write the number of wide characters excluding the null character into ``*size`` - Return ``NULL`` on decoding error or memory allocation error. If *size* is - not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to - ``(size_t)-2`` on decoding error. + On memory allocation failure, set *\*size* to ``(size_t)-1`` and return + ``NULL``. + + On decode error, set *\*size* to ``(size_t)-2`` and return ``NULL``. + Decoding errors should never happen, unless there is a bug in the C + library. The :term:`filesystem encoding and error handler` are selected by :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. - Decoding errors should never happen, unless there is a bug in the C - library. - Use the :c:func:`Py_EncodeLocale` function to encode the character string back to a byte string. @@ -195,17 +196,19 @@ Operating System Utilities .. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) - Encode a wide character string to the :term:`filesystem encoding and error - handler`. If the error handler is :ref:`surrogateescape error handler - `, surrogate characters in the range U+DC80..U+DCFF are - converted to bytes 0x80..0xFF. + Encode a wide character string to the :term:`filesystem encoding ` with the :ref:`surrogateescape error handler + `. Surrogate characters in the range U+DC80..U+DCFF are + encoded to bytes 0x80..0xFF. Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free` - to free the memory. Return ``NULL`` on encoding error or memory allocation - error. + to free the memory. + + On memory allocation failure, set *\*error_pos* to ``(size_t)-1`` and return + ``NULL``. - If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on - success, or set to the index of the invalid character on encoding error. + On encoding error, set *\*error_pos* to the index of the first unencodable + character and return ``NULL``. The :term:`filesystem encoding and error handler` are selected by :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 128790823aa879..a765eb5fe2d322 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -24,27 +24,22 @@ extern "C" { PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); // Export for '_testinternalcapi' shared extension -PyAPI_FUNC(int) _Py_DecodeLocaleEx( +PyAPI_FUNC(int) _Py_DecodeLocale( const char *arg, wchar_t **wstr, size_t *wlen, - const char **reason, int current_locale, _Py_error_handler errors); // Export for '_testinternalcapi' shared extension -PyAPI_FUNC(int) _Py_EncodeLocaleEx( +PyAPI_FUNC(int) _Py_EncodeLocale( const wchar_t *text, char **str, + size_t *output_length, size_t *error_pos, - const char **reason, int current_locale, _Py_error_handler errors); -extern char* _Py_EncodeLocaleRaw( - const wchar_t *text, - size_t *error_pos); - extern PyObject* _Py_device_encoding(int); #if defined(MS_WINDOWS) || defined(__APPLE__) @@ -190,19 +185,23 @@ extern int _Py_open_osfhandle(void *handle, int flags); ? _PyStatus_ERR("cannot decode " NAME) \ : _PyStatus_NO_MEMORY() -extern int _Py_DecodeUTF8Ex( +#define _Py_CODEC_MEMORY_ERROR -1 +#define _Py_CODEC_DECODE_ERROR -2 +#define _Py_CODEC_ENCODE_ERROR -2 +#define _Py_CODEC_UNSUPPORTED_ERROR_HANDLER -3 + +extern int _Py_DecodeUTF8( const char *arg, Py_ssize_t arglen, wchar_t **wstr, size_t *wlen, - const char **reason, _Py_error_handler errors); -extern int _Py_EncodeUTF8Ex( +extern int _Py_EncodeUTF8( const wchar_t *text, char **str, + size_t *output_length, size_t *error_pos, - const char **reason, int raw_malloc, _Py_error_handler errors); diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 1916ee5507e972..2715ad6d7b3f8a 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -4057,34 +4057,67 @@ def test_pickle(self): pickle.dumps(sr, proto) +@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') @unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') class LocaleCodecTest(unittest.TestCase): """ - Test indirectly _Py_DecodeUTF8Ex() and _Py_EncodeUTF8Ex(). + Test public Py_EncodeLocale() and Py_DecodeLocale() C API. + + Test internal _Py_EncodeLocale() and _Py_DecodeLocale() C API. + + Test indirectly _Py_DecodeUTF8() and _Py_EncodeUTF8(). """ ENCODING = sys.getfilesystemencoding() STRINGS = ("ascii", "ulatin1:\xa7\xe9", "u255:\xff", "UCS:\xe9\u20ac\U0010ffff", - "surrogates:\uDC80\uDCFF") + "surrogates:\uDC80\uDCFF", + "embed\0char") BYTES_STRINGS = (b"blatin1:\xa7\xe9", b"b255:\xff") - SURROGATES = "\uDC80\uDCFF" - def encode(self, text, errors="strict"): - return _testinternalcapi.EncodeLocaleEx(text, 0, errors) + def encode_locale_surrogateescape(self, text): + # Test public Py_EncodeLocale() C API: + # use the "surrogateescape" error handler + return _testlimitedcapi.encode_locale(text) + + def encode_locale(self, text, errors="strict"): + # Test internal _Py_EncodeLocale() C API + return _testinternalcapi.encode_locale(text, 0, errors) def check_encode_strings(self, errors): for text in self.STRINGS: with self.subTest(text=text): try: expected = text.encode(self.ENCODING, errors) + if b"\0" in expected: + # Py_EncodeLocale() and _Py_EncodeLocale() + # truncate the input string at the first NUL character + expected = expected.partition(b'\0')[0] except UnicodeEncodeError: + for error_pos in range(len(text)): + try: + text[error_pos].encode(self.ENCODING, errors) + except UnicodeEncodeError: + break + else: + self.fail("failed to compute error_pos") + + if errors == "surrogateescape": + with self.assertRaises(RuntimeError) as cm: + self.encode_locale_surrogateescape(text) + errmsg = f"encode error: pos={error_pos}" + self.assertEqual(str(cm.exception), errmsg) + with self.assertRaises(RuntimeError) as cm: - self.encode(text, errors) - errmsg = str(cm.exception) - self.assertRegex(errmsg, r"encode error: pos=[0-9]+, reason=") + self.encode_locale(text, errors) + errmsg = f"encode error: pos={error_pos}" + self.assertEqual(str(cm.exception), errmsg) else: - encoded = self.encode(text, errors) + if errors in ("strict", "surrogateescape"): + encoded = self.encode_locale_surrogateescape(text) + self.assertEqual(encoded, expected) + + encoded = self.encode_locale(text, errors) self.assertEqual(encoded, expected) def test_encode_strict(self): @@ -4095,7 +4128,7 @@ def test_encode_surrogateescape(self): def test_encode_surrogatepass(self): try: - self.encode('', 'surrogatepass') + self.encode_locale('', 'surrogatepass') except ValueError as exc: if str(exc) == 'unsupported error handler': self.skipTest(f"{self.ENCODING!r} encoder doesn't support " @@ -4107,11 +4140,17 @@ def test_encode_surrogatepass(self): def test_encode_unsupported_error_handler(self): with self.assertRaises(ValueError) as cm: - self.encode('', 'backslashreplace') + self.encode_locale('', 'backslashreplace') self.assertEqual(str(cm.exception), 'unsupported error handler') - def decode(self, encoded, errors="strict"): - return _testinternalcapi.DecodeLocaleEx(encoded, 0, errors) + def decode_locale(self, encoded, errors="strict"): + # Test internal _Py_DecodeLocale() C API + return _testinternalcapi.decode_locale(encoded, 0, errors) + + def decode_locale_surrogateescape(self, encoded): + # Test the public Py_DecodeLocale() C API: + # use the "surrogateescape" error handler + return _testlimitedcapi.decode_locale(encoded) def check_decode_strings(self, errors): is_utf8 = (self.ENCODING == "utf-8") @@ -4138,13 +4177,37 @@ def check_decode_strings(self, errors): with self.subTest(encoded=encoded): try: expected = encoded.decode(self.ENCODING, errors) + if "\0" in expected: + # Py_DecodeLocale() and _Py_DecodeLocale() truncate + # the input string at the first NUL byte + expected = expected.partition('\0')[0] except UnicodeDecodeError: + for error_pos in range(len(encoded) - 1, -1, -1): + try: + encoded[:error_pos].decode(self.ENCODING, errors) + except UnicodeDecodeError: + pass + else: + break + else: + self.fail("failed to compute error_pos") + + if errors == "surrogateescape": + with self.assertRaises(RuntimeError) as cm: + self.decode_locale_surrogateescape(encoded) + errmsg = f"Py_DecodeLocale failed: error_pos={error_pos}" + self.assertEqual(str(cm.exception), errmsg) + with self.assertRaises(RuntimeError) as cm: - self.decode(encoded, errors) - errmsg = str(cm.exception) - self.assertStartsWith(errmsg, "decode error: ") + self.decode_locale(encoded, errors) + errmsg = f"decode error: pos={error_pos}" + self.assertEqual(str(cm.exception), errmsg) else: - decoded = self.decode(encoded, errors) + if errors == ("strict", "surrogateescape"): + decoded = self.decode_locale_surrogateescape(encoded) + self.assertEqual(decoded, expected) + + decoded = self.decode_locale(encoded, errors) self.assertEqual(decoded, expected) def test_decode_strict(self): @@ -4155,7 +4218,7 @@ def test_decode_surrogateescape(self): def test_decode_surrogatepass(self): try: - self.decode(b'', 'surrogatepass') + self.decode_locale(b'', 'surrogatepass') except ValueError as exc: if str(exc) == 'unsupported error handler': self.skipTest(f"{self.ENCODING!r} decoder doesn't support " @@ -4167,7 +4230,7 @@ def test_decode_surrogatepass(self): def test_decode_unsupported_error_handler(self): with self.assertRaises(ValueError) as cm: - self.decode(b'', 'backslashreplace') + self.decode_locale(b'', 'backslashreplace') self.assertEqual(str(cm.exception), 'unsupported error handler') diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a2ce266eb35a65..e1f87eb3b41c9f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1046,92 +1046,119 @@ get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) { } +// Test _Py_EncodeLocale() static PyObject * -encode_locale_ex(PyObject *self, PyObject *args) +encode_locale(PyObject *self, PyObject *args) { PyObject *unicode; int current_locale = 0; - wchar_t *wstr; PyObject *res = NULL; const char *errors = NULL; if (!PyArg_ParseTuple(args, "U|is", &unicode, ¤t_locale, &errors)) { return NULL; } - wstr = PyUnicode_AsWideCharString(unicode, NULL); + + // Accept embedded null characters + Py_ssize_t unused_wlen; + wchar_t *wstr = PyUnicode_AsWideCharString(unicode, &unused_wlen); if (wstr == NULL) { return NULL; } _Py_error_handler error_handler = _Py_GetErrorHandler(errors); - char *str = NULL; - size_t error_pos; - const char *reason = NULL; - int ret = _Py_EncodeLocaleEx(wstr, - &str, &error_pos, &reason, - current_locale, error_handler); + const char *str_canary = (const char*)0x1234; + char *str = (char*)str_canary; + size_t error_pos_canary = (size_t)-123; + size_t error_pos = error_pos_canary; + const size_t output_length_canary = (size_t)-456; + size_t output_length = output_length_canary; + int ret = _Py_EncodeLocale(wstr, + &str, &output_length, &error_pos, + current_locale, error_handler); PyMem_Free(wstr); switch(ret) { case 0: - res = PyBytes_FromString(str); + assert(str != NULL && str != str_canary); + assert(output_length != output_length_canary); + assert(error_pos == error_pos_canary); + res = PyBytes_FromStringAndSize(str, output_length); PyMem_RawFree(str); break; - case -1: + case _Py_CODEC_MEMORY_ERROR: + assert(str == NULL); + assert(output_length == 0); + assert(error_pos == 0); PyErr_NoMemory(); break; - case -2: - PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu, reason=%s", - error_pos, reason); + case _Py_CODEC_ENCODE_ERROR: + assert(str == NULL); + assert(output_length == 0); + assert(error_pos != error_pos_canary); + PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu", error_pos); break; - case -3: + case _Py_CODEC_UNSUPPORTED_ERROR_HANDLER: + assert(str == NULL); + assert(output_length == 0); + assert(error_pos == 0); PyErr_SetString(PyExc_ValueError, "unsupported error handler"); break; default: - PyErr_SetString(PyExc_ValueError, "unknown error code"); + PyErr_SetString(PyExc_SystemError, "unknown error code"); break; } return res; } +// Test _Py_DecodeLocale() static PyObject * -decode_locale_ex(PyObject *self, PyObject *args) +decode_locale(PyObject *self, PyObject *args) { char *str; + Py_ssize_t unused_len; int current_locale = 0; PyObject *res = NULL; const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y|is", &str, ¤t_locale, &errors)) { + // Accept embedded null bytes in str + if (!PyArg_ParseTuple(args, "y#|is", + &str, &unused_len, ¤t_locale, &errors)) { return NULL; } _Py_error_handler error_handler = _Py_GetErrorHandler(errors); - wchar_t *wstr = NULL; - size_t wlen = 0; - const char *reason = NULL; - int ret = _Py_DecodeLocaleEx(str, - &wstr, &wlen, &reason, - current_locale, error_handler); + const wchar_t *wstr_canary = (const wchar_t*)0x12345; + wchar_t *wstr = (wchar_t*)wstr_canary; + const size_t wlen_canary = (size_t)-123; + size_t wlen = wlen_canary; + int ret = _Py_DecodeLocale(str, &wstr, &wlen, + current_locale, error_handler); switch(ret) { case 0: + assert(wstr != NULL && wstr != wstr_canary); + assert(wlen != wlen_canary); res = PyUnicode_FromWideChar(wstr, wlen); PyMem_RawFree(wstr); break; - case -1: + case _Py_CODEC_MEMORY_ERROR: + assert(wstr == NULL); + assert(wlen == 0); PyErr_NoMemory(); break; - case -2: - PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu, reason=%s", - wlen, reason); + case _Py_CODEC_DECODE_ERROR: + assert(wstr == NULL); + assert(wlen != wlen_canary); + PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu", wlen); break; - case -3: + case _Py_CODEC_UNSUPPORTED_ERROR_HANDLER: + assert(wstr == NULL); + assert(wlen == 0); PyErr_SetString(PyExc_ValueError, "unsupported error handler"); break; default: - PyErr_SetString(PyExc_ValueError, "unknown error code"); + PyErr_SetString(PyExc_SystemError, "unknown error code"); break; } return res; @@ -3331,8 +3358,8 @@ static PyMethodDef module_functions[] = { {"test_bytes_find", test_bytes_find, METH_NOARGS}, {"normalize_path", normalize_path, METH_O, NULL}, {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL}, - {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, - {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, + {"encode_locale", encode_locale, METH_VARARGS}, + {"decode_locale", decode_locale, METH_VARARGS}, {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_interp", set_eval_frame_interp, METH_VARARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, diff --git a/Modules/_testlimitedcapi/codec.c b/Modules/_testlimitedcapi/codec.c index 44eecf99f678bb..61e5d4708c71d6 100644 --- a/Modules/_testlimitedcapi/codec.c +++ b/Modules/_testlimitedcapi/codec.c @@ -2,8 +2,8 @@ #ifdef Py_GIL_DISABLED # define Py_TARGET_ABI3T 0x030f0000 #else - // Need limited C API version 3.5 for PyCodec_NameReplaceErrors() -# define Py_LIMITED_API 0x03050000 + // Need limited C API version 3.13 for PyMem_RawFree() +# define Py_LIMITED_API 0x030d0000 #endif #include "parts.h" @@ -15,16 +15,93 @@ codec_namereplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) return PyCodec_NameReplaceErrors(exc); } + +// Test Py_DecodeLocale() +static PyObject * +decode_locale(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *str; + Py_ssize_t unused_len; + // Accept embedded null bytes + if (PyArg_Parse(arg, "y#", &str, &unused_len) < 0) { + return NULL; + } + + const size_t size_canary = (size_t)-123; + size_t size = size_canary; + wchar_t *wstr = Py_DecodeLocale(str, &size); + + if (str == NULL) { + if (size == (size_t)-1) { + PyErr_NoMemory(); + } + else if (size == (size_t)-2) { + PyErr_SetString(PyExc_RuntimeError, "decode error"); + } + else { + PyErr_Format(PyExc_SystemError, + "unknown Py_DecodeLocale() return value: %zd", + (Py_ssize_t)size); + } + return NULL; + } + assert(wstr != NULL); + assert(size != size_canary); + + PyObject *result = PyUnicode_FromWideChar(wstr, size); + PyMem_RawFree(wstr); + return result; +} + + +// Test Py_EncodeLocale() +static PyObject * +encode_locale(PyObject *Py_UNUSED(module), PyObject *arg) +{ + PyObject *unicode; + if (PyArg_Parse(arg, "U", &unicode) < 0) { + return NULL; + } + + // Accept embedded null characters + Py_ssize_t unused_wlen; + wchar_t *wstr = PyUnicode_AsWideCharString(unicode, &unused_wlen); + if (wstr == NULL) { + return NULL; + } + + const size_t error_pos_canary = (size_t)-123; + size_t error_pos = error_pos_canary; + char *str = Py_EncodeLocale(wstr, &error_pos); + PyMem_Free(wstr); + + if (str == NULL) { + if (error_pos == (size_t)-1) { + return PyErr_NoMemory(); + } + else { + assert(error_pos != error_pos_canary); + return PyErr_Format(PyExc_RuntimeError, + "encode error: pos=%zd", error_pos); + } + } + assert(error_pos == error_pos_canary); + + PyObject *result = PyBytes_FromString(str); + PyMem_Free(str); + return result; +} + + static PyMethodDef test_methods[] = { {"codec_namereplace_errors", codec_namereplace_errors, METH_O}, + {"decode_locale", decode_locale, METH_O}, + {"encode_locale", encode_locale, METH_O}, {NULL}, }; int _PyTestLimitedCAPI_Init_Codec(PyObject *module) { - if (PyModule_AddFunctions(module, test_methods) < 0) { - return -1; - } - return 0; + return PyModule_AddFunctions(module, test_methods); } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e630dbb77f8725..157e005c3c0b60 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3764,35 +3764,37 @@ unicode_encode_locale(PyObject *unicode, _Py_error_handler error_handler, } char *str; + size_t str_len; size_t error_pos; - const char *reason; - int res = _Py_EncodeLocaleEx(wstr, &str, &error_pos, &reason, - current_locale, error_handler); + int res = _Py_EncodeLocale(wstr, &str, &str_len, &error_pos, + current_locale, error_handler); PyMem_Free(wstr); if (res != 0) { - if (res == -2) { + if (res == _Py_CODEC_ENCODE_ERROR) { PyObject *exc; + assert(error_pos <= (size_t)(PY_SSIZE_T_MAX - 1)); exc = PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", "locale", unicode, (Py_ssize_t)error_pos, (Py_ssize_t)(error_pos+1), - reason); + "encode error"); if (exc != NULL) { PyCodec_StrictErrors(exc); Py_DECREF(exc); } } - else if (res == -3) { + else if (res == _Py_CODEC_UNSUPPORTED_ERROR_HANDLER) { PyErr_SetString(PyExc_ValueError, "unsupported error handler"); } else { + assert(res == _Py_CODEC_MEMORY_ERROR); PyErr_NoMemory(); } return NULL; } - PyObject *bytes = PyBytes_FromString(str); + PyObject *bytes = PyBytes_FromStringAndSize(str, str_len); PyMem_RawFree(str); return bytes; } @@ -3983,26 +3985,26 @@ unicode_decode_locale(const char *str, Py_ssize_t len, wchar_t *wstr; size_t wlen; - const char *reason; - int res = _Py_DecodeLocaleEx(str, &wstr, &wlen, &reason, - current_locale, errors); + int res = _Py_DecodeLocale(str, &wstr, &wlen, current_locale, errors); if (res != 0) { - if (res == -2) { + if (res == _Py_CODEC_DECODE_ERROR) { PyObject *exc; + assert(wlen <= (size_t)(PY_SSIZE_T_MAX - 1)); exc = PyObject_CallFunction(PyExc_UnicodeDecodeError, "sy#nns", "locale", str, len, (Py_ssize_t)wlen, (Py_ssize_t)(wlen + 1), - reason); + "decode error"); if (exc != NULL) { PyCodec_StrictErrors(exc); Py_DECREF(exc); } } - else if (res == -3) { + else if (res == _Py_CODEC_UNSUPPORTED_ERROR_HANDLER) { PyErr_SetString(PyExc_ValueError, "unsupported error handler"); } else { + assert(res == _Py_CODEC_MEMORY_ERROR); PyErr_NoMemory(); } return NULL; @@ -5481,26 +5483,31 @@ PyUnicode_DecodeUTF8Stateful(const char *s, } -/* UTF-8 decoder: use surrogateescape error handler if 'surrogateescape' is - non-zero, use strict error handler otherwise. - - On success, write a pointer to a newly allocated wide character string into - *wstr (use PyMem_RawFree() to free the memory) and write the output length - (in number of wchar_t units) into *wlen (if wlen is set). - - On memory allocation failure, return -1. - - On decoding error (if surrogateescape is zero), return -2. If wlen is - non-NULL, write the start of the illegal byte sequence into *wlen. If reason - is not NULL, write the decoding error message into *reason. */ +// UTF-8 decoder. +// +// Supported error handlers: "strict", "surrogateescape" and "surrogatepass". +// +// On success, write a pointer to a newly allocated wide character string into +// *wstr (use PyMem_RawFree() to free the memory), write the output length +// (in number of wchar_t units) into *wlen (if wlen is set), and return 0. +// +// On error, return a negative number. +// +// On memory allocation failure, return _Py_CODEC_MEMORY_ERROR. +// +// On decoding error (if errors is "strict"), return _Py_CODEC_DECODE_ERROR. +// If wlen is non-NULL, write the start of the illegal byte sequence into +// *wlen. +// +// Return _Py_CODEC_UNSUPPORTED_ERROR_HANDLER if errors error handler is not +// supported. int -_Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, - const char **reason, _Py_error_handler errors) +_Py_DecodeUTF8(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, + _Py_error_handler errors) { - const char *orig_s = s; - const char *e; - wchar_t *unicode; - Py_ssize_t outpos; + assert(0 <= size); + assert(s != NULL); + assert(wstr != NULL); int surrogateescape = 0; int surrogatepass = 0; @@ -5515,23 +5522,23 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, surrogatepass = 1; break; default: - return -3; + return _Py_CODEC_UNSUPPORTED_ERROR_HANDLER; } /* Note: size will always be longer than the resulting Unicode character count */ - if (PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t) - 1 < size) { - return -1; + if ((size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1 < (size_t)size) { + return _Py_CODEC_MEMORY_ERROR; } - - unicode = PyMem_RawMalloc((size + 1) * sizeof(wchar_t)); + wchar_t *unicode = PyMem_RawMalloc((size + 1) * sizeof(wchar_t)); if (!unicode) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } /* Unpack UTF-8 encoded data */ - e = s + size; - outpos = 0; + const char *orig_s = s; + const char *e = s + size; + Py_ssize_t outpos = 0; while (s < e) { Py_UCS4 ch; #if SIZEOF_WCHAR_T == 4 @@ -5571,24 +5578,10 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, } else { PyMem_RawFree(unicode ); - if (reason != NULL) { - switch (ch) { - case 0: - *reason = "unexpected end of data"; - break; - case 1: - *reason = "invalid start byte"; - break; - /* 2, 3, 4 */ - default: - *reason = "invalid continuation byte"; - break; - } - } if (wlen != NULL) { *wlen = s - orig_s; } - return -2; + return _Py_CODEC_DECODE_ERROR; } } } @@ -5602,17 +5595,20 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, } +// Decode from UTF-8 with the "surrogateescape" error handler. +// On success, set *wlen and return a newly allocated string. +// On error, set *wlen to the error (_Py_CODEC_MEMORY_ERROR or +// _Py_CODEC_DECODE_ERROR) and the return NULL wchar_t* _Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen, size_t *wlen) { wchar_t *wstr; - int res = _Py_DecodeUTF8Ex(arg, arglen, - &wstr, wlen, - NULL, _Py_ERROR_SURROGATEESCAPE); + int res = _Py_DecodeUTF8(arg, arglen, &wstr, wlen, + _Py_ERROR_SURROGATEESCAPE); if (res != 0) { - /* _Py_DecodeUTF8Ex() must support _Py_ERROR_SURROGATEESCAPE */ - assert(res != -3); + /* _Py_DecodeUTF8() must support _Py_ERROR_SURROGATEESCAPE */ + assert(res != _Py_CODEC_UNSUPPORTED_ERROR_HANDLER); if (wlen) { *wlen = (size_t)res; } @@ -5627,19 +5623,24 @@ _Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen, On success, return 0 and write the newly allocated character string (use PyMem_Free() to free the memory) into *str. - On encoding failure, return -2 and write the position of the invalid - surrogate character into *error_pos (if error_pos is set) and the decoding - error message into *reason (if reason is set). + On encoding failure, return _Py_CODEC_ENCODE_ERROR (-2) and write the + position of the invalid surrogate character into *error_pos (if error_pos is + set). - On memory allocation failure, return -1. */ + On memory allocation failure, return _Py_CODEC_MEMORY_ERROR (-1). + + str and output_length must not be NULL +*/ int -_Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, - const char **reason, int raw_malloc, _Py_error_handler errors) +_Py_EncodeUTF8(const wchar_t *text, char **str, size_t *output_length, + size_t *error_pos, int raw_malloc, _Py_error_handler errors) { - const Py_ssize_t max_char_size = 4; - Py_ssize_t len = wcslen(text); + assert(str != NULL); + assert(output_length != NULL); - assert(len >= 0); + // U+10ffff encoded to UTF-8 takes 4 bytes + const size_t max_char_size = 4; + size_t len = wcslen(text); int surrogateescape = 0; int surrogatepass = 0; @@ -5654,11 +5655,11 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, surrogatepass = 1; break; default: - return -3; + return _Py_CODEC_UNSUPPORTED_ERROR_HANDLER; } - if (len > PY_SSIZE_T_MAX / max_char_size - 1) { - return -1; + if (len > (size_t)PY_SSIZE_T_MAX / max_char_size - 1) { + return _Py_CODEC_MEMORY_ERROR; } char *bytes; if (raw_malloc) { @@ -5668,12 +5669,11 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, bytes = PyMem_Malloc((len + 1) * max_char_size); } if (bytes == NULL) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } char *p = bytes; - Py_ssize_t i; - for (i = 0; i < len; ) { + for (size_t i = 0; i < len; ) { Py_ssize_t ch_pos = i; Py_UCS4 ch = text[i]; i++; @@ -5689,7 +5689,6 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, if (ch < 0x80) { /* Encode ASCII */ *p++ = (char) ch; - } else if (ch < 0x0800) { /* Encode Latin-1 */ @@ -5702,16 +5701,13 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, if (error_pos != NULL) { *error_pos = (size_t)ch_pos; } - if (reason != NULL) { - *reason = "encoding error"; - } if (raw_malloc) { PyMem_RawFree(bytes); } else { PyMem_Free(bytes); } - return -2; + return _Py_CODEC_ENCODE_ERROR; } *p++ = (char)(ch & 0xff); } @@ -5729,29 +5725,29 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, *p++ = (char)(0x80 | (ch & 0x3f)); } } - *p++ = '\0'; + *p++ = '\0'; // trailing NUL byte size_t final_size = (p - bytes); - char *bytes2; + assert(final_size >= 1); + *output_length = final_size - 1; // -1 for the trailing NUL byte + + char *result; if (raw_malloc) { - bytes2 = PyMem_RawRealloc(bytes, final_size); + result = PyMem_RawRealloc(bytes, final_size); } else { - bytes2 = PyMem_Realloc(bytes, final_size); + result = PyMem_Realloc(bytes, final_size); } - if (bytes2 == NULL) { - if (error_pos != NULL) { - *error_pos = (size_t)-1; - } + if (result == NULL) { if (raw_malloc) { PyMem_RawFree(bytes); } else { PyMem_Free(bytes); } - return -1; + return _Py_CODEC_MEMORY_ERROR; } - *str = bytes2; + *str = result; return 0; } @@ -15229,8 +15225,11 @@ unicode_iter(PyObject *seq) static int encode_wstr_utf8(wchar_t *wstr, char **str, const char *name) { - int res; - res = _Py_EncodeUTF8Ex(wstr, str, NULL, NULL, 1, _Py_ERROR_STRICT); + assert(str != NULL); + + size_t unused_output_length; + int res = _Py_EncodeUTF8(wstr, str, &unused_output_length, + NULL, 1, _Py_ERROR_STRICT); if (res == -2) { PyErr_Format(PyExc_RuntimeError, "cannot encode %s", name); return -1; diff --git a/Python/fileutils.c b/Python/fileutils.c index 404fec83385e97..8ed88047b35e4f 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -51,13 +51,19 @@ extern int winerror_to_errno(int); int _Py_open_cloexec_works = -1; #endif +// wcstombs() error +static const size_t ENCODE_ERROR = (size_t)-1; // mbstowcs() and mbrtowc() errors -static const size_t DECODE_ERROR = ((size_t)-1); +static const size_t DECODE_ERROR = (size_t)-1; #ifdef HAVE_MBRTOWC static const size_t INCOMPLETE_CHARACTER = (size_t)-2; #endif +// Get the error handler as 'int surrogateescape'. +// Set '*surrogateescape' and return 0 on success. +// Return -1 if the error handler is not supported: other than "strict" and +// "surrogateescape". static int get_surrogateescape(_Py_error_handler errors, int *surrogateescape) { @@ -227,8 +233,7 @@ extern int _Py_normalize_encoding(const char *, char *, size_t, int); Values of force_ascii: 1: the workaround is used: Py_EncodeLocale() uses - encode_ascii_surrogateescape() and Py_DecodeLocale() uses - decode_ascii() + encode_ascii() and Py_DecodeLocale() uses decode_ascii() 0: the workaround is not used: Py_EncodeLocale() uses wcstombs() and Py_DecodeLocale() uses mbstowcs() -1: unknown, need to call check_force_ascii() to get the value @@ -353,36 +358,41 @@ _Py_ResetForceASCII(void) } +// Encode a wide string to the ASCII encoding. If errors is +// _Py_ERROR_SURROGATEESCAPE, handle non-ASCII character using the +// surrogateescape error handler. In that case, the output is non-ASCII. +// +// Set *str to a newly allocated string on success. +// Return a negative number on error: _Py_CODEC_MEMORY_ERROR, +// _Py_CODEC_ENCODE_ERROR or _Py_CODEC_UNSUPPORTED_ERROR_HANDLER. +// +// str and output_length must not be NULL static int -encode_ascii(const wchar_t *text, char **str, - size_t *error_pos, const char **reason, - int raw_malloc, _Py_error_handler errors) +encode_ascii(const wchar_t *text, char **str, size_t *output_length, + size_t *error_pos, int raw_malloc, _Py_error_handler errors) { - char *result = NULL, *out; - size_t len, i; - wchar_t ch; + assert(str != NULL); + assert(output_length != NULL); int surrogateescape; if (get_surrogateescape(errors, &surrogateescape) < 0) { - return -3; + // Only support "strict" and "surrogateescape" + return _Py_CODEC_UNSUPPORTED_ERROR_HANDLER; } - len = wcslen(text); + size_t len = wcslen(text); + *output_length = len; /* +1 for NULL byte */ - if (raw_malloc) { - result = PyMem_RawMalloc(len + 1); - } - else { - result = PyMem_Malloc(len + 1); - } + char *result; + result = raw_malloc ? PyMem_RawMalloc(len + 1) : PyMem_Malloc(len + 1); if (result == NULL) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } - out = result; - for (i=0; i PY_SSIZE_T_MAX / sizeof(wchar_t)) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } res = PyMem_RawMalloc(argsize * sizeof(wchar_t)); if (!res) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } out = res; for (in = (unsigned char*)arg; *in; in++) { unsigned char ch = *in; - if (ch < 128) { + if (ch <= 127) { *out++ = ch; } else { @@ -462,10 +483,7 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, if (wlen) { *wlen = in - (unsigned char*)arg; } - if (reason) { - *reason = "decoding error"; - } - return -2; + return _Py_CODEC_DECODE_ERROR; } *out++ = 0xdc00 + ch; } @@ -480,22 +498,22 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, } #endif /* !HAVE_MBRTOWC */ +// Decode a bytes string from the current locale encoding. +// On success, set *wstr and *wlen (if set) and return 0. +// On error, return a negative number: _Py_CODEC_MEMORY_ERROR, +// _Py_CODEC_DECODE_ERROR or _Py_CODEC_UNSUPPORTED_ERROR_HANDLER. +// On decode, set *wlen (if set). static int decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, - const char **reason, _Py_error_handler errors) + _Py_error_handler errors) { - wchar_t *res; - size_t argsize; - size_t count; -#ifdef HAVE_MBRTOWC - unsigned char *in; - wchar_t *out; - mbstate_t mbs; -#endif + assert(arg != NULL); + assert(wstr != NULL); int surrogateescape; if (get_surrogateescape(errors, &surrogateescape) < 0) { - return -3; + // Only support "strict" and "surrogateescape" + return _Py_CODEC_UNSUPPORTED_ERROR_HANDLER; } #ifdef HAVE_BROKEN_MBSTOWCS @@ -503,21 +521,25 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, * mbstowcs which does not count the characters that * would result from conversion. Use an upper bound. */ - argsize = strlen(arg); + size_t argsize = strlen(arg); #else - argsize = _Py_mbstowcs(NULL, arg, 0); + size_t argsize = _Py_mbstowcs(NULL, arg, 0); #endif + wchar_t *res; if (argsize != DECODE_ERROR) { if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } res = (wchar_t *)PyMem_RawMalloc((argsize + 1) * sizeof(wchar_t)); if (!res) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } - count = _Py_mbstowcs(res, arg, argsize + 1); + // +1 to write also the trailing NUL character + size_t count = _Py_mbstowcs(res, arg, argsize + 1); if (count != DECODE_ERROR) { + // Success + assert(count == argsize); *wstr = res; if (wlen != NULL) { *wlen = count; @@ -535,15 +557,16 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, actual output could use less memory. */ argsize = strlen(arg) + 1; if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } res = (wchar_t*)PyMem_RawMalloc(argsize * sizeof(wchar_t)); if (!res) { - return -1; + return _Py_CODEC_MEMORY_ERROR; } - in = (unsigned char*)arg; - out = res; + unsigned char *in = (unsigned char*)arg; + wchar_t *out = res; + mbstate_t mbs; memset(&mbs, 0, sizeof mbs); while (argsize) { size_t converted = _Py_mbrtowc(out, (char*)in, argsize, &mbs); @@ -575,6 +598,7 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, } if (wlen != NULL) { *wlen = out - res; + assert(res[*wlen] == 0); } *wstr = res; return 0; @@ -584,66 +608,37 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, if (wlen) { *wlen = in - (unsigned char*)arg; } - if (reason) { - *reason = "decoding error"; - } - return -2; + return _Py_CODEC_DECODE_ERROR; #else /* HAVE_MBRTOWC */ /* Cannot use C locale for escaping; manually escape as if charset is ASCII (i.e. escape all bytes > 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ - return decode_ascii(arg, wstr, wlen, reason, errors); + return decode_ascii(arg, wstr, wlen, errors); #endif /* HAVE_MBRTOWC */ } -/* Decode a byte string from the locale encoding. - - Use the strict error handler if 'surrogateescape' is zero. Use the - surrogateescape error handler if 'surrogateescape' is non-zero: undecodable - bytes are decoded as characters in range U+DC80..U+DCFF. If a byte sequence - can be decoded as a surrogate character, escape the bytes using the - surrogateescape error handler instead of decoding them. - - On success, return 0 and write the newly allocated wide character string into - *wstr (use PyMem_RawFree() to free the memory). If wlen is not NULL, write - the number of wide characters excluding the null character into *wlen. - - On memory allocation failure, return -1. - - On decoding error, return -2. If wlen is not NULL, write the start of - invalid byte sequence in the input string into *wlen. If reason is not NULL, - write the decoding error message into *reason. - - Return -3 if the error handler 'errors' is not supported. - - Use the Py_EncodeLocaleEx() function to encode the character string back to - a byte string. */ -int -_Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen, - const char **reason, +static int +decode_locale_impl(const char* arg, wchar_t **wstr, size_t *wlen, int current_locale, _Py_error_handler errors) { if (current_locale) { #ifdef _Py_FORCE_UTF8_LOCALE - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, - errors); + return _Py_DecodeUTF8(arg, strlen(arg), wstr, wlen, errors); #else - return decode_current_locale(arg, wstr, wlen, reason, errors); + return decode_current_locale(arg, wstr, wlen, errors); #endif } #ifdef _Py_FORCE_UTF8_FS_ENCODING - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, - errors); + return _Py_DecodeUTF8(arg, strlen(arg), wstr, wlen, errors); #else int use_utf8 = (_PyRuntime.preconfig.utf8_mode >= 1); #ifdef MS_WINDOWS use_utf8 |= (_PyRuntime.preconfig.legacy_windows_fs_encoding == 0); #endif if (use_utf8) { - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, - errors); + return _Py_DecodeUTF8(arg, strlen(arg), wstr, wlen, errors); } #ifdef USE_FORCE_ASCII @@ -653,45 +648,112 @@ _Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen, if (force_ascii) { /* force ASCII encoding to workaround mbstowcs() issue */ - return decode_ascii(arg, wstr, wlen, reason, errors); + return decode_ascii(arg, wstr, wlen, errors); } #endif - return decode_current_locale(arg, wstr, wlen, reason, errors); + return decode_current_locale(arg, wstr, wlen, errors); #endif /* !_Py_FORCE_UTF8_FS_ENCODING */ } -/* Decode a byte string from the locale encoding with the - surrogateescape error handler: undecodable bytes are decoded as characters - in range U+DC80..U+DCFF. If a byte sequence can be decoded as a surrogate - character, escape the bytes using the surrogateescape error handler instead - of decoding them. +// Decode a byte string from the locale encoding. +// +// Supported error handlers are _Py_ERROR_STRICT and _Py_ERROR_SURROGATEESCAPE. +// The UTF-8 decoder also supports _Py_ERROR_SURROGATEPASS. +// +// If errors is _Py_ERROR_SURROGATEESCAPE, undecodable bytes are decoded as +// characters in range U+DC80..U+DCFF. If a byte sequence can be decoded as a +// surrogate character, escape the bytes using the surrogateescape error +// handler instead of decoding them. +// +// On success, return 0 and write the newly allocated wide character string into +// *wstr (use PyMem_RawFree() to free the memory). If wlen is not NULL, write +// the number of wide characters excluding the null character into *wlen. +// +// On error, return a negative number. +// +// On memory allocation failure, return _Py_CODEC_MEMORY_ERROR. +// +// On decoding error, return _Py_CODEC_DECODE_ERROR. If wlen is not NULL, +// write the start of invalid byte sequence in the input string into *wlen. If +// reason is not NULL, write the decoding error message into *reason. +// +// Return _Py_CODEC_UNSUPPORTED_ERROR_HANDLER if the 'errors' error +// handler is not supported: other than "strict" and "surrogateescape". +// +// Use the _Py_EncodeLocale() function to encode the character string back to +// a byte string. +// +// arg and wstr must not be NULL. +int +_Py_DecodeLocale(const char* arg, wchar_t **wstr, size_t *wlen, + int current_locale, _Py_error_handler errors) +{ + assert(arg != NULL); + assert(wstr != NULL); - Return a pointer to a newly allocated wide character string, use - PyMem_RawFree() to free the memory. If size is not NULL, write the number of - wide characters excluding the null character into *size +#ifdef Py_DEBUG + size_t wlen_canary = (size_t)-2; + if (wlen) { + *wlen = wlen_canary; + } +#endif - Return NULL on decoding error or memory allocation error. If *size* is not - NULL, *size is set to (size_t)-1 on memory error or set to (size_t)-2 on - decoding error. + int res = decode_locale_impl(arg, wstr, wlen, current_locale, errors); + if (res < 0) { + // Error + *wstr = NULL; + if (res == _Py_CODEC_DECODE_ERROR) { +#ifdef Py_DEBUG + assert(wlen == NULL || *wlen != wlen_canary); +#endif + } + else { + if (wlen) { + *wlen = 0; + } + } + } + else { + // Success + assert(*wstr != NULL); +#ifdef Py_DEBUG + if (wlen != NULL) { + assert(*wlen == wcslen(*wstr)); + } +#endif + } + return res; +} - Decoding errors should never happen, unless there is a bug in the C - library. - Use the Py_EncodeLocale() function to encode the character string back to a - byte string. */ +// Decode a byte string from the locale encoding with the +// surrogateescape error handler. Undecodable bytes are decoded as characters +// in range U+DC80..U+DCFF. If a byte sequence can be decoded as a surrogate +// character, escape the bytes using the surrogateescape error handler instead +// of decoding them. +// +// Return a pointer to a newly allocated wide character string, use +// PyMem_RawFree() to free the memory. If size is not NULL, write the number of +// wide characters excluding the null character into *size. +// +// On memory allocation failure, set *size to (size_t)-1 and return NULL. +// +// On decode error, set *size to (size_t)-2 and return NULL. +// Decoding errors should never happen, unless there is a bug in the C library. +// +// Use the Py_EncodeLocale() function to encode the character string back to a +// byte string. wchar_t* -Py_DecodeLocale(const char* arg, size_t *wlen) +Py_DecodeLocale(const char* arg, size_t *size) { wchar_t *wstr; - int res = _Py_DecodeLocaleEx(arg, &wstr, wlen, - NULL, 0, - _Py_ERROR_SURROGATEESCAPE); + int res = _Py_DecodeLocale(arg, &wstr, size, 0, _Py_ERROR_SURROGATEESCAPE); if (res != 0) { - assert(res != -3); - if (wlen != NULL) { - *wlen = (size_t)res; + assert(res == _Py_CODEC_MEMORY_ERROR || res == _Py_CODEC_DECODE_ERROR); + if (size != NULL) { + *size = (size_t)res; } return NULL; } @@ -699,143 +761,154 @@ Py_DecodeLocale(const char* arg, size_t *wlen) } -static int -encode_current_locale(const wchar_t *text, char **str, - size_t *error_pos, const char **reason, - int raw_malloc, _Py_error_handler errors) +// If bytes is NULL, compute the length in 'bytes' of the encoded string. +// Otherwise, encode the wide string into 'bytes', decrements 'size', and +// return 0 on success. +// On encoding error, set 'error_pos' (if set) and return ENCODE_ERROR. +static size_t +encode_current_locale_impl(const wchar_t *text, const size_t len, + int surrogateescape, + char *bytes, size_t size, + size_t *error_pos) { - const size_t len = wcslen(text); - char *result = NULL, *bytes = NULL; - size_t i, size, converted; - wchar_t c, buf[2]; - - int surrogateescape; - if (get_surrogateescape(errors, &surrogateescape) < 0) { - return -3; - } - - /* The function works in two steps: - 1. compute the length of the output buffer in bytes (size) - 2. outputs the bytes */ - size = 0; + wchar_t buf[2]; + // The second character is always the NUL character buf[1] = 0; - while (1) { - for (i=0; i < len; i++) { - c = text[i]; - if (c >= 0xdc80 && c <= 0xdcff) { - if (!surrogateescape) { - goto encode_error; - } - /* UTF-8b surrogate */ - if (bytes != NULL) { - *bytes++ = c - 0xdc00; - size--; - } - else { - size++; + + for (size_t i=0; i < len; i++) { + wchar_t c = text[i]; + if (c >= 0xdc80 && c <= 0xdcff) { + if (!surrogateescape) { + if (error_pos != NULL) { + *error_pos = i; } - continue; + return ENCODE_ERROR; + } + /* UTF-8b surrogate */ + if (bytes != NULL) { + *bytes++ = c - 0xdc00; + size--; } else { - buf[0] = c; - if (bytes != NULL) { - converted = wcstombs(bytes, buf, size); - } - else { - converted = wcstombs(NULL, buf, 0); - } - if (converted == DECODE_ERROR) { - goto encode_error; - } - if (bytes != NULL) { - bytes += converted; - size -= converted; - } - else { - size += converted; - } + size++; } } - if (result != NULL) { - *bytes = '\0'; - break; + else { + // Encode a single character using wcstombs() + buf[0] = c; + size_t converted; + if (bytes != NULL) { + converted = wcstombs(bytes, buf, size); + } + else { + converted = wcstombs(NULL, buf, 0); + } + if (converted == ENCODE_ERROR) { + if (error_pos != NULL) { + *error_pos = i; + } + return ENCODE_ERROR; + } + if (bytes != NULL) { + bytes += converted; + size -= converted; + } + else { + size += converted; + } } + } + if (bytes) { + *bytes = '\0'; + } + // Sanity check, it cannot happen in practice + assert(size != ENCODE_ERROR); + return size; +} - size += 1; /* nul byte at the end */ + +// Encode a wide string to the current locale encoding. +// Set *str to a newly allocated string on success. +// Return a negative number on error: _Py_CODEC_MEMORY_ERROR, +// _Py_CODEC_ENCODE_ERROR or _Py_CODEC_UNSUPPORTED_ERROR_HANDLER. +// +// str and output_length must not be NULL +static int +encode_current_locale(const wchar_t *text, char **str, size_t *output_length, + size_t *error_pos, int raw_malloc, + _Py_error_handler errors) +{ + assert(str != NULL); + assert(output_length != NULL); + + int surrogateescape; + if (get_surrogateescape(errors, &surrogateescape) < 0) { + // Only support "strict" and "surrogateescape" + return _Py_CODEC_UNSUPPORTED_ERROR_HANDLER; + } + + // First, compute the output length + const size_t len = wcslen(text); + // Sanity check, it cannot happen in practice + assert(len != ENCODE_ERROR); + size_t size = encode_current_locale_impl(text, len, surrogateescape, + NULL, 0, error_pos); + if (size == ENCODE_ERROR) { + return _Py_CODEC_ENCODE_ERROR; + } + + *output_length = size; + char *result; + result = raw_malloc ? PyMem_RawMalloc(size + 1) : PyMem_Malloc(size + 1); + if (result == NULL) { + return _Py_CODEC_MEMORY_ERROR; + } + + // Second, encode characters + size = encode_current_locale_impl(text, len, surrogateescape, + result, size, error_pos); + if (size == ENCODE_ERROR) { if (raw_malloc) { - result = PyMem_RawMalloc(size); + PyMem_RawFree(result); } else { - result = PyMem_Malloc(size); - } - if (result == NULL) { - return -1; + PyMem_Free(result); } - bytes = result; + return _Py_CODEC_ENCODE_ERROR; } + assert(size == 0); + *str = result; return 0; - -encode_error: - if (raw_malloc) { - PyMem_RawFree(result); - } - else { - PyMem_Free(result); - } - if (error_pos != NULL) { - *error_pos = i; - } - if (reason) { - *reason = "encoding error"; - } - return -2; } -/* Encode a string to the locale encoding. - - Parameters: - - * raw_malloc: if non-zero, allocate memory using PyMem_RawMalloc() instead - of PyMem_Malloc(). - * current_locale: if non-zero, use the current LC_CTYPE, otherwise use - Python filesystem encoding. - * errors: error handler like "strict" or "surrogateescape". - - Return value: - - 0: success, *str is set to a newly allocated decoded string. - -1: memory allocation failure - -2: encoding error, set *error_pos and *reason (if set). - -3: the error handler 'errors' is not supported. - */ static int -encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos, - const char **reason, - int raw_malloc, int current_locale, _Py_error_handler errors) +encode_locale_inner(const wchar_t *text, char **str, size_t *output_length, + size_t *error_pos, int raw_malloc, + int current_locale, _Py_error_handler errors) { if (current_locale) { #ifdef _Py_FORCE_UTF8_LOCALE - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, - raw_malloc, errors); + return _Py_EncodeUTF8(text, str, output_length, + error_pos, raw_malloc, errors); #else - return encode_current_locale(text, str, error_pos, reason, - raw_malloc, errors); + return encode_current_locale(text, str, output_length, + error_pos, raw_malloc, errors); #endif } #ifdef _Py_FORCE_UTF8_FS_ENCODING - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, - raw_malloc, errors); + return _Py_EncodeUTF8(text, str, output_length, + error_pos, raw_malloc, errors); #else int use_utf8 = (_PyRuntime.preconfig.utf8_mode >= 1); #ifdef MS_WINDOWS use_utf8 |= (_PyRuntime.preconfig.legacy_windows_fs_encoding == 0); #endif if (use_utf8) { - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, - raw_malloc, errors); + return _Py_EncodeUTF8(text, str, output_length, + error_pos, raw_malloc, errors); } #ifdef USE_FORCE_ASCII @@ -844,45 +917,101 @@ encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos, } if (force_ascii) { - return encode_ascii(text, str, error_pos, reason, - raw_malloc, errors); + return encode_ascii(text, str, output_length, + error_pos, raw_malloc, errors); } #endif - return encode_current_locale(text, str, error_pos, reason, - raw_malloc, errors); + return encode_current_locale(text, str, output_length, + error_pos, raw_malloc, errors); #endif /* _Py_FORCE_UTF8_FS_ENCODING */ } + +// Encode a wide string to the locale encoding. +// +// Parameters: +// +// * raw_malloc: if non-zero, allocate memory using PyMem_RawMalloc() instead +// of PyMem_Malloc(). +// * current_locale: if non-zero, use the current LC_CTYPE, otherwise use +// Python filesystem encoding. +// * errors: supported error handlers are _Py_ERROR_STRICT and +// _Py_ERROR_SURROGATEESCAPE. The UTF-8 encoder also supports +// _Py_ERROR_SURROGATEPASS. +// +// Set *str to a newly allocated decoded string and return 0 on success. +// Return a negative result on error: +// +// * _Py_CODEC_MEMORY_ERROR: memory allocation failure +// * _Py_CODEC_ENCODE_ERROR: encoding error, set *error_pos. +// * _Py_CODEC_UNSUPPORTED_ERROR_HANDLER: the 'errors' error handler +// is not supported. +// +// text, str and output_length must not be NULL. +static int +encode_locale_impl(const wchar_t *text, char **str, size_t *output_length, + size_t *error_pos, int raw_malloc, + int current_locale, _Py_error_handler errors) +{ + assert(text != NULL); + assert(str != NULL); + assert(output_length != NULL); + + int res = encode_locale_inner(text, str, output_length, + error_pos, raw_malloc, + current_locale, errors); + if (res < 0) { + // Error + *str = NULL; + *output_length = 0; + if (res != _Py_CODEC_ENCODE_ERROR) { + if (error_pos) { + *error_pos = 0; + } + } + } + else { + assert(*str != NULL); + assert(*output_length == strlen(*str)); + // *error_pos is left unchanged + } + return res; +} + static char* encode_locale(const wchar_t *text, size_t *error_pos, int raw_malloc, int current_locale) { char *str; - int res = encode_locale_ex(text, &str, error_pos, NULL, - raw_malloc, current_locale, - _Py_ERROR_SURROGATEESCAPE); - if (res != -2 && error_pos) { - *error_pos = (size_t)-1; - } + size_t unused_output_length; + int res = encode_locale_impl(text, &str, &unused_output_length, + error_pos, raw_malloc, current_locale, + _Py_ERROR_SURROGATEESCAPE); if (res != 0) { + assert(res == _Py_CODEC_MEMORY_ERROR || res == _Py_CODEC_ENCODE_ERROR); + if (res == _Py_CODEC_MEMORY_ERROR && error_pos != NULL) { + *error_pos = (size_t)-1; + } return NULL; } return str; } -/* Encode a wide character string to the locale encoding with the - surrogateescape error handler: surrogate characters in the range - U+DC80..U+DCFF are converted to bytes 0x80..0xFF. - - Return a pointer to a newly allocated byte string, use PyMem_Free() to free - the memory. Return NULL on encoding or memory allocation error. - - If error_pos is not NULL, *error_pos is set to (size_t)-1 on success, or set - to the index of the invalid character on encoding error. - - Use the Py_DecodeLocale() function to decode the bytes string back to a wide - character string. */ +// Encode a wide character string to the locale encoding with the +// surrogateescape error handler. Surrogate characters in the range +// U+DC80..U+DCFF are encoded to bytes 0x80..0xFF. +// +// Return a pointer to a newly allocated byte string, use PyMem_Free() to free +// the memory. Return NULL on encoding or memory allocation error. +// +// On memory allocation failure, set *error_pos to (size_t)-1 and return NULL. +// +// On encoding error, set *error_pos to the index of the first unencodable +// character and return NULL. +// +// Use the Py_DecodeLocale() function to decode the bytes string back to a wide +// character string. char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) { @@ -892,7 +1021,7 @@ Py_EncodeLocale(const wchar_t *text, size_t *error_pos) /* Similar to Py_EncodeLocale(), but result must be freed by PyMem_RawFree() instead of PyMem_Free(). */ -char* +static char* _Py_EncodeLocaleRaw(const wchar_t *text, size_t *error_pos) { return encode_locale(text, error_pos, 1, 0); @@ -900,12 +1029,12 @@ _Py_EncodeLocaleRaw(const wchar_t *text, size_t *error_pos) int -_Py_EncodeLocaleEx(const wchar_t *text, char **str, - size_t *error_pos, const char **reason, - int current_locale, _Py_error_handler errors) +_Py_EncodeLocale(const wchar_t *text, char **str, size_t *output_length, + size_t *error_pos, int current_locale, + _Py_error_handler errors) { - return encode_locale_ex(text, str, error_pos, reason, 1, - current_locale, errors); + return encode_locale_impl(text, str, output_length, + error_pos, 1, current_locale, errors); } @@ -944,9 +1073,8 @@ _Py_GetLocaleEncoding(void) } wchar_t *wstr; - int res = decode_current_locale(encoding, &wstr, NULL, - NULL, _Py_ERROR_SURROGATEESCAPE); - if (res < 0) { + if (decode_current_locale(encoding, &wstr, NULL, + _Py_ERROR_SURROGATEESCAPE) < 0) { return NULL; } return wstr; diff --git a/Python/initconfig.c b/Python/initconfig.c index 178e6f992fa11f..363739a1a1b43c 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -4153,7 +4153,9 @@ static char* wstr_to_utf8(PyInitConfig *config, wchar_t *wstr) { char *utf8; - int res = _Py_EncodeUTF8Ex(wstr, &utf8, NULL, NULL, 1, _Py_ERROR_STRICT); + size_t utf8_len; + int res = _Py_EncodeUTF8(wstr, &utf8, &utf8_len, + NULL, 1, _Py_ERROR_STRICT); if (res == -2) { initconfig_set_error(config, "encoding error"); return NULL; @@ -4164,7 +4166,7 @@ wstr_to_utf8(PyInitConfig *config, wchar_t *wstr) } // Copy to use the malloc() memory allocator - size_t size = strlen(utf8) + 1; + size_t size = utf8_len + 1; char *str = malloc(size); if (str == NULL) { PyMem_RawFree(utf8); @@ -4323,12 +4325,13 @@ utf8_to_wstr(PyInitConfig *config, const char *str) { wchar_t *wstr; size_t wlen; - int res = _Py_DecodeUTF8Ex(str, strlen(str), &wstr, &wlen, NULL, _Py_ERROR_STRICT); - if (res == -2) { + int res = _Py_DecodeUTF8(str, strlen(str), &wstr, &wlen, _Py_ERROR_STRICT); + if (res == _Py_CODEC_DECODE_ERROR) { initconfig_set_error(config, "decoding error"); return NULL; } if (res < 0) { + assert(res == _Py_CODEC_MEMORY_ERROR); config->status = _PyStatus_NO_MEMORY(); return NULL; } From 72f821fdb81aa8f6bb5d4443bd61d774c9cefb93 Mon Sep 17 00:00:00 2001 From: Shankar Rao S Patil Date: Mon, 5 Oct 2026 02:59:13 +0530 Subject: [PATCH 063/240] gh-149302: Fix incorrect docstring in `RemoteUnwinder.get_all_awaited_by` (#149312) * Fix incorrect docstring for get_all_awaited_by in _remote_debugging * Regenerate Argument Clinic output after docstring fix * Clarify task waiter relationships and fallback task list --------- Co-authored-by: Pablo Galindo Salgado --- Modules/_remote_debugging/clinic/module.c.h | 58 +++++++++---------- Modules/_remote_debugging/module.c | 62 ++++++++++----------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 4d63beb91515f8..349113c36fb5f0 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -288,23 +288,25 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, "get_all_awaited_by($self, /)\n" "--\n" "\n" -"Get all tasks and their awaited_by relationships from the remote process.\n" +"Get awaited_by relationships for tasks in the remote process.\n" "\n" -"This provides a tree structure showing which tasks are waiting for\n" -"other tasks.\n" +"Returns:\n" +" A list of AwaitedInfo objects, where each object contains:\n" +"\n" +" - thread_id (int): Identifier of the thread, or 0 for tasks in the\n" +" interpreter\'s fallback task list.\n" +" - awaited_by (list[TaskInfo]): Tasks registered with this thread.\n" "\n" -"For each task, returns:\n" -"1. The call stack frames leading to where the task is currently\n" -" executing\n" -"2. The name of the task\n" -"3. A list of tasks that this task is waiting for, with their own\n" -" frames/names/etc\n" +"Each TaskInfo contains:\n" +" - task_id (int): Identifier of the task.\n" +" - task_name (str): Name of the task.\n" +" - coroutine_stack (list[CoroInfo]): Stack of coroutine frames.\n" +" - awaited_by (list[CoroInfo]): Coroutine information for tasks or futures\n" +" awaiting this task.\n" "\n" -"Returns a list of [frames, task_name, subtasks] where:\n" -"- frames: List of (func_name, filename, lineno) showing the call\n" -" stack\n" -"- task_name: String identifier for the task\n" -"- subtasks: List of tasks being awaited by this task, in same format\n" +"Each CoroInfo contains:\n" +" - call_stack (list[FrameInfo]): Call stack frames for the coroutine.\n" +" - task_name (int): Identifier of the task or future.\n" "\n" "Raises:\n" " RuntimeError: If AsyncioDebug section is not available in the\n" @@ -313,20 +315,20 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, " OSError: If reading from the remote process fails\n" "\n" "Example output:\n" -"[\n" +"\n" " [\n" -" [(\"c5\", \"script.py\", 10), (\"c4\", \"script.py\", 14)],\n" -" \"c2_root\",\n" -" [\n" -" [\n" -" [(\"c1\", \"script.py\", 23)],\n" -" \"sub_main_2\",\n" -" [...]\n" -" ],\n" -" [...]\n" -" ]\n" -" ]\n" -"]"); +" AwaitedInfo(\n" +" thread_id=12345,\n" +" awaited_by=[\n" +" TaskInfo(\n" +" task_id=1,\n" +" task_name=\"Task-1\",\n" +" coroutine_stack=[...],\n" +" awaited_by=[]\n" +" )\n" +" ]\n" +" )\n" +" ]"); #define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF \ {"get_all_awaited_by", (PyCFunction)_remote_debugging_RemoteUnwinder_get_all_awaited_by, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__}, @@ -1709,4 +1711,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=8ad64b51aaffc3ad input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8424a993a85ef2bd input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 8513bf0e4e65a5..c06d7ca635b167 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -904,27 +904,29 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } /*[clinic input] -@permit_long_summary +@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_all_awaited_by -Get all tasks and their awaited_by relationships from the remote process. +Get awaited_by relationships for tasks in the remote process. + +Returns: + A list of AwaitedInfo objects, where each object contains: -This provides a tree structure showing which tasks are waiting for -other tasks. + - thread_id (int): Identifier of the thread, or 0 for tasks in the + interpreter's fallback task list. + - awaited_by (list[TaskInfo]): Tasks registered with this thread. -For each task, returns: -1. The call stack frames leading to where the task is currently - executing -2. The name of the task -3. A list of tasks that this task is waiting for, with their own - frames/names/etc +Each TaskInfo contains: + - task_id (int): Identifier of the task. + - task_name (str): Name of the task. + - coroutine_stack (list[CoroInfo]): Stack of coroutine frames. + - awaited_by (list[CoroInfo]): Coroutine information for tasks or futures + awaiting this task. -Returns a list of [frames, task_name, subtasks] where: -- frames: List of (func_name, filename, lineno) showing the call - stack -- task_name: String identifier for the task -- subtasks: List of tasks being awaited by this task, in same format +Each CoroInfo contains: + - call_stack (list[FrameInfo]): Call stack frames for the coroutine. + - task_name (int): Identifier of the task or future. Raises: RuntimeError: If AsyncioDebug section is not available in the @@ -933,29 +935,25 @@ Returns a list of [frames, task_name, subtasks] where: OSError: If reading from the remote process fails Example output: -[ - # Task c2_root waiting for two subtasks + [ - # Call stack of c2_root - [("c5", "script.py", 10), ("c4", "script.py", 14)], - "c2_root", - [ - # First subtask (sub_main_2) and what it's waiting for - [ - [("c1", "script.py", 23)], - "sub_main_2", - [...] - ], - # Second subtask and its waiters - [...] - ] + AwaitedInfo( + thread_id=12345, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[...], + awaited_by=[] + ) + ] + ) ] -] [clinic start generated code]*/ static PyObject * _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6a49cd345e8aec53 input=c22bfee0612e0b69]*/ +/*[clinic end generated code: output=6a49cd345e8aec53 input=79e556b3973d21a8]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL; From 587b7a5b315a971ecd376184a1f1cbecaa9e6f21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2026 00:07:31 +0200 Subject: [PATCH 064/240] gh-158585: Fix refleak in _testcapi.bytes_resize() (#158806) --- Modules/_testcapi/bytes.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index e9ed765850a907..605df203d6fc9a 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -42,6 +42,7 @@ bytes_resize(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) if (compute_hash) { if (PyObject_Hash(obj) == -1) { + Py_DECREF(obj); return NULL; } } @@ -56,7 +57,7 @@ bytes_resize(PyObject *Py_UNUSED(module), PyObject *args, PyObject *kwargs) } Py_ssize_t refcnt = Py_REFCNT(obj); - return Py_BuildValue("Onp", obj, refcnt, obj != old_obj); + return Py_BuildValue("Nnp", obj, refcnt, obj != old_obj); } From e0861c6ae70c7e6f16c28399bd2b28c521e11e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 5 Oct 2026 01:57:47 +0200 Subject: [PATCH 065/240] gh-153364: Make frame, coroutine, and task-waiter chain walks iterative and bounded (#153365) * let me declare single limit * use our new limit in process_frame_chain() * add it in parse_async_frame_chain() * parse_coro_chain() * NEWS * async in the message? * test * no race * process_task_awaited_by * process_task_awaited_by limit test * NEWS * MAX_TASK_WAITER_CHAIN_DEPTH * TASK_WAITER_CHAIN_DEPTH in test * TASK_WAITER_CHAIN_DEPTH 256 * prevent the drift with the comment * better naming, better style * MAX_TASK_WAITER_CHAIN_DEPTH comment * task-waiter iterative bfs walk * iterative coro-walk * nicer news * 1 << 14 * comment * unused read_Py_ssize_t * fix tombstones * simplify * correct msg * better test * news for tombstones * left-over from when testing buggy version * redundant new line --- Lib/test/test_external_inspection.py | 291 ++++++++++++++++-- ...-07-08-22-18-04.gh-issue-153364.JBFHEg.rst | 3 + Modules/_remote_debugging/_remote_debugging.h | 37 +-- Modules/_remote_debugging/asyncio.c | 214 +++++++------ Modules/_remote_debugging/frames.c | 6 +- Modules/_remote_debugging/object_reading.c | 1 - 6 files changed, 370 insertions(+), 182 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-08-22-18-04.gh-issue-153364.JBFHEg.rst diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index c83d2cb2abeac8..f487c5943352b2 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -340,6 +340,40 @@ def _run_script_and_get_trace( finally: _cleanup_sockets(client_socket, server_socket) + @contextmanager + def _target_process(self, script_body): + """Context manager for running a target process with socket sync.""" + port = find_unused_port() + script = f"""\ +import socket +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +sock.connect(('localhost', {port})) +{textwrap.dedent(script_body)} +""" + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + def make_unwinder(cache_frames=True): + return RemoteUnwinder( + p.pid, all_threads=True, cache_frames=cache_frames + ) + + yield p, client_socket, make_unwinder + finally: + _cleanup_sockets(client_socket, server_socket) + def _find_frame_in_trace(self, stack_trace, predicate): """ Find a frame matching predicate in stack trace. @@ -1505,6 +1539,71 @@ def matches_awaited_by_pattern(task): finally: _cleanup_sockets(client_socket, server_socket) + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_awaited_by_skips_set_tombstones(self): + script_body = """\ + import asyncio + + class RemovedTask(asyncio.Task): + def __hash__(self): + return 0 + + class RemainingTask(asyncio.Task): + def __hash__(self): + return 1 + + async def main(): + victim = asyncio.current_task() + victim.set_name("victim") + removed = RemovedTask( + asyncio.sleep(10_000), name="removed" + ) + remaining = RemainingTask( + asyncio.sleep(10_000), name="remaining" + ) + + asyncio.future_add_to_awaited_by(victim, removed) + asyncio.future_add_to_awaited_by(victim, remaining) + + # Removing hash 0 leaves a dummy in slot 0 before the only + # active entry in slot 1. It must not count toward the set's + # used entries. + asyncio.future_discard_from_awaited_by(victim, removed) + + sock.sendall(b"ready") + sock.recv(16) + + asyncio.run(main()) + """ + + with self._target_process(script_body) as ( + _, + client_socket, + make_unwinder, + ): + _wait_for_signal(client_socket, b"ready") + + for method_name in ( + "get_async_stack_trace", + "get_all_awaited_by", + ): + with self.subTest(method=method_name): + unwinder = make_unwinder(cache_frames=False) + stack_trace = getattr(unwinder, method_name)() + relationships = self._get_awaited_by_relationships( + stack_trace + ) + self.assertEqual( + relationships["victim"], + {"remaining"}, + ) + + client_socket.sendall(b"done") + @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, @@ -3190,40 +3289,6 @@ class TestFrameCaching(RemoteInspectionTestBase): All tests verify cache reuse via object identity checks (assertIs). """ - @contextmanager - def _target_process(self, script_body): - """Context manager for running a target process with socket sync.""" - port = find_unused_port() - script = f"""\ -import socket -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.connect(('localhost', {port})) -{textwrap.dedent(script_body)} -""" - - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - - server_socket = _create_server_socket(port) - script_name = _make_test_script(script_dir, "script", script) - client_socket = None - - try: - with _managed_subprocess([sys.executable, script_name]) as p: - client_socket, _ = server_socket.accept() - server_socket.close() - server_socket = None - - def make_unwinder(cache_frames=True): - return RemoteUnwinder( - p.pid, all_threads=True, cache_frames=cache_frames - ) - - yield p, client_socket, make_unwinder - finally: - _cleanup_sockets(client_socket, server_socket) - def _get_frames_with_retry(self, unwinder, required_funcs): """Get frames containing required_funcs, with retry for transient errors.""" for _ in range(MAX_TRIES): @@ -4067,5 +4132,163 @@ def test_get_stats_disabled_raises(self): client_socket.sendall(b"done") +@requires_remote_subprocess_debugging() +class TestFrameChainLimits(RemoteInspectionTestBase): + """Frame chain walks abort instead of looping/overflowing on deep chains.""" + + # Limits plus one, to exceed them (must match MAX_FRAME_CHAIN_DEPTH / + # MAX_TASK_WAITER_WALK_TASKS from _remote_debugging.h) + FRAME_CHAIN_DEPTH = 1024 + 512 + 1 + TASK_WAITER_WALK_TASKS = 2**14 + 1 + + def _assert_unwinder_limit_error(self, unwind, expected_substring): + """Call unwind() until it raises the frame chain limit error. + + unwind must construct the RemoteUnwinder and call it, so that + transient RuntimeErrors from either step are retried; a successful + call means the limit never triggered and fails immediately. + """ + last_error = None + for _ in busy_retry(SHORT_TIMEOUT, error=False): + try: + unwind() + except TRANSIENT_ERRORS as e: + if expected_substring in str(e): + return + last_error = e + continue + self.fail( + "frame chain limit did not trigger; call returned a result" + ) + self.fail( + f"frame chain limit never raised; last transient error: " + f"{last_error!r}" + ) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_get_stack_trace_deep_frame_chain_aborts(self): + """Test that a frame chain deeper than the limit aborts the + synchronous stack walk instead of walking it indefinitely.""" + script_body = f"""\ + import sys + sys.setrecursionlimit({self.FRAME_CHAIN_DEPTH * 2}) + + def recurse(n): + if n <= 0: + sock.sendall(b"ready") + sock.recv(16) + return + recurse(n - 1) + + recurse({self.FRAME_CHAIN_DEPTH}) + """ + with self._target_process(script_body) as (p, client_socket, _): + _wait_for_signal(client_socket, b"ready") + self._assert_unwinder_limit_error( + lambda: RemoteUnwinder(p.pid).get_stack_trace(), + "Too many stack frames", + ) + client_socket.sendall(b"done") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_get_async_stack_trace_deep_task_waiter_chain_aborts(self): + """Test that a task waiter chain deeper than the limit aborts + the walk instead of overflowing the C stack.""" + script_body = f"""\ + import asyncio + + async def chain(n): + if n <= 0: + sock.sendall(b"ready") + sock.recv(16) + return + + task = asyncio.create_task(chain(n - 1)) + await task + + asyncio.run(chain({self.TASK_WAITER_WALK_TASKS})) + """ + with self._target_process(script_body) as (p, client_socket, _): + _wait_for_signal(client_socket, b"ready") + self._assert_unwinder_limit_error( + lambda: RemoteUnwinder(p.pid).get_async_stack_trace(), + "Too many task waiters", + ) + client_socket.sendall(b"done") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_get_async_stack_trace_deep_frame_chain_aborts(self): + """Test that a frame chain deeper than the limit aborts the async + stack walk instead of walking it indefinitely.""" + script_body = f"""\ + import sys, asyncio + sys.setrecursionlimit({self.FRAME_CHAIN_DEPTH * 2}) + + def recurse(n): + if n <= 0: + sock.sendall(b"ready") + sock.recv(16) + return + recurse(n - 1) + + async def deep(): + recurse({self.FRAME_CHAIN_DEPTH}) + + asyncio.run(deep()) + """ + with self._target_process(script_body) as (p, client_socket, _): + _wait_for_signal(client_socket, b"ready") + self._assert_unwinder_limit_error( + lambda: RemoteUnwinder(p.pid).get_async_stack_trace(), + "Too many async stack frames", + ) + client_socket.sendall(b"done") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_get_all_awaited_by_deep_coro_chain_aborts(self): + """Test that a coroutine await chain deeper than the limit aborts + the walk instead of overflowing the C stack.""" + script_body = f"""\ + import sys, asyncio + sys.setrecursionlimit({self.FRAME_CHAIN_DEPTH * 2}) + + async def chain(n): + if n <= 0: + await asyncio.sleep(10_000) + return + await chain(n - 1) + + async def main(): + task = asyncio.create_task(chain({self.FRAME_CHAIN_DEPTH})) + await asyncio.sleep(0) + sock.sendall(b"ready") + await task + + asyncio.run(main()) + """ + with self._target_process(script_body) as (p, client_socket, _): + _wait_for_signal(client_socket, b"ready") + self._assert_unwinder_limit_error( + lambda: RemoteUnwinder(p.pid).get_all_awaited_by(), + "Too many coroutine frames", + ) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-07-08-22-18-04.gh-issue-153364.JBFHEg.rst b/Misc/NEWS.d/next/Library/2026-07-08-22-18-04.gh-issue-153364.JBFHEg.rst new file mode 100644 index 00000000000000..b60a75b1c59738 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-08-22-18-04.gh-issue-153364.JBFHEg.rst @@ -0,0 +1,3 @@ +Make frame, coroutine and task-waiter walks iterative and bounded, avoiding +potential hangs and stack overflows. Fix asyncio task inspection when +awaited-by sets contain removed entries. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index fa37fb7b2167ec..3aecf3982b95d7 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -147,6 +147,8 @@ typedef enum _WIN32_THREADSTATE { #define MAX_STACK_CHUNK_SIZE (16 * 1024 * 1024) /* 16 MB max for stack chunks */ #define MAX_LONG_DIGITS 64 /* Allows values up to ~2^1920 */ #define MAX_SET_TABLE_SIZE (1 << 20) /* 1 million entries max for set iteration */ +#define MAX_FRAME_CHAIN_DEPTH (1024 + 512) /* Iteration bound for frame chain walks */ +#define MAX_TASK_WAITER_WALK_TASKS (1 << 14) /* Total-task bound for waiter walks */ #ifndef MAX #define MAX(a, b) ((a) > (b) ? (a) : (b)) @@ -478,12 +480,6 @@ typedef int (*thread_processor_func)( void *context ); -typedef int (*set_entry_processor_func)( - RemoteUnwinderObject *unwinder, - uintptr_t key_addr, - void *context -); - typedef int (*interpreter_processor_func)( RuntimeOffsets *offsets, uintptr_t interpreter_state_addr, @@ -528,7 +524,6 @@ extern int validate_debug_offsets(struct _Py_DebugOffsets *debug_offsets); * ============================================================================ */ extern int read_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *result); -extern int read_Py_ssize_t(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t *result); extern int read_char(RemoteUnwinderObject *unwinder, uintptr_t address, char *result); extern int read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr); @@ -729,12 +724,6 @@ extern int parse_task( PyObject *render_to ); -extern int parse_coro_chain( - RemoteUnwinderObject *unwinder, - uintptr_t coro_address, - PyObject *render_to -); - extern int parse_async_frame_chain( RemoteUnwinderObject *unwinder, PyObject *calls, @@ -742,22 +731,6 @@ extern int parse_async_frame_chain( uintptr_t running_task_code_obj ); -/* Set iteration */ -extern int iterate_set_entries( - RemoteUnwinderObject *unwinder, - uintptr_t set_addr, - set_entry_processor_func processor, - void *context -); - -/* Task awaited_by processing */ -extern int process_task_awaited_by( - RemoteUnwinderObject *unwinder, - uintptr_t task_address, - set_entry_processor_func processor, - void *context -); - extern int process_single_task_node( RemoteUnwinderObject *unwinder, uintptr_t task_addr, @@ -765,12 +738,6 @@ extern int process_single_task_node( PyObject *result ); -extern int process_task_and_waiters( - RemoteUnwinderObject *unwinder, - uintptr_t task_addr, - PyObject *result -); - extern int find_running_task_in_thread( RemoteUnwinderObject *unwinder, uintptr_t thread_state_addr, diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 44a9a3cbce0061..aa3f5b638931f8 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -116,12 +116,11 @@ ensure_async_debug_offsets(RemoteUnwinderObject *unwinder) * SET ITERATION FUNCTIONS * ============================================================================ */ -int +static int iterate_set_entries( RemoteUnwinderObject *unwinder, uintptr_t set_addr, - set_entry_processor_func processor, - void *context + PyObject *awaited_by ) { char set_object[SIZEOF_SET_OBJ]; if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, set_addr, @@ -146,28 +145,22 @@ iterate_set_entries( Py_ssize_t i = 0; Py_ssize_t els = 0; while (i < set_len && els < num_els) { - uintptr_t key_addr; - if (read_py_ptr(unwinder, table_ptr, &key_addr) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry key"); + setentry entry; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, table_ptr, sizeof(entry), &entry) < 0) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry"); return -1; } - if ((void*)key_addr != NULL) { - Py_ssize_t ref_cnt; - if (read_Py_ssize_t(unwinder, table_ptr, &ref_cnt) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry ref count"); + uintptr_t key_addr = (uintptr_t)entry.key; + if (key_addr != 0 && entry.hash != -1) { + if (parse_task(unwinder, key_addr, awaited_by) < 0) { return -1; } - - if (ref_cnt) { - // Process this valid set entry - if (processor(unwinder, key_addr, context) < 0) { - return -1; - } - els++; - } + els++; } - table_ptr += sizeof(void*) * 2; + table_ptr += sizeof(entry); i++; } @@ -248,12 +241,14 @@ parse_task_name( * ============================================================================ */ static int -handle_yield_from_frame( +get_awaited_coro_address( RemoteUnwinderObject *unwinder, uintptr_t gi_iframe_addr, uintptr_t gen_type_addr, - PyObject *render_to + uintptr_t *next_coro ) { + *next_coro = 0; + // Read the entire interpreter frame at once char iframe[SIZEOF_INTERP_FRAME]; int err = _Py_RemoteDebug_PagedReadRemoteMemory( @@ -309,11 +304,7 @@ handle_yield_from_frame( doesn't match the type of whatever it points to in its cr_await. */ - err = parse_coro_chain(unwinder, gi_await_addr, render_to); - if (err) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse coroutine chain in yield_from"); - return -1; - } + *next_coro = gi_await_addr; } } } @@ -321,7 +312,7 @@ handle_yield_from_frame( return 0; } -int +static int parse_coro_chain( RemoteUnwinderObject *unwinder, uintptr_t coro_address, @@ -329,49 +320,64 @@ parse_coro_chain( ) { assert((void*)coro_address != NULL); - // Read the entire generator object at once - char gen_object[SIZEOF_GEN_OBJ]; - int err = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - coro_address, - SIZEOF_GEN_OBJ, - gen_object); - if (err < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read generator object in coro chain"); - return -1; - } + for (size_t depth = 0; (void*)coro_address != NULL; depth++) { + if (depth >= MAX_FRAME_CHAIN_DEPTH) { + PyErr_SetString(PyExc_RuntimeError, + "Too many coroutine frames (possible infinite loop)"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Coroutine chain depth limit exceeded"); + return -1; + } - int8_t frame_state = GET_MEMBER(int8_t, gen_object, unwinder->debug_offsets.gen_object.gi_frame_state); - if (frame_state == FRAME_CLEARED) { - return 0; - } + // Read the entire generator object at once + char gen_object[SIZEOF_GEN_OBJ]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + coro_address, + SIZEOF_GEN_OBJ, + gen_object); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read generator object in coro chain"); + return -1; + } - uintptr_t gen_type_addr = GET_MEMBER(uintptr_t, gen_object, unwinder->debug_offsets.pyobject.ob_type); + int8_t frame_state = GET_MEMBER(int8_t, gen_object, unwinder->debug_offsets.gen_object.gi_frame_state); + if (frame_state == FRAME_CLEARED) { + return 0; + } - PyObject* name = NULL; + uintptr_t gen_type_addr = GET_MEMBER(uintptr_t, gen_object, unwinder->debug_offsets.pyobject.ob_type); - // Parse the previous frame using the gi_iframe from local copy - uintptr_t prev_frame; - uintptr_t gi_iframe_addr = coro_address + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe; - uintptr_t address_of_code_object = 0; - if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); - return -1; - } + PyObject* name = NULL; - if (!name) { - return 0; - } + // Parse the previous frame using the gi_iframe from local copy + uintptr_t prev_frame; + uintptr_t gi_iframe_addr = coro_address + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe; + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); + return -1; + } + + if (!name) { + return 0; + } - if (PyList_Append(render_to, name)) { + if (PyList_Append(render_to, name)) { + Py_DECREF(name); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to coro chain"); + return -1; + } Py_DECREF(name); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to coro chain"); - return -1; - } - Py_DECREF(name); - if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { - return handle_yield_from_frame(unwinder, gi_iframe_addr, gen_type_addr, render_to); + if (frame_state != FRAME_SUSPENDED_YIELD_FROM) { + return 0; + } + + if (get_awaited_coro_address(unwinder, gi_iframe_addr, gen_type_addr, + &coro_address) < 0) { + return -1; + } } return 0; @@ -513,35 +519,11 @@ parse_task( * TASK AWAITED_BY PROCESSING * ============================================================================ */ -// Forward declaration for mutual recursion -static int process_waiter_task(RemoteUnwinderObject *unwinder, uintptr_t key_addr, void *context); - -// Processor function for parsing tasks in sets -static int -process_task_parser( - RemoteUnwinderObject *unwinder, - uintptr_t key_addr, - void *context -) { - PyObject *awaited_by = (PyObject *)context; - return parse_task(unwinder, key_addr, awaited_by); -} - static int parse_task_awaited_by( RemoteUnwinderObject *unwinder, uintptr_t task_address, PyObject *awaited_by -) { - return process_task_awaited_by(unwinder, task_address, process_task_parser, awaited_by); -} - -int -process_task_awaited_by( - RemoteUnwinderObject *unwinder, - uintptr_t task_address, - set_entry_processor_func processor, - void *context ) { // Read the entire TaskObj at once char task_obj[SIZEOF_TASK_OBJ]; @@ -560,10 +542,10 @@ process_task_awaited_by( char awaited_by_is_a_set = GET_MEMBER(char, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set); if (awaited_by_is_a_set) { - return iterate_set_entries(unwinder, task_ab_addr, processor, context); + return iterate_set_entries(unwinder, task_ab_addr, awaited_by); } else { // Single task waiting - return processor(unwinder, task_ab_addr, context); + return parse_task(unwinder, task_ab_addr, awaited_by); } } @@ -658,30 +640,40 @@ process_single_task_node( return -1; } -int -process_task_and_waiters( +static int +process_task_waiters( RemoteUnwinderObject *unwinder, - uintptr_t task_addr, PyObject *result ) { - // First, add this task to the result - if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { - return -1; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(result); i++) { + PyObject *task_info = PyList_GET_ITEM(result, i); + PyObject *waiters = PyStructSequence_GET_ITEM(task_info, 3); + for (Py_ssize_t j = 0; j < PyList_GET_SIZE(waiters); j++) { + if (PyList_GET_SIZE(result) >= MAX_TASK_WAITER_WALK_TASKS) { + PyErr_SetString(PyExc_RuntimeError, + "Too many task waiters (possible infinite loop)"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Task waiter walk size limit exceeded"); + return -1; + } + PyObject *waiter = PyList_GET_ITEM(waiters, j); + // CoroInfo item 1 holds the waiter task address stored by parse_task(). + PyObject *task_id = PyStructSequence_GET_ITEM(waiter, 1); + void *task_ptr = PyLong_AsVoidPtr(task_id); + if (task_ptr == NULL && PyErr_Occurred()) { + set_exception_cause(unwinder, PyExc_RuntimeError, + "Failed to parse waiter task ID"); + return -1; + } + if (process_single_task_node( + unwinder, (uintptr_t)task_ptr, NULL, result) < 0) + { + return -1; + } + } } - // Now find all tasks that are waiting for this task and process them - return process_task_awaited_by(unwinder, task_addr, process_waiter_task, result); -} - -// Processor function for task waiters -static int -process_waiter_task( - RemoteUnwinderObject *unwinder, - uintptr_t key_addr, - void *context -) { - PyObject *result = (PyObject *)context; - return process_task_and_waiters(unwinder, key_addr, result); + return 0; } /* ============================================================================ @@ -776,7 +768,13 @@ parse_async_frame_chain( return -1; } + size_t frame_count = 0; while ((void*)address_of_current_frame != NULL) { + if (++frame_count > MAX_FRAME_CHAIN_DEPTH) { + PyErr_SetString(PyExc_RuntimeError, "Too many async stack frames (possible infinite loop)"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Async frame chain iteration limit exceeded"); + return -1; + } PyObject* frame_info = NULL; uintptr_t address_of_code_object; int res = parse_frame_object( @@ -978,7 +976,7 @@ process_running_task_chain( } // Now find all tasks that are waiting for this task and process them - if (process_task_awaited_by(unwinder, running_task_addr, process_waiter_task, result) < 0) { + if (process_task_waiters(unwinder, result) < 0) { return -1; } diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index 9febe6a472071b..f5b9ed72f1f98e 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -307,9 +307,7 @@ process_frame_chain( uintptr_t frame_addr = ctx->frame_addr; uintptr_t prev_frame_addr = 0; uintptr_t last_frame_addr = 0; - const size_t MAX_FRAMES = 1024 + 512; size_t frame_count = 0; - assert(MAX_FRAMES > 0 && MAX_FRAMES < 10000); ctx->stopped_at_cached_frame = 0; ctx->last_frame_visited = 0; @@ -321,12 +319,12 @@ process_frame_chain( uintptr_t stackpointer = 0; last_frame_addr = frame_addr; - if (++frame_count > MAX_FRAMES) { + if (++frame_count > MAX_FRAME_CHAIN_DEPTH) { PyErr_SetString(PyExc_RuntimeError, "Too many stack frames (possible infinite loop)"); set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain iteration limit exceeded"); return -1; } - assert(frame_count <= MAX_FRAMES); + assert(frame_count <= MAX_FRAME_CHAIN_DEPTH); if (ctx->chunks && ctx->chunks->count > 0) { parse_result = parse_frame_from_chunks( diff --git a/Modules/_remote_debugging/object_reading.c b/Modules/_remote_debugging/object_reading.c index 56d9f80a80fd0f..743970db4ddc88 100644 --- a/Modules/_remote_debugging/object_reading.c +++ b/Modules/_remote_debugging/object_reading.c @@ -25,7 +25,6 @@ read_##type_name(RemoteUnwinderObject *unwinder, uintptr_t address, c_type *resu } DEFINE_MEMORY_READER(ptr, uintptr_t, "Failed to read pointer from remote memory") -DEFINE_MEMORY_READER(Py_ssize_t, Py_ssize_t, "Failed to read Py_ssize_t from remote memory") DEFINE_MEMORY_READER(char, char, "Failed to read char from remote memory") int From 7d25916b41de4fc28e04b4944864ad1094a9e8e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 5 Oct 2026 02:37:21 +0200 Subject: [PATCH 066/240] gh-154194: Degrade frames in Tachyon instead of failing the sample (#154195) * degrade gracefully * news * better NEWS wording * do not raise on MAX_REMOTE_STR_READ * bye MAX_REMOTE_STR_READ * fix -m asyncio ps|pstree * test truncation and linetable sentinel * simpler * simpler * redundant now * respect #157790 in the news --------- Co-authored-by: Pablo Galindo Salgado --- .../pycore_global_objects_fini_generated.h | 3 + Include/internal/pycore_global_strings.h | 3 + .../internal/pycore_runtime_init_generated.h | 3 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/asyncio/tools.py | 10 ++- Lib/test/test_asyncio/test_tools.py | 76 ++++++++++++++++ Lib/test/test_external_inspection.py | 87 +++++++++++++++++++ .../test_binary_format.py | 20 +++++ ...-07-19-22-08-05.gh-issue-154194.1bqRdx.rst | 3 + Modules/_remote_debugging/_remote_debugging.h | 2 +- Modules/_remote_debugging/code_objects.c | 53 +++++++++-- Python/remote_debug.h | 11 ++- 12 files changed, 260 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-19-22-08-05.gh-issue-154194.1bqRdx.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index bcbfcdd9eaaec5..0e2fab84aec0ae 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -45,6 +45,9 @@ _PyStaticObjects_CheckAll(PyInterpreterState *interp) { _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(native), "", 8); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(str_replace_inf), "1e309", 5); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(type_params), ".type_params", 12); + _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(unknown_file), "", 14); + _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(unknown_function), "", 18); + _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(unreadable_frame), "", 18); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(utf_8), "utf-8", 5); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(AGEN_CLOSED), "AGEN_CLOSED", 11); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_ID(AGEN_CREATED), "AGEN_CREATED", 12); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 2e5020578ad69b..cbfa80f76b64cd 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -54,6 +54,9 @@ struct _Py_global_strings { STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") + STRUCT_FOR_STR(unknown_file, "") + STRUCT_FOR_STR(unknown_function, "") + STRUCT_FOR_STR(unreadable_frame, "") STRUCT_FOR_STR(utf_8, "utf-8") } literals; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index c95b2d4f95a50d..34dc6228a0bcb0 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1329,6 +1329,9 @@ extern "C" { INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ + INIT_STR(unknown_file, ""), \ + INIT_STR(unknown_function, ""), \ + INIT_STR(unreadable_frame, ""), \ INIT_STR(utf_8, "utf-8"), \ } diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 795515260daecb..f2f3b9de4a0d87 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -892,7 +892,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { offsetof(struct _Py_global_strings, literals._py_native._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_anon_setcomp._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_anon_string._ascii.ob_base), + offsetof(struct _Py_global_strings, literals._py_unknown_file._ascii.ob_base), + offsetof(struct _Py_global_strings, literals._py_unknown_function._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_anon_unknown._ascii.ob_base), + offsetof(struct _Py_global_strings, literals._py_unreadable_frame._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_json_decoder._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_list_err._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_utf_8._ascii.ob_base), diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 2ac1738d15c6c7..fd494912549bf8 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -27,6 +27,10 @@ def __init__( # ─── indexing helpers ─────────────────────────────────────────── def _format_stack_entry(elem: str|FrameInfo) -> str: if not isinstance(elem, str): + if elem.location is None: + if elem.filename in ("", "~"): + return f"{elem.funcname}" + return f"{elem.funcname} {elem.filename}" if elem.location.lineno == 0 and elem.filename == "": return f"{elem.funcname}" else: @@ -190,8 +194,7 @@ def build_task_table(result): # Build coroutine stack string frames = [frame for coro in task_info.coroutine_stack for frame in coro.call_stack] - coro_stack = " -> ".join(_format_stack_entry(x).split(" ")[0] - for x in frames) + coro_stack = " -> ".join(x.funcname for x in frames) # Handle tasks with no awaiters if not task_info.awaited_by: @@ -202,8 +205,7 @@ def build_task_table(result): # Handle tasks with awaiters for coro_info in task_info.awaited_by: parent_id = coro_info.task_name - awaiter_frames = [_format_stack_entry(x).split(" ")[0] - for x in coro_info.call_stack] + awaiter_frames = [x.funcname for x in coro_info.call_stack] awaiter_chain = " -> ".join(awaiter_frames) awaiter_name = id2name.get(parent_id, "Unknown") parent_id_str = (hex(parent_id) if isinstance(parent_id, int) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index df934164eb9fd6..2b8e4940333196 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -1558,6 +1558,82 @@ def test_table_output_format(self): class TestAsyncioToolsEdgeCases(unittest.TestCase): + def test_frames_without_location_tree(self): + """Frames the unwinder could not fully read - should not crash.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[ + CoroInfo( + call_stack=[ + FrameInfo("", "~", None), + FrameInfo("", "app.py", None), + FrameInfo("big", "big.py", None), + ], + task_name=1 + ) + ], + awaited_by=[] + ) + ] + ) + ] + self.assertEqual( + tools.build_async_tree(input_), + [[ + "└── (T) Task-A", + " └── big big.py", + " └── app.py", + " └── ", + ]], + ) + + def test_frames_without_location_table(self): + """Frame names are not truncated at the first space.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[ + CoroInfo( + call_stack=[ + FrameInfo("", "~", None) + ], + task_name=1 + ) + ], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("", "app.py", None) + ], + task_name=2 + ) + ] + ) + ] + ) + ] + self.assertEqual( + tools.build_task_table(input_), + [[ + 1, + "0x1", + "Task-A", + "", + "", + "Unknown", + "0x2", + ]], + ) + def test_task_awaits_self(self): """A task directly awaits itself - should raise a cycle.""" input_ = [ diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index f487c5943352b2..6b055ec6b84268 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -4132,6 +4132,93 @@ def test_get_stats_disabled_raises(self): client_socket.sendall(b"done") +@requires_remote_subprocess_debugging() +@skip_if_not_supported +@unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", +) +class TestMetadataDegradation(RemoteInspectionTestBase): + """Tests for graceful degradation of oversized code-object metadata.""" + + def test_long_qualname_truncated_not_dropped(self): + """A qualname longer than 1024 chars is truncated instead of + failing the whole sample.""" + name = "f" * 1100 + src = f"def {name}(sample):\n return sample()\n" + ns = {} + exec(src, ns) + + trace = ns[name](RemoteUnwinder(os.getpid()).get_stack_trace) + frame = self._find_frame_in_trace( + trace, lambda f: f.funcname.startswith("fff") + ) + self.assertIsNotNone(frame) + self.assertEqual(frame.funcname, "f" * 1024) + + def test_long_filename_truncated(self): + """A filename longer than 1024 chars is truncated instead of + failing the whole sample.""" + src = "def g(sample):\n return sample()\n" + ns = {} + exec(compile(src, "x" * 1500 + ".py", "exec"), ns) + + trace = ns["g"](RemoteUnwinder(os.getpid()).get_stack_trace) + frame = self._find_frame_in_trace(trace, lambda f: f.funcname == "g") + self.assertIsNotNone(frame) + self.assertEqual(frame.filename, "x" * 1024) + + def test_oversized_linetable_degrades_to_no_location(self): + """A linetable over MAX_LINETABLE_SIZE degrades to a frame without + location instead of failing the whole sample.""" + src = ( + "def big(sample):\n" + + " x = 1\n" * 20_000 + + " return sample()\n" + ) + ns = {} + exec(compile(src, "big_linetable.py", "exec"), ns) + big = ns["big"] + self.assertGreater(len(big.__code__.co_linetable), 64 * 1024) + + trace = big(RemoteUnwinder(os.getpid()).get_stack_trace) + frame = self._find_frame_in_trace( + trace, lambda f: f.funcname == "big" + ) + self.assertIsNone(frame.location) + self.assertEqual(frame.filename, "big_linetable.py") + + @unittest.skipIf( + sys.platform == "win32", + "Process death maps to ProcessLookupError only on POSIX platforms", + ) + def test_dead_process_raises_not_degrades(self): + """Death of the target raises ProcessLookupError instead of + degrading to synthetic frames.""" + script_body = """\ + import time + sock.sendall(b"ready") + time.sleep(10_000) + """ + with self._target_process(script_body) as (p, client_socket, make_unwinder): + _wait_for_signal(client_socket, b"ready") + unwinder = make_unwinder() + _get_stack_trace_with_retry(unwinder) + + p.kill() + p.wait() + + for _ in busy_retry(SHORT_TIMEOUT, error=False): + try: + unwinder.get_stack_trace() + except ProcessLookupError: + break + except RuntimeError: + continue + else: + self.fail("ProcessLookupError never raised for dead process") + + @requires_remote_subprocess_debugging() class TestFrameChainLimits(RemoteInspectionTestBase): """Frame chain walks abort instead of looping/overflowing on deep chains.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index ff944b3163ec3e..d33cc8b619adb1 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -665,6 +665,26 @@ def test_same_line_different_columns(self): collector, count = self.roundtrip(samples) self.assertEqual(count, 3) + def test_synthetic_frames_roundtrip(self): + """Degraded/sentinel frames (location=None) survive the binary format.""" + frames = [ + FrameInfo(("~", None, name, None)) + for name in ( + "", + "", + "", + "", + "", + ) + ] + frames.append(FrameInfo(("app.py", None, "", None))) + frames.append(FrameInfo(("", None, "real_func", None))) + samples = [[make_interpreter(0, [make_thread(1, frames)])]] + + collector, count = self.roundtrip(samples) + self.assertEqual(count, 1) + self.assert_samples_equal(samples, collector) + class TestBinaryEdgeCases(BinaryFormatTestBase): """Tests for edge cases in binary format.""" diff --git a/Misc/NEWS.d/next/Library/2026-07-19-22-08-05.gh-issue-154194.1bqRdx.rst b/Misc/NEWS.d/next/Library/2026-07-19-22-08-05.gh-issue-154194.1bqRdx.rst new file mode 100644 index 00000000000000..9971cd3f8dc25b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-22-08-05.gh-issue-154194.1bqRdx.rst @@ -0,0 +1,3 @@ +Fix the sampling profiler dropping entire samples when a non-fatal read fails; +frames now keep any readable metadata, and long funcnames and filenames are +truncated instead. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 3aecf3982b95d7..cf923014445f55 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -182,7 +182,7 @@ typedef enum _WIN32_THREADSTATE { #define set_exception_cause(unwinder, exc_type, message) \ do { \ assert(PyErr_Occurred() && "function returned -1 without setting exception"); \ - if (unwinder->debug && !_Py_RemoteDebug_HasPermissionError()) { \ + if (unwinder->debug && !_Py_RemoteDebug_IsFatalReadError()) { \ _set_debug_exception_cause(exc_type, message); \ } \ } while (0) diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index 6ad05885019626..081aaa49fb49ff 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -341,11 +341,16 @@ parse_code_object(RemoteUnwinderObject *unwinder, PyObject **result, const CodeObjectContext *ctx) { + _Py_DECLARE_STR(unknown_function, ""); + _Py_DECLARE_STR(unknown_file, ""); + _Py_DECLARE_STR(unreadable_frame, ""); + void *key = (void *)ctx->code_addr; CachedCodeMetadata *meta = NULL; PyObject *func = NULL; PyObject *file = NULL; PyObject *linetable = NULL; + int code_metadata_incomplete = 0; #ifdef Py_GIL_DISABLED // In free threading builds, code object addresses might have the low bit set @@ -369,30 +374,50 @@ parse_code_object(RemoteUnwinderObject *unwinder, if (_Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, real_address, SIZEOF_CODE_OBJ, code_object) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read code object"); - goto error; + if (_Py_RemoteDebug_IsFatalReadError()) { + goto error; + } + PyErr_Clear(); + func = Py_NewRef(&_Py_STR(unreadable_frame)); + file = Py_NewRef(_Py_LATIN1_CHR('~')); + goto degraded; } func = read_py_str(unwinder, GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.qualname), 1024); if (!func) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read function name from code object"); - goto error; + if (_Py_RemoteDebug_IsFatalReadError()) { + goto error; + } + PyErr_Clear(); + func = Py_NewRef(&_Py_STR(unknown_function)); + code_metadata_incomplete = 1; } file = read_py_str(unwinder, GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.filename), 1024); if (!file) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read filename from code object"); - goto error; + if (_Py_RemoteDebug_IsFatalReadError()) { + goto error; + } + PyErr_Clear(); + file = Py_NewRef(&_Py_STR(unknown_file)); + code_metadata_incomplete = 1; + } + + if (code_metadata_incomplete) { + goto degraded; } linetable = read_py_bytes(unwinder, GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.linetable), MAX_LINETABLE_SIZE); if (!linetable) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read linetable from code object"); - goto error; + if (_Py_RemoteDebug_IsFatalReadError()) { + goto error; + } + PyErr_Clear(); + goto degraded; } meta = PyMem_RawMalloc(sizeof(CachedCodeMetadata)); @@ -561,6 +586,18 @@ parse_code_object(RemoteUnwinderObject *unwinder, *result = tuple; return 0; +degraded: { + PyObject *degraded_tuple = make_frame_info(unwinder, file, Py_None, + func, Py_None); + Py_CLEAR(func); + Py_CLEAR(file); + if (!degraded_tuple) { + return -1; + } + *result = degraded_tuple; + return 0; +} + error: Py_XDECREF(func); Py_XDECREF(file); diff --git a/Python/remote_debug.h b/Python/remote_debug.h index caea3d5c062e5f..a279cd9dfd0305 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -107,9 +107,18 @@ _Py_RemoteDebug_HasPermissionError(void) && PyErr_ExceptionMatches(PyExc_PermissionError); } +static inline int +_Py_RemoteDebug_IsFatalReadError(void) +{ + return _Py_RemoteDebug_HasPermissionError() + || PyErr_ExceptionMatches(PyExc_MemoryError) + || PyErr_ExceptionMatches(PyExc_ProcessLookupError) + || (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_Exception)); +} + #define _set_debug_exception_cause(exception, format, ...) \ do { \ - if (!_Py_RemoteDebug_HasPermissionError()) { \ + if (!_Py_RemoteDebug_IsFatalReadError()) { \ PyThreadState *tstate = _PyThreadState_GET(); \ if (!_PyErr_Occurred(tstate)) { \ _PyErr_Format(tstate, exception, format, ##__VA_ARGS__); \ From 0e75a0555731b9d7713d473e65b991f77316838c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 5 Oct 2026 02:03:03 +0100 Subject: [PATCH 067/240] gh-158574: Stabilize TLBC cache refresh tests (#158801) * gh-158574: Stabilize TLBC cache refresh tests * Simplify synchronization in TLBC cache refresh tests --- Lib/test/test_external_inspection.py | 126 +++++++++++++++------------ 1 file changed, 69 insertions(+), 57 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 6b055ec6b84268..c2157f73389059 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -2437,42 +2437,50 @@ def test_tlbc_cache_refresh_after_growth(self): script = textwrap.dedent("""\ import os, threading from _remote_debugging import RemoteUnwinder + from _queue import SimpleQueue from test import support - go = threading.Event() - stop = threading.Event() + go = threading.Lock() + stop = threading.Lock() + go.acquire() + stop.acquire() + ready = SimpleQueue() def leaf(): - stop.wait() - - def wait_for_leaf_frames(u, expected_count): - for _ in support.sleeping_retry( - support.SHORT_TIMEOUT, - f"Expected {expected_count} leaf frames", - ): - try: - traces = u.get_stack_trace() - except RuntimeError as exc: - if str(exc) != "Failed to parse initial frame in chain": - raise - continue - count = sum( - f.funcname == "leaf" - for i in traces - for t in i.threads for f in t.frame_info - ) - if count == expected_count: - return + ready.put(None) + stop.acquire() + + def start_leaf(): + ready.put(None) + go.acquire() + leaf() + + def park(): + ready.put(None) + stop.acquire() + + def leaf_count(u): + return sum( + f.funcname == "leaf" + for i in u.get_stack_trace() + for t in i.threads for f in t.frame_info + ) + # SimpleQueue.put() and Lock.acquire() do not push Python frames. + # Once notified, the worker's stack stays stable until go is released. threading.Thread(target=leaf, daemon=True).start() + ready.get(timeout=support.SHORT_TIMEOUT) for _ in range(16): - threading.Thread(target=stop.wait, daemon=True).start() - threading.Thread(target=lambda: (go.wait(), leaf()), daemon=True).start() + threading.Thread(target=park, daemon=True).start() + ready.get(timeout=support.SHORT_TIMEOUT) + threading.Thread(target=start_leaf, daemon=True).start() + ready.get(timeout=support.SHORT_TIMEOUT) u = RemoteUnwinder(os.getpid(), all_threads=True, cache_frames=False) - wait_for_leaf_frames(u, 1) - go.set() - wait_for_leaf_frames(u, 2) + assert leaf_count(u) == 1 + go.release() + ready.get(timeout=support.SHORT_TIMEOUT) + assert leaf_count(u) == 2 """) result = subprocess.run( [sys.executable, "-X", "gil=0", "-X", "tlbc=1", "-c", script], @@ -2496,45 +2504,49 @@ def test_tlbc_cache_refresh_after_slot_fill(self): script = textwrap.dedent("""\ import os, threading from _remote_debugging import RemoteUnwinder + from _queue import SimpleQueue - go = threading.Event() - stop = threading.Event() + go = threading.Lock() + stop = threading.Lock() + go.acquire() + stop.acquire() + ready = SimpleQueue() def leaf(): - stop.wait() + ready.put(None) + stop.acquire() + + def start_leaf(): + ready.put(None) + go.acquire() + leaf() from test import support - def lines(u, expected_count): - for _ in support.sleeping_retry( - support.SHORT_TIMEOUT, - f"Expected {expected_count} leaf frames", - ): - try: - traces = u.get_stack_trace() - except RuntimeError as exc: - if str(exc) != "Failed to parse initial frame in chain": - raise - continue - result = sorted( - f.location.lineno - for i in traces - for t in i.threads for f in t.frame_info - if f.funcname == "leaf" - ) - # A new frame can still point at the function definition. - if (len(result) == expected_count and - leaf.__code__.co_firstlineno not in result): - return result + def lines(u): + return sorted( + f.location.lineno + for i in u.get_stack_trace() + for t in i.threads for f in t.frame_info + if f.funcname == "leaf" + ) + # SimpleQueue.put() and Lock.acquire() do not push Python frames. + # Once notified, the worker's stack stays stable until go is released. threading.Thread(target=leaf, daemon=True).start() - threading.Thread(target=lambda: (go.wait(), leaf()), daemon=True).start() + ready.get(timeout=support.SHORT_TIMEOUT) + threading.Thread(target=start_leaf, daemon=True).start() + ready.get(timeout=support.SHORT_TIMEOUT) u = RemoteUnwinder(os.getpid(), all_threads=True, cache_frames=False) - before = lines(u, 1) - assert before == [8], before - go.set() - cached = lines(u, 2) - assert cached == [8, 8], cached + before = lines(u) + # The notification can be observed before put() returns, so either + # line in leaf() is a valid sample. + assert before in ([12], [13]), before + go.release() + ready.get(timeout=support.SHORT_TIMEOUT) + cached = lines(u) + assert len(cached) == 2, cached + assert all(line in (12, 13) for line in cached), cached """) result = subprocess.run( [sys.executable, "-X", "gil=0", "-X", "tlbc=1", "-c", script], From f839c061d383bc87c3456d19bbdcbc1c30ed399b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 5 Oct 2026 03:04:20 +0200 Subject: [PATCH 068/240] gh-151292: `_remote_debugging`: Do not corrupt the binary file when hitting `OverflowError` (#152892) * the kolektor * test * better test * news * =Q, move const to the base, not self.running * gh-151292: Track binary writer finalization state --------- Co-authored-by: Pablo Galindo Salgado --- Lib/profiling/sampling/binary_collector.py | 22 ++- .../test_binary_format.py | 138 ++++++++++++++++++ ...-07-02-15-26-51.gh-issue-151292.nmnQlp.rst | 3 + Modules/_remote_debugging/binary_io.h | 9 ++ Modules/_remote_debugging/binary_io_writer.c | 30 +++- Modules/_remote_debugging/module.c | 25 +++- 6 files changed, 213 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-02-15-26-51.gh-issue-151292.nmnQlp.rst diff --git a/Lib/profiling/sampling/binary_collector.py b/Lib/profiling/sampling/binary_collector.py index 7a35044b1ee1d7..3d6d988077cfa2 100644 --- a/Lib/profiling/sampling/binary_collector.py +++ b/Lib/profiling/sampling/binary_collector.py @@ -1,5 +1,6 @@ """Thin Python wrapper around C binary writer for profiling data.""" +import sys import time import _remote_debugging @@ -81,6 +82,7 @@ def __init__(self, filename, sample_interval_usec, *, skip_idle=False, self.filename = filename self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle + self.running = True compression_type = _resolve_compression(compression) start_time_us = int(time.monotonic() * 1_000_000) @@ -102,9 +104,19 @@ def collect(self, stack_frames, timestamp_us=None): timestamp_us: Optional timestamp in microseconds. If not provided, uses time.monotonic() to generate one. """ + if not self.running: + return if timestamp_us is None: timestamp_us = int(time.monotonic() * 1_000_000) - self._writer.write_sample(stack_frames, timestamp_us) + try: + self._writer.write_sample(stack_frames, timestamp_us) + except OverflowError as e: + if not self._writer.limit_reached: + raise + self.running = False + print(f"Warning: {e}; stopping early and keeping the data " + "collected so far.", + file=sys.stderr) def collect_failed_sample(self): """Record a failed sample attempt (no-op for binary format).""" @@ -143,9 +155,5 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit - finalize unless there was an error.""" - if exc_type is None: - self._writer.finalize() - else: - self._writer.close() - return False + """Finalize if the writer can still produce a valid file.""" + return self._writer.__exit__(exc_type, exc_val, exc_tb) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index d33cc8b619adb1..97075a1e2fed80 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -9,6 +9,8 @@ import unittest from collections import defaultdict +from test.support import captured_stderr + try: import _remote_debugging from _remote_debugging import ( @@ -1051,6 +1053,142 @@ def test_writer_total_samples_after_close_returns_zero(self): w.close() self.assertEqual(w.total_samples, 0) + def test_binary_collector_stops_gracefully_on_overflow(self): + """OverflowError from the writer stops collection via the running + protocol instead of propagating and corrupting the file. + See gh-151292.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + collector = BinaryCollector(filename, 1000, compression="none") + self.assertTrue(collector.running) + + sample = [ + make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + + # Collect real samples first, then hit the limit. + for i in range(3): + collector.collect(sample, timestamp_us=(i + 1) * 1000) + self.assertTrue(collector.running) + + bad = [make_interpreter(2**32, sample[0].threads)] + with captured_stderr() as stderr: + collector.collect(bad, timestamp_us=4000) + collector.collect(sample, timestamp_us=5000) + + self.assertFalse(collector.running) + self.assertTrue(collector._writer.limit_reached) + self.assertEqual(stderr.getvalue().count("Warning:"), 1) + self.assertIn("interpreter_id", stderr.getvalue()) + + collector.export(None) + + self.assertEqual(collector.total_samples, 3) + + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(reader_collector), 3) + + def test_interpreter_id_overflow_rejected(self): + """An interpreter_id wider than u32 raises OverflowError before any + writer state is mutated: subsequent valid samples are still accepted + and finalize produces a readable file.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + good = [ + make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + bad = [ + make_interpreter(2**32, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + + writer = _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) + writer.write_sample(good, 1000) + with self.assertRaises(OverflowError): + writer.write_sample(bad, 2000) + writer.write_sample(good, 3000) + writer.finalize() + self.assertEqual(writer.total_samples, 2) + + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(reader_collector), 2) + + def test_writer_finalizes_after_format_limit(self): + for compression in (0, 1) if ZSTD_AVAILABLE else (0,): + with self.subTest(compression=compression): + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + good = [make_interpreter(0, [ + make_thread(1, [make_frame("a.py", 1, "f")]) + ])] + bad = [make_interpreter(2**32, good[0].threads)] + writer = _remote_debugging.BinaryWriter( + filename, 1000, 0, compression=compression + ) + with self.assertRaises(OverflowError): + with writer: + writer.write_sample(good, 1000) + writer.write_sample(good, 2000) + # The first interpreter is committed before the limit. + writer.write_sample(good + bad, 3000) + self.assertEqual(writer.total_samples, 3) + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(RawCollector()), 3) + + def test_collector_does_not_swallow_unrelated_overflow(self): + class BadStatus: + def __index__(self): + raise OverflowError("status conversion failed") + + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + collector = BinaryCollector(filename, 1000, compression="none") + self.addCleanup(collector._writer.close) + sample = [make_interpreter(0, [make_thread(1, [], BadStatus())])] + with captured_stderr() as stderr: + with self.assertRaisesRegex(OverflowError, "status conversion failed"): + collector.collect(sample, timestamp_us=1000) + self.assertEqual(stderr.getvalue(), "") + self.assertFalse(collector._writer.limit_reached) + with self.assertRaisesRegex(ValueError, "broken"): + collector.export() + with self.assertRaisesRegex(ValueError, "broken"): + collector._writer.write_sample([], 2000) + # Closing a broken writer must not attempt to finalize it. + collector.__exit__(None, None, None) + + def test_collector_finalizes_after_external_exception(self): + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + with self.assertRaisesRegex(RuntimeError, "sampling failed"): + with BinaryCollector(filename, 1000, compression="none") as collector: + collector.collect([make_interpreter(0, [make_thread(1, [])])]) + raise RuntimeError("sampling failed") + self.assertEqual(collector.total_samples, 1) + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(RawCollector()), 1) + + @unittest.skipUnless(os.path.exists("/dev/full"), "requires /dev/full") + def test_finalize_failure_breaks_writer(self): + writer = _remote_debugging.BinaryWriter("/dev/full", 1000, 0) + self.addCleanup(writer.close) + writer.write_sample([make_interpreter(0, [make_thread(1, [])])], 1000) + with self.assertRaises(OSError): + writer.finalize() + self.assertFalse(writer.limit_reached) + with self.assertRaisesRegex(ValueError, "broken"): + writer.finalize() + with self.assertRaisesRegex(ValueError, "broken"): + writer.write_sample([], 2000) + class TestBinaryFormatValidation(BinaryFormatTestBase): """Tests for malformed binary files.""" diff --git a/Misc/NEWS.d/next/Library/2026-07-02-15-26-51.gh-issue-151292.nmnQlp.rst b/Misc/NEWS.d/next/Library/2026-07-02-15-26-51.gh-issue-151292.nmnQlp.rst new file mode 100644 index 00000000000000..8825a70047eedd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-02-15-26-51.gh-issue-151292.nmnQlp.rst @@ -0,0 +1,3 @@ +Fix ``profiling.sampling --binary`` leaving unreadable profile files when +the binary format reaches a size limit. Preserve collected samples when the +writer can still finalize safely. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index c936d3372e5acd..6a2c5b795823e1 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -290,9 +290,18 @@ typedef struct { size_t pending_rle_samples; } ThreadEntry; +/* Limit errors occur before emitting an incomplete sample. Other write + * failures may leave partial records and must prevent finalization. */ +typedef enum { + BINARY_WRITER_OPEN, + BINARY_WRITER_LIMIT_REACHED, + BINARY_WRITER_BROKEN, +} BinaryWriterState; + /* Main binary writer structure */ typedef struct { FILE *fp; + BinaryWriterState state; /* Write buffer for batched I/O */ uint8_t *write_buffer; diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 6af81515e7131d..4cf81ca3cccd41 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -371,6 +371,7 @@ writer_intern_string(BinaryWriter *writer, PyObject *string, uint32_t *index) } if (writer->string_count >= UINT32_MAX) { + writer->state = BINARY_WRITER_LIMIT_REACHED; PyErr_SetString(PyExc_OverflowError, "too many strings for binary format"); return -1; @@ -380,6 +381,9 @@ writer_intern_string(BinaryWriter *writer, PyObject *string, uint32_t *index) (void **)&writer->string_lengths, &writer->string_capacity, sizeof(char *), sizeof(size_t)) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + writer->state = BINARY_WRITER_LIMIT_REACHED; + } return -1; } } @@ -390,6 +394,7 @@ writer_intern_string(BinaryWriter *writer, PyObject *string, uint32_t *index) return -1; } if ((uintmax_t)str_len > UINT32_MAX) { + writer->state = BINARY_WRITER_LIMIT_REACHED; PyErr_Format(PyExc_OverflowError, "string length %zd exceeds binary format maximum %u", str_len, UINT32_MAX); @@ -438,12 +443,16 @@ writer_intern_frame(BinaryWriter *writer, const FrameEntry *entry, uint32_t *ind } if (writer->frame_count >= UINT32_MAX) { + writer->state = BINARY_WRITER_LIMIT_REACHED; PyErr_SetString(PyExc_OverflowError, "too many frames for binary format"); return -1; } if (GROW_ARRAY(writer->frame_entries, writer->frame_count, writer->frame_capacity, FrameEntry) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + writer->state = BINARY_WRITER_LIMIT_REACHED; + } return -1; } @@ -487,6 +496,7 @@ writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, } if (writer->thread_count >= UINT32_MAX) { + writer->state = BINARY_WRITER_LIMIT_REACHED; PyErr_SetString(PyExc_OverflowError, "too many threads for binary format"); return NULL; @@ -496,6 +506,9 @@ writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, &writer->thread_capacity, sizeof(ThreadEntry)); if (!new_entries) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + writer->state = BINARY_WRITER_LIMIT_REACHED; + } return NULL; } writer->thread_entries = new_entries; @@ -928,6 +941,12 @@ static int process_thread_sample(BinaryWriter *writer, PyObject *thread_info, uint32_t interpreter_id, uint64_t timestamp_us) { + if (writer->total_samples == UINT64_MAX) { + writer->state = BINARY_WRITER_LIMIT_REACHED; + PyErr_SetString(PyExc_OverflowError, "too many samples for binary format"); + return -1; + } + PyObject *thread_id_obj = PyStructSequence_GET_ITEM(thread_info, 0); PyObject *status_obj = PyStructSequence_GET_ITEM(thread_info, 1); PyObject *frame_list = PyStructSequence_GET_ITEM(thread_info, 2); @@ -950,7 +969,6 @@ process_thread_sample(BinaryWriter *writer, PyObject *thread_info, /* Calculate timestamp delta */ uint64_t delta = timestamp_us - entry->prev_timestamp; - entry->prev_timestamp = timestamp_us; /* Process frames and build current stack */ uint32_t curr_stack[MAX_STACK_DEPTH]; @@ -1006,6 +1024,7 @@ process_thread_sample(BinaryWriter *writer, PyObject *thread_info, entry->prev_stack_depth = curr_depth; } + entry->prev_timestamp = timestamp_us; writer->total_samples++; return 0; } @@ -1025,15 +1044,16 @@ binary_writer_write_sample(BinaryWriter *writer, PyObject *stack_frames, uint64_ PyObject *interp_id_obj = PyStructSequence_GET_ITEM(interp_info, 0); PyObject *threads = PyStructSequence_GET_ITEM(interp_info, 1); - unsigned long interp_id_long = PyLong_AsUnsignedLong(interp_id_obj); - if (interp_id_long == (unsigned long)-1 && PyErr_Occurred()) { + unsigned long long interp_id_long = PyLong_AsUnsignedLongLong(interp_id_obj); + if (interp_id_long == (unsigned long long)-1 && PyErr_Occurred()) { return -1; } /* Bounds check: interpreter_id is stored as uint32_t in binary format */ if (interp_id_long > UINT32_MAX) { + writer->state = BINARY_WRITER_LIMIT_REACHED; PyErr_Format(PyExc_OverflowError, - "interpreter_id %lu exceeds maximum value %lu", - interp_id_long, (unsigned long)UINT32_MAX); + "interpreter_id %llu exceeds maximum value %u", + interp_id_long, UINT32_MAX); return -1; } uint32_t interpreter_id = (uint32_t)interp_id_long; diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index c06d7ca635b167..6fa277d1065ed9 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -1789,7 +1789,15 @@ _remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self, return NULL; } + if (self->writer->state == BINARY_WRITER_BROKEN) { + PyErr_SetString(PyExc_ValueError, "Writer is broken"); + return NULL; + } + self->writer->state = BINARY_WRITER_OPEN; if (binary_writer_write_sample(self->writer, stack_frames, timestamp_us) < 0) { + if (self->writer->state != BINARY_WRITER_LIMIT_REACHED) { + self->writer->state = BINARY_WRITER_BROKEN; + } return NULL; } @@ -1852,7 +1860,12 @@ _remote_debugging_BinaryWriter_set_stats_impl(BinaryWriterObject *self, static int binary_writer_finalize_and_cache(BinaryWriterObject *self) { + if (self->writer->state == BINARY_WRITER_BROKEN) { + PyErr_SetString(PyExc_ValueError, "Writer is broken"); + return -1; + } if (binary_writer_finalize(self->writer) < 0) { + self->writer->state = BINARY_WRITER_BROKEN; return -1; } self->cached_total_samples = self->writer->total_samples; @@ -1933,8 +1946,7 @@ _remote_debugging_BinaryWriter___exit___impl(BinaryWriterObject *self, /*[clinic end generated code: output=61831f47c72a53c6 input=12334ce1009af37f]*/ { if (self->writer) { - /* Only finalize on normal exit (no exception) */ - if (exc_type == Py_None) { + if (self->writer->state != BINARY_WRITER_BROKEN) { if (binary_writer_finalize_and_cache(self) < 0) { if (self->writer) { binary_writer_destroy(self->writer); @@ -1983,8 +1995,17 @@ BinaryWriter_get_total_samples(PyObject *op, void *closure) return PyLong_FromUnsignedLongLong(self->writer->total_samples); } +static PyObject * +BinaryWriter_get_limit_reached(PyObject *op, void *closure) +{ + BinaryWriter *writer = BinaryWriter_CAST(op)->writer; + return PyBool_FromLong(writer && writer->state == BINARY_WRITER_LIMIT_REACHED); +} + static PyGetSetDef BinaryWriter_getset[] = { {"total_samples", BinaryWriter_get_total_samples, NULL, "Total samples written", NULL}, + {"limit_reached", BinaryWriter_get_limit_reached, NULL, + "A format limit was reached; the collected samples can still be finalized", NULL}, {NULL} }; From b93fb19a6e3118857d9a1dc4ffcc179b632a88c1 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 5 Oct 2026 02:11:42 +0100 Subject: [PATCH 069/240] Run a job that the test suite with MSan to the CI (#158625) * Run the test suite with MSan in CI * Additional fixes * Add `_Py_MSAN_UNPOISON_STRING` * Apply Victor's suggestions Co-authored-by: Victor Stinner * Apply Victor's suggestions Co-authored-by: Victor Stinner --------- Co-authored-by: Victor Stinner --- .github/workflows/build.yml | 3 +++ .github/workflows/reusable-san.yml | 22 ++++++++++++++++++-- Doc/using/configure.rst | 4 ++++ Include/pyport.h | 4 ++++ Lib/test/_test_multiprocessing.py | 2 ++ Lib/test/test_cext/__init__.py | 1 + Lib/test/test_faulthandler.py | 4 ++-- Modules/_remote_debugging/binary_io_reader.c | 5 +++++ Modules/_remote_debugging/binary_io_writer.c | 6 ++++++ Modules/_testinternalcapi.c | 6 +++--- Modules/posixmodule.c | 1 + Modules/socketmodule.c | 9 ++++++-- Python/instrumentation.c | 1 + configure | 2 +- configure.ac | 2 +- 15 files changed, 61 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ab0473f4b9c59..00d64b3df7e965 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -547,6 +547,9 @@ jobs: - check-name: Undefined behavior sanitizer: UBSan free-threading: false + - check-name: Memory + sanitizer: MSan + free-threading: false uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index ad3232743874d6..da6306a50cf7bc 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -60,7 +60,7 @@ jobs: || '' }} - name: UBSan option setup - if: inputs.sanitizer != 'TSan' + if: inputs.sanitizer == 'UBSan' run: >- echo "UBSAN_OPTIONS=${SAN_LOG_OPTION} @@ -69,6 +69,20 @@ jobs: >> "$GITHUB_ENV" env: SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: MSan option setup + if: inputs.sanitizer == 'MSan' + run: | + echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV" + # MSan reports false positives for memory initialized by libraries + # that are not built with MSan, so disable modules that use them. + # _remote_debugging links to libzstd directly, but we unpoision the memory. + { + echo '*disabled*' + echo '_bz2 _ctypes _curses _curses_panel _dbm _decimal _gdbm _hashlib' + echo '_lzma _sqlite3 _ssl _tkinter _uuid _zstd readline zlib' + } > Modules/Setup.local + env: + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" @@ -93,6 +107,8 @@ jobs: # gh-157958: -O2 instead of the pydebug default -Og to avoid a clang 21 # compile-time blowup on some interpreter files. # (https://github.com/llvm/llvm-project/issues/179695) + # MSan uses --with-assertions instead of --with-pydebug because its + # hooks on the Python memory allocators hide uninitialized reads. - name: Configure CPython run: >- ./configure @@ -101,9 +117,11 @@ jobs: ${{ inputs.sanitizer == 'TSan' && '--with-thread-sanitizer' + || inputs.sanitizer == 'MSan' + && '--with-memory-sanitizer' || '--with-undefined-behavior-sanitizer --with-strict-overflow' }} - --with-pydebug + ${{ inputs.sanitizer == 'MSan' && '--with-assertions' || '--with-pydebug' }} ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }} ${{ inputs.free-threading && '--disable-gil' || '' }} - name: Build CPython diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 5b1676e504a628..d6f21712b103ba 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -1026,6 +1026,10 @@ Debug options Enable MemorySanitizer allocation error detector, ``msan`` (default is no). + MSan reports false positives for memory initialized by libraries that are + not built with MSan, so either build all dependencies with MSan or disable + the extension modules that use them in :file:`Modules/Setup.local`. + .. versionadded:: 3.6 .. option:: --with-undefined-behavior-sanitizer diff --git a/Include/pyport.h b/Include/pyport.h index 9cfdd09689d5c8..2206beaa77221c 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -558,6 +558,7 @@ extern "C" { # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) # define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) +# define _Py_MSAN_UNPOISON_STRING(STR) (__msan_unpoison_string(STR)) # endif # endif # if __has_feature(address_sanitizer) @@ -599,6 +600,9 @@ extern "C" { #ifndef _Py_MSAN_UNPOISON # define _Py_MSAN_UNPOISON(PTR, SIZE) #endif +#ifndef _Py_MSAN_UNPOISON_STRING +# define _Py_MSAN_UNPOISON_STRING(STR) +#endif /* AIX has __bool__ redefined in it's system header file. */ #if defined(_AIX) && defined(__bool__) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 46ed8843fcd051..7292128fb78870 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3159,6 +3159,7 @@ def test_imap_and_imap_unordered_with_buffersize_type_validation( with self.assertRaisesRegex(expected_exception, expected_regex): method(str, range(4), buffersize=buffersize) + @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.subTests('method_name', ("imap", "imap_unordered")) def test_imap_and_imap_unordered_when_buffer_is_full(self, method_name): @@ -3194,6 +3195,7 @@ def produce_args(): p.terminate() p.join() + @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.subTests('method_name', ("imap", "imap_unordered")) def test_imap_and_imap_unordered_with_buffersize_when_buffer_is_full( diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index c4fd2a1e044d89..9bd602ca2e4af4 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -192,6 +192,7 @@ def test_build(self): self.check_build('_test_cppext_internal') +@support.requires_venv_with_pip() def setUpModule(): global VENV_CONTEXT, PYTHON_EXE VENV_CONTEXT = support.setup_venv_with_pip_setuptools('env') diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 5a493a4fd95680..82b347c8f8c045 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -34,8 +34,8 @@ def skip_if_sanitizer_signal(signame): - return support.skip_if_sanitizer(f"TSAN/UBSan itercepts {signame}", - thread=True, ub=True) + return support.skip_if_sanitizer(f"TSan/UBSan/MSan intercepts {signame}", + thread=True, ub=True, memory=True) def expected_traceback(lineno1, lineno2, header, min_count=1): diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 9625ee6f301f05..8af1d281cee6b6 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -19,6 +19,10 @@ #include #endif +#ifdef _Py_MEMORY_SANITIZER +# include +#endif + /* ============================================================================ * CONSTANTS FOR BINARY FORMAT SIZES * ============================================================================ */ @@ -315,6 +319,7 @@ reader_decompress_samples(BinaryReader *reader, const uint8_t *data) return -1; } + _Py_MSAN_UNPOISON(output.dst, output.pos); total_output += output.pos; } diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 4cf81ca3cccd41..1794017053da50 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -19,6 +19,10 @@ #include #endif +#ifdef _Py_MEMORY_SANITIZER +# include +#endif + /* ============================================================================ * CONSTANTS FOR BINARY FORMAT SIZES * ============================================================================ */ @@ -235,6 +239,7 @@ writer_flush_buffer(BinaryWriter *writer) return -1; } + _Py_MSAN_UNPOISON(writer->zstd.compressed_buffer, output.pos); if (output.pos > 0) { if (fwrite_checked_allow_threads(writer->zstd.compressed_buffer, output.pos, writer->fp) < 0) { return -1; @@ -1104,6 +1109,7 @@ binary_writer_finalize(BinaryWriter *writer) return -1; } + _Py_MSAN_UNPOISON(writer->zstd.compressed_buffer, output.pos); if (output.pos > 0) { if (fwrite_checked_allow_threads(writer->zstd.compressed_buffer, output.pos, writer->fp) < 0) { return -1; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e1f87eb3b41c9f..f629b8e528faca 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -469,7 +469,7 @@ next_frame_pointer_is_valid(uintptr_t *frame_pointer, uintptr_t *next_fp, #endif } -static PyObject * +static PyObject * _Py_NO_SANITIZE_MEMORY manual_unwind_from_fp(uintptr_t *frame_pointer) { uintptr_t stack_min = 0; @@ -2110,8 +2110,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self, static PyObject * check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) { - /* ASan or TSan would report an use-after-free error */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) + /* ASan, MSan or TSan would report an error. */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) || defined(_Py_MEMORY_SANITIZER) Py_RETURN_NONE; #else PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0a451b8a833e67..eacf6556c1ffa4 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10138,6 +10138,7 @@ os_getlogin_impl(PyObject *module) errno = old_errno; } else { + _Py_MSAN_UNPOISON(name, sizeof(name)); result = PyUnicode_DecodeFSDefault(name); } #else diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 61505c2603f22c..5fec77a2b82bf7 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -774,7 +774,9 @@ set_herror(socket_state *state, int h_error) PyObject *v; #ifdef HAVE_HSTRERROR - v = Py_BuildValue("(iN)", h_error, decode_error_message(hstrerror(h_error))); + const char *errmsg = hstrerror(h_error); + _Py_MSAN_UNPOISON_STRING(errmsg); + v = Py_BuildValue("(iN)", h_error, decode_error_message(errmsg)); #else v = Py_BuildValue("(is)", h_error, "host not found"); #endif @@ -801,7 +803,9 @@ set_gaierror(socket_state *state, int error) #endif #ifdef HAVE_GAI_STRERROR - v = Py_BuildValue("(iN)", error, decode_error_message(gai_strerror(error))); + const char *errmsg = gai_strerror(error); + _Py_MSAN_UNPOISON_STRING(errmsg); + v = Py_BuildValue("(iN)", error, decode_error_message(errmsg)); #else v = Py_BuildValue("(is)", error, "getaddrinfo failed"); #endif @@ -6522,6 +6526,7 @@ _socket_getservbyport_impl(PyObject *module, int port, const char *proto) PyErr_SetString(PyExc_OSError, "port/proto not found"); return NULL; } + _Py_MSAN_UNPOISON_STRING(sp->s_name); return PyUnicode_FromString(sp->s_name); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 806d3fbf5d6b19..25663ce5430d2e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1690,6 +1690,7 @@ allocate_instrumentation_data(PyCodeObject *code) } monitoring->local_monitors = (_Py_LocalMonitors){ 0 }; monitoring->active_monitors = (_Py_LocalMonitors){ 0 }; + memset(monitoring->tool_versions, 0, sizeof(monitoring->tool_versions)); monitoring->tools = NULL; monitoring->lines = NULL; monitoring->line_tools = NULL; diff --git a/configure b/configure index 7d1003a0e0e985..e5f33f0f4e5ac9 100755 --- a/configure +++ b/configure @@ -16680,7 +16680,7 @@ int main(void) { return 2; } - ffi_arg rc; + ffi_arg rc = 0; ffi_call(&cif, FFI_FN(z_is_expected), &rc, values); return !rc; } diff --git a/configure.ac b/configure.ac index e1d55a7a3cef90..86227335731371 100644 --- a/configure.ac +++ b/configure.ac @@ -4458,7 +4458,7 @@ int main(void) { return 2; } - ffi_arg rc; + ffi_arg rc = 0; ffi_call(&cif, FFI_FN(z_is_expected), &rc, values); return !rc; } From 5fecd448bb120378978a37dde65dfce233d88c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 5 Oct 2026 03:49:56 +0200 Subject: [PATCH 070/240] gh-155811: Add a seqcount to `gc_stats` to prevent torn reads (#155828) * update_seq * no need for XCHGL, MOVL is enough? * gh-155811: Retry an inconsistent GC snapshot once --------- Co-authored-by: Pablo Galindo Salgado --- Include/internal/pycore_interp_structs.h | 1 + ...-08-15-10-20-40.gh-issue-155811.knP-YB.rst | 3 ++ Modules/_remote_debugging/gc_stats.c | 42 ++++++++++++++++--- Python/gc.c | 10 ++++- Python/gc_free_threading.c | 7 ++++ 5 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-15-10-20-40.gh-issue-155811.knP-YB.rst diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index fe501be1afc59b..9cc25eda44f9de 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -219,6 +219,7 @@ struct gc_old_stats_buffer { struct gc_stats { struct gc_young_stats_buffer young; struct gc_old_stats_buffer old[2]; + uint32_t update_seq; }; struct _gc_runtime_state { diff --git a/Misc/NEWS.d/next/Library/2026-08-15-10-20-40.gh-issue-155811.knP-YB.rst b/Misc/NEWS.d/next/Library/2026-08-15-10-20-40.gh-issue-155811.knP-YB.rst new file mode 100644 index 00000000000000..2032fd74380db8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-15-10-20-40.gh-issue-155811.knP-YB.rst @@ -0,0 +1,3 @@ +Add a sequence counter to GC statistics to prevent :mod:`!_remote_debugging` +returning inconsistent snapshots caused by non-atomic reads. Patch by Maurycy +Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/gc_stats.c b/Modules/_remote_debugging/gc_stats.c index d5d05edb8ecf5e..23fa879b503283 100644 --- a/Modules/_remote_debugging/gc_stats.c +++ b/Modules/_remote_debugging/gc_stats.c @@ -103,12 +103,42 @@ get_gc_stats_from_interpreter_state(RuntimeOffsets *offsets, } struct gc_stats stats; - if (_Py_RemoteDebug_ReadRemoteMemory(&offsets->handle, - gc_stats_addr, - sizeof(stats), - &stats) < 0) { - set_exception_cause(offsets, PyExc_RuntimeError, "Failed to read GC state"); - return -1; + uintptr_t sequence_address = gc_stats_addr + + offsetof(struct gc_stats, update_seq); + /* A short GC update may finish before a second attempt. */ + for (int attempt = 0; attempt < 2; attempt++) { + uint32_t before; + if (_Py_RemoteDebug_ReadRemoteMemory(&offsets->handle, + sequence_address, + sizeof(before), &before) < 0) { + set_exception_cause(offsets, PyExc_RuntimeError, + "Failed to read GC update sequence"); + return -1; + } + if (_Py_RemoteDebug_ReadRemoteMemory(&offsets->handle, + gc_stats_addr, + sizeof(stats), + &stats) < 0) { + set_exception_cause(offsets, PyExc_RuntimeError, "Failed to read GC state"); + return -1; + } + + uint32_t after; + if (_Py_RemoteDebug_ReadRemoteMemory(&offsets->handle, + sequence_address, + sizeof(after), &after) < 0) { + set_exception_cause(offsets, PyExc_RuntimeError, + "Failed to read GC update sequence"); + return -1; + } + if (before == after && before == stats.update_seq && !(after & 1)) { + break; + } + if (attempt == 1) { + PyErr_SetString(PyExc_RuntimeError, + "GC stats changed while being read; retry later"); + return -1; + } } if (read_gc_stats(&stats, iid, ctx->result, diff --git a/Python/gc.c b/Python/gc.c index 201c621bcc3cb9..bb20dae5a6543f 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1399,6 +1399,13 @@ gc_get_prev_stats(GCState *gcstate, int gen) static void add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats) { + struct gc_stats *generation_stats = gcstate->generation_stats; + uint32_t seq = _Py_atomic_load_uint32_relaxed(&generation_stats->update_seq); + assert((seq & 1) == 0); + /* Odd seq tells the reader that an update is in progress. */ + _Py_atomic_store_uint32_relaxed(&generation_stats->update_seq, seq + 1); + _Py_atomic_fence_seq_cst(); + struct gc_generation_stats *prev_stats = gc_get_prev_stats(gcstate, gen); struct gc_generation_stats *cur_stats = gc_get_stats(gcstate, gen); @@ -1412,9 +1419,8 @@ add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats) cur_stats->duration += stats->duration; cur_stats->heap_size = stats->heap_size; - /* Publish ts_stop last so remote readers do not select a partially - updated stats record as the latest collection. */ cur_stats->ts_stop = stats->ts_stop; + _Py_atomic_store_uint32_release(&generation_stats->update_seq, seq + 2); } /* This is the main function. Read this to understand how the diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 004b5871d196e4..42a28b8fb2a5ee 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2287,6 +2287,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) /* Update stats. */ PyMutex_Lock(&gcstate->stats_mutex); + struct gc_stats *generation_stats = gcstate->generation_stats; + uint32_t seq = _Py_atomic_load_uint32_relaxed(&generation_stats->update_seq); + assert((seq & 1) == 0); + /* Odd seq tells the reader that an update is in progress. */ + _Py_atomic_store_uint32_relaxed(&generation_stats->update_seq, seq + 1); + _Py_atomic_fence_seq_cst(); struct gc_generation_stats *stats = get_stats(gcstate, generation); stats->ts_start = start; stats->ts_stop = stop; @@ -2295,6 +2301,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) stats->uncollectable += n; stats->duration += duration; stats->candidates += state.candidates; + _Py_atomic_store_uint32_release(&generation_stats->update_seq, seq + 2); PyMutex_Unlock(&gcstate->stats_mutex); GC_STAT_ADD(generation, objects_collected, m); From d95b7ae6dc8201a1793e2bf09bfa03ca0bc97fa4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2026 04:41:29 +0200 Subject: [PATCH 071/240] Test MemoryError on locale encoding C functions (#158808) Fix also two typos. --- Lib/test/test_codecs.py | 20 +++++++++++++++++++- Modules/_testlimitedcapi/codec.c | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 2715ad6d7b3f8a..37ac57063c3539 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -4203,7 +4203,7 @@ def check_decode_strings(self, errors): errmsg = f"decode error: pos={error_pos}" self.assertEqual(str(cm.exception), errmsg) else: - if errors == ("strict", "surrogateescape"): + if errors in ("strict", "surrogateescape"): decoded = self.decode_locale_surrogateescape(encoded) self.assertEqual(decoded, expected) @@ -4233,6 +4233,24 @@ def test_decode_unsupported_error_handler(self): self.decode_locale(b'', 'backslashreplace') self.assertEqual(str(cm.exception), 'unsupported error handler') + def test_memory_error(self): + # Make sure that MemoryError is handled properly + with self.assertRaises(MemoryError): + with support.inject_memory_error_cm(): + self.decode_locale(b'short ascii string', 0, 'strict') + + with self.assertRaises(MemoryError): + with support.inject_memory_error_cm(): + self.decode_locale_surrogateescape(b'short ascii string') + + with self.assertRaises(MemoryError): + with support.inject_memory_error_cm(): + self.encode_locale('short string', 0, 'strict') + + with self.assertRaises(MemoryError): + with support.inject_memory_error_cm(): + self.encode_locale_surrogateescape('short string') + class Rot13Test(unittest.TestCase): """Test the educational ROT-13 codec.""" diff --git a/Modules/_testlimitedcapi/codec.c b/Modules/_testlimitedcapi/codec.c index 61e5d4708c71d6..e9c8fed198ebfe 100644 --- a/Modules/_testlimitedcapi/codec.c +++ b/Modules/_testlimitedcapi/codec.c @@ -31,7 +31,7 @@ decode_locale(PyObject *Py_UNUSED(module), PyObject *arg) size_t size = size_canary; wchar_t *wstr = Py_DecodeLocale(str, &size); - if (str == NULL) { + if (wstr == NULL) { if (size == (size_t)-1) { PyErr_NoMemory(); } From 30ce3d462dc97c8f351964e872c418abacaaabfd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2026 04:47:04 +0200 Subject: [PATCH 072/240] gh-158585: Simplify PyBytesWriter_Finish() (#158822) PyBytesWriter_Finish() doesn't need to check the size. It's known to be valid. PyBytesWriter_Create() now only calls byteswriter_resize() if the size is larger than the small buffer. Remove byteswriter_data() function: call static inline _PyBytesWriter_GetData() function directly. --- Modules/_struct.c | 3 +- Objects/bytesobject.c | 101 +++++++++++++++++++++------------------- Objects/unicodeobject.c | 7 +-- Python/assemble.c | 2 +- 4 files changed, 57 insertions(+), 56 deletions(-) diff --git a/Modules/_struct.c b/Modules/_struct.c index fb2f40ab9766c5..1e5d52b758f7e2 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2498,7 +2498,8 @@ Struct_pack_impl(PyStructObject *self, PyObject * const *values, return NULL; } - return PyBytesWriter_FinishWithSize(writer, self->s_size); + assert(PyBytesWriter_GetSize(writer) == self->s_size); + return PyBytesWriter_Finish(writer); } /*[clinic input] diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index af5dce763e1f25..9bd9afc9b51ba4 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3731,12 +3731,6 @@ _PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, // one extra NUL byte which is a common error. #define PyBytesWriter_CANARY_BYTE PYMEM_DEADBYTE -static inline char* -byteswriter_data(PyBytesWriter *writer) -{ - return _PyBytesWriter_GetData(writer); -} - static inline Py_ssize_t byteswriter_allocated(PyBytesWriter *writer) @@ -3758,7 +3752,7 @@ byteswriter_allocated(PyBytesWriter *writer) static void byteswriter_write_canary_byte(PyBytesWriter *writer) { - unsigned char *data = (unsigned char*)byteswriter_data(writer); + unsigned char *data = (unsigned char*)_PyBytesWriter_GetData(writer); data[writer->size] = PyBytesWriter_CANARY_BYTE; } @@ -3770,7 +3764,7 @@ byteswriter_reset_trailing_byte(PyBytesWriter *writer) // bytes/bytearray expects the last byte to be a null byte. // Reset the last byte to null for bytes/bytearray. Py_ssize_t allocated = byteswriter_allocated(writer); - char *data = byteswriter_data(writer); + char *data = _PyBytesWriter_GetData(writer); data[allocated] = '\0'; } #endif @@ -3801,7 +3795,7 @@ byteswriter_check_consistency(PyBytesWriter *writer) } #ifdef Py_DEBUG - const unsigned char *data = (const unsigned char*)byteswriter_data(writer); + const unsigned char *data = (const unsigned char*)_PyBytesWriter_GetData(writer); unsigned char canary = data[writer->size]; if (canary != PyBytesWriter_CANARY_BYTE) { _Py_FatalErrorFormat(__func__, @@ -3895,7 +3889,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t new_size, int resize) if (resize) { Py_ssize_t old_size = writer->size; assert(allocated > old_size); - memset(byteswriter_data(writer) + old_size, PyBytesWrite_NEW_BYTE, + memset(_PyBytesWriter_GetData(writer) + old_size, PyBytesWrite_NEW_BYTE, allocated - old_size); } #endif @@ -3921,24 +3915,25 @@ byteswriter_create(Py_ssize_t size, int use_bytearray) } } writer->obj = NULL; - writer->size = 0; + writer->size = size; writer->use_bytearray = use_bytearray; writer->overallocate = !use_bytearray; - if (size >= 1) { + Py_ssize_t allocated = sizeof(writer->small_buffer) - 1; + if (size > allocated) { if (byteswriter_resize(writer, size, 0) < 0) { #ifdef Py_DEBUG // Write the canary byte so byteswriter_check_consistency() // doesn't fail in PyBytesWriter_Discard() + writer->size = 0; byteswriter_write_canary_byte(writer); #endif PyBytesWriter_Discard(writer); return NULL; } - writer->size = size; } #ifdef Py_DEBUG - memset(byteswriter_data(writer), PyBytesWrite_NEW_BYTE, + memset(_PyBytesWriter_GetData(writer), PyBytesWrite_NEW_BYTE, byteswriter_allocated(writer)); byteswriter_write_canary_byte(writer); #endif @@ -3967,34 +3962,23 @@ PyBytesWriter_Discard(PyBytesWriter *writer) } assert(byteswriter_check_consistency(writer)); -#ifdef Py_DEBUG if (writer->obj != NULL) { +#ifdef Py_DEBUG byteswriter_reset_trailing_byte(writer); - } #endif - - Py_XDECREF(writer->obj); + Py_DECREF(writer->obj); + } _Py_FREELIST_FREE(bytes_writers, writer, PyMem_Free); } -PyObject* -PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) +static inline PyObject* +byteswriter_finish_with_size(PyBytesWriter *writer, Py_ssize_t final_size) { assert(byteswriter_check_consistency(writer)); - if (size < 0) { - PyErr_Format(PyExc_ValueError, "size must be positive"); - goto error; - } - - if (size > writer->size) { - PyErr_SetString(PyExc_ValueError, "size larger than allocated size"); - goto error; - } - PyObject *result; - if (size == 0 && !writer->use_bytearray) { + if (final_size == 0 && !writer->use_bytearray) { result = bytes_get_empty(); if (writer->obj != NULL) { #ifdef Py_DEBUG @@ -4011,19 +3995,19 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) #endif if (writer->use_bytearray) { - if (size != PyByteArray_GET_SIZE(writer->obj)) { - if (PyByteArray_Resize(writer->obj, size)) { + if (final_size != PyByteArray_GET_SIZE(writer->obj)) { + if (PyByteArray_Resize(writer->obj, final_size)) { goto error; } } } else { - if (size == 1) { + if (final_size == 1) { unsigned char ch = PyBytes_AS_STRING(writer->obj)[0]; Py_SETREF(writer->obj, bytes_get_char(ch)); } - else if (size != PyBytes_GET_SIZE(writer->obj)) { - if (bytes_resize_inplace(&writer->obj, size)) { + else if (final_size != PyBytes_GET_SIZE(writer->obj)) { + if (bytes_resize_inplace(&writer->obj, final_size)) { goto error; } } @@ -4036,18 +4020,18 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) // Create an object from the small buffer const char *buffer = (const char *)writer->small_buffer; if (writer->use_bytearray) { - result = PyByteArray_FromStringAndSize(buffer, size); + result = PyByteArray_FromStringAndSize(buffer, final_size); } else { - if (size == 1) { + if (final_size == 1) { result = bytes_get_char((uint8_t)buffer[0]); } else { - result = bytes_alloc(size); + result = bytes_alloc(final_size); if (result == NULL) { goto error; } - memcpy(PyBytes_AS_STRING(result), buffer, size); + memcpy(PyBytes_AS_STRING(result), buffer, final_size); } } } @@ -4061,17 +4045,36 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) return NULL; } +PyObject* +PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) +{ + if (size < 0) { + PyErr_Format(PyExc_ValueError, "size must be positive"); + goto error; + } + if (size > writer->size) { + PyErr_SetString(PyExc_ValueError, "size larger than allocated size"); + goto error; + } + return byteswriter_finish_with_size(writer, size); + +error: + PyBytesWriter_Discard(writer); + return NULL; +} + PyObject* PyBytesWriter_Finish(PyBytesWriter *writer) { - return PyBytesWriter_FinishWithSize(writer, writer->size); + return byteswriter_finish_with_size(writer, writer->size); } PyObject* PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) { - Py_ssize_t size = (char*)buf - byteswriter_data(writer); + Py_ssize_t size = (char*)buf - _PyBytesWriter_GetData(writer); + // Call PyBytesWriter_FinishWithSize() to check size return PyBytesWriter_FinishWithSize(writer, size); } @@ -4081,7 +4084,7 @@ PyBytesWriter_GetData(PyBytesWriter *writer) { assert(byteswriter_check_consistency(writer)); - return byteswriter_data(writer); + return _PyBytesWriter_GetData(writer); } @@ -4125,11 +4128,11 @@ static void* _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *data) { - Py_ssize_t pos = (char*)data - byteswriter_data(writer); + Py_ssize_t pos = (char*)data - _PyBytesWriter_GetData(writer); if (PyBytesWriter_Resize(writer, size) < 0) { return NULL; } - return byteswriter_data(writer) + pos; + return _PyBytesWriter_GetData(writer) + pos; } @@ -4176,11 +4179,11 @@ void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf) { - Py_ssize_t pos = (char*)buf - byteswriter_data(writer); + Py_ssize_t pos = (char*)buf - _PyBytesWriter_GetData(writer); if (PyBytesWriter_Grow(writer, size) < 0) { return NULL; } - return byteswriter_data(writer) + pos; + return _PyBytesWriter_GetData(writer) + pos; } @@ -4201,7 +4204,7 @@ PyBytesWriter_WriteBytes(PyBytesWriter *writer, if (PyBytesWriter_Grow(writer, size) < 0) { return -1; } - char *buf = byteswriter_data(writer); + char *buf = _PyBytesWriter_GetData(writer); memcpy(buf + pos, bytes, size); assert(byteswriter_check_consistency(writer)); @@ -4232,7 +4235,7 @@ PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) return -1; } - Py_ssize_t size = buf - byteswriter_data(writer); + Py_ssize_t size = buf - _PyBytesWriter_GetData(writer); return PyBytesWriter_Resize(writer, size); } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 157e005c3c0b60..ded13cc37f69f1 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4880,7 +4880,7 @@ _PyUnicode_EncodeUTF7(PyObject *str, else { /* not in a shift sequence */ if (ch == '+') { *out++ = '+'; - *out++ = '-'; + *out++ = '-'; } else if (ENCODE_DIRECT(ch)) { *out++ = (char) ch; @@ -8644,10 +8644,7 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject *unicode, iconv(cd, NULL, NULL, NULL, NULL); } - if (PyBytesWriter_Resize(writer, out - (char *)PyBytesWriter_GetData(writer)) < 0) { - goto done; - } - result = PyBytesWriter_Finish(writer); + result = PyBytesWriter_FinishWithPointer(writer, out); writer = NULL; done: diff --git a/Python/assemble.c b/Python/assemble.c index 2c8abb116b2e33..8f68e233ab7b93 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -102,7 +102,7 @@ assemble_free(struct assembler *a) static inline void write_except_byte(struct assembler *a, int byte) { - unsigned char *p = (unsigned char *) PyBytesWriter_GetData(a->a_except_table_writer); + unsigned char *p = PyBytesWriter_GetData(a->a_except_table_writer); p[a->a_except_table_off++] = byte; } From 182f3231542e84fd1b0c795898f69f61fe35df0e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2026 05:09:25 +0200 Subject: [PATCH 073/240] gh-158585: Adjust _PyBytes_FormatEx() allocation (#158823) Count properly the number of preallocated bytes when growing the buffer. --- Lib/test/test_bytes.py | 2 +- Objects/bytesobject.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index f87a924c57016e..96190a7f582170 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -894,7 +894,7 @@ def __int__(self): def test_memory_leak_gh_140939(self): # gh-140939: MemoryError is raised without leaking _testcapi = import_helper.import_module('_testcapi') - with self.assertRaises(MemoryError): + with self.assertRaises((MemoryError, OverflowError)): b = self.type2test(b'%*b') b % (_testcapi.PY_SSIZE_T_MAX, b'abc') diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 9bd9afc9b51ba4..1917055d25198f 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -747,6 +747,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, char onechar; /* For byte_converter() */ Py_ssize_t alloc; + const char *start = fmt; fmt++; if (*fmt == '%') { *res++ = '%'; @@ -1108,9 +1109,10 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, alloc = width; if (sign != 0 && len == width) alloc++; - /* 2: size preallocated for %s */ - if (alloc > 2) { - res = PyBytesWriter_GrowAndUpdatePointer(writer, alloc - 2, res); + /* size preallocated for the format */ + Py_ssize_t prealloc = fmt - start; + if (alloc > prealloc) { + res = PyBytesWriter_GrowAndUpdatePointer(writer, alloc - prealloc, res); if (res == NULL) { Py_XDECREF(temp); goto error; From 84b0669f81c0cf9b0b1accdbf02cf06747bccb04 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2026 09:16:37 +0200 Subject: [PATCH 074/240] gh-157710: Soft deprecate C API modifying str objects (#157711) Soft deprecate PyUnicode_New(), PyUnicode_CopyCharacters(), PyUnicode_Fill(), PyUnicode_Resize(), PyUnicode_WRITE() and PyUnicode_WriteChar() functions. Use the PyUnicodeWriter API instead. * Add tests on PyUnicode_New() and PyUnicode_Resize(). Check that the result is either a mutable string or the empty string singleton. * Check that PyUnicode_Fill(), PyUnicode_CopyCharacters() and PyUnicode_WriteChar() fail to modify a string with 2 references. Co-authored-by: Peter Bierma --- Doc/c-api/unicode.rst | 45 ++++++++++++++++--- Doc/whatsnew/3.16.rst | 7 +++ Lib/test/test_capi/test_unicode.py | 45 ++++++++++++++++--- ...-09-17-23-18-12.gh-issue-157710.1xKQAH.rst | 5 +++ Modules/_testcapi/unicode.c | 40 ++++++++++++++--- Modules/_testlimitedcapi/unicode.c | 18 ++++++-- 6 files changed, 138 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-17-23-18-12.gh-issue-157710.1xKQAH.rst diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index fcdcd26ff70cb0..584732b05c811f 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -168,8 +168,14 @@ access to internal read-only data of Unicode objects: The function performs no checks for any of its requirements, and is intended for usage in loops. + While :class:`str` objects are usually immutable in Python, this special C API allows + mutating a fresh :class:`str` object if the string has not been "used" yet. + .. versionadded:: 3.3 + .. soft-deprecated:: next + Use the :c:type:`PyUnicodeWriter` API instead. + .. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, Py_ssize_t index) @@ -407,9 +413,15 @@ APIs: using the :c:type:`PyUnicodeWriter` API, or one of the ``PyUnicode_From*`` functions below. + While :class:`str` objects are usually immutable in Python, this special C API + returns a :class:`str` object that can be mutated, except if *size* is zero, in which + case it returns the immutable empty string constant. .. versionadded:: 3.3 + .. soft-deprecated:: next + Use the :c:type:`PyUnicodeWriter` API instead. + .. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \ Py_ssize_t size) @@ -754,11 +766,16 @@ APIs: possible. Returns ``-1`` and sets an exception on error, otherwise returns the number of copied characters. - The string must not have been “used” yet. + While :class:`str` objects are usually immutable in Python, this special C API allows + mutating a fresh :class:`str` object if the string has not been "used" yet. + See :c:func:`PyUnicode_New` for details. .. versionadded:: 3.3 + .. soft-deprecated:: next + Use the :c:type:`PyUnicodeWriter` API instead. + .. c:function:: int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length); @@ -774,6 +791,14 @@ APIs: The function doesn't check string content, the result may not be a string in canonical representation. + While :class:`str` objects are usually immutable in Python, this special C API + can resize a :class:`str` object in-place if the string has not been "used" yet. + It returns a :class:`str` object which can be mutated, except if *size* is zero, in + which case it returns the immutable empty string constant. + + .. soft-deprecated:: next + Use the :c:type:`PyUnicodeWriter` API instead. + .. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \ Py_ssize_t length, Py_UCS4 fill_char) @@ -784,14 +809,19 @@ APIs: Fail if *fill_char* is bigger than the string maximum character, or if the string has more than 1 reference. - The string must not have been “used” yet. - See :c:func:`PyUnicode_New` for details. - Return the number of written characters, or return ``-1`` and raise an exception on error. + While :class:`str` objects are usually immutable in Python, this special C API allows + mutating a fresh :class:`str` object if the string has not been "used" yet. + + See :c:func:`PyUnicode_New` for details. + .. versionadded:: 3.3 + .. soft-deprecated:: next + Use the :c:type:`PyUnicodeWriter` API instead. + .. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \ Py_UCS4 character) @@ -804,11 +834,16 @@ APIs: See :c:func:`PyUnicode_WRITE` for a version that skips these checks, making them your responsibility. - The string must not have been “used” yet. + While :class:`str` objects are usually immutable in Python, this special C API allows + mutating a fresh :class:`str` object if the string has not been "used" yet. + See :c:func:`PyUnicode_New` for details. .. versionadded:: 3.3 + .. soft-deprecated:: next + Use the :c:type:`PyUnicodeWriter` API instead. + .. c:function:: Py_UCS4 PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 16e03e9ef06532..65db9bd6bf70ba 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1224,6 +1224,13 @@ Deprecated C APIs :c:func:`PyModule_GetFilenameObject` instead is still recommended. (Contributed by Victor Stinner in :gh:`154757`.) +* Soft deprecate functions modifying Unicode strings: + :c:func:`PyUnicode_New`, :c:func:`PyUnicode_CopyCharacters`, + :c:func:`PyUnicode_Fill`, :c:func:`PyUnicode_Resize`, + :c:func:`PyUnicode_WRITE` and :c:func:`PyUnicode_WriteChar`. + Use the safer :c:type:`PyUnicodeWriter` API instead. + (Contributed by Victor Stinner in :gh:`157710`.) + .. Add C API deprecations above alphabetically, not here at the end. .. include:: ../deprecations/c-api-pending-removal-in-3.18.rst diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index d43318770a4e62..9237809e7f0dcc 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -25,6 +25,7 @@ # Maximum invalid character which fits into 32-bit Py_UCS4 MAX_INVALID_CHAR = 0xFFFF_FFFF NULL = None +USED_STR_ERROR = 'Cannot modify a string currently used' class Str(str): pass @@ -76,9 +77,27 @@ def test_checkexact(self): # Test PyUnicode_CheckExact() self._test_check(_testlimitedcapi.unicode_checkexact, exact=True) + def assert_is_mutable(self, result, refcnt): + # Check that result is a "mutable" Unicode string + self.assertEqual(refcnt, 1) + self.assertFalse(sys._is_immortal(result)) + + def assert_is_empty_singleton(self, result): + # Check that result is the empty string singleton + self.assertEqual(result, '') + self.assertTrue(sys._is_immortal(result)) + def test_new(self): """Test PyUnicode_New()""" - new = _testcapi.unicode_new + _unicode_new = _testcapi.unicode_new + + def new(size, maxchar): + result = _unicode_new(size, maxchar) + if size != 0: + self.assert_is_mutable(result, sys.getrefcount(result)) + else: + self.assert_is_empty_singleton(result) + return result for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff: self.assertEqual(new(0, maxchar), '') @@ -123,6 +142,10 @@ def test_fill(self): self.assertEqual(fill(to, start, length, fill_char), (expected, filled)) + # A string with 2 references cannot be modified + with self.assertRaisesRegex(SystemError, USED_STR_ERROR): + fill('abc', 0, 3, ord('x'), incref=True) + s = strings[0] self.assertRaises(IndexError, fill, s, -1, 0, 0x78) self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78) @@ -162,7 +185,12 @@ def _test_writechar(self, writechar, *, check): def test_writechar(self): """Test PyUnicode_WriteChar()""" - self._test_writechar(_testlimitedcapi.unicode_writechar, check=True) + writechar = _testlimitedcapi.unicode_writechar + self._test_writechar(writechar, check=True) + + # A string with 2 references cannot be modified + with self.assertRaisesRegex(SystemError, USED_STR_ERROR): + writechar('abc', 1, ord('x'), incref=True) def test_write_macro(self): """Test PyUnicode_WRITE()""" @@ -187,18 +215,16 @@ def resize(s, length, new=True, compute_hash=False): self.assertFalse(is_new_obj) elif length == 0: # Get the empty Unicode string - self.assertEqual(result, '') - self.assertTrue(sys._is_immortal(result)) + self.assert_is_empty_singleton(result) self.assertTrue(is_new_obj) elif (not new) or compute_hash: # Get a fresh copy - self.assertEqual(refcnt, 1) + self.assert_is_mutable(result, refcnt) self.assertTrue(is_new_obj) - self.assertFalse(sys._is_immortal(result)) else: # In-size replace can return the same address, or not. # So 'is_new_obj' cannot be tested. - self.assertFalse(sys._is_immortal(result)) + self.assert_is_mutable(result, refcnt) return result @@ -1793,6 +1819,11 @@ def test_copycharacters(self): self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, PY_SSIZE_T_MIN) self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) self.assertRaises(SystemError, unicode_copycharacters, s, 0, [], 0, 0) + + # A string with 2 references cannot be modified + with self.assertRaisesRegex(SystemError, USED_STR_ERROR): + unicode_copycharacters('abc', 0, 'abc', 0, 1, incref=True) + # CRASHES unicode_copycharacters(s, 0, NULL, 0, 0) # TODO: Test PyUnicode_CopyCharacters() with non-unicode and # non-modifiable unicode as "to". diff --git a/Misc/NEWS.d/next/C_API/2026-09-17-23-18-12.gh-issue-157710.1xKQAH.rst b/Misc/NEWS.d/next/C_API/2026-09-17-23-18-12.gh-issue-157710.1xKQAH.rst new file mode 100644 index 00000000000000..4b203e430e94d8 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-17-23-18-12.gh-issue-157710.1xKQAH.rst @@ -0,0 +1,5 @@ +Soft deprecate functions modifying Unicode strings: +:c:func:`PyUnicode_New`, :c:func:`PyUnicode_CopyCharacters`, +:c:func:`PyUnicode_Fill`, :c:func:`PyUnicode_Resize`, +:c:func:`PyUnicode_WRITE` and :c:func:`PyUnicode_WriteChar`. Use +the :c:type:`PyUnicodeWriter` API instead. Patch by Victor Stinner. diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 5138fbab1f4ed6..ec3e9a623519f6 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -59,13 +59,20 @@ unicode_copy(PyObject *unicode) /* Test PyUnicode_Fill() */ static PyObject * -unicode_fill(PyObject *self, PyObject *args) +unicode_fill(PyObject *self, PyObject *args, PyObject *kwargs) { + static char *kwlist[] = {"to", "start", "length", "fill_char", + "incref", NULL}; PyObject *to, *to_copy; Py_ssize_t start, length, filled; unsigned int fill_char; + int incref = 0; - if (!PyArg_ParseTuple(args, "OnnI", &to, &start, &length, &fill_char)) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "OnnI|p", kwlist, + &to, &start, &length, + &fill_char, &incref)) + { return NULL; } @@ -74,7 +81,14 @@ unicode_fill(PyObject *self, PyObject *args) return NULL; } + if (incref) { + Py_INCREF(to_copy); + } filled = PyUnicode_Fill(to_copy, start, length, (Py_UCS4)fill_char); + if (incref) { + Py_DECREF(to_copy); + } + if (filled == -1 && PyErr_Occurred()) { Py_DECREF(to_copy); return NULL; @@ -190,13 +204,18 @@ unicode_asutf8(PyObject *self, PyObject *args) /* Test PyUnicode_CopyCharacters() */ static PyObject * -unicode_copycharacters(PyObject *self, PyObject *args) +unicode_copycharacters(PyObject *self, PyObject *args, PyObject *kwargs) { + static char *kwlist[] = {"to", "to_start", "from", "from_start", + "howmany", "incref", NULL}; PyObject *from, *to, *to_copy; Py_ssize_t from_start, to_start, how_many, copied; + int incref = 0; - if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, - &from, &from_start, &how_many)) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "UnOnn|p", kwlist, + &to, &to_start, &from, &from_start, + &how_many, &incref)) { return NULL; } @@ -210,8 +229,15 @@ unicode_copycharacters(PyObject *self, PyObject *args) return NULL; } + if (incref) { + Py_INCREF(to_copy); + } copied = PyUnicode_CopyCharacters(to_copy, to_start, from, from_start, how_many); + if (incref) { + Py_DECREF(to_copy); + } + if (copied == -1 && PyErr_Occurred()) { Py_DECREF(to_copy); return NULL; @@ -856,12 +882,12 @@ static PyType_Spec Writer_spec = { static PyMethodDef TestMethods[] = { {"unicode_new", unicode_new, METH_VARARGS}, - {"unicode_fill", unicode_fill, METH_VARARGS}, + {"unicode_fill", _PyCFunction_CAST(unicode_fill), METH_VARARGS | METH_KEYWORDS}, {"unicode_fromkindanddata", unicode_fromkindanddata, METH_VARARGS}, {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, - {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, + {"unicode_copycharacters", _PyCFunction_CAST(unicode_copycharacters), METH_VARARGS | METH_KEYWORDS}, {"unicode_GET_CACHED_HASH", unicode_GET_CACHED_HASH, METH_O}, {"test_py_identifier", test_py_identifier, METH_NOARGS}, {"corrupt_unicode", corrupt_unicode, METH_VARARGS}, diff --git a/Modules/_testlimitedcapi/unicode.c b/Modules/_testlimitedcapi/unicode.c index 53fff1916ac1ab..3d88cbb9fb304d 100644 --- a/Modules/_testlimitedcapi/unicode.c +++ b/Modules/_testlimitedcapi/unicode.c @@ -140,14 +140,19 @@ unicode_copy(PyObject *unicode) /* Test PyUnicode_WriteChar() */ static PyObject * -unicode_writechar(PyObject *self, PyObject *args) +unicode_writechar(PyObject *self, PyObject *args, PyObject *kwargs) { + static char *kwlist[] = {"to", "index", "character", "incref", NULL}; PyObject *to, *to_copy; Py_ssize_t index; unsigned int character; int result; + int incref = 0; - if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "OnI|p", kwlist, + &to, &index, &character, &incref)) + { return NULL; } @@ -156,7 +161,14 @@ unicode_writechar(PyObject *self, PyObject *args) return NULL; } + if (incref) { + Py_INCREF(to_copy); + } result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); + if (incref) { + Py_DECREF(to_copy); + } + if (result == -1 && PyErr_Occurred()) { Py_DECREF(to_copy); return NULL; @@ -1915,7 +1927,7 @@ static PyMethodDef TestMethods[] = { test_unicode_compare_with_ascii, METH_NOARGS}, {"test_string_from_format", test_string_from_format, METH_NOARGS}, {"test_widechar", test_widechar, METH_NOARGS}, - {"unicode_writechar", unicode_writechar, METH_VARARGS}, + {"unicode_writechar", _PyCFunction_CAST(unicode_writechar), METH_VARARGS | METH_KEYWORDS}, {"unicode_resize", unicode_resize, METH_VARARGS}, {"unicode_resize_null", unicode_resize_null, METH_VARARGS}, {"unicode_append", unicode_append, METH_VARARGS}, From 9133d5c2cb4ace240306f151b54c52a47136af81 Mon Sep 17 00:00:00 2001 From: "Jiucheng(Oliver)" Date: Mon, 5 Oct 2026 05:10:14 -0400 Subject: [PATCH 075/240] gh-152107: Fix crash when creating a dict item iterator under MemoryError (#152110) Co-authored-by: Sergey Miryanov --- Lib/test/test_dict.py | 20 +++++++++++++++++++ ...-06-24-14-05-00.gh-issue-152107.K3jX9p.rst | 3 +++ Objects/dictobject.c | 9 +++++---- 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-24-14-05-00.gh-issue-152107.K3jX9p.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 3b9ff672a1aa03..26bdaa9d6a35b7 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -116,6 +116,26 @@ def test_items(self): self.assertRaises(TypeError, d.items, None) self.assertEqual(repr(dict(a=1).items()), "dict_items([('a', 1)])") + @support.cpython_only + def test_item_iterator_oom(self): + import_helper.import_module('_testcapi') + from test.support.script_helper import assert_python_ok + code = """if 1: + import _testcapi + items = {1: 2, 3: 4}.items() + ballast = [(i, i) for i in range(3000)] + held = [] + for start in range(1, 5): + _testcapi.set_nomemory(start) + try: + held.append(iter(items)) + except MemoryError: + pass + finally: + _testcapi.remove_mem_hooks() + """ + assert_python_ok('-c', code) + def test_views_mapping(self): mappingproxy = type(type.__dict__) class Dict(dict): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-24-14-05-00.gh-issue-152107.K3jX9p.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-24-14-05-00.gh-issue-152107.K3jX9p.rst new file mode 100644 index 00000000000000..a8f5a99ee06524 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-24-14-05-00.gh-issue-152107.K3jX9p.rst @@ -0,0 +1,3 @@ +Fix a crash when creating a :class:`dict` item iterator (for example +``iter(d.items())`` or ``reversed(d.items())``) under a memory-allocation +failure. Patch by Jiucheng Zang. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index c6d055a412a515..bab983c2509c2c 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5632,6 +5632,7 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) used = GET_USED(dict); di->di_used = used; di->len = used; + di->di_result = NULL; if (itertype == &PyDictRevIterKey_Type || itertype == &PyDictRevIterItem_Type || itertype == &PyDictRevIterValue_Type) { @@ -5645,6 +5646,10 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) else { di->di_pos = 0; } + /* gh-152107: track before allocating di_result. A dictiter with a NULL + di_result is a valid state for dictiter_traverse()/dictiter_dealloc(), + so a failure of the allocation below can safely DECREF a tracked di. */ + _PyObject_GC_TRACK(di); if (itertype == &PyDictIterItem_Type || itertype == &PyDictRevIterItem_Type) { di->di_result = _PyTuple_FromPairSteal(Py_None, Py_None); @@ -5653,10 +5658,6 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) return NULL; } } - else { - di->di_result = NULL; - } - _PyObject_GC_TRACK(di); return (PyObject *)di; } From b2bfbce9a2cb00cb75c08230698cde408c130c92 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2026 13:30:16 +0200 Subject: [PATCH 076/240] gh-158810: Fix Py_ARRAY_LENGTH() macro on C++ (#158812) --- Include/pymacro.h | 30 +++++++++++-------- Lib/test/test_cext/extension.c | 4 +++ ...-10-05-01-09-32.gh-issue-158810.F1p-PK.rst | 2 ++ 3 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-10-05-01-09-32.gh-issue-158810.F1p-PK.rst diff --git a/Include/pymacro.h b/Include/pymacro.h index f31c9505a6a812..979e6563742b98 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -199,19 +199,23 @@ } while(0) #endif -/* Get the number of elements in a visible array - - This does not work on pointers, or arrays declared as [], or function - parameters. With correct compiler support, such usage will cause a build - error (see Py_BUILD_ASSERT_EXPR). - - Written by Rusty Russell, public domain, http://ccodearchive.net/ - - Requires at GCC 3.1+ */ -#if (defined(__GNUC__) && !defined(__STRICT_ANSI__) && \ - (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) || (__GNUC__ >= 4))) -/* Two gcc extensions. - &a[0] degrades to a pointer: a different type from an array */ +// Get the number of elements in a visible array. +// +// This does not work on pointers, or arrays declared as [], or function +// parameters. With correct compiler support, such usage will cause a build +// error (see Py_BUILD_ASSERT_EXPR). +// +// Written by Rusty Russell, public domain, http://ccodearchive.net/ +// +// Require GCC 4 (it works on GCC 3.1). +// +// Two GCC extensions: &a[0] degrades to a pointer, a different type from an +// array. +// +// gh-158810: Do not use __builtin_types_compatible_p() in strict C ANSI mode +// and on C++. +#if (defined(__GNUC__) && __GNUC__ >= 4 \ + && !defined(__STRICT_ANSI__) && !defined(__cplusplus)) #define Py_ARRAY_LENGTH(array) \ (sizeof(array) / sizeof((array)[0]) \ + Py_BUILD_ASSERT_EXPR(!__builtin_types_compatible_p(typeof(array), \ diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index ea7a1ca3c38a84..e392b66f19cc4a 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -83,6 +83,7 @@ test_macros(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { PyObject *obj, *dict; PyObject *slots[1]; + int small_array[] = {2, 5, 7}; // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); @@ -134,6 +135,9 @@ test_macros(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_END_CRITICAL_SECTION(); Py_DECREF(dict); + // Test Py_ARRAY_LENGTH() + assert(Py_ARRAY_LENGTH(small_array) == 3); + Py_RETURN_NONE; } diff --git a/Misc/NEWS.d/next/C_API/2026-10-05-01-09-32.gh-issue-158810.F1p-PK.rst b/Misc/NEWS.d/next/C_API/2026-10-05-01-09-32.gh-issue-158810.F1p-PK.rst new file mode 100644 index 00000000000000..8d0878fd057a04 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-10-05-01-09-32.gh-issue-158810.F1p-PK.rst @@ -0,0 +1,2 @@ +Fix :c:macro:`Py_ARRAY_LENGTH()` macro on C++: do not use +``__builtin_types_compatible_p()`` on C++. Patch by Victor Stinner. From fc3cd8e3f59fcae208bc2b53014dcb90fdc038cb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2026 13:51:53 +0200 Subject: [PATCH 077/240] gh-158585: Optimize bytes.join() for non-empty separator (#158689) Remove the conditional branch from the loop. --- Objects/stringlib/join.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Objects/stringlib/join.h b/Objects/stringlib/join.h index fe460d41f6700f..5fd2ca70f98dd3 100644 --- a/Objects/stringlib/join.h +++ b/Objects/stringlib/join.h @@ -134,13 +134,15 @@ STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) } } else { - for (i = 0; i < nbufs; i++) { - Py_ssize_t n; - char *q; - if (i) { - memcpy(p, sepstr, seplen); - p += seplen; - } + Py_ssize_t n = buffers[0].len; + char *q = buffers[0].buf; + memcpy(p, q, n); + p += n; + + for (i = 1; i < nbufs; i++) { + memcpy(p, sepstr, seplen); + p += seplen; + n = buffers[i].len; q = buffers[i].buf; memcpy(p, q, n); From 9112daed900c3314fa522043fc64ddbb2ecd4ca5 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 5 Oct 2026 21:18:22 +0900 Subject: [PATCH 078/240] gh-130706: Add a fast path to _Py_Dealloc() for non-GC objects. (gh-158645) --- ...-10-03-19-11-03.gh-issue-130706.01-O6U.rst | 2 ++ Objects/object.c | 26 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-19-11-03.gh-issue-130706.01-O6U.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-19-11-03.gh-issue-130706.01-O6U.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-19-11-03.gh-issue-130706.01-O6U.rst new file mode 100644 index 00000000000000..3351513347daec --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-19-11-03.gh-issue-130706.01-O6U.rst @@ -0,0 +1,2 @@ +Add a fast path to ``_Py_Dealloc()`` for non-GC objects. Patch by Donghee +Na. diff --git a/Objects/object.c b/Objects/object.c index c7aeba0cee22c6..e334097d31c432 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3306,8 +3306,8 @@ next" object in the chain to 0. This can easily lead to stack overflows. To avoid that, if the C stack is nearing its limit, instead of calling dealloc on the object, it is added to a queue to be freed later when the stack is shallower */ -void -_Py_Dealloc(PyObject *op) +static Py_NO_INLINE void +py_dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); unsigned long gc_flag = type->tp_flags & Py_TPFLAGS_HAVE_GC; @@ -3372,6 +3372,28 @@ _Py_Dealloc(PyObject *op) } } +/* + * gh-130706: Keep the GC/reftracer path in the non-inlined py_dealloc(). + * Inlining it makes the compiler save callee-saved registers at entry, + * so the non-GC objects path would have unnecessary register spills. + */ +void +_Py_Dealloc(PyObject *op) +{ +#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) + // gh-130706: Avoid unnecessary register spills for non-GC objects. + PyTypeObject *type = Py_TYPE(op); + if (_PyRuntime.ref_tracer.tracer_func == NULL + && !(type->tp_flags & Py_TPFLAGS_HAVE_GC)) + { + type->tp_dealloc(op); + return; + } +#endif + // GC objects (trashcan), reftracer set, or debug builds. + py_dealloc(op); +} + PyObject ** PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) From 0b8aa89a05e6b0dee93213dae21f230ce688f1ae Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 5 Oct 2026 14:28:51 +0100 Subject: [PATCH 079/240] gh-153569: Simplify tokenizer state (#158124) * Use diagnostic line for missing-brace errors * Keep implicit prepared newline metadata in the reader * Reuse parsed formatted string kind in lexer * Transfer owned tokenizer encoding strings * Remove unused tokenizer state and redundant branches * Avoid rescanning partial decoded tokenizer lines * Reuse prepared tokenizer line terminators for CR detection * Reuse interactive tokenizer chunks without carriage returns * Release tokenizer constructor resources on allocation failure * Narrow interactive newline normalization wrapper * Close duplicated tokenizer descriptors when fdopen fails * Preserve cached tokenizer lines when newline allocation fails * Collect cycles involving tokenizer readline callbacks * Preserve buffered text when decoding source files * Clear the transferred syntax error line before cleanup * Propagate tokenizer column conversion failures * Allocate parser repetition buffers on first match * Allocate type comment storage only when needed * Reuse decoded multiline text when counting token columns * Skip empty decoded tokenizer chunks * Check parser array growth before allocating * Name tokenizer bounds and derive byte lengths from their data * Set the tokenizer exception when file input size overflows * gh-153569: Send exact line endings in the REPL test * Share parser repetition buffer growth * Fold Unicode input coverage into the existing REPL test * Keep parser array growth policy private * Drop obsolete bitmap stress from source discard test * Preserve input failures while scanning string literals * fixup! Preserve input failures while scanning string literals * fixup! fixup! Preserve input failures while scanning string literals * gh-153569: Remove assertion-only lexer bookkeeping --- Lib/test/test_fstring.py | 14 + Lib/test/test_repl.py | 15 +- Lib/test/test_source_encoding.py | 29 + Lib/test/test_tokenize.py | 33 + Lib/test/test_type_comments.py | 10 + ...4-18-20-00.gh-issue-153569.diagnostics.rst | 4 + Modules/_testinternalcapi/tokenizer.c | 67 +- Parser/lexer/lexer.c | 5 +- Parser/lexer/lexer_internal.h | 3 +- Parser/lexer/string.c | 42 +- Parser/parser.c | 1235 +++++------------ Parser/pegen.c | 65 +- Parser/pegen.h | 2 + Parser/pegen_errors.c | 1 + Parser/tokenizer/decoder.c | 195 ++- Parser/tokenizer/reader.c | 105 +- Parser/tokenizer/reader_internal.h | 9 +- Parser/tokenizer/source.c | 70 +- Parser/tokenizer/source.h | 11 +- Parser/tokenizer/tokenizer.h | 1 + Parser/tokenizer/types.h | 6 - Python/Python-tokenize.c | 139 +- .../peg_generator/pegen/c_generator_rules.py | 22 +- 23 files changed, 775 insertions(+), 1308 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 63006266bcd039..72c80b8bfbfecc 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -645,6 +645,20 @@ def test_unclosed_multiline_replacement_field(self): f"{prefix[-1]}-string: expecting '}}' to close '{{' " f"on line {lineno}") + def test_unclosed_replacement_field_quote_line(self): + for prefix in ('f', 't', 'rf', 'rt'): + for quote in ('"', "'"): + triple = quote * 3 + for suffix in ('', '\nx'): + source = prefix + triple + '{1' + triple + suffix + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + self.assertEqual( + cm.exception.msg, + f"{prefix[-1]}-string: expecting '}}'") + self.assertEqual(cm.exception.lineno, 1) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 21e603c561d73c..3525dee84493ac 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -187,16 +187,19 @@ def read_until(marker, start=0): def test_lexer_buffer_realloc_with_null_start(self): # gh-144759: NULL pointer arithmetic when the lexer buffer grows # while parsing long input. - long_value = "a" * 2000 + long_value = "é漢" * 2000 user_input = dedent(f"""\ x = f'{{{long_value!r}}}' print(x) """) - p = spawn_repl() - p.stdin.write(user_input) - output = kill_python(p) - self.assertEqual(p.returncode, 0) - self.assertIn(long_value, output) + for newline in ("\n", "\r\n"): + with self.subTest(newline=newline): + p = spawn_repl(encoding="utf-8") + # Bypass Windows text-mode translation of CRLF to CRCRLF. + p.stdin.buffer.write(user_input.replace("\n", newline).encode("utf-8")) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(long_value, output) @cpython_only def test_multiline_fstring_source_reallocation(self): diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index ec98e609c4e98f..3c29002025575e 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -29,6 +29,30 @@ def test_compilestring(self): exec(c, d) self.assertEqual(d['u'], '\xf3') + def test_compilestring_line_endings(self): + for newline in ('\n', '\r', '\r\n', '\r\r\n'): + expected = 'é\n\ntext' if newline == '\r\r\n' else 'é\ntext' + for suffix in ('', newline): + for mode in ('exec', 'eval'): + source = f"'''é{newline}text'''{suffix}" + if mode == 'exec': + source = 'value = ' + source + for encoding in (None, 'utf-8', 'latin-1'): + with self.subTest(newline=newline, suffix=suffix, + mode=mode, encoding=encoding): + input = source + if encoding is not None: + input = (f'# coding: {encoding}{newline}' + + source).encode(encoding) + code = compile(input, '', mode) + if mode == 'exec': + namespace = {} + exec(code, namespace) + value = namespace['value'] + else: + value = eval(code) + self.assertEqual(value, expected) + def test_issue2301(self): try: compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec") @@ -135,6 +159,11 @@ def test_stateful_file_decoder_spans_lines(self): ) self._assert_python_file_ok(source) + @support.requires_subprocess() + def test_stateful_file_decoder_preserves_buffered_text(self): + source = b"# coding: hz\nx~\ny = 1\nassert xy == 1\n" + self._assert_python_file_ok(source) + @support.requires_subprocess() def test_stateful_file_decoder_finalizes_before_implicit_newline(self): source = b"# coding: hz\n# ~{1dA?" diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index cb8f58511aab87..9fa294aa361884 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -8,6 +8,7 @@ import token import tokenize import unittest +import weakref from io import BytesIO, StringIO from textwrap import dedent from unittest import TestCase, mock @@ -2268,6 +2269,18 @@ def check_tokenize(self, s, expected): ) self.assertEqual(result, expected.rstrip().splitlines()) + def test_readline_reference_cycle(self): + class Readline: + def __call__(self): + return "" + + readline = Readline() + readline.iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + ref = weakref.ref(readline) + del readline + support.gc_collect() + self.assertIsNone(ref()) + def test_encoding(self): def readline(encoding): yield "1+1".encode(encoding) @@ -2341,6 +2354,20 @@ def test_utf8_decoder_spans_readline_calls(self): tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""), ]) + def test_utf8_decoder_spans_many_readline_calls(self): + for prefix in (b"", b"previous\n"): + with self.subTest(prefix=prefix): + chunks = ([prefix + b"x\xc3"] + [b"\xa9\xc3"] * 100 + + [b"\xa9\n", b"z\xc3", b"\xa9\n", b""]) + source = b"".join(chunks) + expected = list(_tokenize.TokenizerIter( + BytesIO(source).readline, encoding="utf-8", extra_tokens=True + )) + tokens = list(_tokenize.TokenizerIter( + iter(chunks).__next__, encoding="utf-8", extra_tokens=True + )) + self.assertEqual(tokens, expected) + def test_utf8_decoder_replaces_incomplete_input_at_eof(self): expected = [ tokenize.TokenInfo(token.NAME, "x�", (1, 0), (1, 2), "x�"), @@ -2396,6 +2423,12 @@ def test_multiline_readline_chunk_with_unterminated_tail(self): ) self.assertEqual(readline.call_count, 2) + def test_readline_memory_error_in_string(self): + readline = mock.Mock(side_effect=['"""first\n', MemoryError]) + iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + with self.assertRaises(MemoryError): + next(iterator) + def test_readline_callback_is_not_read_ahead(self): readline = mock.Mock(side_effect=["x\n", "y\n", ""]) iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index d827ac271085bd..832a1e283e0c27 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -338,6 +338,16 @@ def test_ignores(self): tree = self.classic_parse(ignores) self.assertEqual(tree.type_ignores, []) + def test_many_ignores(self): + comment_count = 25 + tags = [f"[tag_{index}]" for index in range(comment_count)] + source = "".join(f"pass # type: ignore{tag}\n" for tag in tags) + for tree in self.parse_all(source): + self.assertEqual( + [(item.lineno, item.tag) for item in tree.type_ignores], + list(enumerate(tags, start=1)), + ) + def test_longargs(self): for tree in self.parse_all(longargs, minver=8): for t in tree.body: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst new file mode 100644 index 00000000000000..7bd6b023fb32da --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -0,0 +1,4 @@ +Correct missing-brace error messages for multiline f-strings and t-strings. +Improve tokenization of UTF-8 input split across many readline calls. +Preserve source order when reading files with stateful encodings. +Preserve input errors when scanning string literals. diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 1f89c12f223c7d..0c2bdee2e803b2 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -38,6 +38,12 @@ static PyObject * test_tokenizer_source(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + const char multiple_lines[] = "a\nb\n"; + const char first_line[] = "alpha\n"; + const char second_line[] = "\xce\xb2\n"; + const char expected[] = "alpha\n\xce\xb2\n"; + const char tail[] = "tail"; + const char terminated_line[] = "x\n"; _PyTok_SourceText source; _PyTok_SourceInit(&source); @@ -46,23 +52,19 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), } if (check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + _PyTok_SourceAppendLine(&source, "", 0) < 0, "accepted empty source line") < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, + _PyTok_SourceAppendLine( + &source, multiple_lines, sizeof(multiple_lines) - 1) < 0, "accepted multiple source lines") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0 || check(_PyTok_SourceAppendLine( - &source, "alpha\n", 6, 0) == 0, + &source, first_line, sizeof(first_line) - 1) == 0, "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( - &source, "\xce\xb2\n", 3, 1) == 6, - "wrong second source offset") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 1) && - _PyTok_SourceLineIsImplicit(&source, 2), - "wrong implicit newline flags") < 0) { + &source, second_line, sizeof(second_line) - 1) == + (Py_ssize_t)sizeof(first_line) - 1, + "wrong second source offset") < 0) { goto error; } @@ -74,16 +76,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - if (check(source.len == 9 && - memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + if (check(source.len == (Py_ssize_t)sizeof(expected) - 1 && + memcmp(source.bytes, expected, sizeof(expected)) == 0, "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || + if (_PyTok_SourceAppendLine(&source, tail, sizeof(tail) - 1) < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, + _PyTok_SourceAppendLine( + &source, terminated_line, sizeof(terminated_line) - 1) < 0, "appended after unterminated source line") < 0) { goto error; } @@ -104,27 +107,35 @@ static PyObject * test_tokenizer_source_discard(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + enum { LINE_COUNT = 2 }; + const char first_line[] = "x\n"; + const char second_line[] = "y\n"; + const char tail[] = "tail"; + const char final_line[] = "z\n"; + const _PyTok_Off first_batch_len = LINE_COUNT * (sizeof(first_line) - 1); + const _PyTok_Off second_batch_len = LINE_COUNT * (sizeof(second_line) - 1); + const _PyTok_Off discarded_len = first_batch_len + second_batch_len; _PyTok_SourceText source; _PyTok_SourceInit(&source); - for (int i = 0; i < 260; i++) { - if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) { + for (int i = 0; i < LINE_COUNT; i++) { + if (_PyTok_SourceAppendLine(&source, first_line, sizeof(first_line) - 1) < 0) { goto error; } } char *bytes = source.bytes; _PyTok_Off capacity = source.cap; _PyTok_SourceDiscard(&source); - if (check(source.base_offset == 520 && source.len == 0 && + if (check(source.base_offset == first_batch_len && source.len == 0 && source.nlines == 0 && source.bytes == bytes && source.cap == capacity && source.bytes[0] == '\0', "discard did not preserve source allocation") < 0) { goto error; } - for (int i = 0; i < 260; i++) { - if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i, - "wrong source offset after discard") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, i + 1), - "discard preserved implicit newline flag") < 0) { + for (int i = 0; i < LINE_COUNT; i++) { + if (check(_PyTok_SourceAppendLine( + &source, second_line, sizeof(second_line) - 1) == + first_batch_len + ((Py_ssize_t)sizeof(second_line) - 1) * i, + "wrong source offset after discard") < 0) { goto error; } } @@ -133,18 +144,22 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040, + if (check(_PyTok_SourceAppendLine( + &source, tail, sizeof(tail) - 1) == discarded_len, "wrong source offset after repeated discard") < 0) { goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044, + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) == + discarded_len + (Py_ssize_t)sizeof(tail) - 1, "cannot append after discarding unterminated line") < 0) { goto error; } _PyTok_SourceDiscard(&source); source.base_offset = PY_SSIZE_T_MAX - 1; - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 && + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) < 0 && PyErr_ExceptionMatches(PyExc_MemoryError), "accepted overflowing logical source offset") < 0) { goto error; diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 4652103107367e..a4fdb9484abd8d 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -314,7 +314,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token // Handle valid f or t string creation: if (saw_f || saw_t) { - return _PyLexer_scan_fstring_start(tok, token, c); + ftstring_kind kind = saw_t + ? (saw_r ? RAW_TSTRING : TSTRING) + : (saw_r ? RAW_FSTRING : FSTRING); + return _PyLexer_scan_fstring_start(tok, token, c, kind); } return _PyLexer_scan_string(tok, token, c); } diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index f0d9576d95f8ff..652512a3e88c22 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -66,7 +66,8 @@ int _PyLexer_close_ftstring_expr( void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); -int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); +int _PyLexer_scan_fstring_start( + struct tok_state *, struct token *, int, ftstring_kind); int _PyLexer_scan_string(struct tok_state *, struct token *, int); int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *); int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index a48cedf270362a..99e45578b6ca16 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -13,7 +13,7 @@ string_error_token(struct tok_state *tok, struct token *token, { tok->diagnostic = (_PyTokenizer_Diagnostic){ .location = {location.lineno, location.byte_col + 1}, - .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp), + .text_span = {start - location.byte_col, tok->inp}, }; int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1); token->start_loc = location; @@ -83,18 +83,17 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state *state, PyObject *res; if (comments != NULL && comments->count > 0) { Py_ssize_t stripped_size = expr_len; - _PyTok_Off previous_end = state->expr_span.start; Py_ssize_t comment_count = 0; for (Py_ssize_t i = 0; i < comments->count; i++) { _PyTok_Span comment = comments->spans[i]; assert(_PyTok_SpanIsValid(comment)); - assert(comment.start >= previous_end); + assert(comment.start >= (i == 0 ? state->expr_span.start + : comments->spans[i - 1].end)); if (comment.start >= state->expr_span.end) { break; } assert(comment.end <= state->expr_span.end); stripped_size -= comment.end - comment.start; - previous_end = comment.end; comment_count++; } char *stripped = PyMem_Malloc((size_t)stripped_size); @@ -254,7 +253,8 @@ _PyLexer_check_string_prefixes(struct tok_state *tok, } int -_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) +_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, + int c, ftstring_kind kind) { _PyTok_Off p_start = -1; _PyTok_Off p_end = -1; @@ -293,30 +293,9 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) state->start_loc = tok->start_loc; state->expr_span = (_PyTok_Span){-1, -1}; - int raw = 0; - int tstring = 0; - switch (*_PyLexer_BufferPointer(tok, tok->start)) { - case 'T': - case 't': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - tstring = 1; - break; - case 'F': - case 'f': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - break; - case 'R': - case 'r': - raw = 1; - tstring = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 't'; - break; - default: - Py_UNREACHABLE(); - } - state->kind = tstring - ? (raw ? RAW_TSTRING : TSTRING) - : (raw ? RAW_FSTRING : FSTRING); - return tstring ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); + state->kind = kind; + return _PyLexer_IsTString(kind) + ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); } int @@ -355,6 +334,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) break; } if (c == EOF || (quote_size == 1 && c == '\n')) { + if (tok_failed(tok)) { + return MAKE_TOKEN(ERRORTOKEN); + } int end_lineno = tok->lineno; _PyTok_Loc location = tok->start_loc; const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col; @@ -370,7 +352,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) assert(level >= 0 && level < tok->level); assert(tok->parenstack[level] == '{'); int lineno = tok->parenlinenostack[level]; - if (lineno != tok->lineno) { + if (lineno != location.lineno) { _PyTokenizer_syntaxerror_at( tok, line, cursor_offset, location.lineno, -1, -1, "%c-string: expecting '}' to close '{' on line %d", diff --git a/Parser/parser.c b/Parser/parser.c index 896c089e336003..979907f4873b9d 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27350,14 +27350,8 @@ _loop0_1_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // NEWLINE if (p->error_indicator) { @@ -27372,16 +27366,13 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27392,9 +27383,9 @@ _loop0_1_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -27417,14 +27408,8 @@ _loop1_2_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // statement if (p->error_indicator) { @@ -27439,16 +27424,13 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27464,9 +27446,9 @@ _loop1_2_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -27489,14 +27471,8 @@ _loop0_3_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ';' simple_stmt if (p->error_indicator) { @@ -27520,16 +27496,13 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27540,9 +27513,9 @@ _loop0_3_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28043,14 +28016,8 @@ _loop1_12_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -28065,16 +28032,13 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28090,9 +28054,9 @@ _loop1_12_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28115,14 +28079,8 @@ _loop0_13_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' NAME if (p->error_indicator) { @@ -28146,16 +28104,13 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28166,9 +28121,9 @@ _loop0_13_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28335,14 +28290,8 @@ _loop0_17_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28357,16 +28306,13 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28377,9 +28323,9 @@ _loop0_17_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28402,14 +28348,8 @@ _loop1_18_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28424,16 +28364,13 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28449,9 +28386,9 @@ _loop1_18_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28474,14 +28411,8 @@ _loop0_19_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' import_from_as_name if (p->error_indicator) { @@ -28505,16 +28436,13 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28525,9 +28453,9 @@ _loop0_19_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28637,14 +28565,8 @@ _loop0_22_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_as_name if (p->error_indicator) { @@ -28668,16 +28590,13 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28688,9 +28607,9 @@ _loop0_22_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28754,14 +28673,8 @@ _loop1_24_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('@' named_expression NEWLINE) if (p->error_indicator) { @@ -28776,16 +28689,13 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28801,9 +28711,9 @@ _loop1_24_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28921,14 +28831,8 @@ _loop0_27_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -28943,16 +28847,13 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28963,9 +28864,9 @@ _loop0_27_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28988,14 +28889,8 @@ _loop0_28_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -29010,16 +28905,13 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29030,9 +28922,9 @@ _loop0_28_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29055,14 +28947,8 @@ _loop1_29_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -29077,16 +28963,13 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29102,9 +28985,9 @@ _loop1_29_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29127,14 +29010,8 @@ _loop1_30_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -29149,16 +29026,13 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29174,9 +29048,9 @@ _loop1_30_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29199,14 +29073,8 @@ _loop0_31_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29221,16 +29089,13 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29241,9 +29106,9 @@ _loop0_31_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29266,14 +29131,8 @@ _loop1_32_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29288,16 +29147,13 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29313,9 +29169,9 @@ _loop1_32_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29338,14 +29194,8 @@ _loop0_33_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' with_item if (p->error_indicator) { @@ -29369,16 +29219,13 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29389,9 +29236,9 @@ _loop0_33_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29531,14 +29378,8 @@ _loop1_36_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_block if (p->error_indicator) { @@ -29553,16 +29394,13 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29578,9 +29416,9 @@ _loop1_36_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29603,14 +29441,8 @@ _loop1_37_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_star_block if (p->error_indicator) { @@ -29625,16 +29457,13 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29650,9 +29479,9 @@ _loop1_37_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29675,14 +29504,8 @@ _loop1_38_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // case_block if (p->error_indicator) { @@ -29697,16 +29520,13 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29722,9 +29542,9 @@ _loop1_38_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29747,14 +29567,8 @@ _loop0_39_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // '|' closed_pattern if (p->error_indicator) { @@ -29778,16 +29592,13 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29798,9 +29609,9 @@ _loop0_39_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30073,14 +29884,8 @@ _loop0_44_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' maybe_star_pattern if (p->error_indicator) { @@ -30104,16 +29909,13 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30124,9 +29926,9 @@ _loop0_44_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30190,14 +29992,8 @@ _loop0_46_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' key_value_pattern if (p->error_indicator) { @@ -30221,16 +30017,13 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30241,9 +30034,9 @@ _loop0_46_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30364,14 +30157,8 @@ _loop0_49_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' pattern if (p->error_indicator) { @@ -30395,16 +30182,13 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30415,9 +30199,9 @@ _loop0_49_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30481,14 +30265,8 @@ _loop0_51_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' keyword_pattern if (p->error_indicator) { @@ -30512,16 +30290,13 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30532,9 +30307,9 @@ _loop0_51_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30598,14 +30373,8 @@ _loop0_53_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' type_param if (p->error_indicator) { @@ -30629,16 +30398,13 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30649,9 +30415,9 @@ _loop0_53_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30715,14 +30481,8 @@ _loop1_55_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' expression) if (p->error_indicator) { @@ -30737,16 +30497,13 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30762,9 +30519,9 @@ _loop1_55_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30787,14 +30544,8 @@ _loop1_56_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_expression) if (p->error_indicator) { @@ -30809,16 +30560,13 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30834,9 +30582,9 @@ _loop1_56_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30859,14 +30607,8 @@ _loop0_57_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' star_named_expression if (p->error_indicator) { @@ -30890,16 +30632,13 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30910,9 +30649,9 @@ _loop0_57_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30976,14 +30715,8 @@ _loop1_59_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('or' conjunction) if (p->error_indicator) { @@ -30998,16 +30731,13 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31023,9 +30753,9 @@ _loop1_59_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31048,14 +30778,8 @@ _loop1_60_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('and' inversion) if (p->error_indicator) { @@ -31070,16 +30794,13 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31095,9 +30816,9 @@ _loop1_60_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31120,14 +30841,8 @@ _loop1_61_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // compare_op_bitwise_or_pair if (p->error_indicator) { @@ -31142,16 +30857,13 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31167,9 +30879,9 @@ _loop1_61_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31235,14 +30947,8 @@ _loop0_63_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (slice | starred_expression) if (p->error_indicator) { @@ -31266,16 +30972,13 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31286,9 +30989,9 @@ _loop0_63_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31683,14 +31386,8 @@ _loop0_70_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31705,16 +31402,13 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31725,9 +31419,9 @@ _loop0_70_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31750,14 +31444,8 @@ _loop0_71_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31772,16 +31460,13 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31792,9 +31477,9 @@ _loop0_71_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31817,14 +31502,8 @@ _loop1_72_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31839,16 +31518,13 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31864,9 +31540,9 @@ _loop1_72_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31889,14 +31565,8 @@ _loop1_73_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31911,16 +31581,13 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31936,9 +31603,9 @@ _loop1_73_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31961,14 +31628,8 @@ _loop0_74_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -31983,16 +31644,13 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32003,9 +31661,9 @@ _loop0_74_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32028,14 +31686,8 @@ _loop1_75_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -32050,16 +31702,13 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32075,9 +31724,9 @@ _loop1_75_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32100,14 +31749,8 @@ _loop0_76_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_format_spec if (p->error_indicator) { @@ -32122,16 +31765,13 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32142,9 +31782,9 @@ _loop0_76_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32167,14 +31807,8 @@ _loop0_77_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_middle if (p->error_indicator) { @@ -32189,16 +31823,13 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32209,9 +31840,9 @@ _loop0_77_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32234,14 +31865,8 @@ _loop0_78_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_format_spec if (p->error_indicator) { @@ -32256,16 +31881,13 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32276,9 +31898,9 @@ _loop0_78_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32301,14 +31923,8 @@ _loop0_79_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_middle if (p->error_indicator) { @@ -32323,16 +31939,13 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32343,9 +31956,9 @@ _loop0_79_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32368,14 +31981,8 @@ _loop1_80_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (fstring | string) if (p->error_indicator) { @@ -32390,16 +31997,13 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32415,9 +32019,9 @@ _loop1_80_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32440,14 +32044,8 @@ _loop1_81_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring if (p->error_indicator) { @@ -32462,16 +32060,13 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32487,9 +32082,9 @@ _loop1_81_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32512,14 +32107,8 @@ _loop0_82_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' display_item if (p->error_indicator) { @@ -32543,16 +32132,13 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32563,9 +32149,9 @@ _loop0_82_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32678,14 +32264,8 @@ _loop0_85_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' double_starred_kvpair if (p->error_indicator) { @@ -32709,16 +32289,13 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32729,9 +32306,9 @@ _loop0_85_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32795,14 +32372,8 @@ _loop1_87_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // for_if_clause if (p->error_indicator) { @@ -32817,16 +32388,13 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32842,9 +32410,9 @@ _loop1_87_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32867,14 +32435,8 @@ _loop0_88_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('if' disjunction) if (p->error_indicator) { @@ -32889,16 +32451,13 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32909,9 +32468,9 @@ _loop0_88_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33012,14 +32571,8 @@ _loop0_90_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression | (assignment_expression | expression !':=') !'=') if (p->error_indicator) { @@ -33043,16 +32596,13 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33063,9 +32613,9 @@ _loop0_90_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33176,14 +32726,8 @@ _loop0_93_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_starred if (p->error_indicator) { @@ -33207,16 +32751,13 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33227,9 +32768,9 @@ _loop0_93_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33339,14 +32880,8 @@ _loop0_96_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_double_starred if (p->error_indicator) { @@ -33370,16 +32905,13 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33390,9 +32922,9 @@ _loop0_96_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33456,14 +32988,8 @@ _loop0_98_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_target) if (p->error_indicator) { @@ -33478,16 +33004,13 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33498,9 +33021,9 @@ _loop0_98_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33639,14 +33162,8 @@ _loop0_101_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' del_target if (p->error_indicator) { @@ -33670,16 +33187,13 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33690,9 +33204,9 @@ _loop0_101_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33756,14 +33270,8 @@ _loop0_103_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' expression if (p->error_indicator) { @@ -33787,16 +33295,13 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33807,9 +33312,9 @@ _loop0_103_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33973,14 +33478,8 @@ _loop0_107_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression !'=') if (p->error_indicator) { @@ -34004,16 +33503,13 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34024,9 +33520,9 @@ _loop0_107_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34289,14 +33785,8 @@ _loop1_113_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (!STRING expression_without_invalid) if (p->error_indicator) { @@ -34311,16 +33801,13 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34336,9 +33823,9 @@ _loop1_113_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34744,14 +34231,8 @@ _loop0_119_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // star_named_expressions if (p->error_indicator) { @@ -34766,16 +34247,13 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34786,9 +34264,9 @@ _loop0_119_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34811,14 +34289,8 @@ _loop0_120_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -34833,16 +34305,13 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34853,9 +34322,9 @@ _loop0_120_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35356,14 +34825,8 @@ _loop0_129_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' lambda_param if (p->error_indicator) { @@ -35387,16 +34850,13 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35407,9 +34867,9 @@ _loop0_129_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35692,14 +35152,8 @@ _loop0_135_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_name if (p->error_indicator) { @@ -35723,16 +35177,13 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35743,9 +35194,9 @@ _loop0_135_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35850,14 +35301,8 @@ _loop0_138_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expression ['as' star_target]) if (p->error_indicator) { @@ -35881,16 +35326,13 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35901,9 +35343,9 @@ _loop0_138_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35967,14 +35409,8 @@ _loop0_140_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expressions ['as' star_target]) if (p->error_indicator) { @@ -35998,16 +35434,13 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -36018,9 +35451,9 @@ _loop0_140_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -36141,14 +35574,8 @@ _loop0_143_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // block if (p->error_indicator) { @@ -36163,16 +35590,13 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -36183,9 +35607,9 @@ _loop0_143_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -37794,14 +37218,8 @@ _loop0_172_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' bitwise_or) if (p->error_indicator) { @@ -37816,16 +37234,13 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -37836,9 +37251,9 @@ _loop0_172_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 7b228ca874bc0b..4ca42a407bcde8 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -13,6 +13,8 @@ #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. #define IDENTIFIER_CACHE_MAX_PROBES 8 +#define PEGEN_ARRAY_GROWTH_FACTOR 2 +#define TYPE_IGNORE_INITIAL_CAPACITY 10 struct _identifier_cache_entry { const char *key; // Borrowed from arena-owned token bytes. @@ -36,6 +38,9 @@ Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset) { const unsigned char *data = (const unsigned char*)PyUnicode_AsUTF8(line); + if (data == NULL) { + return -1; + } Py_ssize_t len = 0; while (col_offset < end_col_offset) { @@ -144,21 +149,37 @@ init_normalization(Parser *p) return 1; } -static int -growable_comment_array_init(growable_comment_array *arr, size_t initial_size) { - assert(initial_size > 0); - arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items)); - arr->size = initial_size; - arr->num_items = 0; - - return arr->items != NULL; +int +_PyPegen_grow_loop_buffer(void ***buffer, Py_ssize_t *capacity) +{ + if ((size_t)*capacity > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(**buffer))) { + return -1; + } + Py_ssize_t new_capacity = *capacity == 0 + ? 1 : *capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **new_buffer = PyMem_Realloc(*buffer, new_capacity * sizeof(**buffer)); + if (new_buffer == NULL) { + return -1; + } + *buffer = new_buffer; + *capacity = new_capacity; + return 0; } static int -growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { +growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) +{ if (arr->num_items >= arr->size) { - size_t new_size = arr->size * 2; - void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items)); + if (arr->size > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*arr->items))) { + return 0; + } + size_t new_size = arr->size == 0 + ? TYPE_IGNORE_INITIAL_CAPACITY + : arr->size * PEGEN_ARRAY_GROWTH_FACTOR; + void *new_items_array = PyMem_Realloc( + arr->items, new_size * sizeof(*arr->items)); if (!new_items_array) { return 0; } @@ -173,8 +194,9 @@ growable_comment_array_add(growable_comment_array *arr, int lineno, char *commen } static void -growable_comment_array_deallocate(growable_comment_array *arr) { - for (unsigned i = 0; i < arr->num_items; i++) { +growable_comment_array_deallocate(growable_comment_array *arr) +{ + for (size_t i = 0; i < arr->num_items; i++) { PyMem_Free(arr->items[i].comment); } PyMem_Free(arr->items); @@ -274,12 +296,13 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to static int _resize_tokens_array(Parser *p) { - if (p->size > INT_MAX / 2 || - (size_t)p->size > PY_SSIZE_T_MAX / (2 * sizeof(*p->tokens))) { + if (p->size > INT_MAX / PEGEN_ARRAY_GROWTH_FACTOR || + (size_t)p->size > PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*p->tokens))) { PyErr_NoMemory(); return -1; } - int newsize = p->size * 2; + int newsize = p->size * PEGEN_ARRAY_GROWTH_FACTOR; Token **new_tokens = PyMem_Realloc(p->tokens, (size_t)newsize * sizeof(Token *)); if (new_tokens == NULL) { PyErr_NoMemory(); @@ -321,6 +344,7 @@ _PyPegen_fill_token(Parser *p) tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, new_token.end_loc.lineno, tag)) { + PyMem_Free(tag); PyErr_NoMemory(); goto error; } @@ -944,12 +968,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, PyMem_Free(p); return (Parser *) PyErr_NoMemory(); } - if (!growable_comment_array_init(&p->type_ignore_comments, 10)) { - PyMem_Free(p->tokens[0]); - PyMem_Free(p->tokens); - PyMem_Free(p); - return (Parser *) PyErr_NoMemory(); - } + p->type_ignore_comments = (growable_comment_array){0}; p->mark = 0; p->fill = 0; @@ -996,7 +1015,7 @@ _PyPegen_Parser_Free(Parser *p) Py_XDECREF(p->normalize); // Resizes allocate blocks starting at indices 1, 2, 4, and so on. PyMem_Free(p->tokens[0]); - for (int i = 1; i < p->size; i *= 2) { + for (int i = 1; i < p->size; i *= PEGEN_ARRAY_GROWTH_FACTOR) { PyMem_Free(p->tokens[i]); } PyMem_Free(p->tokens); diff --git a/Parser/pegen.h b/Parser/pegen.h index eb3f1d8a3041a7..7454289ef6df03 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -190,6 +190,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p); expr_ty _PyPegen_fstring_middle_token(Parser* p); Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); int _PyPegen_fill_token(Parser *p); +// Return -1 on failure, leaving the buffer and capacity unchanged. +int _PyPegen_grow_loop_buffer(void ***buffer, Py_ssize_t *capacity); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 74ab56c912ae1a..6997a1158e9310 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -327,6 +327,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, } tmp = Py_BuildValue("(OnnNnn)", info.filename, lineno, col_number, error_line, end_lineno, end_col_number); + error_line = NULL; if (!tmp) { goto error; } diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 69c3bb371add96..220fed65ad4e45 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -8,6 +8,14 @@ #include "helpers.h" #include "../lexer/state.h" +#define UNICODE_BOM 0xFEFF +#define UTF16_LE_BOM "\xff\xfe" +#define UTF16_BE_BOM "\xfe\xff" +#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1)) +#define UTF8_BOM "\xef\xbb\xbf" +#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1)) +#define NORMALIZED_LINE_INITIAL_CAPACITY 256 + char * _PyTok_CopyBytes(const char *data, Py_ssize_t len) { @@ -60,9 +68,9 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return -1; } if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 && - PyUnicode_ReadChar(unicode, 0) == 0xFEFF) { - utf8 += 3; - utf8_len -= 3; + PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) { + utf8 += UTF8_BOM_LENGTH; + utf8_len -= UTF8_BOM_LENGTH; } chunk_release_data(chunk); chunk->owner = unicode; @@ -72,58 +80,32 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return 0; } -// The caller must provide len + 2 bytes: normalization can append a final -// newline and always writes a NUL terminator. -static void -normalize_newlines_into(char *result, const char *data, Py_ssize_t len, - int preserve_crlf, int add_final_newline, - Py_ssize_t *out_len, int *implicit_newline) -{ - Py_ssize_t write = 0; - if (preserve_crlf || memchr(data, '\r', len) == NULL) { - // No translation needed: copy verbatim. - memcpy(result, data, len); - write = len; - } - else { - for (Py_ssize_t read = 0; read < len; read++) { - char c = data[read]; - if (c == '\r') { - if (read + 1 < len && data[read + 1] == '\n') { - read++; - } - c = '\n'; - } - result[write++] = c; - } - } - int implicit = add_final_newline && write > 0 && result[write - 1] != '\n'; - if (implicit) { - result[write++] = '\n'; - } - result[write] = '\0'; - *out_len = write; - if (implicit_newline != NULL) { - *implicit_newline = implicit; - } -} - char * -_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, - int add_final_newline, Py_ssize_t *out_len, - int *implicit_newline) +_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, + Py_ssize_t *out_len) { - if (len > PY_SSIZE_T_MAX - 2) { + if (len < 0 || len == PY_SSIZE_T_MAX) { PyErr_NoMemory(); return NULL; } - char *result = PyMem_Malloc((size_t)len + 2); + char *result = PyMem_Malloc((size_t)len + 1); if (result == NULL) { PyErr_NoMemory(); return NULL; } - normalize_newlines_into(result, data, len, preserve_crlf, - add_final_newline, out_len, implicit_newline); + Py_ssize_t write = 0; + for (Py_ssize_t read = 0; read < len; read++) { + char c = data[read]; + if (c == '\r') { + if (read + 1 < len && data[read + 1] == '\n') { + read++; + } + c = '\n'; + } + result[write++] = c; + } + result[write] = '\0'; + *out_len = write; return result; } @@ -158,11 +140,12 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) return 0; } } - for (; i + 6 < len; i++) { - if (memcmp(line + i, "coding", 6) != 0) { + const Py_ssize_t marker_len = sizeof("coding") - 1; + for (; i < len - marker_len; i++) { + if (memcmp(line + i, "coding", marker_len) != 0) { continue; } - const char *cursor = line + i + 6; + const char *cursor = line + i + marker_len; if (*cursor != ':' && *cursor != '=') { continue; } @@ -179,37 +162,36 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) if (cursor == start) { continue; } - char *found = _PyTok_CopyBytes(start, cursor - start); - if (found == NULL) { - return -1; - } - char normalized[13]; - int n; - for (n = 0; n < 12 && found[n] != '\0'; n++) { - normalized[n] = found[n] == '_' ? '-' : Py_TOLOWER(found[n]); + Py_ssize_t encoding_len = cursor - start; + // Include the longest recognized prefix and its NUL terminator. + char normalized[sizeof("iso-latin-1-")]; + const Py_ssize_t prefix_len = Py_MIN( + encoding_len, (Py_ssize_t)sizeof(normalized) - 1); + Py_ssize_t n; + for (n = 0; n < prefix_len; n++) { + normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]); } normalized[n] = '\0'; - const char *canonical = found; + const char *canonical = start; if (strcmp(normalized, "utf-8") == 0 || - strncmp(normalized, "utf-8-", 6) == 0) { + strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) { canonical = "utf-8"; } else if (strcmp(normalized, "latin-1") == 0 || strcmp(normalized, "iso-8859-1") == 0 || strcmp(normalized, "iso-latin-1") == 0 || - strncmp(normalized, "latin-1-", 8) == 0 || - strncmp(normalized, "iso-8859-1-", 11) == 0 || - strncmp(normalized, "iso-latin-1-", 12) == 0) { + strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 || + strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 || + strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) { canonical = "iso-8859-1"; } - if (canonical != found) { - PyMem_Free(found); - found = _PyTok_CopyBytes(canonical, strlen(canonical)); - if (found == NULL) { - return -1; - } + if (canonical != start) { + encoding_len = strlen(canonical); + } + *encoding = _PyTok_CopyBytes(canonical, encoding_len); + if (*encoding == NULL) { + return -1; } - *encoding = found; *scan_next = 0; return 0; } @@ -221,17 +203,15 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, const _PyTok_Chunk *second, int final, Py_ssize_t *bom_len) { - int bom = first->len >= 3 && - (unsigned char)first->data[0] == 0xEF && - (unsigned char)first->data[1] == 0xBB && - (unsigned char)first->data[2] == 0xBF; - *bom_len = bom ? 3 : 0; + int bom = first->len >= UTF8_BOM_LENGTH && + memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0; + *bom_len = bom ? UTF8_BOM_LENGTH : 0; char *cookie = NULL; int scan_next = 0; int cookie_line = 1; - const char *first_data = first->data + (bom ? 3 : 0); - Py_ssize_t first_len = first->len - (bom ? 3 : 0); + const char *first_data = first->data + *bom_len; + Py_ssize_t first_len = first->len - *bom_len; if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) { return _PYTOK_ENCODING_ERROR; } @@ -245,19 +225,16 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, return _PYTOK_ENCODING_NEED_SECOND_LINE; } - if (bom) { - if (_PyTok_SetEncoding(tok, "utf-8") < 0) { - PyMem_Free(cookie); + if (cookie == NULL) { + if (bom && _PyTok_SetEncoding(tok, "utf-8") < 0) { return _PYTOK_ENCODING_ERROR; } - } - if (cookie == NULL) { return _PYTOK_ENCODING_DONE; } if (bom && strcmp(cookie, "utf-8") != 0) { const _PyTok_Chunk *line = cookie_line == 2 ? second : first; - const char *line_data = line->data + (cookie_line == 1 ? 3 : 0); - Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0); + const char *line_data = line->data + (cookie_line == 1 ? *bom_len : 0); + Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0); int end_col = (int)Py_MIN(line_len, INT_MAX); if (end_col > 0 && (line_data[end_col - 1] == '\n' || line_data[end_col - 1] == '\r')) { @@ -269,11 +246,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } - if (!bom && _PyTok_SetEncoding(tok, cookie) < 0) { - PyMem_Free(cookie); - return _PYTOK_ENCODING_ERROR; - } - PyMem_Free(cookie); + PyMem_Free(tok->encoding); + tok->encoding = cookie; return _PYTOK_ENCODING_DONE; } @@ -329,37 +303,46 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, data[pos + raw_line_len - 1] == '\r'; int add_newline = add_final_newline && pos + raw_line_len == len && !terminated; - int normalize = add_newline || - (!preserve_crlf && - memchr(data + pos, '\r', raw_line_len) != NULL); - const char *line = data + pos; Py_ssize_t line_len = raw_line_len; - int implicit = 0; - if (normalize) { - if (line_len > PY_SSIZE_T_MAX - 2) { + // raw_line_length stops at the first CR or LF, so any CR is in + // the line terminator. + Py_ssize_t terminator_len = 0; + if (!preserve_crlf) { + if (line[line_len - 1] == '\r') { + terminator_len = 1; + } + else if (line_len > 1 && line[line_len - 2] == '\r') { + terminator_len = 2; + } + } + if (terminator_len || add_newline) { + Py_ssize_t payload_len = line_len - terminator_len; + if (payload_len > PY_SSIZE_T_MAX - 2) { PyErr_NoMemory(); tok->done = E_NOMEM; goto error; } - // Reserve space for an optional final '\n' and the NUL terminator. - Py_ssize_t needed = line_len + 2; - if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, 256) < 0) { + Py_ssize_t needed = payload_len + 2; + if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, + NORMALIZED_LINE_INITIAL_CAPACITY) < 0) { tok->done = E_NOMEM; goto error; } - normalize_newlines_into(normalized, line, line_len, - preserve_crlf, add_newline, - &line_len, &implicit); + memcpy(normalized, line, payload_len); + normalized[payload_len] = '\n'; + line_len = payload_len + 1; + normalized[line_len] = '\0'; line = normalized; } _PyTok_Off appended = _PyTok_SourceAppendLine( - &tok->source, line, line_len, implicit); + &tok->source, line, line_len); if (appended < 0) { tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; goto error; } + tok->reader->prepared_final_newline_is_implicit = add_newline; pos += raw_line_len; } PyMem_Free(normalized); @@ -472,11 +455,9 @@ _PyTok_DecodeChunk(struct tok_state *tok, _PyTok_Chunk *chunk, int final) return 0; } int strip_bom = reader->kind == _PYTOK_READER_READLINE && - chunk->len >= 2 && - (((unsigned char)chunk->data[0] == 0xFF && - (unsigned char)chunk->data[1] == 0xFE) || - ((unsigned char)chunk->data[0] == 0xFE && - (unsigned char)chunk->data[1] == 0xFF)); + chunk->len >= UTF16_BOM_LENGTH && + (memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 || + memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0); PyObject *input; if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT && PyBytes_Check(chunk->owner) && diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index c003deba18dd8f..36f910e3199756 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_fileutils.h" #include "pycore_pystate.h" +#include "pycore_runtime.h" #include "errcode.h" #include "helpers.h" @@ -8,6 +9,8 @@ #include "reader_internal.h" #include "../lexer/state.h" +#define READER_BUFFER_GROWTH_FACTOR 2 + #ifdef HAVE_UNISTD_H # include #endif @@ -37,6 +40,24 @@ _PyTok_ReaderFree(struct tok_state *tok) tok->reader = NULL; } +int +_PyTokenizer_Traverse(struct tok_state *tok, visitproc visit, void *arg) +{ + Py_VISIT(tok->filename); + Py_VISIT(tok->module); + _PyTok_Reader *reader = tok->reader; + Py_VISIT(reader->readline); + Py_VISIT(reader->decoder); + for (int i = 0; + i < (int)Py_ARRAY_LENGTH(reader->prefetched_lines); i++) { + _PyTok_Chunk *chunk = &reader->prefetched_lines[i]; + if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT) { + Py_VISIT(chunk->owner); + } + } + return 0; +} + int _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, Py_ssize_t initial_capacity) @@ -48,11 +69,11 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, // Grow geometrically to avoid reallocating for every longer line. Py_ssize_t cap = *capacity > 0 ? *capacity : initial_capacity; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / READER_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= READER_BUFFER_GROWTH_FACTOR; } char *resized = PyMem_Realloc(*buffer, cap); if (resized == NULL) { @@ -67,10 +88,14 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, static int append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) { + if (len == 0) { + return 0; + } if (reader->decoded_pos > 0) { Py_ssize_t remaining = reader->decoded_len - reader->decoded_pos; memmove(reader->decoded, reader->decoded + reader->decoded_pos, (size_t)remaining); + reader->decoded_scan -= reader->decoded_pos; reader->decoded_pos = 0; reader->decoded_len = remaining; } @@ -104,13 +129,17 @@ static int pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) { assert(reader->decoded_pos >= 0 && reader->decoded_pos <= reader->decoded_len); + assert(reader->decoded_scan >= reader->decoded_pos && + reader->decoded_scan <= reader->decoded_len); if (reader->decoded_pos == reader->decoded_len) { return 0; } char *start = reader->decoded + reader->decoded_pos; - char *newline = memchr(start, '\n', - reader->decoded_len - reader->decoded_pos); + // Previously scanned bytes cannot contain a newline. + char *newline = memchr(reader->decoded + reader->decoded_scan, '\n', + reader->decoded_len - reader->decoded_scan); if (newline == NULL) { + reader->decoded_scan = reader->decoded_len; return 0; } Py_ssize_t len = newline - start + 1; @@ -118,10 +147,12 @@ pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) chunk->len = len; chunk->ownership = _PYTOK_CHUNK_BORROWED; reader->decoded_pos += len; + reader->decoded_scan = reader->decoded_pos; chunk->implicit_newline = reader->decoded_pos == reader->decoded_len && reader->decoded_tail_is_implicit; if (reader->decoded_pos == reader->decoded_len) { reader->decoded_pos = reader->decoded_len = 0; + reader->decoded_scan = 0; reader->decoded_tail_is_implicit = 0; } return 1; @@ -142,7 +173,6 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) if (tok->lineno >= tok->source.nlines) { return _PYTOK_READ_EOF; } - int lineno = tok->lineno + 1; const char *start = _PyLexer_BufferPointer(tok, tok->inp); const char *newline = memchr( start, '\n', tok->source.bytes + tok->source.len - start); @@ -151,8 +181,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) chunk->data = (char *)start; chunk->len = tok->source.bytes + end - start; chunk->ownership = _PYTOK_CHUNK_BORROWED; - chunk->implicit_newline = _PyTok_SourceLineIsImplicit( - &tok->source, lineno); + chunk->implicit_newline = end == tok->source.len && + tok->reader->prepared_final_newline_is_implicit; return _PYTOK_READ_LINE; } @@ -162,9 +192,12 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_Reader *reader = tok->reader; Py_ssize_t len = 0; for (;;) { - if (len > PY_SSIZE_T_MAX - BUFSIZ || - _PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, - len + BUFSIZ, BUFSIZ) < 0) { + if (len > PY_SSIZE_T_MAX - BUFSIZ) { + PyErr_NoMemory(); + return _PYTOK_READ_ERROR; + } + if (_PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, + len + BUFSIZ, BUFSIZ) < 0) { return _PYTOK_READ_ERROR; } int available = (int)Py_MIN(reader->file_buffer_cap - len, INT_MAX); @@ -182,7 +215,7 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) break; } } - int implicit = len == 0 || reader->file_buffer[len - 1] != '\n'; + int implicit = reader->file_buffer[len - 1] != '\n'; chunk->data = reader->file_buffer; chunk->len = len; chunk->implicit_newline = implicit; @@ -281,8 +314,7 @@ finalize_decoding(struct tok_state *tok) return -1; } } - if (reader->decoded_pos < reader->decoded_len && - append_implicit_newline(reader) < 0) { + if (append_implicit_newline(reader) < 0) { tok->done = E_NOMEM; return -1; } @@ -325,7 +357,8 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } int decoded = _PyTok_DecodeChunk(tok, &input, 0); - if (decoded == 0 && chunk_is_line(&input)) { + if (decoded == 0 && reader->decoded_pos == reader->decoded_len && + chunk_is_line(&input)) { *chunk = input; return _PYTOK_READ_LINE; } @@ -466,9 +499,6 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_ERROR; } } - if (pop_decoded_line(reader, chunk)) { - return _PYTOK_READ_LINE; - } } } @@ -503,12 +533,14 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_ChunkClear(&decoded); return _PYTOK_READ_ERROR; } + if (memchr(decoded.data, '\r', decoded.len) == NULL) { + *chunk = decoded; + return _PYTOK_READ_LINE; + } chunk->data = _PyTok_NormalizeNewlines( - decoded.data, decoded.len, 0, 0, - &chunk->len, NULL); + decoded.data, decoded.len, &chunk->len); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { - PyErr_NoMemory(); tok->done = E_NOMEM; return _PYTOK_READ_ERROR; } @@ -618,8 +650,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) reset_streaming_buffer(tok); } _PyTok_Off source_start = _PyTok_SourceAppendLine( - &tok->source, chunk.data, chunk.len, - chunk.implicit_newline); + &tok->source, chunk.data, chunk.len); if (source_start < 0) { _PyTok_ChunkClear(&chunk); tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) @@ -780,7 +811,16 @@ static FILE * fdopen_borrow(int fd) { int copy = _Py_dup(fd); - return copy < 0 ? NULL : fdopen(copy, "r"); + if (copy < 0) { + return NULL; + } + FILE *fp = fdopen(copy, "r"); + if (fp == NULL) { + int saved_errno = errno; + close(copy); + errno = saved_errno; + } + return fp; } #endif @@ -796,21 +836,14 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) fclose(fp); return NULL; } - tok->filename = filename != NULL - ? Py_NewRef(filename) : PyUnicode_FromString(""); - if (tok->filename == NULL) { - fclose(fp); - _PyTokenizer_Free(tok); - return NULL; - } - if (initialize_file(tok) < 0) { - fclose(fp); - _PyTokenizer_Free(tok); - return NULL; + _Py_DECLARE_STR(anon_string, ""); + tok->filename = Py_NewRef(filename != NULL ? filename : &_Py_STR(anon_string)); + char *encoding = NULL; + if (initialize_file(tok) == 0) { + encoding = tok->encoding; + tok->encoding = NULL; } fclose(fp); - char *encoding = tok->encoding == NULL - ? NULL : _PyTok_CopyBytes(tok->encoding, strlen(tok->encoding)); _PyTokenizer_Free(tok); return encoding; } diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 390b6ed9d8581a..f74063de8d653a 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -3,6 +3,8 @@ #include "Python.h" +#define _PYTOK_ENCODING_COOKIE_MAX_LINES 2 + typedef enum { _PYTOK_READER_PREPARED, _PYTOK_READER_FILE, @@ -46,13 +48,15 @@ typedef struct _PyTok_Reader { char *file_buffer; Py_ssize_t file_buffer_cap; - _PyTok_Chunk prefetched_lines[2]; + _PyTok_Chunk prefetched_lines[_PYTOK_ENCODING_COOKIE_MAX_LINES]; char *decoded; Py_ssize_t decoded_pos; + Py_ssize_t decoded_scan; // First byte not yet scanned for a newline. Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; + unsigned char prepared_final_newline_is_implicit; unsigned char decoded_tail_is_implicit; unsigned char file_initialized; unsigned char file_eof; @@ -67,8 +71,7 @@ int _PyTok_ReserveBuffer(char **, Py_ssize_t *, Py_ssize_t, Py_ssize_t); char *_PyTok_CopyBytes(const char *, Py_ssize_t); int _PyTok_DecodeOnce( struct tok_state *, _PyTok_Chunk *, const char *, const char *); -char *_PyTok_NormalizeNewlines( - const char *, Py_ssize_t, int, int, Py_ssize_t *, int *); +char *_PyTok_NormalizeNewlines(const char *, Py_ssize_t, Py_ssize_t *); void _PyTok_ChunkClear(_PyTok_Chunk *); int _PyTok_SetEncoding(struct tok_state *, const char *); _PyTok_EncodingResult _PyTok_DetectEncoding( diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index d69eab93923e81..ebb7b05507c56e 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -1,7 +1,10 @@ #include "Python.h" +#include "pycore_pymem.h" // PYMEM_DEADBYTE #include "source.h" +#define SOURCE_BUFFER_GROWTH_FACTOR 2 + void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -12,7 +15,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -25,10 +27,6 @@ _PyTok_SourceDiscard(_PyTok_SourceText *source) if (source->bytes != NULL) { source->bytes[0] = '\0'; } - if (source->implicit_lines != NULL) { - Py_ssize_t used = source->nlines / 8 + (source->nlines % 8 != 0); - memset(source->implicit_lines, 0, Py_MIN(used, source->implicit_cap)); - } source->nlines = 0; } @@ -40,11 +38,11 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } Py_ssize_t cap = source->cap > 0 ? source->cap : BUFSIZ; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / SOURCE_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= SOURCE_BUFFER_GROWTH_FACTOR; } char *bytes; #ifdef Py_DEBUG @@ -62,7 +60,7 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } #ifdef Py_DEBUG if (source->bytes != NULL) { - memset(source->bytes, 0xDD, source->cap); + memset(source->bytes, PYMEM_DEADBYTE, source->cap); PyMem_Free(source->bytes); } #endif @@ -71,35 +69,9 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_implicit_lines(_PyTok_SourceText *source, int nlines) -{ - Py_ssize_t needed = nlines / 8 + (nlines % 8 != 0); - if (needed <= source->implicit_cap) { - return 0; - } - Py_ssize_t cap = source->implicit_cap > 0 ? source->implicit_cap : 16; - while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { - cap = needed; - break; - } - cap *= 2; - } - unsigned char *lines = PyMem_Realloc(source->implicit_lines, cap); - if (lines == NULL) { - PyErr_NoMemory(); - return -1; - } - memset(lines + source->implicit_cap, 0, cap - source->implicit_cap); - source->implicit_lines = lines; - source->implicit_cap = cap; - return 0; -} - static int validate_line(const _PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { if (len <= 0 || bytes == NULL || (source->nlines > 0 && @@ -108,8 +80,7 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, return -1; } const char *newline = memchr(bytes, '\n', len); - if ((newline != NULL && newline != bytes + len - 1) || - (implicit_newline && newline == NULL)) { + if (newline != NULL && newline != bytes + len - 1) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); return -1; } @@ -123,9 +94,9 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, _PyTok_Off _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { - if (validate_line(source, bytes, len, implicit_newline) < 0) { + if (validate_line(source, bytes, len) < 0) { return -1; } if (source->len > PY_SSIZE_T_MAX - len - 1 || @@ -133,9 +104,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, PyErr_NoMemory(); return -1; } - int nlines = source->nlines + 1; - if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || - reserve_bytes(source, source->len + len + 1) < 0) { + if (reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -143,11 +112,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (implicit_newline) { - source->implicit_lines[(nlines - 1) / 8] |= - (unsigned char)(1U << ((nlines - 1) & 7)); - } - source->nlines = nlines; + source->nlines++; return source->base_offset + start; } @@ -170,14 +135,3 @@ _PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, *len = (newline != NULL ? newline : end) - line; return line; } - -int -_PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) -{ - if (lineno < 1 || lineno > source->nlines || - (lineno - 1) / 8 >= source->implicit_cap) { - return 0; - } - return (source->implicit_lines[(lineno - 1) / 8] >> - ((lineno - 1) & 7)) & 1; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 9576419aade1d4..66980a41fb7c48 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -10,9 +10,7 @@ typedef struct { _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - unsigned char *implicit_lines; int nlines; - Py_ssize_t implicit_cap; } _PyTok_SourceText; static inline const char * @@ -29,19 +27,14 @@ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may contain one newline, as its final byte. An unterminated line must be the - final line. implicit_newline means that the final newline was synthesized. - The input must not point into source storage. */ + final line. The input must not point into source storage. */ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( - _PyTok_SourceText *source, const char *bytes, Py_ssize_t len, - int implicit_newline); + _PyTok_SourceText *source, const char *bytes, Py_ssize_t len); /* Return borrowed bytes excluding '\n', writing the byte length to *len. Line numbers are 1-based and clamp to the first or final line; a trailing '\n' adds an empty final line. The view need not be NUL-terminated. This does not set an exception. Append, discard, and clear invalidate the view. */ PyAPI_FUNC(const char *) _PyTok_SourceLineView( const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); -/* Return false for invalid line numbers and the virtual EOF line. */ -PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( - const _PyTok_SourceText *, int); #endif diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index 82a84830d7cbfe..2ef87add3646c3 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -58,6 +58,7 @@ typedef struct { Errors are returned as ERRORTOKEN, with or without a Python exception. */ void _PyTokenizer_Get(struct tok_state *, struct token *); void _PyTokenizer_Free(struct tok_state *); +int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *); void _PyTokenizer_raise_init_error(PyObject *filename); void _PyToken_Init(struct token *); static inline void diff --git a/Parser/tokenizer/types.h b/Parser/tokenizer/types.h index c12b6de449c700..b488cb01618d35 100644 --- a/Parser/tokenizer/types.h +++ b/Parser/tokenizer/types.h @@ -18,12 +18,6 @@ typedef struct { int byte_col; } _PyTok_Loc; -static inline _PyTok_Span -_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end) -{ - return (_PyTok_Span){start, end}; -} - static inline int _PyTok_SpanIsValid(_PyTok_Span span) { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 3802709732e29d..e91141bd1b9e61 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -1,7 +1,6 @@ #include "Python.h" #include "errcode.h" #include "internal/pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION -#include "internal/pycore_tuple.h" // _PyTuple_FromPair #include "../Parser/tokenizer/tokenizer.h" #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() @@ -37,7 +36,6 @@ typedef struct /* Needed to cache line for performance */ PyObject *last_line; Py_ssize_t last_lineno; - Py_ssize_t last_end_lineno; Py_ssize_t byte_col_offset_diff; } tokenizeriterobject; @@ -57,21 +55,18 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, int extra_tokens, const char *encoding) /*[clinic end generated code: output=7501a1211683ce16 input=f7dddf8a613ae8bd]*/ { - tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); - if (self == NULL) { + struct tok_state *tok = _PyTokenizer_FromReadline(readline, encoding); + if (tok == NULL) { return NULL; } - PyObject *filename = PyUnicode_FromString(""); - if (filename == NULL) { - return NULL; - } - self->tok = _PyTokenizer_FromReadline(readline, encoding); - if (self->tok == NULL) { - Py_DECREF(filename); + _Py_DECLARE_STR(anon_string, ""); + _PyTokenizer_SetContext(tok, &_Py_STR(anon_string), NULL); + tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); + if (self == NULL) { + _PyTokenizer_Free(tok); return NULL; } - _PyTokenizer_SetContext(self->tok, filename, NULL); - Py_DECREF(filename); + self->tok = tok; _PyTokenizer_SetOptions(self->tok, extra_tokens, 0); self->extra_tokens = extra_tokens; self->done = 0; @@ -79,18 +74,15 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, self->last_line = NULL; self->byte_col_offset_diff = 0; self->last_lineno = 0; - self->last_end_lineno = 0; return (PyObject *)self; } -static int +static void _tokenizer_error(tokenizeriterobject *it) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); - if (PyErr_Occurred()) { - return -1; - } + assert(!PyErr_Occurred()); const char *msg = NULL; PyObject* errtype = PyExc_SyntaxError; @@ -105,19 +97,17 @@ _tokenizer_error(tokenizeriterobject *it) PyErr_SyntaxLocationObject( info.filename, info.location.lineno, (int)Py_MAX(0, info.input_span.end - info.input_span.start)); - return -1; + return; case E_DEDENT: msg = "unindent does not match any outer indentation level"; errtype = PyExc_IndentationError; break; case E_INTR: - if (!PyErr_Occurred()) { - PyErr_SetNone(PyExc_KeyboardInterrupt); - } - return -1; + PyErr_SetNone(PyExc_KeyboardInterrupt); + return; case E_NOMEM: PyErr_NoMemory(); - return -1; + return; case E_TABSPACE: errtype = PyExc_TabError; msg = "inconsistent use of tabs and spaces in indentation"; @@ -134,11 +124,8 @@ _tokenizer_error(tokenizeriterobject *it) msg = "unknown tokenization error"; } - PyObject* errstr = NULL; PyObject* error_line = NULL; - PyObject* tmp = NULL; PyObject* value = NULL; - int result = 0; Py_ssize_t input_size; const char *input = _PyTokenizer_SpanView( @@ -148,41 +135,25 @@ _tokenizer_error(tokenizeriterobject *it) size -= 1; // Remove the newline character from the end of the line error_line = PyUnicode_DecodeUTF8(input, size, "replace"); if (!error_line) { - result = -1; goto exit; } Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, input_size); if (offset == -1) { - result = -1; - goto exit; - } - tmp = Py_BuildValue("(OnnOOO)", info.filename, info.location.lineno, offset, error_line, Py_None, Py_None); - if (!tmp) { - result = -1; - goto exit; - } - - errstr = PyUnicode_FromString(msg); - if (!errstr) { - result = -1; goto exit; } - - value = _PyTuple_FromPair(errstr, tmp); + value = Py_BuildValue("(s(OinOOO))", msg, info.filename, + info.location.lineno, offset, error_line, + Py_None, Py_None); if (!value) { - result = -1; goto exit; } PyErr_SetObject(errtype, value); exit: - Py_XDECREF(errstr); Py_XDECREF(error_line); - Py_XDECREF(tmp); Py_XDECREF(value); - return result; } static PyObject * @@ -190,38 +161,38 @@ _get_current_line(tokenizeriterobject *it, int current_lineno, const char *line_start, Py_ssize_t size, int *line_changed) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); - PyObject *line; if (current_lineno != it->last_lineno) { // Line has changed since last token, so we fetch the new line and cache it // in the iter object. Py_XDECREF(it->last_line); - line = PyUnicode_DecodeUTF8(line_start, size, "replace"); - it->last_line = line; + it->last_line = PyUnicode_DecodeUTF8(line_start, size, "replace"); it->byte_col_offset_diff = 0; } else { - line = it->last_line; *line_changed = 0; } - return line; + return it->last_line; } -static void +static int _get_col_offsets(tokenizeriterobject *it, const struct token *token, - const char *token_start, const char *line_start, - const char *end_line_start, PyObject *line, int line_changed, + const _PyToken_View *view, PyObject *line, int line_changed, Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + const char *token_start = view->text; const char *token_end = token_start == NULL - ? NULL : token_start + token->span.end - token->span.start; + ? NULL : token_start + view->length; Py_ssize_t lineno = token->start_loc.lineno; Py_ssize_t end_lineno = token->end_loc.lineno; Py_ssize_t byte_offset = -1; - if (token_start != NULL && token_start >= line_start) { - byte_offset = token_start - line_start; + if (token_start != NULL && token_start >= view->line) { + byte_offset = token_start - view->line; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); + if (*col_offset < 0) { + return -1; + } it->byte_col_offset_diff = byte_offset - *col_offset; } else { @@ -229,21 +200,28 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, } } - if (token_end != NULL && token_end >= end_line_start) { - Py_ssize_t end_byte_offset = token_end - end_line_start; + if (token_end != NULL && token_end >= view->end_line) { + Py_ssize_t end_byte_offset = token_end - view->end_line; if (lineno == end_lineno) { // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); + if (token_col_offset < 0) { + return -1; + } *end_col_offset = *col_offset + token_col_offset; it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { - *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset); + *end_col_offset = _PyPegen_byte_offset_to_character_offset_line( + line, view->end_line - view->line, token_end - view->line); + if (*end_col_offset < 0) { + return -1; + } it->byte_col_offset_diff += end_byte_offset - *end_col_offset; } } it->last_lineno = lineno; - it->last_end_lineno = end_lineno; + return 0; } static PyObject * @@ -254,6 +232,11 @@ tokenizeriter_next(PyObject *op) Py_BEGIN_CRITICAL_SECTION(it); + if (it->done) { + PyErr_SetString(PyExc_StopIteration, "EOF"); + goto unlock; + } + struct token token; _PyToken_Init(&token); @@ -266,11 +249,6 @@ tokenizeriter_next(PyObject *op) } goto exit; } - if (it->done || type == ERRORTOKEN) { - PyErr_SetString(PyExc_StopIteration, "EOF"); - it->done = 1; - goto exit; - } _PyToken_View view; _PyToken_GetView(it->tok, &token, &view); const char *token_start = view.text; @@ -286,10 +264,7 @@ tokenizeriter_next(PyObject *op) goto exit; } - int is_trailing_token = 0; - if (type == ENDMARKER || (type == DEDENT && view.at_eof)) { - is_trailing_token = 1; - } + int is_trailing_token = type == ENDMARKER || (type == DEDENT && view.at_eof); PyObject* line = NULL; int line_changed = 1; @@ -313,8 +288,11 @@ tokenizeriter_next(PyObject *op) Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, &token, token_start, view.line, view.end_line, line, - line_changed, &col_offset, &end_col_offset); + if (_get_col_offsets(it, &token, &view, line, + line_changed, &col_offset, &end_col_offset) < 0) { + Py_DECREF(str); + goto exit; + } if (it->extra_tokens) { if (is_trailing_token) { @@ -327,8 +305,8 @@ tokenizeriter_next(PyObject *op) type = OP; } else if (type == NEWLINE) { - Py_DECREF(str); if (!view.implicit_newline) { + Py_DECREF(str); assert(token_start != NULL); if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); @@ -346,7 +324,6 @@ tokenizeriter_next(PyObject *op) } if (str == NULL) { - Py_DECREF(line); goto exit; } } @@ -358,6 +335,7 @@ tokenizeriter_next(PyObject *op) it->done = 1; } +unlock:; Py_END_CRITICAL_SECTION(); return result; } @@ -367,15 +345,26 @@ tokenizeriter_dealloc(PyObject *op) { tokenizeriterobject *it = (tokenizeriterobject*)op; PyTypeObject *tp = Py_TYPE(it); + PyObject_GC_UnTrack(it); Py_XDECREF(it->last_line); _PyTokenizer_Free(it->tok); tp->tp_free(it); Py_DECREF(tp); } +static int +tokenizeriter_traverse(PyObject *op, visitproc visit, void *arg) +{ + tokenizeriterobject *it = (tokenizeriterobject *)op; + Py_VISIT(Py_TYPE(it)); + Py_VISIT(it->last_line); + return _PyTokenizer_Traverse(it->tok, visit, arg); +} + static PyType_Slot tokenizeriter_slots[] = { {Py_tp_new, tokenizeriter_new}, {Py_tp_dealloc, tokenizeriter_dealloc}, + {Py_tp_traverse, tokenizeriter_traverse}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, tokenizeriter_next}, @@ -385,7 +374,7 @@ static PyType_Slot tokenizeriter_slots[] = { static PyType_Spec tokenizeriter_spec = { .name = "_tokenize.TokenizerIter", .basicsize = sizeof(tokenizeriterobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), .slots = tokenizeriter_slots, }; diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 6dd598beb1b39c..f6cf2d954e4e7b 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -31,12 +31,6 @@ def emit(self, value: str) -> None: self.writer.print("p->level--;") self.writer.print(f"return {value};") - def check_memory(self, expr: str) -> None: - self.writer.print(f"if ({expr}) {{") - with self.writer.indent(): - self.no_memory() - self.writer.print("}") - def no_memory(self) -> None: self.writer.print("p->error_indicator = 1;") self.writer.print("PyErr_NoMemory();") @@ -51,24 +45,19 @@ class _LoopBuffer: def __init__(self, writer: CWriter, returns: _CReturnEmitter): self._print = writer.print self._indent = writer.indent - self._returns = returns self.error_returns = returns.with_cleanup(self._release) def initialize(self) -> None: - self._print("void **_children = PyMem_Malloc(sizeof(void *));") - self._returns.check_memory("!_children") - self._print("Py_ssize_t _children_capacity = 1;") + self._print("void **_children = NULL;") + self._print("Py_ssize_t _children_capacity = 0;") self._print("Py_ssize_t _n = 0;") def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._print("_children_capacity *= 2;") - self._print( - "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" + self._check_memory( + "_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0" ) - self._check_memory("!_new_children") - self._print("_children = _new_children;") self._print("}") self._print(f"_children[_n++] = {value};") @@ -87,8 +76,7 @@ def finish(self, *, require_one: bool) -> str: def _check_memory(self, expr: str) -> None: self._print(f"if ({expr}) {{") with self._indent(): - self._print(self._release) - self._returns.no_memory() + self.error_returns.no_memory() self._print("}") From 3f02aab9faa338853425da23541e44e4bd24d478 Mon Sep 17 00:00:00 2001 From: he_tao <53343436+hetaozdh@users.noreply.github.com> Date: Mon, 5 Oct 2026 08:51:20 -0500 Subject: [PATCH 080/240] gh-158540: Add the profiled script's directory to sys.path (#158548) * gh-158540: Add the profiled script's directory to sys.path When a script is profiled with ``python -m profiling.sampling run script.py`` from another directory, the script cannot import modules placed next to it, because ``_sync_coordinator._execute_script()`` executes it with the working directory (added by ``_setup_environment()`` for the module case) as ``sys.path[0]`` instead of the script's own directory. Make the script's directory importable in ``_execute_script()``, matching the behavior of ``python script.py``. Add a regression test that runs the coordinator on a script importing a sibling module. * Update _sync_coordinator.py Comment simplified Co-authored-by: Eduardo Villalpando Mello * gh-158540: Resolve symlinks when adding the script directory to sys.path ``python script.py`` resolves symlinks when computing ``sys.path[0]``, so a script reached through a symlink (``link.py -> sub/where.py``) imports modules from the real script's directory. Apply ``os.path.realpath()`` before taking the directory name, and make sure the result is placed at the front of ``sys.path`` even if it was already listed. Add a regression test for a symlinked script. --------- Co-authored-by: Eduardo Villalpando Mello --- Lib/profiling/sampling/_sync_coordinator.py | 5 + .../test_sync_coordinator.py | 113 ++++++++++++++++++ ...-09-30-22-10-00.gh-issue-158540.kQ9vXm.rst | 4 + 3 files changed, 122 insertions(+) create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_sync_coordinator.py create mode 100644 Misc/NEWS.d/next/Library/2026-09-30-22-10-00.gh-issue-158540.kQ9vXm.rst diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index a1cce314b33b19..bb0ee5b5317d69 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -168,6 +168,11 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: if not os.path.isfile(script_path): raise TargetError(f"Script not found: {script_path}") + script_dir = os.path.dirname(os.path.realpath(script_path)) + if script_dir in sys.path: + sys.path.remove(script_dir) + sys.path.insert(0, script_dir) + # Replace sys.argv to match original script call sys.argv = [script_path] + script_args diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_sync_coordinator.py b/Lib/test/test_profiling/test_sampling_profiler/test_sync_coordinator.py new file mode 100644 index 00000000000000..936c01e206f9fb --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_sync_coordinator.py @@ -0,0 +1,113 @@ +"""Tests for the sampling profiler's synchronization coordinator.""" + +import os +import socket +import subprocess +import sys +import tempfile +import unittest + +from test.support import SHORT_TIMEOUT, os_helper, requires_subprocess + + +@requires_subprocess() +class TestSyncCoordinatorScriptExecution(unittest.TestCase): + """Tests for how the coordinator executes a target script.""" + + def run_coordinator(self, cwd, target, *target_args): + """Execute *target* from *cwd* with the sync coordinator. + + Returns the ``(stdout, stderr)`` captured from the coordinator. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(1) + server.settimeout(SHORT_TIMEOUT) + port = server.getsockname()[1] + + cmd = ( + sys.executable, + "-m", + "profiling.sampling._sync_coordinator", + str(port), + cwd, + target, + ) + tuple(target_args) + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + conn, _ = server.accept() + with conn: + # _signal_readiness() sends b"ready" before running target. + self.assertEqual(conn.recv(64), b"ready") + stdout, stderr = process.communicate(timeout=SHORT_TIMEOUT) + finally: + if process.poll() is None: + process.kill() + process.wait() + return stdout, stderr + + def test_script_directory_on_sys_path(self): + # gh-158540: The coordinator must put the directory containing the + # script (not the working directory) on sys.path, matching + # ``python script.py``, so that modules next to the script can be + # imported even when the coordinator runs from another directory. + with tempfile.TemporaryDirectory() as tmpdir: + script_dir = os.path.join(tmpdir, "sub") + os.mkdir(script_dir) + with open(os.path.join(script_dir, "helper.py"), "w") as f: + f.write("message = 'helper imported'\n") + with open(os.path.join(script_dir, "where.py"), "w") as f: + f.write( + "import os\n" + "import sys\n" + "print('PATH0:', os.path.realpath(sys.path[0]))\n" + "import helper\n" + "print('HELPER:', helper.message)\n" + ) + + stdout, stderr = self.run_coordinator( + tmpdir, os.path.join("sub", "where.py") + ) + + self.assertNotIn("ModuleNotFoundError", stderr) + self.assertIn("HELPER: helper imported", stdout) + self.assertIn(f"PATH0: {os.path.realpath(script_dir)}", stdout) + + @os_helper.skip_unless_symlink + def test_symlinked_script_uses_real_directory(self): + # gh-158540: ``python script.py`` resolves symlinks when computing + # sys.path[0], so a symlinked script must import modules next to the + # real script, not next to the link. + with tempfile.TemporaryDirectory() as tmpdir: + script_dir = os.path.join(tmpdir, "sub") + os.mkdir(script_dir) + with open(os.path.join(script_dir, "helper.py"), "w") as f: + f.write("message = 'helper imported'\n") + with open(os.path.join(script_dir, "where.py"), "w") as f: + f.write( + "import os\n" + "import sys\n" + "print('PATH0:', os.path.realpath(sys.path[0]))\n" + "import helper\n" + "print('HELPER:', helper.message)\n" + ) + os.symlink( + os.path.join("sub", "where.py"), + os.path.join(tmpdir, "link.py"), + ) + + stdout, stderr = self.run_coordinator(tmpdir, "link.py") + + self.assertNotIn("ModuleNotFoundError", stderr) + self.assertIn("HELPER: helper imported", stdout) + self.assertIn(f"PATH0: {os.path.realpath(script_dir)}", stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-09-30-22-10-00.gh-issue-158540.kQ9vXm.rst b/Misc/NEWS.d/next/Library/2026-09-30-22-10-00.gh-issue-158540.kQ9vXm.rst new file mode 100644 index 00000000000000..9cbd698cf9a96e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-30-22-10-00.gh-issue-158540.kQ9vXm.rst @@ -0,0 +1,4 @@ +Fix :mod:`profiling.sampling` not adding the directory containing the +profiled script to :data:`sys.path`, which prevented the script from +importing modules placed next to it. The script's directory is now added, +matching the behavior of ``python script.py``. From 1643525f0f89438d87daf4cdbb1cbb1de9df08e8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 5 Oct 2026 14:51:26 +0100 Subject: [PATCH 081/240] gh-158552: Wait for Windows threads to suspend before blocking sampling (#158802) * gh-158552: Wait for Windows threads to suspend before blocking sampling * Use a named Windows thread enumeration status constant --- .../test_sampling_profiler/test_blocking.py | 57 ++++++++++++ ...10-04-20-00-00.gh-issue-158552.windows.rst | 3 + Modules/_remote_debugging/_remote_debugging.h | 1 + Modules/_remote_debugging/threads.c | 89 +++++++++++++++++++ 4 files changed, 150 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-10-04-20-00-00.gh-issue-158552.windows.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py b/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py index 0a5541c733d4c7..84b9e7aca0ffbd 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py @@ -5,6 +5,7 @@ import subprocess import sys import textwrap +import time import unittest from unittest import mock @@ -213,3 +214,59 @@ def test_run_blocking_exits_after_target_process_exits(self): replay.returncode, 0, f"stdout:\n{replay.stdout}\nstderr:\n{replay.stderr}", ) + + +@requires_remote_subprocess_debugging() +@unittest.skipUnless(sys.platform == "win32", "Windows only") +class TestBlockingModeSuspension(unittest.TestCase): + def test_all_threads_stop_before_pause_returns(self): + import mmap + + tag = f"cpython_blocking_{os.getpid()}_{id(self)}" + script = textwrap.dedent(f''' + import mmap + import struct + import threading + + memory = mmap.mmap(-1, 16, tagname={tag!r}) + + def worker(offset): + counter = 0 + while True: + counter += 1 + struct.pack_into("q", memory, offset, counter) + + for offset in (0, 8): + threading.Thread(target=worker, args=(offset,), daemon=True).start() + _test_sock.sendall(b"working") + _test_sock.recv(1) + ''') + with mmap.mmap(-1, 16, tagname=tag) as memory: + with test_subprocess(script, wait_for_working=True) as subproc: + unwinder = _remote_debugging.RemoteUnwinder( + subproc.process.pid, all_threads=True) + deadline = time.monotonic() + SHORT_TIMEOUT + while not all(memory[offset:offset + 8] != bytes(8) + for offset in (0, 8)): + self.assertLess(time.monotonic(), deadline, + "Worker threads did not start") + time.sleep(0.001) + for _ in range(100): + self.assertTrue(unwinder.pause_threads()) + try: + before = memory[:] + self.assertFalse(unwinder.pause_threads()) + unwinder.get_stack_trace() + time.sleep(0.001) + self.assertEqual(memory[:], before, + "Target memory changed while paused") + finally: + unwinder.resume_threads() + self.assertFalse(unwinder.resume_threads()) + before = memory[:] + deadline = time.monotonic() + SHORT_TIMEOUT + while any(memory[offset:offset + 8] == before[offset:offset + 8] + for offset in (0, 8)): + self.assertLess(time.monotonic(), deadline, + "Worker threads did not resume") + time.sleep(0.001) diff --git a/Misc/NEWS.d/next/Library/2026-10-04-20-00-00.gh-issue-158552.windows.rst b/Misc/NEWS.d/next/Library/2026-10-04-20-00-00.gh-issue-158552.windows.rst new file mode 100644 index 00000000000000..e46f9edcebe09b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-04-20-00-00.gh-issue-158552.windows.rst @@ -0,0 +1,3 @@ +Wait for Windows threads to finish suspending before reading target memory +in the sampling profiler's blocking mode. Resume the target if waiting for +suspension fails. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index cf923014445f55..c57b30533ce1c1 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -105,6 +105,7 @@ typedef struct { #ifdef MS_WINDOWS #define STATUS_SUCCESS ((NTSTATUS)0x00000000L) #define STATUS_INFO_LENGTH_MISMATCH ((NTSTATUS)0xC0000004L) +#define STATUS_NO_MORE_ENTRIES ((NTSTATUS)0x8000001AL) typedef enum _WIN32_THREADSTATE { WIN32_THREADSTATE_INITIALIZED = 0, WIN32_THREADSTATE_READY = 1, diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 04c70cc96d6bd1..666316ceaa28fc 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -828,6 +828,89 @@ _Py_RemoteDebug_ResumeAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug #elif defined(MS_WINDOWS) +static int +wait_for_threads_to_stop(RemoteUnwinderObject *unwinder) +{ + typedef NTSTATUS (NTAPI *NtGetNextThreadFunc)( + HANDLE, HANDLE, ACCESS_MASK, ULONG, ULONG, PHANDLE); + static NtGetNextThreadFunc pNtGetNextThread = NULL; + static int tried_load = 0; + + if (!tried_load) { + HMODULE hNtdll = GetModuleHandleW(L"ntdll.dll"); + if (hNtdll) { + pNtGetNextThread = (NtGetNextThreadFunc)GetProcAddress( + hNtdll, "NtGetNextThread"); + } + tried_load = 1; + } + if (pNtGetNextThread == NULL) { + PyErr_SetString(PyExc_RuntimeError, "NtGetNextThread not available"); + return -1; + } + + HANDLE previous = NULL; + int result = -1; + for (;;) { + HANDLE next = NULL; + // Enumerate with the available access, then obtain context access + // separately so threads that deny it are not silently skipped. + NTSTATUS status = pNtGetNextThread(unwinder->handle.hProcess, + previous, MAXIMUM_ALLOWED, 0, 0, &next); + if (previous != NULL) { + CloseHandle(previous); + } + previous = next; + if (status == STATUS_NO_MORE_ENTRIES) { + break; + } + if (status < 0) { + if (!is_process_alive(unwinder->handle.hProcess)) { + PyErr_Format(PyExc_ProcessLookupError, + "Process %d has terminated", unwinder->handle.pid); + } + else { + PyErr_Format(PyExc_RuntimeError, + "NtGetNextThread failed: 0x%lx", status); + } + goto done; + } + HANDLE thread; + if (!DuplicateHandle(GetCurrentProcess(), next, GetCurrentProcess(), + &thread, THREAD_GET_CONTEXT | SYNCHRONIZE, + FALSE, 0)) { + PyErr_SetFromWindowsErr(GetLastError()); + goto done; + } + // Suspension is asynchronous. Reading the context waits for the + // thread to stop before we start reading the target's memory. + CONTEXT context = {.ContextFlags = CONTEXT_CONTROL}; + if (!GetThreadContext(thread, &context)) { + DWORD error = GetLastError(); + int exited = WaitForSingleObject(thread, 0) == WAIT_OBJECT_0; + CloseHandle(thread); + if (exited) { + continue; + } + PyErr_SetFromWindowsErr(error); + goto done; + } + CloseHandle(thread); + } + if (!is_process_alive(unwinder->handle.hProcess)) { + PyErr_Format(PyExc_ProcessLookupError, + "Process %d has terminated", unwinder->handle.pid); + goto done; + } + result = 0; + +done: + if (previous != NULL) { + CloseHandle(previous); + } + return result; +} + void _Py_RemoteDebug_InitThreadsState(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st) { @@ -858,6 +941,12 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T if (status >= 0) { st->hProcess = unwinder->handle.hProcess; st->suspended = 1; + if (wait_for_threads_to_stop(unwinder) < 0) { + // pause_threads() has not yet set threads_stopped, so its caller + // will not resume the process when we return an error. + _Py_RemoteDebug_ResumeAllThreads(unwinder, st); + return -1; + } _Py_RemoteDebug_ClearCache(&unwinder->handle); return 0; } From 8df25809e928b88db5ec16c73470dd6bd4ed3653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 5 Oct 2026 16:13:14 +0200 Subject: [PATCH 082/240] gh-154060: Fix the `Profile Stats` offsets in the profiling binary format docs and a missing flag (#157871) * correct profile stats size and paddings * missing flag * gh-154060: Clarify the main-thread status flag --------- Co-authored-by: Pablo Galindo Salgado --- InternalDocs/profiling_binary_format.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/InternalDocs/profiling_binary_format.md b/InternalDocs/profiling_binary_format.md index 0947e946f41bd0..a3d86b1a4ab569 100644 --- a/InternalDocs/profiling_binary_format.md +++ b/InternalDocs/profiling_binary_format.md @@ -47,8 +47,8 @@ The file consists of five required sections and one optional extension: | String Table | Variable size +------------------+ frame_table_offset | Frame Table | Variable size -+------------------+ file_size - 64 (when stats are present) -| Profile Stats | 32 bytes (optional) ++------------------+ file_size - 88 (when stats are present) +| Profile Stats | 56 bytes (optional) +------------------+ file_size - 32 | Footer | 32 bytes (fixed) +------------------+ file_size @@ -220,6 +220,7 @@ The status byte is a bitfield encoding thread state at sample time: | 2 | THREAD_STATUS_UNKNOWN | Thread state could not be determined | | 3 | THREAD_STATUS_GIL_REQUESTED | Thread is waiting to acquire the GIL | | 4 | THREAD_STATUS_HAS_EXCEPTION | Thread has a pending exception | +| 5 | THREAD_STATUS_MAIN_THREAD | Thread is the interpreter's main thread | Multiple flags can be set simultaneously (e.g., a thread can hold the GIL while also running on CPU). Analysis tools use these to filter samples or From e4a24732eb5829302461069f7b005b2e1cb6f76b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Mon, 5 Oct 2026 22:37:01 +0800 Subject: [PATCH 083/240] gh-152721: Fix quadratic RLE replay time in the profiling binary reader (#152722) * gh-152721: Fix quadratic RLE replay time in the profiling binary reader * gh-152721: Clarify the batch-list comment Reword per review: the list is built with append per element, not pre-sized; note the old alloc(count - i) + trim per batch was O(count^2). * Cap RLE batch size to bound the per-batch timestamp list * Move MAX_RLE_BATCH_SAMPLES comment to its own line (keep under 79 cols) * Drop the now-redundant emit_batch trim and fix the review nits The batch list is built to its exact size, so the PyList_SetSlice trim was a no-op; call emit_sample directly. Also point the cap comment at the issue (gh-151378) and drop the stale over-long comment. * gh-152721: Test replay batches beyond the RLE limit --------- Co-authored-by: Pablo Galindo Salgado --- .../test_binary_format.py | 72 +++++++++++++++++++ ...07-01-18-00-00.gh-issue-152721.rlequad.rst | 2 + Modules/_remote_debugging/binary_io_reader.c | 43 +++++------ 3 files changed, 93 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-01-18-00-00.gh-issue-152721.rlequad.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 97075a1e2fed80..66a9c68a61c47d 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -7,6 +7,7 @@ import struct import tempfile import unittest +from unittest import mock from collections import defaultdict from test.support import captured_stderr @@ -1503,6 +1504,77 @@ def test_alternating_threads_status_changes(self): self.assertEqual(count, 100) self.assert_samples_equal(samples, collector) + def test_rle_alternating_status_batches_correctly(self): + """A repeat record whose status alternates every sample replays as N + single-status batches with the right cumulative timestamps.""" + class BatchCollector: + def __init__(self): + self.batches = [] + + def collect(self, stack_frames, timestamps_us): + for interp in stack_frames: + for thread in interp.threads: + self.batches.append( + (thread.status, list(timestamps_us)) + ) + + def export(self, filename): + pass + + num_samples = 2000 + frame = make_frame("rle.py", 42, "rle_func") + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + writer = BinaryCollector(filename, 1000, compression="none") + expected = [] + for i in range(num_samples): + status = THREAD_STATUS_HAS_GIL if i % 2 else 0 + ts = 1000 + i + expected.append((status, [ts])) + sample = [ + make_interpreter(0, [make_thread(1, [frame], status)]) + ] + writer.collect(sample, timestamp_us=ts) + writer.export(None) + + collector = BatchCollector() + with BinaryReader(filename) as reader: + count = reader.replay_samples(collector) + + self.assertEqual(count, num_samples) + self.assertEqual(len(collector.batches), num_samples) + self.assertEqual(collector.batches, expected) + + + def test_rle_long_run_splits_batches(self): + # Construct a single repeat record larger than the writer's buffer. + num_samples = 8193 + filename = self.create_binary_file([], compression="none") + data = bytearray(pathlib.Path(filename).read_bytes()) + record = (struct.pack("=QIB", 1, 0, 0) # STACK_REPEAT + + b"\x81\x40" # 8193 as a varint + + b"\x01\x00" * num_samples) # delta=1, status=0 + data[64:64] = record + struct.pack_into("=Q", data, 12, 0) # start timestamp + struct.pack_into("=Q", data, 28, num_samples) + struct.pack_into("=I", data, 36, 1) # thread count + for offset in (40, 48): # string and frame table offsets + old_offset = struct.unpack_from("=Q", data, offset)[0] + struct.pack_into("=Q", data, offset, old_offset + len(record)) + struct.pack_into("=Q", data, len(data) - 24, len(data)) + pathlib.Path(filename).write_bytes(data) + + collector = mock.Mock() + with BinaryReader(filename) as reader: + count = reader.replay_samples(collector) + batches = [call.args[1] for call in collector.collect.call_args_list] + self.assertEqual(count, num_samples) + self.assertEqual([len(batch) for batch in batches], [8192, 1]) + self.assertEqual([ts for batch in batches for ts in batch], + list(range(1, num_samples + 1))) + class TestBinaryStress(BinaryFormatTestBase): """Randomized stress tests for binary format.""" diff --git a/Misc/NEWS.d/next/Library/2026-07-01-18-00-00.gh-issue-152721.rlequad.rst b/Misc/NEWS.d/next/Library/2026-07-01-18-00-00.gh-issue-152721.rlequad.rst new file mode 100644 index 00000000000000..4dac0ed245bd67 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-01-18-00-00.gh-issue-152721.rlequad.rst @@ -0,0 +1,2 @@ +Fix quadratic replay time in the :mod:`profiling.sampling` binary reader when a +profile's run-length-encoded samples alternate thread status. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 8af1d281cee6b6..80627db913ea21 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -33,6 +33,9 @@ /* Progress callback frequency */ #define PROGRESS_CALLBACK_INTERVAL 1000 +/* Cap per-batch RLE samples to bound the timestamp list (gh-151378) */ +#define MAX_RLE_BATCH_SAMPLES 8192 + /* ============================================================================ * BINARY READER IMPLEMENTATION * ============================================================================ */ @@ -1083,21 +1086,6 @@ emit_sample(RemoteDebuggingState *state, PyObject *collector, return 0; } -/* Helper to trim timestamp list and emit batch. Returns 0 on success, -1 on error. */ -static int -emit_batch(RemoteDebuggingState *state, PyObject *collector, - uint64_t thread_id, uint32_t interpreter_id, uint8_t status, - const uint32_t *frame_indices, size_t stack_depth, - BinaryReader *reader, PyObject *timestamps_list, Py_ssize_t actual_size) -{ - /* Trim list to actual size */ - if (PyList_SetSlice(timestamps_list, actual_size, PyList_GET_SIZE(timestamps_list), NULL) < 0) { - return -1; - } - return emit_sample(state, collector, thread_id, interpreter_id, status, - frame_indices, stack_depth, reader, timestamps_list); -} - /* Helper to invoke progress callback, returns -1 on error */ static inline int invoke_progress_callback(PyObject *callback, Py_ssize_t current, uint64_t total) @@ -1226,17 +1214,18 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre ts->prev_timestamp += delta; /* Start new batch on first sample or status change */ - if (i == 0 || status != batch_status) { + if (i == 0 || status != batch_status + || batch_idx >= MAX_RLE_BATCH_SAMPLES) { if (timestamps_list) { - int rc = emit_batch(state, collector, thread_id, interpreter_id, - batch_status, ts->current_stack, ts->current_stack_depth, - reader, timestamps_list, batch_idx); + int rc = emit_sample(state, collector, thread_id, interpreter_id, + batch_status, ts->current_stack, ts->current_stack_depth, + reader, timestamps_list); Py_DECREF(timestamps_list); if (rc < 0) { return -1; } } - timestamps_list = PyList_New(count - i); + timestamps_list = PyList_New(0); if (!timestamps_list) { return -1; } @@ -1249,14 +1238,20 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre Py_DECREF(timestamps_list); return -1; } - PyList_SET_ITEM(timestamps_list, batch_idx++, ts_obj); + int append_rc = PyList_Append(timestamps_list, ts_obj); + Py_DECREF(ts_obj); + if (append_rc < 0) { + Py_DECREF(timestamps_list); + return -1; + } + batch_idx++; } /* Emit final batch */ if (timestamps_list) { - int rc = emit_batch(state, collector, thread_id, interpreter_id, - batch_status, ts->current_stack, ts->current_stack_depth, - reader, timestamps_list, batch_idx); + int rc = emit_sample(state, collector, thread_id, interpreter_id, + batch_status, ts->current_stack, ts->current_stack_depth, + reader, timestamps_list); Py_DECREF(timestamps_list); if (rc < 0) { return -1; From 1441e51dbc4ae92566c703b17daf7f99a3e371bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Mon, 5 Oct 2026 22:37:06 +0800 Subject: [PATCH 084/240] gh-156545: Fix flamegraph export RecursionError on deeply recursive programs (#156546) * gh-156545: Fix flamegraph export RecursionError on deeply recursive programs * Tighten comments * gh-156545: Verify recursion limit restoration during export --------- Co-authored-by: Pablo Galindo Salgado --- Lib/profiling/sampling/stack_collector.py | 57 +++++++++++-------- .../test_sampling_profiler/test_collectors.py | 35 +++++++++++- ...-08-29-11-37-40.gh-issue-156545.fLaMe1.rst | 3 + 3 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-29-11-37-40.gh-issue-156545.fLaMe1.rst diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 7410d07c7e0162..e420bb6d2e9b87 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -68,6 +68,10 @@ def export(self, filename): return True +# Bounded by the unwinder's maximum captured stack depth (MAX_FRAMES). +_FLAMEGRAPH_RECURSION_MARGIN = 2000 + + class FlamegraphCollector(StackTraceCollector): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -166,34 +170,41 @@ def set_mode(self, mode): self.stats["mode"] = mode def export(self, filename): - flamegraph_data = self._convert_to_flamegraph_format() - - # Debug output with string table statistics - num_functions = len(flamegraph_data.get("children", [])) - total_time = flamegraph_data.get("value", 0) - string_count = len(self._string_table) - s1 = "" if num_functions == 1 else "s" - s2 = "" if total_time == 1 else "s" - s3 = "" if string_count == 1 else "s" - print( - f"Flamegraph data: {num_functions} root function{s1}, " - f"{total_time} total sample{s2}, " - f"{string_count} unique string{s3}" - ) - - if num_functions == 0: + # Converting the call tree recurses to the sampled stack depth. + old_limit = sys.getrecursionlimit() + sys.setrecursionlimit(old_limit + _FLAMEGRAPH_RECURSION_MARGIN) + try: + flamegraph_data = self._convert_to_flamegraph_format() + + # Debug output with string table statistics + num_functions = len(flamegraph_data.get("children", [])) + total_time = flamegraph_data.get("value", 0) + string_count = len(self._string_table) + s1 = "" if num_functions == 1 else "s" + s2 = "" if total_time == 1 else "s" + s3 = "" if string_count == 1 else "s" print( - "Warning: No functions found in profiling data. Check if sampling captured any data." + f"Flamegraph data: {num_functions} root function{s1}, " + f"{total_time} total sample{s2}, " + f"{string_count} unique string{s3}" ) - return False - html_content = self._create_flamegraph_html(flamegraph_data) + if num_functions == 0: + print( + "Warning: No functions found in profiling data. " + "Check if sampling captured any data." + ) + return False - with open(filename, "w", encoding="utf-8") as f: - f.write(html_content) + html_content = self._create_flamegraph_html(flamegraph_data) - print(f"Flamegraph saved to: {filename}") - return True + with open(filename, "w", encoding="utf-8") as f: + f.write(html_content) + + print(f"Flamegraph saved to: {filename}") + return True + finally: + sys.setrecursionlimit(old_limit) @staticmethod @functools.lru_cache(maxsize=None) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 282f97e5259e09..eb58c29dd361d3 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -4,10 +4,11 @@ import marshal import opcode import os +import sys import tempfile import unittest -from test.support import is_emscripten +from test.support import is_emscripten, set_recursion_limit try: import _remote_debugging # noqa: F401 @@ -610,6 +611,38 @@ def test_flamegraph_collector_empty_export_fails(self): self.assertFalse(export_ok) self.assertEqual(os.path.getsize(flamegraph_out.name), 0) + def test_flamegraph_deep_stack_export(self): + flamegraph_out = tempfile.NamedTemporaryFile( + suffix=".html", delete=False + ) + self.addCleanup(close_and_unlink, flamegraph_out) + + collector = FlamegraphCollector(1000) + # Deeper than the default recursion limit. + frames = [MockFrameInfo("f.py", i + 1, f"f{i}") for i in range(1536)] + collector.collect( + [MockInterpreterInfo(0, [MockThreadInfo(1, frames)])]) + + with set_recursion_limit(1000), captured_stdout(), captured_stderr(): + export_ok = collector.export(flamegraph_out.name) + self.assertEqual(sys.getrecursionlimit(), 1000) + + self.assertTrue(export_ok) + self.assertGreater(os.path.getsize(flamegraph_out.name), 0) + + def test_flamegraph_export_restores_recursion_limit(self): + collector = FlamegraphCollector(1000) + frame = MockFrameInfo("f.py", 1, "f") + with set_recursion_limit(500), captured_stdout(), captured_stderr(): + self.assertFalse(collector.export(None)) + self.assertEqual(sys.getrecursionlimit(), 500) + collector.collect([ + MockInterpreterInfo(0, [MockThreadInfo(1, [ + frame, MockFrameInfo("f.py", 2, "caller")])])]) + with self.assertRaises(TypeError): + collector.export(None) + self.assertEqual(sys.getrecursionlimit(), 500) + def test_gecko_collector_basic(self): """Test basic GeckoCollector functionality.""" collector = GeckoCollector(1000) diff --git a/Misc/NEWS.d/next/Library/2026-08-29-11-37-40.gh-issue-156545.fLaMe1.rst b/Misc/NEWS.d/next/Library/2026-08-29-11-37-40.gh-issue-156545.fLaMe1.rst new file mode 100644 index 00000000000000..cc4f57ff42adab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-29-11-37-40.gh-issue-156545.fLaMe1.rst @@ -0,0 +1,3 @@ +Fix the sampling profiler's flamegraph export so that profiling a deeply +recursive program no longer fails with :exc:`RecursionError` instead of +producing a flamegraph. Patch by tonghuaroot. From 722687092f8aaa005708302a502e930a729e8660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Mon, 5 Oct 2026 22:50:03 +0800 Subject: [PATCH 085/240] gh-153838: Skip non-regular source files in the heatmap exporter (#153839) * gh-153838: Skip non-regular and oversized files in heatmap exporter * Drop the size cap, keep the is_file() guard only * gh-153838: Verify non-regular sources are never opened --------- Co-authored-by: Pablo Galindo Salgado --- Lib/profiling/sampling/heatmap_collector.py | 14 +++---- Lib/test/test_profiling/test_heatmap.py | 41 ++++++++++++++++--- ...-07-17-12-00-00.gh-issue-153838.HmCp5s.rst | 2 + 3 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 220b6b8150ac98..62f42755827f06 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -783,14 +783,14 @@ def _generate_file_html(self, output_path: Path, filename: str, line_counts: Dict[int, int], self_counts: Dict[int, int], file_stat: FileStats): """Generate HTML for a single source file with heatmap coloring.""" - # Read source file + source_lines = [f"# Source file not available: {filename}"] try: - source_lines = Path(filename).read_text(encoding='utf-8', errors='replace').splitlines() - except (IOError, OSError) as e: - if not (filename.startswith('<') or filename.startswith('[') or - filename in ('~', '...', '.') or len(filename) < 2): - print(f"Warning: Could not read source file {filename}: {e}") - source_lines = [f"# Source file not available: {filename}"] + path = Path(filename) + if path.is_file(): + source_lines = path.read_text( + encoding='utf-8', errors='replace').splitlines() + except (IOError, OSError): + pass # Generate HTML for each line max_samples = max(line_counts.values()) if line_counts else 1 diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index ee27fdd3fa3053..1b4de0caa3d00f 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -4,6 +4,7 @@ import shutil import tempfile import unittest +from unittest import mock from collections import namedtuple from pathlib import Path @@ -615,13 +616,43 @@ def test_export_file_html_has_line_numbers(self): html_files = [f for f in os.listdir(output_path) if f.startswith('file_') and f.endswith('.html')] - if html_files: - with open(os.path.join(output_path, html_files[0]), 'r', encoding='utf-8') as f: - content = f.read() + self.assertEqual(len(html_files), 1) + with open(os.path.join(output_path, html_files[0]), 'r', encoding='utf-8') as f: + content = f.read() + + # Should have line-related content + self.assertIn('line-', content) + + def test_export_skips_nonexistent_source(self): + self.check_unavailable_source(os.path.join(self.test_dir, 'missing.py')) - # Should have line-related content - self.assertIn('line-', content) + def test_export_skips_directory_source(self): + self.check_unavailable_source(self.test_dir) + @unittest.skipUnless(hasattr(os, 'mkfifo'), 'requires os.mkfifo') + def test_export_skips_fifo_source(self): + filename = os.path.join(self.test_dir, 'source.fifo') + os.mkfifo(filename) + self.check_unavailable_source(filename) + + def check_unavailable_source(self, filename): + collector = HeatmapCollector(sample_interval_usec=100) + frames = [(filename, (1, 1, -1, -1), 'f', None)] + collector.process_frames(frames, thread_id=1) + output_path = Path(self.test_dir) / 'unavailable_source' + read_text = Path.read_text + + def check_read(path, *args, **kwargs): + self.assertNotEqual(path, Path(filename), + 'Non-regular source must not be opened') + return read_text(path, *args, **kwargs) + + with (captured_stdout(), captured_stderr(), + mock.patch.object(Path, 'read_text', check_read)): + collector.export(output_path) + content = (output_path / collector.file_index[filename]).read_text( + encoding='utf-8') + self.assertIn('Source file not available', content) class MockFrameInfo: """Mock FrameInfo for testing. diff --git a/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst b/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst new file mode 100644 index 00000000000000..faecab8cff3e7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-17-12-00-00.gh-issue-153838.HmCp5s.rst @@ -0,0 +1,2 @@ +Skip non-regular files in the ``profiling.sampling`` heatmap exporter +instead of reading them unconditionally. Patch by tonghuaroot. From 802f1457ed6088a1687907593f8cbfff102c6925 Mon Sep 17 00:00:00 2001 From: Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:58:35 +0200 Subject: [PATCH 086/240] gh-79459 tempfile: Raise a ValueError if the prefix or suffix contains a directory (#150477) Co-authored-by: Victor Stinner Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/tempfile.py | 4 ++ Lib/test/test_tempfile.py | 41 +++++++++++++++++++ ...6-10-05-13-57-08.gh-issue-79459.2RgBDL.rst | 2 + 3 files changed, 47 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-10-05-13-57-08.gh-issue-79459.2RgBDL.rst diff --git a/Lib/tempfile.py b/Lib/tempfile.py index ad81c5d3417b4c..82c1ee353117e9 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -119,11 +119,15 @@ def _sanitize_params(prefix, suffix, dir): output_type = _infer_return_type(prefix, suffix, dir) if suffix is None: suffix = output_type() + if _os.path.dirname(suffix): + raise ValueError("suffix can't contain a directory component") if prefix is None: if output_type is str: prefix = template else: prefix = _os.fsencode(template) + if _os.path.dirname(prefix): + raise ValueError("prefix can't contain a directory component") if dir is None: if output_type is str: dir = gettempdir() diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index cd960ed99117b6..2f7b8798daa8c9 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -2154,5 +2154,46 @@ def test_cleanup_safe(self): # platforms, but don't forget to update the docs. self.assertTrue(tempfile._rmtree_use_dir_fd) + +class TestMisc(BaseTestCase): + def test_prefix_suffix_error(self): + tests = [ + f"dir{os.sep}name", + f"{os.sep}abs_name", + ] + if os.altsep is not None: + tests.extend(( + f"dir{os.altsep}name", + f"{os.altsep}abs_name", + )) + if support.MS_WINDOWS: + tests.append('C:name') + tests.extend(tuple(os.fsencode(path) for path in tests)) + + PREFIX_ERR = "prefix can't contain a directory component" + SUFFIX_ERR = "suffix can't contain a directory component" + for value in tests: + with self.subTest(value): + # test prefix + with self.assertRaisesRegex(ValueError, PREFIX_ERR): + tempfile.mkstemp(prefix=value) + with self.assertRaisesRegex(ValueError, PREFIX_ERR): + tempfile.mkdtemp(prefix=value) + with self.assertRaisesRegex(ValueError, PREFIX_ERR): + tempfile.TemporaryFile(prefix=value) + with self.assertRaisesRegex(ValueError, PREFIX_ERR): + tempfile.NamedTemporaryFile(prefix=value) + + # test suffix + with self.assertRaisesRegex(ValueError, SUFFIX_ERR): + tempfile.mkstemp(suffix=value) + with self.assertRaisesRegex(ValueError, SUFFIX_ERR): + tempfile.mkdtemp(suffix=value) + with self.assertRaisesRegex(ValueError, SUFFIX_ERR): + tempfile.TemporaryFile(suffix=value) + with self.assertRaisesRegex(ValueError, SUFFIX_ERR): + tempfile.NamedTemporaryFile(suffix=value) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-10-05-13-57-08.gh-issue-79459.2RgBDL.rst b/Misc/NEWS.d/next/Library/2026-10-05-13-57-08.gh-issue-79459.2RgBDL.rst new file mode 100644 index 00000000000000..2bbe77d6972e22 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-05-13-57-08.gh-issue-79459.2RgBDL.rst @@ -0,0 +1,2 @@ +:mod:`tempfile` functions that take a ``prefix`` or ``suffix`` argument now raise a :exc:`ValueError` if they contain a directory component: +:func:`tempfile.mkstemp`, :func:`tempfile.mkdtemp`, :func:`tempfile.TemporaryFile`, :func:`tempfile.NamedTemporaryFile`. From 38416fedef5496cb10223cfecf28e3952c47e1d7 Mon Sep 17 00:00:00 2001 From: he_tao <53343436+hetaozdh@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:16:14 -0500 Subject: [PATCH 087/240] gh-158539: Fix exception mode missing handlers in generators/coroutines (#158581) * gh-158539: Fix exception mode missing handlers in generators/coroutines The sampling profiler's exception mode decided whether a thread was handling an exception by reading the embedded PyThreadState.exc_state. Generators, coroutines and async generators repoint tstate->exc_info at their own _PyErr_StackItem while they run, so an except block running in one of them (or in a function they call) stored the exception in that item instead, and was never sampled. Follow tstate->exc_info and its previous_item chain, mirroring _PyErr_GetTopmostException(), and export the two debug offsets needed to walk the chain from remote memory. The common case where exc_info points at the embedded exc_state keeps the existing zero-extra-read fast path. * gh-158539: Handle deep exception chains without changing debug offsets * gh-158539: Use portable static assertion messages * gh-158539: Keep layout assertions with debug-offset validation * gh-158539: Use the platform guard for in-process inspection tests --------- Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_external_inspection.py | 236 ++++++++++++++++++ ...-10-01-14-00-00.gh-issue-158539.q1w2e3.rst | 5 + .../debug_offsets_validation.h | 15 +- Modules/_remote_debugging/threads.c | 54 +++- 4 files changed, 299 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index c2157f73389059..0c189e60a93b8e 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -3293,6 +3293,242 @@ def test_finally_no_exception_no_flag(self): self._check_exception_status(p, thread_tid, expect_exception=False) +@skip_if_not_supported +class TestExceptionDetectionInProcess(RemoteInspectionTestBase): + """gh-158539: HAS_EXCEPTION for handlers running in generators/coroutines. + + ``TestExceptionDetectionScenarios`` samples a child process and therefore + needs subprocess debugging permissions. These tests inspect the current + process with ``RemoteUnwinder`` and only need self-inspection, so they also + run on macOS without special entitlements. + """ + + @classmethod + def setUpClass(cls): + try: + RemoteUnwinder(os.getpid(), all_threads=True).get_stack_trace() + except PermissionError as exc: + raise unittest.SkipTest(f"self-inspection is unavailable: {exc}") + + def _check_running_handler( + self, target, expect_exception, *, mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False, + ): + """Run *target* in a thread and check its HAS_EXCEPTION flag. + + *target* receives ``(ready, stop)`` events and must signal ``ready`` + only once it is executing inside the code region under test, then keep + running until ``stop`` is set. + """ + stop = threading.Event() + ready = threading.Event() + failure = [] + + def runner(): + try: + target(ready, stop) + except BaseException as exc: + failure.append(exc) + ready.set() + + thread = threading.Thread(target=runner, daemon=True) + thread.start() + try: + self.assertTrue(ready.wait(SHORT_TIMEOUT), "handler never started") + self.assertFalse(failure, f"handler raised {failure!r}") + + unwinder = RemoteUnwinder( + os.getpid(), + all_threads=True, + mode=mode, + skip_non_matching_threads=skip_non_matching_threads, + ) + observed = [] + for _ in busy_retry(SHORT_TIMEOUT): + with contextlib.suppress(*TRANSIENT_ERRORS): + statuses = self._get_thread_statuses(unwinder.get_stack_trace()) + status = statuses.get(thread.native_id) + if status is None: + continue + has_exception = bool(status & THREAD_STATUS_HAS_EXCEPTION) + observed.append(has_exception) + if has_exception == expect_exception: + break + self.assertTrue( + observed, "target thread status was never observed" + ) + self.assertIn( + expect_exception, + observed, + f"HAS_EXCEPTION was never {expect_exception} while the " + f"handler was running (observed {observed})", + ) + finally: + stop.set() + thread.join(SHORT_TIMEOUT) + + def _busy_until_stopped(self, ready, stop): + ready.set() + while not stop.is_set(): + time.sleep(0.001) + + def test_handler_in_function(self): + def target(ready, stop): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + + self._check_running_handler(target, expect_exception=True) + + def test_handler_in_generator(self): + def target(ready, stop): + def gen(): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + yield + + for _ in gen(): + pass + + self._check_running_handler(target, expect_exception=True) + + def test_handler_in_genexpr_callee(self): + def target(ready, stop): + def callee(): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + + list(callee() for _ in range(1)) + + self._check_running_handler(target, expect_exception=True) + + def test_handler_in_coroutine(self): + async def coro(ready, stop): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + + def target(ready, stop): + asyncio.run(coro(ready, stop)) + + self._check_running_handler(target, expect_exception=True) + + def test_handler_in_callee_from_coroutine(self): + def callee(ready, stop): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + + async def coro(ready, stop): + callee(ready, stop) + + def target(ready, stop): + asyncio.run(coro(ready, stop)) + + self._check_running_handler(target, expect_exception=True) + + def test_outer_handler_while_generator_runs(self): + """A generator with no handler of its own must not hide the outer one. + + ``exc_info`` points at the generator's empty ``_PyErr_StackItem`` whose + ``previous_item`` is the thread's ``exc_state``, so the profiler has to + walk the chain to find the exception ``sys.exception()`` reports. + """ + def target(ready, stop): + def gen(): + self._busy_until_stopped(ready, stop) + yield + + try: + raise ValueError("outer") + except ValueError: + for _ in gen(): + pass + + self._check_running_handler(target, expect_exception=True) + + def test_generator_without_exception(self): + def target(ready, stop): + def gen(): + self._busy_until_stopped(ready, stop) + yield + + for _ in gen(): + pass + + self._check_running_handler(target, expect_exception=False) + + def test_outer_handler_while_nested_generators_run(self): + def target(ready, stop): + def gen(depth): + if depth: + yield from gen(depth - 1) + else: + self._busy_until_stopped(ready, stop) + yield + + try: + raise ValueError("outer") + except ValueError: + for _ in gen(32): + pass + + self._check_running_handler( + target, + expect_exception=True, + mode=PROFILING_MODE_EXCEPTION, + skip_non_matching_threads=True, + ) + + def test_generator_finally_after_except(self): + """The handled exception is cleared before the generator's finally.""" + def target(ready, stop): + def gen(): + try: + raise ValueError("test") + except ValueError: + pass + finally: + self._busy_until_stopped(ready, stop) + yield + + for _ in gen(): + pass + + self._check_running_handler(target, expect_exception=False) + + def test_exception_mode_filter_keeps_generator_handler(self): + """The exception-mode thread filter must not drop a generator handler. + + This mirrors what ``--mode=exception`` actually does: threads without + HAS_EXCEPTION are skipped before their stack is unwound. + """ + def target(ready, stop): + def gen(): + try: + raise ValueError("test") + except ValueError: + self._busy_until_stopped(ready, stop) + yield + + for _ in gen(): + pass + + self._check_running_handler( + target, + expect_exception=True, + mode=PROFILING_MODE_EXCEPTION, + skip_non_matching_threads=True, + ) + + @requires_remote_subprocess_debugging() class TestFrameCaching(RemoteInspectionTestBase): """Test that frame caching produces correct results. diff --git a/Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst b/Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst new file mode 100644 index 00000000000000..b1ab9c8e017fb2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-01-14-00-00.gh-issue-158539.q1w2e3.rst @@ -0,0 +1,5 @@ +Fix :mod:`profiling.sampling` exception mode discarding samples for code +running inside an ``except`` block in a generator or coroutine. The remote +debugger now follows ``tstate->exc_info`` and its ``previous_item`` chain +instead of only reading the embedded ``exc_state``, matching the exception +that :func:`sys.exception` reports. diff --git a/Modules/_remote_debugging/debug_offsets_validation.h b/Modules/_remote_debugging/debug_offsets_validation.h index c0c01a0a639e19..66c35e5522a9c7 100644 --- a/Modules/_remote_debugging/debug_offsets_validation.h +++ b/Modules/_remote_debugging/debug_offsets_validation.h @@ -48,6 +48,15 @@ static_assert( PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE, "Update _remote_debugging validation for _Py_AsyncioModuleDebugOffsets"); +/* Derive unexported offsets from adjacent fields to keep the debug-offset + * table compatible across patch releases. */ +static_assert(offsetof(PyThreadState, exc_info) == + offsetof(PyThreadState, current_exception) + sizeof(uintptr_t), + "exc_info must immediately follow current_exception"); +static_assert(offsetof(_PyErr_StackItem, previous_item) == + offsetof(_PyErr_StackItem, exc_value) + sizeof(uintptr_t), + "previous_item must immediately follow exc_value"); + /* * This logic lives in a private header because it is shared by module.c and * asyncio.c. Keep the helpers static inline so they stay local to those users @@ -249,6 +258,7 @@ validate_fixed_field( #define PY_REMOTE_DEBUG_RUNTIME_STATE_FIELDS(APPLY, buffer_size) \ APPLY(runtime_state, interpreters_head, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size) +/* current_exception also covers the adjacent exc_info pointer. */ #define PY_REMOTE_DEBUG_THREAD_STATE_FIELDS(APPLY, buffer_size) \ APPLY(thread_state, native_thread_id, sizeof(unsigned long), _Alignof(long), buffer_size); \ APPLY(thread_state, interp, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ @@ -256,7 +266,7 @@ validate_fixed_field( APPLY(thread_state, status, FIELD_SIZE(PyThreadState, _status), _Alignof(unsigned int), buffer_size); \ APPLY(thread_state, holds_gil, sizeof(int), _Alignof(int), buffer_size); \ APPLY(thread_state, gil_requested, sizeof(int), _Alignof(int), buffer_size); \ - APPLY(thread_state, current_exception, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(thread_state, current_exception, 2 * sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ APPLY(thread_state, thread_id, sizeof(unsigned long), _Alignof(long), buffer_size); \ APPLY(thread_state, next, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ APPLY(thread_state, current_frame, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ @@ -351,10 +361,11 @@ _PyRemoteDebug_ValidateDebugOffsetsLayout(struct _Py_DebugOffsets *debug_offsets PY_REMOTE_DEBUG_THREAD_STATE_FIELDS( PY_REMOTE_DEBUG_VALIDATE_FIELD, SIZEOF_THREAD_STATE); + /* exc_value also covers the adjacent previous_item pointer. */ PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD( err_stackitem, exc_value, - sizeof(uintptr_t), + 2 * sizeof(uintptr_t), _Alignof(uintptr_t), sizeof(_PyErr_StackItem)); PY_REMOTE_DEBUG_VALIDATE_NESTED_FIELD( diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 666316ceaa28fc..198134fe6cfbea 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -17,6 +17,9 @@ #include #endif +/* Bound traversal of corrupted remote exception chains. */ +#define MAX_EXCEPTION_CHAIN_DEPTH (2 << 15) + /* ============================================================================ * THREAD ITERATION FUNCTIONS * ============================================================================ */ @@ -436,16 +439,49 @@ unwind_stack_for_thread( has_exception = 1; } - // Check exc_state.exc_value (exception being handled in except block) - // exc_state is embedded in PyThreadState, so we read it directly from - // the thread state buffer. This catches most cases; nested exception - // handlers where exc_info points elsewhere are rare. + // Generators and coroutines use their own exception stack items. + // Follow exc_info to find the innermost handler, as sys.exception() does. if (!has_exception) { - uintptr_t exc_value = GET_MEMBER(uintptr_t, ts, - unwinder->debug_offsets.thread_state.exc_state + - unwinder->debug_offsets.err_stackitem.exc_value); - if (exc_value != 0) { - has_exception = 1; + uintptr_t exc_info = GET_MEMBER(uintptr_t, ts, + unwinder->debug_offsets.thread_state.current_exception + + sizeof(uintptr_t)); + uintptr_t exc_state_addr = + *current_tstate + unwinder->debug_offsets.thread_state.exc_state; + uintptr_t exc_value_offset = + unwinder->debug_offsets.err_stackitem.exc_value; + uintptr_t previous_item_offset = + exc_value_offset + sizeof(uintptr_t); + + for (int depth = 0; exc_info != 0 && depth < MAX_EXCEPTION_CHAIN_DEPTH; + depth++) + { + if (exc_info == exc_state_addr) { + // Bottom of the chain: the stack item embedded in the thread + // state, which is already in the local thread state buffer. + uintptr_t exc_value = GET_MEMBER(uintptr_t, ts, + unwinder->debug_offsets.thread_state.exc_state + + exc_value_offset); + if (exc_value != 0) { + has_exception = 1; + } + break; + } + uintptr_t exc_value = 0; + if (read_ptr(unwinder, exc_info + exc_value_offset, &exc_value) < 0) { + PyErr_Clear(); // Best effort: treat as no active exception + break; + } + if (exc_value != 0) { + has_exception = 1; + break; + } + uintptr_t previous_item = 0; + if (read_ptr(unwinder, exc_info + previous_item_offset, + &previous_item) < 0) { + PyErr_Clear(); + break; + } + exc_info = previous_item; } } From 14641b0badff11e2f90a4e8839287d8c2872bad8 Mon Sep 17 00:00:00 2001 From: Neel Somani Date: Mon, 5 Oct 2026 08:39:11 -0700 Subject: [PATCH 088/240] gh-158486: Fix PowerSort mask calculation for very large lists (#158487) --- .../2026-09-30-01-30-58.gh-issue-158486.cqmMwv.rst | 2 ++ Objects/listobject.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-01-30-58.gh-issue-158486.cqmMwv.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-01-30-58.gh-issue-158486.cqmMwv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-01-30-58.gh-issue-158486.cqmMwv.rst new file mode 100644 index 00000000000000..6ef73833fcdb76 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-01-30-58.gh-issue-158486.cqmMwv.rst @@ -0,0 +1,2 @@ +Fix the PowerSort mask calculation in :meth:`list.sort` for very large +lists. diff --git a/Objects/listobject.c b/Objects/listobject.c index 81eb3e1102159b..962ed70aed6b8d 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2263,7 +2263,7 @@ merge_init(MergeState *ms, Py_ssize_t list_size, int has_keyfunc, while (list_size >> ms->mr_e >= MAX_MINRUN) { ++ms->mr_e; } - ms->mr_mask = (1 << ms->mr_e) - 1; + ms->mr_mask = ((Py_ssize_t)1 << ms->mr_e) - 1; ms->mr_current = 0; } From 05302c0ad6d909badf3c5c4c5745d398901a2465 Mon Sep 17 00:00:00 2001 From: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:46:39 +0300 Subject: [PATCH 089/240] gh-158522: Fix truncated stack for a task whose coroutine recurses (#158526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gh-158522: Fix truncated stack for a task whose coroutine recurses * fix lint * Apply suggestion from @maurycy Co-authored-by: Maurycy Pawłowski-Wieroński * Update Misc/NEWS.d/next/Library/2026-09-30-20-09-46.gh-issue-158522.L6j7tU.rst Co-authored-by: Maurycy Pawłowski-Wieroński * Fix line length in the new test --------- Co-authored-by: Maurycy Pawłowski-Wieroński --- Lib/test/test_external_inspection.py | 16 ++++++++++ ...-09-30-20-09-46.gh-issue-158522.L6j7tU.rst | 2 ++ Modules/_remote_debugging/_remote_debugging.h | 6 ++-- Modules/_remote_debugging/asyncio.c | 32 +++++++------------ 4 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-30-20-09-46.gh-issue-158522.L6j7tU.rst diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 0c189e60a93b8e..9fff8e8bff91f7 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -492,6 +492,22 @@ async def main(): self.assertIn(main_name, names) self.assertEqual([len(n) for n in names if n.startswith("x")], [255]) + @skip_if_not_supported + def test_recursive_coroutine_stack_is_not_truncated(self): + # gh-158522 + async def rec(n): + if n: + return await rec(n - 1) + return [ + frame.funcname.rpartition(".")[2] + for task in RemoteUnwinder( + os.getpid()).get_async_stack_trace()[0].awaited_by + for coro in task.coroutine_stack + for frame in coro.call_stack + ] + + self.assertEqual(asyncio.run(rec(3)), ["rec"] * 4) + @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, diff --git a/Misc/NEWS.d/next/Library/2026-09-30-20-09-46.gh-issue-158522.L6j7tU.rst b/Misc/NEWS.d/next/Library/2026-09-30-20-09-46.gh-issue-158522.L6j7tU.rst new file mode 100644 index 00000000000000..30c2df4bacd4a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-30-20-09-46.gh-issue-158522.L6j7tU.rst @@ -0,0 +1,2 @@ +Fix :mod:`profiling.sampling` showing a single frame for a task whose +coroutine recurses. Patch by Timofei Ivankov. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index c57b30533ce1c1..79fa4a92e745bd 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -729,7 +729,7 @@ extern int parse_async_frame_chain( RemoteUnwinderObject *unwinder, PyObject *calls, uintptr_t address_of_thread, - uintptr_t running_task_code_obj + uintptr_t running_task_frame ); extern int process_single_task_node( @@ -745,10 +745,10 @@ extern int find_running_task_in_thread( uintptr_t *running_task_addr ); -extern int get_task_code_object( +extern int get_task_frame( RemoteUnwinderObject *unwinder, uintptr_t task_addr, - uintptr_t *code_obj_addr + uintptr_t *frame_addr ); extern int append_awaited_by( diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index aa3f5b638931f8..c8d79774a47a73 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -716,7 +716,7 @@ find_running_task_in_thread( } int -get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *code_obj_addr) { +get_task_frame(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *frame_addr) { uintptr_t running_coro_addr = 0; if(read_py_ptr( @@ -733,21 +733,7 @@ get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintpt return -1; } - // note: genobject's gi_iframe is an embedded struct so the address to - // the offset leads directly to its first field: f_executable - if (read_py_ptr( - unwinder, - running_coro_addr + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); - return -1; - } - - if (*code_obj_addr == 0) { - PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task code object address is NULL"); - return -1; - } - + *frame_addr = running_coro_addr + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe; return 0; } @@ -760,7 +746,7 @@ parse_async_frame_chain( RemoteUnwinderObject *unwinder, PyObject *calls, uintptr_t address_of_thread, - uintptr_t running_task_code_obj + uintptr_t running_task_frame ) { uintptr_t address_of_current_frame; if (find_running_frame(unwinder, address_of_thread, &address_of_current_frame) < 0) { @@ -777,6 +763,8 @@ parse_async_frame_chain( } PyObject* frame_info = NULL; uintptr_t address_of_code_object; + + uintptr_t this_frame = address_of_current_frame; int res = parse_frame_object( unwinder, &frame_info, @@ -802,7 +790,9 @@ parse_async_frame_chain( Py_DECREF(frame_info); - if (address_of_code_object == running_task_code_obj) { + // Stop at the task's own frame. Code objects are shared by + // recursive calls, so they cannot identify it. + if (this_frame == running_task_frame) { break; } } @@ -938,8 +928,8 @@ process_running_task_chain( uintptr_t thread_state_addr, PyObject *result ) { - uintptr_t running_task_code_obj = 0; - if(get_task_code_object(unwinder, running_task_addr, &running_task_code_obj) < 0) { + uintptr_t running_task_frame = 0; + if(get_task_frame(unwinder, running_task_addr, &running_task_frame) < 0) { return -1; } @@ -971,7 +961,7 @@ process_running_task_chain( } // Add the chain from the current frame to this task - if (parse_async_frame_chain(unwinder, frame_chain, thread_state_addr, running_task_code_obj) < 0) { + if (parse_async_frame_chain(unwinder, frame_chain, thread_state_addr, running_task_frame) < 0) { return -1; } From 64bad2354042fa75ca2b4a8594de6e252142509e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 5 Oct 2026 17:56:52 +0100 Subject: [PATCH 090/240] gh-154062: Stop normalizing differential flamegraph duration (#154082) * gh-154062: Stop normalizing differential flamegraph duration * gh-154032: Scale elided flamegraph values * gh-154062: Keep elided metadata in raw sample units --- Lib/profiling/sampling/stack_collector.py | 22 ++-- .../test_sampling_profiler/mocks.py | 10 +- .../test_sampling_profiler/test_collectors.py | 124 ++++++++++++++---- ...-07-19-11-00-00.gh-issue-154062.N5ks3A.rst | 2 + 4 files changed, 119 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-19-11-00-00.gh-issue-154062.N5ks3A.rst diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index e420bb6d2e9b87..72a8cae8807890 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -676,16 +676,16 @@ def _convert_to_flamegraph_format(self): current_stats = self._aggregate_path_samples(self._root) baseline_stats = self._aggregate_path_samples(self._baseline_collector._root) - # Scale baseline values to make them comparable, accounting for both - # sample count differences and sample interval differences. + # Express baseline samples in units of the current sample interval. + # Do not normalize by total profile duration: doing so makes unchanged + # functions appear different when another function becomes faster or + # slower. baseline_total = self._baseline_collector._total_samples - if baseline_total > 0 and self._total_samples > 0: - current_time = self._total_samples * self.sample_interval_usec - baseline_time = baseline_total * self._baseline_collector.sample_interval_usec - scale = current_time / baseline_time - elif baseline_total > 0: - # Current profile is empty - use interval-based scale for elided display - scale = self.sample_interval_usec / self._baseline_collector.sample_interval_usec + if baseline_total > 0: + scale = ( + self._baseline_collector.sample_interval_usec + / self.sample_interval_usec + ) else: scale = 1.0 @@ -902,6 +902,10 @@ def _add_elided_metadata(self, node, baseline_stats, scale, path): else: node["diff_pct"] = 0.0 + # Scale geometry after computing metadata from raw baseline counts. + node["value"] = node.get("value", 0) * scale + node["self"] = node.get("self", 0) * scale + if "children" in node and node["children"]: for child in node["children"]: self._add_elided_metadata(child, baseline_stats, scale, current_path) diff --git a/Lib/test/test_profiling/test_sampling_profiler/mocks.py b/Lib/test/test_profiling/test_sampling_profiler/mocks.py index 6ac2d08e898d81..128870ffd4d5e4 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/mocks.py +++ b/Lib/test/test_profiling/test_sampling_profiler/mocks.py @@ -93,7 +93,9 @@ def __repr__(self): return f"MockAwaitedInfo(thread_id={self.thread_id}, awaited_by={len(self.awaited_by)} tasks)" -def make_diff_collector_with_mock_baseline(baseline_samples): +def make_diff_collector_with_mock_baseline( + baseline_samples, *, baseline_interval=1000, current_interval=1000 +): """Create a DiffFlamegraphCollector with baseline injected directly, skipping the binary round-trip that _load_baseline normally does.""" from profiling.sampling.stack_collector import ( @@ -101,12 +103,14 @@ def make_diff_collector_with_mock_baseline(baseline_samples): FlamegraphCollector, ) - baseline = FlamegraphCollector(1000) + baseline = FlamegraphCollector(baseline_interval) for sample in baseline_samples: baseline.collect(sample) # Path is unused since we inject _baseline_collector directly; # use __file__ as a dummy path that passes the existence check. - diff = DiffFlamegraphCollector(1000, baseline_binary_path=__file__) + diff = DiffFlamegraphCollector( + current_interval, baseline_binary_path=__file__ + ) diff._baseline_collector = baseline return diff diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index eb58c29dd361d3..36713ff1314ff9 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -1663,7 +1663,8 @@ def test_diff_flamegraph_changed_functions(self): ]) ] - # Baseline: 2 samples, current: 4, scale = 2.0 + # Baseline: 2 samples, current: 4. Profiles are compared in absolute + # time rather than normalized to the same total duration. diff = make_diff_collector_with_mock_baseline( [hot_leaf_sample, cold_leaf_sample] ) @@ -1673,7 +1674,7 @@ def test_diff_flamegraph_changed_functions(self): data = diff._convert_to_flamegraph_format() strings = data.get("strings", []) - self.assertAlmostEqual(data["stats"]["baseline_scale"], 2.0) + self.assertAlmostEqual(data["stats"]["baseline_scale"], 1.0) children = data.get("children", []) hot_node = find_child_by_name(children, strings, "hot_leaf") @@ -1681,17 +1682,17 @@ def test_diff_flamegraph_changed_functions(self): self.assertIsNotNone(hot_node) self.assertIsNotNone(cold_node) - # hot_leaf regressed (+50%) - self.assertAlmostEqual(hot_node["baseline"], 2.0) + # hot_leaf regressed (+200%) + self.assertAlmostEqual(hot_node["baseline"], 1.0) self.assertEqual(hot_node["self_time"], 3) - self.assertAlmostEqual(hot_node["diff"], 1.0) - self.assertAlmostEqual(hot_node["diff_pct"], 50.0) + self.assertAlmostEqual(hot_node["diff"], 2.0) + self.assertAlmostEqual(hot_node["diff_pct"], 200.0) - # cold_leaf improved (-50%) - self.assertAlmostEqual(cold_node["baseline"], 2.0) + # cold_leaf is unchanged + self.assertAlmostEqual(cold_node["baseline"], 1.0) self.assertEqual(cold_node["self_time"], 1) - self.assertAlmostEqual(cold_node["diff"], -1.0) - self.assertAlmostEqual(cold_node["diff_pct"], -50.0) + self.assertAlmostEqual(cold_node["diff"], 0.0) + self.assertAlmostEqual(cold_node["diff_pct"], 0.0) def test_diff_flamegraph_rejects_mismatched_profiling_modes(self): from profiling.sampling.binary_collector import BinaryCollector @@ -1737,8 +1738,8 @@ def test_diff_flamegraph_rejects_mismatched_capture_config(self): with self.assertRaisesRegex(ValueError, "all_threads"): diff._convert_to_flamegraph_format() - def test_diff_flamegraph_scale_factor(self): - """Scale factor adjusts when sample counts differ.""" + def test_diff_flamegraph_does_not_normalize_duration(self): + """A longer current run is compared in absolute time.""" baseline_frames = [ MockInterpreterInfo(0, [ MockThreadInfo(1, [ @@ -1753,15 +1754,74 @@ def test_diff_flamegraph_scale_factor(self): diff.collect(baseline_frames) data = diff._convert_to_flamegraph_format() - self.assertAlmostEqual(data["stats"]["baseline_scale"], 4.0) + self.assertAlmostEqual(data["stats"]["baseline_scale"], 1.0) children = data.get("children", []) self.assertEqual(len(children), 1) func1_node = children[0] self.assertEqual(func1_node["self_time"], 4) - self.assertAlmostEqual(func1_node["baseline"], 4.0) - self.assertAlmostEqual(func1_node["diff"], 0.0) - self.assertAlmostEqual(func1_node["diff_pct"], 0.0) + self.assertAlmostEqual(func1_node["baseline"], 1.0) + self.assertAlmostEqual(func1_node["diff"], 3.0) + self.assertAlmostEqual(func1_node["diff_pct"], 300.0) + + def test_diff_flamegraph_scale_factor_uses_sample_intervals(self): + """Baseline samples are converted to current sample units.""" + frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [MockFrameInfo("file.py", 10, "func1")]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline( + [frames] * 10, + baseline_interval=1000, + current_interval=10000, + ) + diff.collect(frames) + + data = diff._convert_to_flamegraph_format() + self.assertAlmostEqual(data["stats"]["baseline_scale"], 0.1) + self.assertAlmostEqual(data["baseline"], 1.0) + self.assertEqual(data["self_time"], 1) + self.assertAlmostEqual(data["diff"], 0.0) + self.assertAlmostEqual(data["diff_pct"], 0.0) + + def test_diff_flamegraph_elided_values_use_current_interval(self): + """Elided geometry and metadata use the same sample units.""" + baseline_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "old_func"), + MockFrameInfo("file.py", 30, "parent"), + ]) + ]) + ] + current_frames = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 20, "new_func"), + MockFrameInfo("file.py", 30, "parent"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline( + [baseline_frames] * 10, + baseline_interval=1000, + current_interval=10000, + ) + diff.collect(current_frames) + + data = diff._convert_to_flamegraph_format() + elided = data["stats"]["elided_flamegraph"] + self.assertAlmostEqual(elided["value"], 1.0) + self.assertEqual(elided["self"], 0) + self.assertAlmostEqual(elided["baseline_total"], 1.0) + child, = elided["children"] + self.assertAlmostEqual(child["value"], 1.0) + self.assertAlmostEqual(child["self"], 1.0) + self.assertAlmostEqual(child["baseline"], 1.0) + self.assertAlmostEqual(child["diff"], -1.0) def test_diff_flamegraph_elided_stacks(self): """Paths in baseline but not current produce elided stacks.""" @@ -2064,7 +2124,11 @@ def test_diff_flamegraph_empty_current(self): ]) ] - diff = make_diff_collector_with_mock_baseline([baseline_frames]) + diff = make_diff_collector_with_mock_baseline( + [baseline_frames] * 10, + baseline_interval=1000, + current_interval=10000, + ) # Don't collect anything in current data = diff._convert_to_flamegraph_format() @@ -2074,6 +2138,11 @@ def test_diff_flamegraph_empty_current(self): self.assertTrue(data["stats"]["is_differential"]) # All baseline paths should be elided since current is empty self.assertGreater(data["stats"]["elided_count"], 0) + self.assertAlmostEqual(data["stats"]["baseline_scale"], 0.1) + elided = data["stats"]["elided_flamegraph"] + self.assertAlmostEqual(elided["value"], 1.0) + self.assertAlmostEqual(elided["baseline"], 1.0) + self.assertAlmostEqual(elided["diff"], -1.0) def test_diff_flamegraph_empty_baseline(self): """Empty baseline with non-empty current uses scale=1.0 fallback.""" @@ -2240,7 +2309,8 @@ def test_diff_flamegraph_load_baseline(self): make_frame("file.py", 20, "caller"), ])])] - # Baseline: 2 samples, current: 4, scale = 2.0 + # Baseline: 2 samples, current: 4. Profiles are compared in absolute + # time rather than normalized to the same total duration. bin_file = tempfile.NamedTemporaryFile(suffix=".bin", delete=False) self.addCleanup(close_and_unlink, bin_file) @@ -2270,7 +2340,7 @@ def test_diff_flamegraph_load_baseline(self): strings = data.get("strings", []) self.assertTrue(data["stats"]["is_differential"]) - self.assertAlmostEqual(data["stats"]["baseline_scale"], 2.0) + self.assertAlmostEqual(data["stats"]["baseline_scale"], 1.0) children = data.get("children", []) hot_node = find_child_by_name(children, strings, "hot_leaf") @@ -2278,17 +2348,17 @@ def test_diff_flamegraph_load_baseline(self): self.assertIsNotNone(hot_node) self.assertIsNotNone(cold_node) - # hot_leaf regressed (+50%) - self.assertAlmostEqual(hot_node["baseline"], 2.0) + # hot_leaf regressed (+200%) + self.assertAlmostEqual(hot_node["baseline"], 1.0) self.assertEqual(hot_node["self_time"], 3) - self.assertAlmostEqual(hot_node["diff"], 1.0) - self.assertAlmostEqual(hot_node["diff_pct"], 50.0) + self.assertAlmostEqual(hot_node["diff"], 2.0) + self.assertAlmostEqual(hot_node["diff_pct"], 200.0) - # cold_leaf improved (-50%) - self.assertAlmostEqual(cold_node["baseline"], 2.0) + # cold_leaf is unchanged + self.assertAlmostEqual(cold_node["baseline"], 1.0) self.assertEqual(cold_node["self_time"], 1) - self.assertAlmostEqual(cold_node["diff"], -1.0) - self.assertAlmostEqual(cold_node["diff_pct"], -50.0) + self.assertAlmostEqual(cold_node["diff"], 0.0) + self.assertAlmostEqual(cold_node["diff_pct"], 0.0) def test_jsonl_collector_export_exact_output(self): jsonl_out = tempfile.NamedTemporaryFile(delete=False) diff --git a/Misc/NEWS.d/next/Library/2026-07-19-11-00-00.gh-issue-154062.N5ks3A.rst b/Misc/NEWS.d/next/Library/2026-07-19-11-00-00.gh-issue-154062.N5ks3A.rst new file mode 100644 index 00000000000000..7c4469aba9bc7e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-11-00-00.gh-issue-154062.N5ks3A.rst @@ -0,0 +1,2 @@ +Stop normalizing differential flamegraph baselines to the duration of the +current profile. From cfd6cf71547ef1b68285f337138937154d6a01c3 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 5 Oct 2026 10:03:27 -0700 Subject: [PATCH 091/240] gh-157407: Use _PyUnicode_AsUTF8NoNUL for AC 's' and 'z' converters (#158210) * gh-157407: Use _PyUnicode_AsUTF8NoNUL for AC 's' and 'z' converters The code was doing the same as the helper added in GH-111957 but inline. That resulted in significantly more `.text` for most functions. Refactor the Arguemnt Clinic converters code to deduplicate a bit while keeping generated code close to original. On my Linux x64 box this reduces `.text` by ~6200 bytes in a clang release build (no PGO or LTO). Performance is flat. The helper was already exported for `_sqlite3`, update the comment that it is used in AC generated code as well. Co-Authored-By: Claude Opus 5 (1M context) * Update Include/internal/pycore_unicodeobject.h Co-authored-by: Pieter Eendebak * Remove NEWS, add period --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Pieter Eendebak --- Include/internal/pycore_unicodeobject.h | 2 +- Modules/_ctypes/clinic/_ctypes.c.h | 10 +- Modules/_ctypes/clinic/callproc.c.h | 10 +- Modules/_io/clinic/_iomodule.c.h | 31 +- Modules/_io/clinic/fileio.c.h | 10 +- Modules/_io/clinic/textio.c.h | 24 +- Modules/_io/clinic/winconsoleio.c.h | 10 +- .../clinic/multiprocessing.c.h | 10 +- Modules/_multiprocessing/clinic/semaphore.c.h | 10 +- Modules/_sqlite/clinic/_sqlite3.connect.c.h | 3 +- Modules/_sqlite/clinic/connection.c.h | 87 +---- Modules/_sqlite/clinic/cursor.c.h | 10 +- Modules/_sqlite/clinic/module.c.h | 10 +- Modules/_testcapi/clinic/exceptions.c.h | 17 +- Modules/cjkcodecs/clinic/multibytecodec.c.h | 17 +- Modules/clinic/_codecsmodule.c.h | 346 +++--------------- Modules/clinic/_cursesmodule.c.h | 52 +-- Modules/clinic/_datetimemodule.c.h | 17 +- Modules/clinic/_dbmmodule.c.h | 10 +- Modules/clinic/_elementtree.c.h | 10 +- Modules/clinic/_gdbmmodule.c.h | 10 +- Modules/clinic/_hashopenssl.c.h | 17 +- Modules/clinic/_interpretersmodule.c.h | 10 +- Modules/clinic/_localemodule.c.h | 73 +--- Modules/clinic/_pickle.c.h | 45 +-- Modules/clinic/_ssl.c.h | 73 +--- Modules/clinic/_testclinic.c.h | 17 +- Modules/clinic/_testclinic_depr.c.h | 3 +- Modules/clinic/_testclinic_kwds.c.h | 3 +- Modules/clinic/_tkinter.c.h | 101 +---- Modules/clinic/arraymodule.c.h | 10 +- Modules/clinic/mmapmodule.c.h | 10 +- Modules/clinic/posixmodule.c.h | 24 +- Modules/clinic/pyexpat.c.h | 38 +- Modules/clinic/socketmodule.c.h | 45 +-- Modules/clinic/symtablemodule.c.h | 10 +- Modules/clinic/syslogmodule.c.h | 10 +- Modules/clinic/timemodule.c.h | 10 +- Objects/clinic/bytearrayobject.c.h | 31 +- Objects/clinic/bytesobject.c.h | 31 +- Objects/clinic/floatobject.c.h | 10 +- Objects/clinic/memoryobject.c.h | 10 +- Objects/clinic/unicodeobject.c.h | 31 +- Python/clinic/Python-tokenize.c.h | 10 +- Python/clinic/bltinmodule.c.h | 10 +- Python/clinic/sysmodule.c.h | 17 +- Tools/clinic/libclinic/converters.py | 69 ++-- 47 files changed, 309 insertions(+), 1115 deletions(-) diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 112337d90968b3..b8e24da693be08 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -468,7 +468,7 @@ extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **); extern void _PyUnicode_ClearInterned(PyInterpreterState *interp); // Like PyUnicode_AsUTF8(), but check for embedded null characters. -// Export for '_sqlite3' shared extension. +// Export for '_sqlite3' shared extension, and for Argument Clinic code. PyAPI_FUNC(const char *) _PyUnicode_AsUTF8NoNUL(PyObject *); diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h index 04e8823b260f88..0bc06695b900e1 100644 --- a/Modules/_ctypes/clinic/_ctypes.c.h +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -5,6 +5,7 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_ctypes_CType_Type___sizeof____doc__, "__sizeof__($self, /)\n" @@ -240,15 +241,10 @@ CDataType_in_dll(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ss _PyArg_BadArgument("in_dll", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[1]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = CDataType_in_dll_impl(type, cls, dll, name); exit: @@ -822,4 +818,4 @@ Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py #define _CTYPES_SIMPLE_VALUE_GETSETDEF {"value", (getter)_ctypes_Simple_value_get, (setter)_ctypes_Simple_value_set, NULL}, -/*[clinic end generated code: output=1d81035cd8b2015d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e7e9bd957c3d7362 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/clinic/callproc.c.h b/Modules/_ctypes/clinic/callproc.c.h index ef68372110416a..82cb08ff00dfb0 100644 --- a/Modules/_ctypes/clinic/callproc.c.h +++ b/Modules/_ctypes/clinic/callproc.c.h @@ -5,6 +5,7 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_ctypes_get_errno__doc__, "get_errno($module, /)\n" @@ -406,15 +407,10 @@ _ctypes_dlsym(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("dlsym", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[1]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _ctypes_dlsym_impl(module, handle, name); exit: @@ -797,4 +793,4 @@ PyDoc_STRVAR(_ctypes_buffer_info__doc__, #ifndef _CTYPES_DLLIST_METHODDEF #define _CTYPES_DLLIST_METHODDEF #endif /* !defined(_CTYPES_DLLIST_METHODDEF) */ -/*[clinic end generated code: output=71a41a6d90e69821 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=615c411ddc01d0a4 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index f03638064385e2..0547982cad48cc 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_io_open__doc__, "open($module, /, file, mode=\'r\', buffering=-1, encoding=None,\n" @@ -192,15 +193,10 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw _PyArg_BadArgument("open", "argument 'mode'", "str", args[1]); goto exit; } - Py_ssize_t mode_length; - mode = PyUnicode_AsUTF8AndSize(args[1], &mode_length); + mode = _PyUnicode_AsUTF8NoNUL(args[1]); if (mode == NULL) { goto exit; } - if (strlen(mode) != (size_t)mode_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -219,15 +215,10 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw encoding = NULL; } else if (PyUnicode_Check(args[3])) { - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[3], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[3]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("open", "argument 'encoding'", "str or None", args[3]); @@ -242,15 +233,10 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw errors = NULL; } else if (PyUnicode_Check(args[4])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[4], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[4]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("open", "argument 'errors'", "str or None", args[4]); @@ -265,15 +251,10 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw newline = NULL; } else if (PyUnicode_Check(args[5])) { - Py_ssize_t newline_length; - newline = PyUnicode_AsUTF8AndSize(args[5], &newline_length); + newline = _PyUnicode_AsUTF8NoNUL(args[5]); if (newline == NULL) { goto exit; } - if (strlen(newline) != (size_t)newline_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("open", "argument 'newline'", "str or None", args[5]); @@ -411,4 +392,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=5190d11f0803bfe8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4570f4466d711d60 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 074a6bb92c7e4a..9db31938f3334f 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_io_FileIO__dealloc_warn__doc__, "_dealloc_warn($self, source, /)\n" @@ -134,15 +135,10 @@ _io_FileIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("FileIO", "argument 'mode'", "str", fastargs[1]); goto exit; } - Py_ssize_t mode_length; - mode = PyUnicode_AsUTF8AndSize(fastargs[1], &mode_length); + mode = _PyUnicode_AsUTF8NoNUL(fastargs[1]); if (mode == NULL) { goto exit; } - if (strlen(mode) != (size_t)mode_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -625,4 +621,4 @@ _io_FileIO__blksize_get(PyObject *self, void *Py_UNUSED(context)) #define _IO_FILEIO__BLKSIZE_GETSETDEF {"_blksize", (getter)_io_FileIO__blksize_get, (setter)NULL, _io_FileIO__blksize__doc__}, -/*[clinic end generated code: output=6ad45da113899288 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b8d6d338bfd0a58f input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 65be4537601da2..6d63d4706f609c 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_io__TextIOBase_detach__doc__, "detach($self, /)\n" @@ -151,15 +152,10 @@ _io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, _PyArg_BadArgument("write", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t s_length; - s = PyUnicode_AsUTF8AndSize(args[0], &s_length); + s = _PyUnicode_AsUTF8NoNUL(args[0]); if (s == NULL) { goto exit; } - if (strlen(s) != (size_t)s_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _io__TextIOBase_write_impl(self, cls, s); exit: @@ -533,15 +529,10 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) encoding = NULL; } else if (PyUnicode_Check(fastargs[1])) { - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(fastargs[1], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(fastargs[1]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("TextIOWrapper", "argument 'encoding'", "str or None", fastargs[1]); @@ -562,15 +553,10 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) newline = NULL; } else if (PyUnicode_Check(fastargs[3])) { - Py_ssize_t newline_length; - newline = PyUnicode_AsUTF8AndSize(fastargs[3], &newline_length); + newline = _PyUnicode_AsUTF8NoNUL(fastargs[3]); if (newline == NULL) { goto exit; } - if (strlen(newline) != (size_t)newline_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("TextIOWrapper", "argument 'newline'", "str or None", fastargs[3]); @@ -1260,4 +1246,4 @@ _io_TextIOWrapper_buffer_get(PyObject *self, void *Py_UNUSED(context)) #define _IO_TEXTIOWRAPPER_BUFFER_GETSETDEF {"buffer", (getter)_io_TextIOWrapper_buffer_get, (setter)NULL, NULL}, -/*[clinic end generated code: output=8b82c8fba47166e9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=89de9ab341df9f66 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 4ad3f1c9c72cff..76baeb79e68cd0 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() #if defined(HAVE_WINDOWS_CONSOLE_IO) @@ -109,15 +110,10 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("_WindowsConsoleIO", "argument 'mode'", "str", fastargs[1]); goto exit; } - Py_ssize_t mode_length; - mode = PyUnicode_AsUTF8AndSize(fastargs[1], &mode_length); + mode = _PyUnicode_AsUTF8NoNUL(fastargs[1]); if (mode == NULL) { goto exit; } - if (strlen(mode) != (size_t)mode_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -555,4 +551,4 @@ _io__WindowsConsoleIO_mode_get(PyObject *self, void *Py_UNUSED(context)) # define _IO__WINDOWSCONSOLEIO_MODE_GETSETDEF #endif -/*[clinic end generated code: output=97863abe2990b665 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7c7499116eefd498 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/multiprocessing.c.h b/Modules/_multiprocessing/clinic/multiprocessing.c.h index 6d4f5c2afcfd39..ea6fffc428919b 100644 --- a/Modules/_multiprocessing/clinic/multiprocessing.c.h +++ b/Modules/_multiprocessing/clinic/multiprocessing.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() #if defined(MS_WINDOWS) @@ -138,15 +139,10 @@ _multiprocessing_sem_unlink(PyObject *module, PyObject *arg) _PyArg_BadArgument("sem_unlink", "argument", "str", arg); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(arg, &name_length); + name = _PyUnicode_AsUTF8NoNUL(arg); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _multiprocessing_sem_unlink_impl(module, name); exit: @@ -164,4 +160,4 @@ _multiprocessing_sem_unlink(PyObject *module, PyObject *arg) #ifndef _MULTIPROCESSING_SEND_METHODDEF #define _MULTIPROCESSING_SEND_METHODDEF #endif /* !defined(_MULTIPROCESSING_SEND_METHODDEF) */ -/*[clinic end generated code: output=73b4cb8428d816da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f5147f6a6e2cbd17 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 6b1c0092ce4816..34dba39dab5414 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() #if defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) @@ -292,15 +293,10 @@ _multiprocessing_SemLock(PyTypeObject *type, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("SemLock", "argument 'name'", "str", fastargs[3]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(fastargs[3], &name_length); + name = _PyUnicode_AsUTF8NoNUL(fastargs[3]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } unlink = PyObject_IsTrue(fastargs[4]); if (unlink < 0) { goto exit; @@ -582,4 +578,4 @@ _multiprocessing_SemLock___exit__(PyObject *self, PyObject *const *args, Py_ssiz #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=d1e349d4ee3d4bbf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c3a00b85537e647 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h index e9d560666c1491..ce8c795ed2d5a0 100644 --- a/Modules/_sqlite/clinic/_sqlite3.connect.c.h +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(pysqlite_connect__doc__, "connect($module, /, database, *, timeout=5.0, detect_types=0,\n" @@ -22,4 +23,4 @@ PyDoc_STRVAR(pysqlite_connect__doc__, #define PYSQLITE_CONNECT_METHODDEF \ {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, -/*[clinic end generated code: output=3d83139ba65e0bb5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=07d5692e22b0fdf2 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index e80c48be0d6fe6..69ef85e4ee5408 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() static int pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, @@ -280,28 +281,18 @@ blobopen(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwna _PyArg_BadArgument("blobopen", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t table_length; - table = PyUnicode_AsUTF8AndSize(args[0], &table_length); + table = _PyUnicode_AsUTF8NoNUL(args[0]); if (table == NULL) { goto exit; } - if (strlen(table) != (size_t)table_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!PyUnicode_Check(args[1])) { _PyArg_BadArgument("blobopen", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t col_length; - col = PyUnicode_AsUTF8AndSize(args[1], &col_length); + col = _PyUnicode_AsUTF8NoNUL(args[1]); if (col == NULL) { goto exit; } - if (strlen(col) != (size_t)col_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!sqlite3_int64_converter(args[2], &row)) { goto exit; } @@ -321,15 +312,10 @@ blobopen(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwna _PyArg_BadArgument("blobopen", "argument 'name'", "str", args[4]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[4], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[4]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_kwonly: return_value = blobopen_impl((pysqlite_Connection *)self, table, col, row, readonly, name); @@ -459,15 +445,10 @@ pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject _PyArg_BadArgument("create_function", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } narg = PyLong_AsInt(args[1]); if (narg == -1 && PyErr_Occurred()) { goto exit; @@ -530,15 +511,10 @@ create_window_function(PyObject *self, PyTypeObject *cls, PyObject *const *args, _PyArg_BadArgument("create_window_function", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } num_params = PyLong_AsInt(args[1]); if (num_params == -1 && PyErr_Occurred()) { goto exit; @@ -585,15 +561,10 @@ pysqlite_connection_create_aggregate(PyObject *self, PyTypeObject *cls, PyObject _PyArg_BadArgument("create_aggregate", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } n_arg = PyLong_AsInt(args[1]); if (n_arg == -1 && PyErr_Occurred()) { goto exit; @@ -841,15 +812,10 @@ pysqlite_connection_load_extension(PyObject *self, PyObject *const *args, Py_ssi _PyArg_BadArgument("load_extension", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t extension_name_length; - extension_name = PyUnicode_AsUTF8AndSize(args[0], &extension_name_length); + extension_name = _PyUnicode_AsUTF8NoNUL(args[0]); if (extension_name == NULL) { goto exit; } - if (strlen(extension_name) != (size_t)extension_name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!noptargs) { goto skip_optional_kwonly; } @@ -857,15 +823,10 @@ pysqlite_connection_load_extension(PyObject *self, PyObject *const *args, Py_ssi entrypoint = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t entrypoint_length; - entrypoint = PyUnicode_AsUTF8AndSize(args[1], &entrypoint_length); + entrypoint = _PyUnicode_AsUTF8NoNUL(args[1]); if (entrypoint == NULL) { goto exit; } - if (strlen(entrypoint) != (size_t)entrypoint_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("load_extension", "argument 'entrypoint'", "str or None", args[1]); @@ -1150,15 +1111,10 @@ pysqlite_connection_backup(PyObject *self, PyObject *const *args, Py_ssize_t nar _PyArg_BadArgument("backup", "argument 'name'", "str", args[3]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[3], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[3]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -1212,15 +1168,10 @@ pysqlite_connection_create_collation(PyObject *self, PyTypeObject *cls, PyObject _PyArg_BadArgument("create_collation", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } callable = args[1]; return_value = pysqlite_connection_create_collation_impl((pysqlite_Connection *)self, cls, name, callable); @@ -1297,15 +1248,10 @@ serialize(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn _PyArg_BadArgument("serialize", "argument 'name'", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_kwonly: return_value = serialize_impl((pysqlite_Connection *)self, name); @@ -1406,15 +1352,10 @@ deserialize(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *k _PyArg_BadArgument("deserialize", "argument 'name'", "str", args[1]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[1]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_kwonly: return_value = deserialize_impl((pysqlite_Connection *)self, &data, name); @@ -1660,4 +1601,4 @@ getconfig(PyObject *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=c3e7157ea43130f8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=faee86c7feaa0937 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index fcefc6535393b2..45780bd6353007 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_long.h" // _PyLong_UInt32_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() static int pysqlite_cursor_init_impl(pysqlite_Cursor *self, @@ -136,15 +137,10 @@ pysqlite_cursor_executescript(PyObject *self, PyObject *arg) _PyArg_BadArgument("executescript", "argument", "str", arg); goto exit; } - Py_ssize_t sql_script_length; - sql_script = PyUnicode_AsUTF8AndSize(arg, &sql_script_length); + sql_script = _PyUnicode_AsUTF8NoNUL(arg); if (sql_script == NULL) { goto exit; } - if (strlen(sql_script) != (size_t)sql_script_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = pysqlite_cursor_executescript_impl((pysqlite_Cursor *)self, sql_script); exit: @@ -364,4 +360,4 @@ _sqlite3_Cursor_arraysize_set(PyObject *self, PyObject *arg, void *Py_UNUSED(con } #define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", (getter)_sqlite3_Cursor_arraysize_get, (setter)_sqlite3_Cursor_arraysize_set, NULL}, -/*[clinic end generated code: output=e920343d84bd4976 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c24b4d93900ed908 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/module.c.h b/Modules/_sqlite/clinic/module.c.h index 82ecd3eea1b39b..79d1043cd729df 100644 --- a/Modules/_sqlite/clinic/module.c.h +++ b/Modules/_sqlite/clinic/module.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(pysqlite_complete_statement__doc__, "complete_statement($module, /, statement)\n" @@ -63,15 +64,10 @@ pysqlite_complete_statement(PyObject *module, PyObject *const *args, Py_ssize_t _PyArg_BadArgument("complete_statement", "argument 'statement'", "str", args[0]); goto exit; } - Py_ssize_t statement_length; - statement = PyUnicode_AsUTF8AndSize(args[0], &statement_length); + statement = _PyUnicode_AsUTF8NoNUL(args[0]); if (statement == NULL) { goto exit; } - if (strlen(statement) != (size_t)statement_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = pysqlite_complete_statement_impl(module, statement); exit: @@ -211,4 +207,4 @@ pysqlite_adapt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=17c4e031680a5168 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=02aee45b7f9f14f0 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 89b78923bdb2d0..359f90a2b63bf8 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_testcapi_err_set_raised__doc__, "err_set_raised($module, exception, /)\n" @@ -146,15 +147,10 @@ _testcapi_make_exception_with_doc(PyObject *module, PyObject *const *args, Py_ss _PyArg_BadArgument("make_exception_with_doc", "argument 'name'", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -163,15 +159,10 @@ _testcapi_make_exception_with_doc(PyObject *module, PyObject *const *args, Py_ss _PyArg_BadArgument("make_exception_with_doc", "argument 'doc'", "str", args[1]); goto exit; } - Py_ssize_t doc_length; - doc = PyUnicode_AsUTF8AndSize(args[1], &doc_length); + doc = _PyUnicode_AsUTF8NoNUL(args[1]); if (doc == NULL) { goto exit; } - if (strlen(doc) != (size_t)doc_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -490,4 +481,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=a21ce5554900dba1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=89fd3856718cf39c input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index 3bdabf901126ea..1229a0fb48fe95 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "encode($self, /, input, errors=None)\n" @@ -77,15 +78,10 @@ _multibytecodec_MultibyteCodec_encode(PyObject *self, PyObject *const *args, Py_ errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("encode", "argument 'errors'", "str or None", args[1]); @@ -169,15 +165,10 @@ _multibytecodec_MultibyteCodec_decode(PyObject *self, PyObject *const *args, Py_ errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("decode", "argument 'errors'", "str or None", args[1]); @@ -672,4 +663,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=b7eb6679d9f627b7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=20fec21d79e87a15 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 5ca9614f7f3216..0b4ec169a2da14 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_codecs_register__doc__, "register($module, search_function, /)\n" @@ -55,15 +56,10 @@ _codecs_lookup(PyObject *module, PyObject *arg) _PyArg_BadArgument("lookup", "argument", "str", arg); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(arg, &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(arg); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _codecs_lookup_impl(module, encoding); exit: @@ -140,15 +136,10 @@ _codecs_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("encode", "argument 'encoding'", "str", args[1]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[1], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[1]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -157,15 +148,10 @@ _codecs_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("encode", "argument 'errors'", "str", args[2]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = _codecs_encode_impl(module, obj, encoding, errors); @@ -243,15 +229,10 @@ _codecs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("decode", "argument 'encoding'", "str", args[1]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[1], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[1]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -260,15 +241,10 @@ _codecs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("decode", "argument 'errors'", "str", args[2]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = _codecs_decode_impl(module, obj, encoding, errors); @@ -320,15 +296,10 @@ _codecs_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("escape_decode", "argument 2", "str or None", args[1]); @@ -380,15 +351,10 @@ _codecs_escape_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("escape_encode", "argument 2", "str or None", args[1]); @@ -434,15 +400,10 @@ _codecs_utf_7_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_7_decode", "argument 2", "str or None", args[1]); @@ -500,15 +461,10 @@ _codecs_utf_8_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_8_decode", "argument 2", "str or None", args[1]); @@ -566,15 +522,10 @@ _codecs_utf_16_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_16_decode", "argument 2", "str or None", args[1]); @@ -632,15 +583,10 @@ _codecs_utf_16_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_16_le_decode", "argument 2", "str or None", args[1]); @@ -698,15 +644,10 @@ _codecs_utf_16_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_16_be_decode", "argument 2", "str or None", args[1]); @@ -766,15 +707,10 @@ _codecs_utf_16_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_16_ex_decode", "argument 2", "str or None", args[1]); @@ -839,15 +775,10 @@ _codecs_utf_32_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_32_decode", "argument 2", "str or None", args[1]); @@ -905,15 +836,10 @@ _codecs_utf_32_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_32_le_decode", "argument 2", "str or None", args[1]); @@ -971,15 +897,10 @@ _codecs_utf_32_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_32_be_decode", "argument 2", "str or None", args[1]); @@ -1039,15 +960,10 @@ _codecs_utf_32_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_32_ex_decode", "argument 2", "str or None", args[1]); @@ -1124,15 +1040,10 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("unicode_escape_decode", "argument 2", "str or None", args[1]); @@ -1202,15 +1113,10 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("raw_unicode_escape_decode", "argument 2", "str or None", args[1]); @@ -1267,15 +1173,10 @@ _codecs_latin_1_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("latin_1_decode", "argument 2", "str or None", args[1]); @@ -1325,15 +1226,10 @@ _codecs_ascii_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("ascii_decode", "argument 2", "str or None", args[1]); @@ -1384,15 +1280,10 @@ _codecs_charmap_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("charmap_decode", "argument 2", "str or None", args[1]); @@ -1449,15 +1340,10 @@ _codecs_mbcs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("mbcs_decode", "argument 2", "str or None", args[1]); @@ -1519,15 +1405,10 @@ _codecs_oem_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("oem_decode", "argument 2", "str or None", args[1]); @@ -1594,15 +1475,10 @@ _codecs_code_page_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[2])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("code_page_decode", "argument 3", "str or None", args[2]); @@ -1659,15 +1535,10 @@ _codecs_iconv_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("iconv_decode", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[0], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[0]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (PyObject_GetBuffer(args[1], &data, PyBUF_SIMPLE) != 0) { goto exit; } @@ -1678,15 +1549,10 @@ _codecs_iconv_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[2])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("iconv_decode", "argument 3", "str or None", args[2]); @@ -1757,15 +1623,10 @@ _codecs_readbuffer_encode(PyObject *module, PyObject *const *args, Py_ssize_t na errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("readbuffer_encode", "argument 2", "str or None", args[1]); @@ -1817,15 +1678,10 @@ _codecs_utf_7_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_7_encode", "argument 2", "str or None", args[1]); @@ -1872,15 +1728,10 @@ _codecs_utf_8_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_8_encode", "argument 2", "str or None", args[1]); @@ -1928,15 +1779,10 @@ _codecs_utf_16_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_16_encode", "argument 2", "str or None", args[1]); @@ -1990,15 +1836,10 @@ _codecs_utf_16_le_encode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_16_le_encode", "argument 2", "str or None", args[1]); @@ -2045,15 +1886,10 @@ _codecs_utf_16_be_encode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_16_be_encode", "argument 2", "str or None", args[1]); @@ -2101,15 +1937,10 @@ _codecs_utf_32_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_32_encode", "argument 2", "str or None", args[1]); @@ -2163,15 +1994,10 @@ _codecs_utf_32_le_encode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_32_le_encode", "argument 2", "str or None", args[1]); @@ -2218,15 +2044,10 @@ _codecs_utf_32_be_encode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("utf_32_be_encode", "argument 2", "str or None", args[1]); @@ -2273,15 +2094,10 @@ _codecs_unicode_escape_encode(PyObject *module, PyObject *const *args, Py_ssize_ errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("unicode_escape_encode", "argument 2", "str or None", args[1]); @@ -2328,15 +2144,10 @@ _codecs_raw_unicode_escape_encode(PyObject *module, PyObject *const *args, Py_ss errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("raw_unicode_escape_encode", "argument 2", "str or None", args[1]); @@ -2383,15 +2194,10 @@ _codecs_latin_1_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("latin_1_encode", "argument 2", "str or None", args[1]); @@ -2438,15 +2244,10 @@ _codecs_ascii_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("ascii_encode", "argument 2", "str or None", args[1]); @@ -2494,15 +2295,10 @@ _codecs_charmap_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("charmap_encode", "argument 2", "str or None", args[1]); @@ -2582,15 +2378,10 @@ _codecs_mbcs_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("mbcs_encode", "argument 2", "str or None", args[1]); @@ -2640,15 +2431,10 @@ _codecs_oem_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("oem_encode", "argument 2", "str or None", args[1]); @@ -2704,15 +2490,10 @@ _codecs_code_page_encode(PyObject *module, PyObject *const *args, Py_ssize_t nar errors = NULL; } else if (PyUnicode_Check(args[2])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("code_page_encode", "argument 3", "str or None", args[2]); @@ -2756,15 +2537,10 @@ _codecs_iconv_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("iconv_encode", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[0], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[0]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!PyUnicode_Check(args[1])) { _PyArg_BadArgument("iconv_encode", "argument 2", "str", args[1]); goto exit; @@ -2777,15 +2553,10 @@ _codecs_iconv_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) errors = NULL; } else if (PyUnicode_Check(args[2])) { - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("iconv_encode", "argument 3", "str or None", args[2]); @@ -2832,15 +2603,10 @@ _codecs_register_error(PyObject *module, PyObject *const *args, Py_ssize_t nargs _PyArg_BadArgument("register_error", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[0], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[0]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } handler = args[1]; return_value = _codecs_register_error_impl(module, errors, handler); @@ -2879,15 +2645,10 @@ _codecs__unregister_error(PyObject *module, PyObject *arg) _PyArg_BadArgument("_unregister_error", "argument", "str", arg); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(arg, &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(arg); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } _return_value = _codecs__unregister_error_impl(module, errors); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -2923,15 +2684,10 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) _PyArg_BadArgument("lookup_error", "argument", "str", arg); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(arg, &name_length); + name = _PyUnicode_AsUTF8NoNUL(arg); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _codecs_lookup_error_impl(module, name); exit: @@ -3033,4 +2789,4 @@ _codecs__normalize_encoding(PyObject *module, PyObject *const *args, Py_ssize_t #ifndef _CODECS_ICONV_ENCODE_METHODDEF #define _CODECS_ICONV_ENCODE_METHODDEF #endif /* !defined(_CODECS_ICONV_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=d6e155d435335482 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2bf386d7e284e6f3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 7b83727a20b68a..1eb42e884a61ec 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(complexchar_new__doc__, "complexchar(text, /, attr=0, pair=0)\n" @@ -3814,15 +3815,10 @@ _curses_define_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs) definition = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t definition_length; - definition = PyUnicode_AsUTF8AndSize(args[0], &definition_length); + definition = _PyUnicode_AsUTF8NoNUL(args[0]); if (definition == NULL) { goto exit; } - if (strlen(definition) != (size_t)definition_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("define_key", "argument 1", "str or None", args[0]); @@ -3870,15 +3866,10 @@ _curses_key_defined(PyObject *module, PyObject *arg) _PyArg_BadArgument("key_defined", "argument", "str", arg); goto exit; } - Py_ssize_t definition_length; - definition = PyUnicode_AsUTF8AndSize(arg, &definition_length); + definition = _PyUnicode_AsUTF8NoNUL(arg); if (definition == NULL) { goto exit; } - if (strlen(definition) != (size_t)definition_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _curses_key_defined_impl(module, definition); exit: @@ -4273,15 +4264,10 @@ _curses_setupterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO term = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t term_length; - term = PyUnicode_AsUTF8AndSize(args[0], &term_length); + term = _PyUnicode_AsUTF8NoNUL(args[0]); if (term == NULL) { goto exit; } - if (strlen(term) != (size_t)term_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("setupterm", "argument 'term'", "str or None", args[0]); @@ -4343,15 +4329,10 @@ _curses_newterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) type = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t type_length; - type = PyUnicode_AsUTF8AndSize(args[0], &type_length); + type = _PyUnicode_AsUTF8NoNUL(args[0]); if (type == NULL) { goto exit; } - if (strlen(type) != (size_t)type_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("newterm", "argument 1", "str or None", args[0]); @@ -5735,15 +5716,10 @@ _curses_tigetflag(PyObject *module, PyObject *arg) _PyArg_BadArgument("tigetflag", "argument", "str", arg); goto exit; } - Py_ssize_t capname_length; - capname = PyUnicode_AsUTF8AndSize(arg, &capname_length); + capname = _PyUnicode_AsUTF8NoNUL(arg); if (capname == NULL) { goto exit; } - if (strlen(capname) != (size_t)capname_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _curses_tigetflag_impl(module, capname); exit: @@ -5778,15 +5754,10 @@ _curses_tigetnum(PyObject *module, PyObject *arg) _PyArg_BadArgument("tigetnum", "argument", "str", arg); goto exit; } - Py_ssize_t capname_length; - capname = PyUnicode_AsUTF8AndSize(arg, &capname_length); + capname = _PyUnicode_AsUTF8NoNUL(arg); if (capname == NULL) { goto exit; } - if (strlen(capname) != (size_t)capname_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _curses_tigetnum_impl(module, capname); exit: @@ -5821,15 +5792,10 @@ _curses_tigetstr(PyObject *module, PyObject *arg) _PyArg_BadArgument("tigetstr", "argument", "str", arg); goto exit; } - Py_ssize_t capname_length; - capname = PyUnicode_AsUTF8AndSize(arg, &capname_length); + capname = _PyUnicode_AsUTF8NoNUL(arg); if (capname == NULL) { goto exit; } - if (strlen(capname) != (size_t)capname_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _curses_tigetstr_impl(module, capname); exit: @@ -6749,4 +6715,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=cc7b5de1c82ae56b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=957470523e2b0eb1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index fac41e7aefc7f4..392a9da93ddbb4 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(delta_new__doc__, "timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0,\n" @@ -957,15 +958,10 @@ datetime_time_isoformat(PyObject *self, PyObject *const *args, Py_ssize_t nargs, _PyArg_BadArgument("isoformat", "argument 'timespec'", "str", args[0]); goto exit; } - Py_ssize_t timespec_length; - timespec = PyUnicode_AsUTF8AndSize(args[0], ×pec_length); + timespec = _PyUnicode_AsUTF8NoNUL(args[0]); if (timespec == NULL) { goto exit; } - if (strlen(timespec) != (size_t)timespec_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = datetime_time_isoformat_impl((PyDateTime_Time *)self, timespec); @@ -1816,15 +1812,10 @@ datetime_datetime_isoformat(PyObject *self, PyObject *const *args, Py_ssize_t na _PyArg_BadArgument("isoformat", "argument 'timespec'", "str", args[1]); goto exit; } - Py_ssize_t timespec_length; - timespec = PyUnicode_AsUTF8AndSize(args[1], ×pec_length); + timespec = _PyUnicode_AsUTF8NoNUL(args[1]); if (timespec == NULL) { goto exit; } - if (strlen(timespec) != (size_t)timespec_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = datetime_datetime_isoformat_impl((PyDateTime_DateTime *)self, sep, timespec); @@ -2091,4 +2082,4 @@ datetime_datetime___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return datetime_datetime___reduce___impl((PyDateTime_DateTime *)self); } -/*[clinic end generated code: output=8f63509398651723 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7bdfdfbd568c63ed input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index d36d72b8652d98..6a61185c032b7c 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -4,6 +4,7 @@ preserve #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_dbm_dbm_close__doc__, "close($self, /)\n" @@ -231,15 +232,10 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("open", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t flags_length; - flags = PyUnicode_AsUTF8AndSize(args[1], &flags_length); + flags = _PyUnicode_AsUTF8NoNUL(args[1]); if (flags == NULL) { goto exit; } - if (strlen(flags) != (size_t)flags_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -253,4 +249,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=336b7a79dd19219c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1ad0e5f0f4c7e9dd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index e3e09e2cb7d071..9b0494f35fae9a 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_elementtree_SubElement__doc__, "SubElement($module, parent, tag, attrib={}, /, **extra)\n" @@ -1352,15 +1353,10 @@ _elementtree_XMLParser___init__(PyObject *self, PyObject *args, PyObject *kwargs encoding = NULL; } else if (PyUnicode_Check(fastargs[1])) { - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(fastargs[1], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(fastargs[1]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("XMLParser", "argument 'encoding'", "str or None", fastargs[1]); @@ -1486,4 +1482,4 @@ _elementtree_XMLParser__setevents(PyObject *self, PyObject *const *args, Py_ssiz exit: return return_value; } -/*[clinic end generated code: output=9f9f792978663b16 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=01693c4bcc6c63b2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 4200983ecf44dd..13cc70390f51e3 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -4,6 +4,7 @@ preserve #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_gdbm_gdbm_get__doc__, "get($self, key, default=None, /)\n" @@ -383,15 +384,10 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("open", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t flags_length; - flags = PyUnicode_AsUTF8AndSize(args[1], &flags_length); + flags = _PyUnicode_AsUTF8NoNUL(args[1]); if (flags == NULL) { goto exit; } - if (strlen(flags) != (size_t)flags_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -405,4 +401,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=841b16bd11ef0066 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a3249cf7b6c4fae9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 17ed37bbd3b619..b99545b0eb3347 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_UnsignedLong_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_hashlib_HASH_copy__doc__, "copy($self, /)\n" @@ -307,15 +308,10 @@ _hashlib_HASH_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO _PyArg_BadArgument("new", "argument 'name'", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -1463,15 +1459,10 @@ pbkdf2_hmac(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject _PyArg_BadArgument("pbkdf2_hmac", "argument 'hash_name'", "str", args[0]); goto exit; } - Py_ssize_t hash_name_length; - hash_name = PyUnicode_AsUTF8AndSize(args[0], &hash_name_length); + hash_name = _PyUnicode_AsUTF8NoNUL(args[0]); if (hash_name == NULL) { goto exit; } - if (strlen(hash_name) != (size_t)hash_name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (PyObject_GetBuffer(args[1], &password, PyBUF_SIMPLE) != 0) { goto exit; } @@ -1986,4 +1977,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #define _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #endif /* !defined(_HASHLIB_OPENSSL_SHAKE_256_METHODDEF) */ -/*[clinic end generated code: output=cf405e652a340bb2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=608e8ac9483c9173 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpretersmodule.c.h b/Modules/clinic/_interpretersmodule.c.h index 406dbca93e8581..f297280279c5cd 100644 --- a/Modules/clinic/_interpretersmodule.c.h +++ b/Modules/clinic/_interpretersmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_interpreters_new_config__doc__, "new_config($module, name=\'isolated\', /, **overrides)\n" @@ -44,15 +45,10 @@ _interpreters_new_config(PyObject *module, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("new_config", "argument 1", "str", PyTuple_GET_ITEM(args, 0)); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(PyTuple_GET_ITEM(args, 0), &name_length); + name = _PyUnicode_AsUTF8NoNUL(PyTuple_GET_ITEM(args, 0)); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional: if (kwargs == NULL) { overrides = PyDict_New(); @@ -1264,4 +1260,4 @@ _interpreters_capture_exception(PyObject *module, PyObject *const *args, Py_ssiz exit: return return_value; } -/*[clinic end generated code: output=38892db3b955efbb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fa37872d6ee729ef input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_localemodule.c.h b/Modules/clinic/_localemodule.c.h index 118946b81474ea..90267389477456 100644 --- a/Modules/clinic/_localemodule.c.h +++ b/Modules/clinic/_localemodule.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_locale_setlocale__doc__, "setlocale($module, category, locale=None, /)\n" @@ -37,15 +38,10 @@ _locale_setlocale(PyObject *module, PyObject *const *args, Py_ssize_t nargs) locale = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t locale_length; - locale = PyUnicode_AsUTF8AndSize(args[1], &locale_length); + locale = _PyUnicode_AsUTF8NoNUL(args[1]); if (locale == NULL) { goto exit; } - if (strlen(locale) != (size_t)locale_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("setlocale", "argument 2", "str or None", args[1]); @@ -230,15 +226,10 @@ _locale_gettext(PyObject *module, PyObject *arg) _PyArg_BadArgument("gettext", "argument", "str", arg); goto exit; } - Py_ssize_t in_length; - in = PyUnicode_AsUTF8AndSize(arg, &in_length); + in = _PyUnicode_AsUTF8NoNUL(arg); if (in == NULL) { goto exit; } - if (strlen(in) != (size_t)in_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _locale_gettext_impl(module, in); exit: @@ -277,15 +268,10 @@ _locale_dgettext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) domain = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t domain_length; - domain = PyUnicode_AsUTF8AndSize(args[0], &domain_length); + domain = _PyUnicode_AsUTF8NoNUL(args[0]); if (domain == NULL) { goto exit; } - if (strlen(domain) != (size_t)domain_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("dgettext", "argument 1", "str or None", args[0]); @@ -295,15 +281,10 @@ _locale_dgettext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("dgettext", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t in_length; - in = PyUnicode_AsUTF8AndSize(args[1], &in_length); + in = _PyUnicode_AsUTF8NoNUL(args[1]); if (in == NULL) { goto exit; } - if (strlen(in) != (size_t)in_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _locale_dgettext_impl(module, domain, in); exit: @@ -342,15 +323,10 @@ _locale_dcgettext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) domain = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t domain_length; - domain = PyUnicode_AsUTF8AndSize(args[0], &domain_length); + domain = _PyUnicode_AsUTF8NoNUL(args[0]); if (domain == NULL) { goto exit; } - if (strlen(domain) != (size_t)domain_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("dcgettext", "argument 1", "str or None", args[0]); @@ -360,15 +336,10 @@ _locale_dcgettext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("dcgettext", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t msgid_length; - msgid = PyUnicode_AsUTF8AndSize(args[1], &msgid_length); + msgid = _PyUnicode_AsUTF8NoNUL(args[1]); if (msgid == NULL) { goto exit; } - if (strlen(msgid) != (size_t)msgid_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } category = PyLong_AsInt(args[2]); if (category == -1 && PyErr_Occurred()) { goto exit; @@ -405,15 +376,10 @@ _locale_textdomain(PyObject *module, PyObject *arg) domain = NULL; } else if (PyUnicode_Check(arg)) { - Py_ssize_t domain_length; - domain = PyUnicode_AsUTF8AndSize(arg, &domain_length); + domain = _PyUnicode_AsUTF8NoNUL(arg); if (domain == NULL) { goto exit; } - if (strlen(domain) != (size_t)domain_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("textdomain", "argument", "str or None", arg); @@ -456,15 +422,10 @@ _locale_bindtextdomain(PyObject *module, PyObject *const *args, Py_ssize_t nargs _PyArg_BadArgument("bindtextdomain", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t domain_length; - domain = PyUnicode_AsUTF8AndSize(args[0], &domain_length); + domain = _PyUnicode_AsUTF8NoNUL(args[0]); if (domain == NULL) { goto exit; } - if (strlen(domain) != (size_t)domain_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } dirname_obj = args[1]; return_value = _locale_bindtextdomain_impl(module, domain, dirname_obj); @@ -503,28 +464,18 @@ _locale_bind_textdomain_codeset(PyObject *module, PyObject *const *args, Py_ssiz _PyArg_BadArgument("bind_textdomain_codeset", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t domain_length; - domain = PyUnicode_AsUTF8AndSize(args[0], &domain_length); + domain = _PyUnicode_AsUTF8NoNUL(args[0]); if (domain == NULL) { goto exit; } - if (strlen(domain) != (size_t)domain_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (args[1] == Py_None) { codeset = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t codeset_length; - codeset = PyUnicode_AsUTF8AndSize(args[1], &codeset_length); + codeset = _PyUnicode_AsUTF8NoNUL(args[1]); if (codeset == NULL) { goto exit; } - if (strlen(codeset) != (size_t)codeset_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("bind_textdomain_codeset", "argument 2", "str or None", args[1]); @@ -595,4 +546,4 @@ _locale_getencoding(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF #define _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF #endif /* !defined(_LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF) */ -/*[clinic end generated code: output=bb987603f9de8824 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b995b35bd3085d5b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index d15c04e295d8cc..ae94e4c1d252ae 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_pickle_Pickler_clear_memo__doc__, "clear_memo($self, /)\n" @@ -479,15 +480,10 @@ _pickle_Unpickler___init__(PyObject *self, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("Unpickler", "argument 'encoding'", "str", fastargs[2]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(fastargs[2], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(fastargs[2]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -497,15 +493,10 @@ _pickle_Unpickler___init__(PyObject *self, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("Unpickler", "argument 'errors'", "str", fastargs[3]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(fastargs[3], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(fastargs[3]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -892,15 +883,10 @@ _pickle_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject _PyArg_BadArgument("load", "argument 'encoding'", "str", args[2]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[2], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[2]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -910,15 +896,10 @@ _pickle_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject _PyArg_BadArgument("load", "argument 'errors'", "str", args[3]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[3], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[3]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -1021,15 +1002,10 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec _PyArg_BadArgument("loads", "argument 'encoding'", "str", args[2]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[2], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[2]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -1039,15 +1015,10 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec _PyArg_BadArgument("loads", "argument 'errors'", "str", args[3]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[3], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[3]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_kwonly; } @@ -1059,4 +1030,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=df25a5fe475b57b0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=df810fb55415996f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index b0dc01adfc999e..d0bce13f2fd945 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -11,6 +11,7 @@ preserve #include "pycore_fileutils.h" // _Py_Off_t_Converter() #include "pycore_long.h" // _PyLong_Size_t_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, "do_handshake($self, /)\n" @@ -771,15 +772,10 @@ _ssl__SSLSocket_get_channel_binding(PyObject *self, PyObject *const *args, Py_ss _PyArg_BadArgument("get_channel_binding", "argument 'cb_type'", "str", args[0]); goto exit; } - Py_ssize_t cb_type_length; - cb_type = PyUnicode_AsUTF8AndSize(args[0], &cb_type_length); + cb_type = _PyUnicode_AsUTF8NoNUL(args[0]); if (cb_type == NULL) { goto exit; } - if (strlen(cb_type) != (size_t)cb_type_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLSocket_get_channel_binding_impl((PySSLSocket *)self, cb_type); @@ -922,15 +918,10 @@ _ssl__SSLContext_set_ciphers(PyObject *self, PyObject *arg) _PyArg_BadArgument("set_ciphers", "argument", "str", arg); goto exit; } - Py_ssize_t cipherlist_length; - cipherlist = PyUnicode_AsUTF8AndSize(arg, &cipherlist_length); + cipherlist = _PyUnicode_AsUTF8NoNUL(arg); if (cipherlist == NULL) { goto exit; } - if (strlen(cipherlist) != (size_t)cipherlist_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_set_ciphers_impl((PySSLContext *)self, cipherlist); Py_END_CRITICAL_SECTION(); @@ -961,15 +952,10 @@ _ssl__SSLContext_set_ciphersuites(PyObject *self, PyObject *arg) _PyArg_BadArgument("set_ciphersuites", "argument", "str", arg); goto exit; } - Py_ssize_t ciphersuites_length; - ciphersuites = PyUnicode_AsUTF8AndSize(arg, &ciphersuites_length); + ciphersuites = _PyUnicode_AsUTF8NoNUL(arg); if (ciphersuites == NULL) { goto exit; } - if (strlen(ciphersuites) != (size_t)ciphersuites_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_set_ciphersuites_impl((PySSLContext *)self, ciphersuites); Py_END_CRITICAL_SECTION(); @@ -1022,15 +1008,10 @@ _ssl__SSLContext_set_groups(PyObject *self, PyObject *arg) _PyArg_BadArgument("set_groups", "argument", "str", arg); goto exit; } - Py_ssize_t grouplist_length; - grouplist = PyUnicode_AsUTF8AndSize(arg, &grouplist_length); + grouplist = _PyUnicode_AsUTF8NoNUL(arg); if (grouplist == NULL) { goto exit; } - if (strlen(grouplist) != (size_t)grouplist_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_set_groups_impl((PySSLContext *)self, grouplist); Py_END_CRITICAL_SECTION(); @@ -1128,15 +1109,10 @@ _ssl__SSLContext_set_client_sigalgs(PyObject *self, PyObject *arg) _PyArg_BadArgument("set_client_sigalgs", "argument", "str", arg); goto exit; } - Py_ssize_t sigalgslist_length; - sigalgslist = PyUnicode_AsUTF8AndSize(arg, &sigalgslist_length); + sigalgslist = _PyUnicode_AsUTF8NoNUL(arg); if (sigalgslist == NULL) { goto exit; } - if (strlen(sigalgslist) != (size_t)sigalgslist_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_set_client_sigalgs_impl((PySSLContext *)self, sigalgslist); Py_END_CRITICAL_SECTION(); @@ -1167,15 +1143,10 @@ _ssl__SSLContext_set_server_sigalgs(PyObject *self, PyObject *arg) _PyArg_BadArgument("set_server_sigalgs", "argument", "str", arg); goto exit; } - Py_ssize_t sigalgslist_length; - sigalgslist = PyUnicode_AsUTF8AndSize(arg, &sigalgslist_length); + sigalgslist = _PyUnicode_AsUTF8NoNUL(arg); if (sigalgslist == NULL) { goto exit; } - if (strlen(sigalgslist) != (size_t)sigalgslist_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(self); return_value = _ssl__SSLContext_set_server_sigalgs_impl((PySSLContext *)self, sigalgslist); Py_END_CRITICAL_SECTION(); @@ -2354,15 +2325,10 @@ _ssl__SSLContext_set_psk_server_callback(PyObject *self, PyObject *const *args, identity_hint = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t identity_hint_length; - identity_hint = PyUnicode_AsUTF8AndSize(args[1], &identity_hint_length); + identity_hint = _PyUnicode_AsUTF8NoNUL(args[1]); if (identity_hint == NULL) { goto exit; } - if (strlen(identity_hint) != (size_t)identity_hint_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("set_psk_server_callback", "argument 'identity_hint'", "str or None", args[1]); @@ -2855,15 +2821,10 @@ _ssl_txt2obj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject _PyArg_BadArgument("txt2obj", "argument 'txt'", "str", args[0]); goto exit; } - Py_ssize_t txt_length; - txt = PyUnicode_AsUTF8AndSize(args[0], &txt_length); + txt = _PyUnicode_AsUTF8NoNUL(args[0]); if (txt == NULL) { goto exit; } - if (strlen(txt) != (size_t)txt_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -2973,15 +2934,10 @@ _ssl_enum_certificates(PyObject *module, PyObject *const *args, Py_ssize_t nargs _PyArg_BadArgument("enum_certificates", "argument 'store_name'", "str", args[0]); goto exit; } - Py_ssize_t store_name_length; - store_name = PyUnicode_AsUTF8AndSize(args[0], &store_name_length); + store_name = _PyUnicode_AsUTF8NoNUL(args[0]); if (store_name == NULL) { goto exit; } - if (strlen(store_name) != (size_t)store_name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_enum_certificates_impl(module, store_name); Py_END_CRITICAL_SECTION(); @@ -3054,15 +3010,10 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("enum_crls", "argument 'store_name'", "str", args[0]); goto exit; } - Py_ssize_t store_name_length; - store_name = PyUnicode_AsUTF8AndSize(args[0], &store_name_length); + store_name = _PyUnicode_AsUTF8NoNUL(args[0]); if (store_name == NULL) { goto exit; } - if (strlen(store_name) != (size_t)store_name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_enum_crls_impl(module, store_name); Py_END_CRITICAL_SECTION(); @@ -3132,4 +3083,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #define _SSL_SSLSESSION_HAS_TICKET_GETSETDEF {"has_ticket", (getter)_ssl_SSLSession_has_ticket_get, (setter)NULL, _ssl_SSLSession_has_ticket__doc__}, -/*[clinic end generated code: output=42b28429643df3e3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=14ced41bd24b60c3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index cc1bb819afce65..3fbf9c836a9abd 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -11,6 +11,7 @@ preserve #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(test_empty_function__doc__, "test_empty_function($module, /)\n" @@ -4043,15 +4044,10 @@ clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw _PyArg_BadArgument("clone_f1", "argument 'path'", "str", args[0]); goto exit; } - Py_ssize_t path_length; - path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + path = _PyUnicode_AsUTF8NoNUL(args[0]); if (path == NULL) { goto exit; } - if (strlen(path) != (size_t)path_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = clone_f1_impl(module, path); exit: @@ -4112,15 +4108,10 @@ clone_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw _PyArg_BadArgument("clone_f2", "argument 'path'", "str", args[0]); goto exit; } - Py_ssize_t path_length; - path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + path = _PyUnicode_AsUTF8NoNUL(args[0]); if (path == NULL) { goto exit; } - if (strlen(path) != (size_t)path_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = clone_f2_impl(module, path); exit: @@ -5318,4 +5309,4 @@ vc_kwonly_vectorcall(PyObject *type, PyObject *const *args, kwnames ? PyTuple_GET_SIZE(kwnames) : 0, NULL, kwnames); } -/*[clinic end generated code: output=5131e55aeb855d6e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e1f97f59fec61b6b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 1b0195185329a2..1652e9223e54f9 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -11,6 +11,7 @@ preserve #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(depr_star_new__doc__, "DeprStarNew(a=None)\n" @@ -2681,4 +2682,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=858abe8a5a885725 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=abe0af8d8c637c36 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_kwds.c.h b/Modules/clinic/_testclinic_kwds.c.h index ce4ee7a850f45b..faae3f3d0c2658 100644 --- a/Modules/clinic/_testclinic_kwds.c.h +++ b/Modules/clinic/_testclinic_kwds.c.h @@ -11,6 +11,7 @@ preserve #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(lone_kwds__doc__, "lone_kwds($module, /, **kwds)\n" @@ -229,4 +230,4 @@ kwds_with_pos_only_and_stararg(PyObject *module, PyObject *args, PyObject *kwarg return return_value; } -/*[clinic end generated code: output=62804c41a11bbd73 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=27a7ba9440b9a884 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 21ae39737a1810..1e197167409a03 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_tkinter_tkapp_call__doc__, "call($self, /, *args)\n" @@ -55,15 +56,10 @@ _tkinter_tkapp_eval(PyObject *self, PyObject *arg) _PyArg_BadArgument("eval", "argument", "str", arg); goto exit; } - Py_ssize_t script_length; - script = PyUnicode_AsUTF8AndSize(arg, &script_length); + script = _PyUnicode_AsUTF8NoNUL(arg); if (script == NULL) { goto exit; } - if (strlen(script) != (size_t)script_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_eval_impl((TkappObject *)self, script); exit: @@ -91,15 +87,10 @@ _tkinter_tkapp_evalfile(PyObject *self, PyObject *arg) _PyArg_BadArgument("evalfile", "argument", "str", arg); goto exit; } - Py_ssize_t fileName_length; - fileName = PyUnicode_AsUTF8AndSize(arg, &fileName_length); + fileName = _PyUnicode_AsUTF8NoNUL(arg); if (fileName == NULL) { goto exit; } - if (strlen(fileName) != (size_t)fileName_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_evalfile_impl((TkappObject *)self, fileName); exit: @@ -127,15 +118,10 @@ _tkinter_tkapp_record(PyObject *self, PyObject *arg) _PyArg_BadArgument("record", "argument", "str", arg); goto exit; } - Py_ssize_t script_length; - script = PyUnicode_AsUTF8AndSize(arg, &script_length); + script = _PyUnicode_AsUTF8NoNUL(arg); if (script == NULL) { goto exit; } - if (strlen(script) != (size_t)script_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_record_impl((TkappObject *)self, script); exit: @@ -163,15 +149,10 @@ _tkinter_tkapp_adderrorinfo(PyObject *self, PyObject *arg) _PyArg_BadArgument("adderrorinfo", "argument", "str", arg); goto exit; } - Py_ssize_t msg_length; - msg = PyUnicode_AsUTF8AndSize(arg, &msg_length); + msg = _PyUnicode_AsUTF8NoNUL(arg); if (msg == NULL) { goto exit; } - if (strlen(msg) != (size_t)msg_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_adderrorinfo_impl((TkappObject *)self, msg); exit: @@ -262,15 +243,10 @@ _tkinter_tkapp_exprstring(PyObject *self, PyObject *arg) _PyArg_BadArgument("exprstring", "argument", "str", arg); goto exit; } - Py_ssize_t s_length; - s = PyUnicode_AsUTF8AndSize(arg, &s_length); + s = _PyUnicode_AsUTF8NoNUL(arg); if (s == NULL) { goto exit; } - if (strlen(s) != (size_t)s_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_exprstring_impl((TkappObject *)self, s); exit: @@ -298,15 +274,10 @@ _tkinter_tkapp_exprlong(PyObject *self, PyObject *arg) _PyArg_BadArgument("exprlong", "argument", "str", arg); goto exit; } - Py_ssize_t s_length; - s = PyUnicode_AsUTF8AndSize(arg, &s_length); + s = _PyUnicode_AsUTF8NoNUL(arg); if (s == NULL) { goto exit; } - if (strlen(s) != (size_t)s_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_exprlong_impl((TkappObject *)self, s); exit: @@ -334,15 +305,10 @@ _tkinter_tkapp_exprdouble(PyObject *self, PyObject *arg) _PyArg_BadArgument("exprdouble", "argument", "str", arg); goto exit; } - Py_ssize_t s_length; - s = PyUnicode_AsUTF8AndSize(arg, &s_length); + s = _PyUnicode_AsUTF8NoNUL(arg); if (s == NULL) { goto exit; } - if (strlen(s) != (size_t)s_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_exprdouble_impl((TkappObject *)self, s); exit: @@ -370,15 +336,10 @@ _tkinter_tkapp_exprboolean(PyObject *self, PyObject *arg) _PyArg_BadArgument("exprboolean", "argument", "str", arg); goto exit; } - Py_ssize_t s_length; - s = PyUnicode_AsUTF8AndSize(arg, &s_length); + s = _PyUnicode_AsUTF8NoNUL(arg); if (s == NULL) { goto exit; } - if (strlen(s) != (size_t)s_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_exprboolean_impl((TkappObject *)self, s); exit: @@ -432,15 +393,10 @@ _tkinter_tkapp_createcommand(PyObject *self, PyObject *const *args, Py_ssize_t n _PyArg_BadArgument("createcommand", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } func = args[1]; return_value = _tkinter_tkapp_createcommand_impl((TkappObject *)self, name, func); @@ -469,15 +425,10 @@ _tkinter_tkapp_deletecommand(PyObject *self, PyObject *arg) _PyArg_BadArgument("deletecommand", "argument", "str", arg); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(arg, &name_length); + name = _PyUnicode_AsUTF8NoNUL(arg); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _tkinter_tkapp_deletecommand_impl((TkappObject *)self, name); exit: @@ -864,15 +815,10 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) screenName = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t screenName_length; - screenName = PyUnicode_AsUTF8AndSize(args[0], &screenName_length); + screenName = _PyUnicode_AsUTF8NoNUL(args[0]); if (screenName == NULL) { goto exit; } - if (strlen(screenName) != (size_t)screenName_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("create", "argument 1", "str or None", args[0]); @@ -885,15 +831,10 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("create", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t baseName_length; - baseName = PyUnicode_AsUTF8AndSize(args[1], &baseName_length); + baseName = _PyUnicode_AsUTF8NoNUL(args[1]); if (baseName == NULL) { goto exit; } - if (strlen(baseName) != (size_t)baseName_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (nargs < 3) { goto skip_optional; } @@ -901,15 +842,10 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("create", "argument 3", "str", args[2]); goto exit; } - Py_ssize_t className_length; - className = PyUnicode_AsUTF8AndSize(args[2], &className_length); + className = _PyUnicode_AsUTF8NoNUL(args[2]); if (className == NULL) { goto exit; } - if (strlen(className) != (size_t)className_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (nargs < 4) { goto skip_optional; } @@ -945,15 +881,10 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) use = NULL; } else if (PyUnicode_Check(args[7])) { - Py_ssize_t use_length; - use = PyUnicode_AsUTF8AndSize(args[7], &use_length); + use = _PyUnicode_AsUTF8NoNUL(args[7]); if (use == NULL) { goto exit; } - if (strlen(use) != (size_t)use_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("create", "argument 8", "str or None", args[7]); @@ -1032,4 +963,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=bb55a7d4c5a8a002 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=229ae8ed00ee4e40 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index e4543c108df2b3..41798e098389c2 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -4,6 +4,7 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(array_array_clear__doc__, "clear($self, /)\n" @@ -631,15 +632,10 @@ array__array_reconstructor(PyObject *module, PyObject *const *args, Py_ssize_t n _PyArg_BadArgument("_array_reconstructor", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t typecode_length; - typecode = PyUnicode_AsUTF8AndSize(args[1], &typecode_length); + typecode = _PyUnicode_AsUTF8NoNUL(args[1]); if (typecode == NULL) { goto exit; } - if (strlen(typecode) != (size_t)typecode_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } mformat_code = PyLong_AsInt(args[2]); if (mformat_code == -1 && PyErr_Occurred()) { goto exit; @@ -726,4 +722,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=54ec10a63676f320 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=172a9af0316afc14 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mmapmodule.c.h b/Modules/clinic/mmapmodule.c.h index 98c5bf6a2fb146..1343eda42e84ec 100644 --- a/Modules/clinic/mmapmodule.c.h +++ b/Modules/clinic/mmapmodule.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(mmap_mmap_close__doc__, "close($self, /)\n" @@ -547,15 +548,10 @@ mmap_mmap_set_name(PyObject *self, PyObject *arg) _PyArg_BadArgument("set_name", "argument", "str", arg); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(arg, &name_length); + name = _PyUnicode_AsUTF8NoNUL(arg); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(self); return_value = mmap_mmap_set_name_impl((mmap_object *)self, name); Py_END_CRITICAL_SECTION(); @@ -881,4 +877,4 @@ mmap_mmap_madvise(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #ifndef MMAP_MMAP_MADVISE_METHODDEF #define MMAP_MMAP_MADVISE_METHODDEF #endif /* !defined(MMAP_MMAP_MADVISE_METHODDEF) */ -/*[clinic end generated code: output=1122b93314aebc5c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d2ae0d1d754f51cc input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 46e223a06626d8..7f4bab7de9488c 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10,6 +10,7 @@ preserve #include "pycore_fileutils.h" // _Py_Off_t_Converter() #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(os_stat__doc__, "stat($module, /, path, *, dir_fd=None, follow_symlinks=True)\n" @@ -5321,15 +5322,10 @@ os_getgrouplist(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("getgrouplist", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t user_length; - user = PyUnicode_AsUTF8AndSize(args[0], &user_length); + user = _PyUnicode_AsUTF8NoNUL(args[0]); if (user == NULL) { goto exit; } - if (strlen(user) != (size_t)user_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } basegid = PyLong_AsInt(args[1]); if (basegid == -1 && PyErr_Occurred()) { goto exit; @@ -5375,15 +5371,10 @@ os_getgrouplist(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("getgrouplist", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t user_length; - user = PyUnicode_AsUTF8AndSize(args[0], &user_length); + user = _PyUnicode_AsUTF8NoNUL(args[0]); if (user == NULL) { goto exit; } - if (strlen(user) != (size_t)user_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!_Py_Gid_Converter(args[1], &basegid)) { goto exit; } @@ -13070,15 +13061,10 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py _PyArg_BadArgument("_emscripten_log", "argument 'arg'", "str", args[0]); goto exit; } - Py_ssize_t arg_length; - arg = PyUnicode_AsUTF8AndSize(args[0], &arg_length); + arg = _PyUnicode_AsUTF8NoNUL(args[0]); if (arg == NULL) { goto exit; } - if (strlen(arg) != (size_t)arg_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = os__emscripten_log_impl(module, arg); exit: @@ -13774,4 +13760,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=38edcd9b6f56b729 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f1113bf7c270d271 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 95f56c88e1aa92..0ce841c83feb20 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_long.h" // _PyLong_UnsignedLongLong_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__, "SetReparseDeferralEnabled($self, enabled, /)\n" @@ -157,15 +158,10 @@ pyexpat_xmlparser_SetBase(PyObject *self, PyObject *arg) _PyArg_BadArgument("SetBase", "argument", "str", arg); goto exit; } - Py_ssize_t base_length; - base = PyUnicode_AsUTF8AndSize(arg, &base_length); + base = _PyUnicode_AsUTF8NoNUL(arg); if (base == NULL) { goto exit; } - if (strlen(base) != (size_t)base_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = pyexpat_xmlparser_SetBase_impl((xmlparseobject *)self, base); exit: @@ -272,15 +268,10 @@ pyexpat_xmlparser_ExternalEntityParserCreate(PyObject *self, PyTypeObject *cls, context = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t context_length; - context = PyUnicode_AsUTF8AndSize(args[0], &context_length); + context = _PyUnicode_AsUTF8NoNUL(args[0]); if (context == NULL) { goto exit; } - if (strlen(context) != (size_t)context_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("ExternalEntityParserCreate", "argument 1", "str or None", args[0]); @@ -293,15 +284,10 @@ pyexpat_xmlparser_ExternalEntityParserCreate(PyObject *self, PyTypeObject *cls, _PyArg_BadArgument("ExternalEntityParserCreate", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[1], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[1]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional: return_value = pyexpat_xmlparser_ExternalEntityParserCreate_impl((xmlparseobject *)self, cls, context, encoding); @@ -670,15 +656,10 @@ pyexpat_ParserCreate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, encoding = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[0], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[0]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("ParserCreate", "argument 'encoding'", "str or None", args[0]); @@ -693,15 +674,10 @@ pyexpat_ParserCreate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, namespace_separator = NULL; } else if (PyUnicode_Check(args[1])) { - Py_ssize_t namespace_separator_length; - namespace_separator = PyUnicode_AsUTF8AndSize(args[1], &namespace_separator_length); + namespace_separator = _PyUnicode_AsUTF8NoNUL(args[1]); if (namespace_separator == NULL) { goto exit; } - if (strlen(namespace_separator) != (size_t)namespace_separator_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("ParserCreate", "argument 'namespace_separator'", "str or None", args[1]); @@ -766,4 +742,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF) */ -/*[clinic end generated code: output=6a2d0b0e35fd07fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f1652fd212bce55f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index fcccf347a0ae9b..09041d92a30579 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_UInt16_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() #if (defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4)) @@ -1485,15 +1486,10 @@ _socket_getservbyname(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("getservbyname", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + name = _PyUnicode_AsUTF8NoNUL(args[0]); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (nargs < 2) { goto skip_optional; } @@ -1501,15 +1497,10 @@ _socket_getservbyname(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("getservbyname", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t proto_length; - proto = PyUnicode_AsUTF8AndSize(args[1], &proto_length); + proto = _PyUnicode_AsUTF8NoNUL(args[1]); if (proto == NULL) { goto exit; } - if (strlen(proto) != (size_t)proto_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional: return_value = _socket_getservbyname_impl(module, name, proto); @@ -1557,15 +1548,10 @@ _socket_getservbyport(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("getservbyport", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t proto_length; - proto = PyUnicode_AsUTF8AndSize(args[1], &proto_length); + proto = _PyUnicode_AsUTF8NoNUL(args[1]); if (proto == NULL) { goto exit; } - if (strlen(proto) != (size_t)proto_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional: return_value = _socket_getservbyport_impl(module, port, proto); @@ -1599,15 +1585,10 @@ _socket_getprotobyname(PyObject *module, PyObject *arg) _PyArg_BadArgument("getprotobyname", "argument", "str", arg); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(arg, &name_length); + name = _PyUnicode_AsUTF8NoNUL(arg); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _socket_getprotobyname_impl(module, name); exit: @@ -1835,15 +1816,10 @@ _socket_inet_aton(PyObject *module, PyObject *arg) _PyArg_BadArgument("inet_aton", "argument", "str", arg); goto exit; } - Py_ssize_t ip_addr_length; - ip_addr = PyUnicode_AsUTF8AndSize(arg, &ip_addr_length); + ip_addr = _PyUnicode_AsUTF8NoNUL(arg); if (ip_addr == NULL) { goto exit; } - if (strlen(ip_addr) != (size_t)ip_addr_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _socket_inet_aton_impl(module, ip_addr); exit: @@ -1920,15 +1896,10 @@ _socket_inet_pton(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("inet_pton", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t ip_length; - ip = PyUnicode_AsUTF8AndSize(args[1], &ip_length); + ip = _PyUnicode_AsUTF8NoNUL(args[1]); if (ip == NULL) { goto exit; } - if (strlen(ip) != (size_t)ip_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = _socket_inet_pton_impl(module, af, ip); exit: @@ -2478,4 +2449,4 @@ _socket_CMSG_SPACE(PyObject *module, PyObject *arg) #ifndef _SOCKET_CMSG_SPACE_METHODDEF #define _SOCKET_CMSG_SPACE_METHODDEF #endif /* !defined(_SOCKET_CMSG_SPACE_METHODDEF) */ -/*[clinic end generated code: output=dfe0ca7c716c1183 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bca43f33969d67ad input=a9049054013a1b77]*/ diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 00ee4315846b60..2df590875955f3 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(_symtable_symtable__doc__, "symtable($module, source, filename, startstr, /, *, module=None)\n" @@ -75,15 +76,10 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py _PyArg_BadArgument("symtable", "argument 3", "str", args[2]); goto exit; } - Py_ssize_t startstr_length; - startstr = PyUnicode_AsUTF8AndSize(args[2], &startstr_length); + startstr = _PyUnicode_AsUTF8NoNUL(args[2]); if (startstr == NULL) { goto exit; } - if (strlen(startstr) != (size_t)startstr_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!noptargs) { goto skip_optional_kwonly; } @@ -97,4 +93,4 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } -/*[clinic end generated code: output=23523cada784726e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=910e4f6ce98ae0a1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/syslogmodule.c.h b/Modules/clinic/syslogmodule.c.h index 1603d8ba9ff1c2..6f878d675045e4 100644 --- a/Modules/clinic/syslogmodule.c.h +++ b/Modules/clinic/syslogmodule.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(syslog_openlog__doc__, "openlog($module, /, ident=, logoption=0,\n" @@ -134,15 +135,10 @@ syslog_syslog(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("syslog", "argument", "str", args[nargs - 1]); goto exit; } - Py_ssize_t message_length; - message = PyUnicode_AsUTF8AndSize(args[nargs - 1], &message_length); + message = _PyUnicode_AsUTF8NoNUL(args[nargs - 1]); if (message == NULL) { goto exit; } - if (strlen(message) != (size_t)message_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_syslog_impl(module, group_left_1, priority, message); Py_END_CRITICAL_SECTION(); @@ -273,4 +269,4 @@ syslog_LOG_UPTO(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=ab818466dcdbbe59 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0c9607f279107505 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/timemodule.c.h b/Modules/clinic/timemodule.c.h index 3b0a8d44129d38..a42c1d7905e811 100644 --- a/Modules/clinic/timemodule.c.h +++ b/Modules/clinic/timemodule.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(time_time__doc__, "time($module, /)\n" @@ -697,15 +698,10 @@ time_get_clock_info(PyObject *module, PyObject *arg) _PyArg_BadArgument("get_clock_info", "argument", "str", arg); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(arg, &name_length); + name = _PyUnicode_AsUTF8NoNUL(arg); if (name == NULL) { goto exit; } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = time_get_clock_info_impl(module, name); exit: @@ -751,4 +747,4 @@ time_get_clock_info(PyObject *module, PyObject *arg) #ifndef TIME_THREAD_TIME_NS_METHODDEF #define TIME_THREAD_TIME_NS_METHODDEF #endif /* !defined(TIME_THREAD_TIME_NS_METHODDEF) */ -/*[clinic end generated code: output=540a094cac69e0d4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d89be8cc2b2a7b05 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 41ce82c05c57d9..b074851f1fbf78 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() static int bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, @@ -72,15 +73,10 @@ bytearray___init__(PyObject *self, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("bytearray", "argument 'encoding'", "str", fastargs[1]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(fastargs[1], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(fastargs[1]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -89,15 +85,10 @@ bytearray___init__(PyObject *self, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("bytearray", "argument 'errors'", "str", fastargs[2]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(fastargs[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(fastargs[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = bytearray___init___impl((PyByteArrayObject *)self, arg, encoding, errors); @@ -1542,15 +1533,10 @@ bytearray_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("decode", "argument 'encoding'", "str", args[0]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[0], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[0]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -1559,15 +1545,10 @@ bytearray_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("decode", "argument 'errors'", "str", args[1]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: Py_BEGIN_CRITICAL_SECTION(self); return_value = bytearray_decode_impl((PyByteArrayObject *)self, encoding, errors); @@ -1882,4 +1863,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=6dc315d35de3e670 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=16fb40de6aefa5f8 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index ee2b737f9e63f9..56e8d089334a7a 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(bytes___bytes____doc__, "__bytes__($self, /)\n" @@ -1136,15 +1137,10 @@ bytes_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject * _PyArg_BadArgument("decode", "argument 'encoding'", "str", args[0]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[0], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[0]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -1153,15 +1149,10 @@ bytes_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject * _PyArg_BadArgument("decode", "argument 'errors'", "str", args[1]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = bytes_decode_impl((PyBytesObject *)self, encoding, errors); @@ -1423,15 +1414,10 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("bytes", "argument 'encoding'", "str", fastargs[1]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(fastargs[1], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(fastargs[1]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -1440,19 +1426,14 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("bytes", "argument 'errors'", "str", fastargs[2]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(fastargs[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(fastargs[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = bytes_new_impl(type, x, encoding, errors); exit: return return_value; } -/*[clinic end generated code: output=c20458db7a2123db input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2de57d656cc2f041 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index 8768555c909257..cc37ea55612a8f 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(float_is_integer__doc__, "is_integer($self, /)\n" @@ -310,15 +311,10 @@ float___getformat__(PyObject *type, PyObject *arg) _PyArg_BadArgument("__getformat__", "argument", "str", arg); goto exit; } - Py_ssize_t typestr_length; - typestr = PyUnicode_AsUTF8AndSize(arg, &typestr_length); + typestr = _PyUnicode_AsUTF8NoNUL(arg); if (typestr == NULL) { goto exit; } - if (strlen(typestr) != (size_t)typestr_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = float___getformat___impl((PyTypeObject *)type, typestr); exit: @@ -353,4 +349,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=5d7b0bf9e47ff997 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e0ffc05aa7fc4b10 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index bccce41b13e137..93daf550b0092c 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(memoryview__doc__, "memoryview(object)\n" @@ -373,15 +374,10 @@ memoryview_tobytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb order = NULL; } else if (PyUnicode_Check(args[0])) { - Py_ssize_t order_length; - order = PyUnicode_AsUTF8AndSize(args[0], &order_length); + order = _PyUnicode_AsUTF8NoNUL(args[0]); if (order == NULL) { goto exit; } - if (strlen(order) != (size_t)order_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } } else { _PyArg_BadArgument("tobytes", "argument 'order'", "str or None", args[0]); @@ -560,4 +556,4 @@ memoryview_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=a597307395a568ef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=889b7cc1a94bfc9b input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index d0753b38843fcc..f214fa42fa1a59 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(EncodingMap_size__doc__, "size($self, /)\n" @@ -263,15 +264,10 @@ unicode_encode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject _PyArg_BadArgument("encode", "argument 'encoding'", "str", args[0]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(args[0], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(args[0]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -280,15 +276,10 @@ unicode_encode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject _PyArg_BadArgument("encode", "argument 'errors'", "str", args[1]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(args[1], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(args[1]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = unicode_encode_impl(self, encoding, errors); @@ -1884,15 +1875,10 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("str", "argument 'encoding'", "str", fastargs[1]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(fastargs[1], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(fastargs[1]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!--noptargs) { goto skip_optional_pos; } @@ -1901,19 +1887,14 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("str", "argument 'errors'", "str", fastargs[2]); goto exit; } - Py_ssize_t errors_length; - errors = PyUnicode_AsUTF8AndSize(fastargs[2], &errors_length); + errors = _PyUnicode_AsUTF8NoNUL(fastargs[2]); if (errors == NULL) { goto exit; } - if (strlen(errors) != (size_t)errors_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_pos: return_value = unicode_new_impl(type, x, encoding, errors); exit: return return_value; } -/*[clinic end generated code: output=9d243c63e951e31d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=80d039a395f835c3 input=a9049054013a1b77]*/ diff --git a/Python/clinic/Python-tokenize.c.h b/Python/clinic/Python-tokenize.c.h index 9e64c2055a8db6..ebdb6dbc73a980 100644 --- a/Python/clinic/Python-tokenize.c.h +++ b/Python/clinic/Python-tokenize.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() static PyObject * tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, @@ -68,19 +69,14 @@ tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("tokenizeriter", "argument 'encoding'", "str", fastargs[2]); goto exit; } - Py_ssize_t encoding_length; - encoding = PyUnicode_AsUTF8AndSize(fastargs[2], &encoding_length); + encoding = _PyUnicode_AsUTF8NoNUL(fastargs[2]); if (encoding == NULL) { goto exit; } - if (strlen(encoding) != (size_t)encoding_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } skip_optional_kwonly: return_value = tokenizeriter_new_impl(type, readline, extra_tokens, encoding); exit: return return_value; } -/*[clinic end generated code: output=4c448f34d9c835c0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f9091945b96e09b7 input=a9049054013a1b77]*/ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 890dee5a51b546..0a32ca28da2ef2 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(builtin___import____doc__, "__import__($module, /, name, globals=None, locals=None, fromlist=(),\n" @@ -415,15 +416,10 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj _PyArg_BadArgument("compile", "argument 'mode'", "str", args[2]); goto exit; } - Py_ssize_t mode_length; - mode = PyUnicode_AsUTF8AndSize(args[2], &mode_length); + mode = _PyUnicode_AsUTF8NoNUL(args[2]); if (mode == NULL) { goto exit; } - if (strlen(mode) != (size_t)mode_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } if (!noptargs) { goto skip_optional_pos; } @@ -1579,4 +1575,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=2454c3867f2a73f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ac4800dc25f87f6e input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 955ced2e493764..11bd8ba5bc2d5f 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() PyDoc_STRVAR(sys_addaudithook__doc__, "addaudithook($module, /, hook)\n" @@ -92,15 +93,10 @@ sys_audit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("audit", "argument 1", "str", args[0]); goto exit; } - Py_ssize_t event_length; - event = PyUnicode_AsUTF8AndSize(args[0], &event_length); + event = _PyUnicode_AsUTF8NoNUL(args[0]); if (event == NULL) { goto exit; } - if (strlen(event) != (size_t)event_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } __clinic_args = PyTuple_FromArray(args + 1, nargs - 1); if (__clinic_args == NULL) { goto exit; @@ -1575,15 +1571,10 @@ sys_activate_stack_trampoline(PyObject *module, PyObject *arg) _PyArg_BadArgument("activate_stack_trampoline", "argument", "str", arg); goto exit; } - Py_ssize_t backend_length; - backend = PyUnicode_AsUTF8AndSize(arg, &backend_length); + backend = _PyUnicode_AsUTF8NoNUL(arg); if (backend == NULL) { goto exit; } - if (strlen(backend) != (size_t)backend_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } return_value = sys_activate_stack_trampoline_impl(module, backend); exit: @@ -2226,4 +2217,4 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=0de5ea347ec24d8a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dbca81c5072bd0e8 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 5f663a50ab8ce8..8db4c5fe22e26d 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -4,6 +4,7 @@ from types import NoneType from typing import Any +import libclinic from libclinic import fail, NullType, unspecified, NULL, c_bytes_repr, c_unichar_repr from libclinic.function import ( Function, Parameter, @@ -835,12 +836,13 @@ def post_parsing(self) -> str: return "" def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 's': - return self.format_code(""" - if (!PyUnicode_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} + if self.format_unit not in ('s', 'z'): + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + + # Decode a value already known to be a str. Shared by 's' and 'z', + # which differ only in whether None is also accepted. + if limited_capi: + decode = """ Py_ssize_t {length_name}; {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); if ({paramname} == NULL) {{{{ @@ -850,35 +852,52 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str', limited_capi=limited_capi), - length_name=self.length_name) + """ + else: + # _PyUnicode_AsUTF8NoNUL() is the same code, but its Py_ssize_t + # stays in the callee's frame. An address-taken local here would + # give every generated parser a stack protector canary. + self.add_include('pycore_unicodeobject.h', + '_PyUnicode_AsUTF8NoNUL()') + decode = """ + {paramname} = _PyUnicode_AsUTF8NoNUL({argname}); + if ({paramname} == NULL) {{{{ + goto exit; + }}}} + """ + if self.format_unit == 'z': - return self.format_code(""" + expected = 'str or None' + code = """ if ({argname} == Py_None) {{{{ {paramname} = NULL; }}}} else if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t {length_name}; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); - if ({paramname} == NULL) {{{{ - goto exit; - }}}} - if (strlen({paramname}) != (size_t){length_name}) {{{{ - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - }}}} + {decode} }}}} else {{{{ {bad_argument} goto exit; }}}} - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'str or None', limited_capi=limited_capi), - length_name=self.length_name) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) + """ + else: + expected = 'str' + code = """ + if (!PyUnicode_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}} + {decode} + """ + + return self.format_code( + libclinic.linear_format( + libclinic.normalize_snippet(code), + decode=libclinic.normalize_snippet(decode)), + argname=argname, + bad_argument=self.bad_argument(displayname, expected, + limited_capi=limited_capi), + length_name=self.length_name) # # This is the fourth or fifth rewrite of registering all the From 3f9118f1d17ae2ff0e25bd07fce9940ba13afdff Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 5 Oct 2026 20:44:28 +0200 Subject: [PATCH 092/240] gh-151157: Speed up PyObject_CallMethod via _PyObject_GetMethod (#151158) * Speed up PyObject_CallMethod via _PyObject_GetMethod PyObject_CallMethod (and _PyObject_CallMethod, PyEval_CallMethod, _PyObject_CallMethodId and the _SizeT variant) resolved the method with PyObject_GetAttr, which builds a temporary bound-method object on every call, then called it. Resolve the method with _PyObject_GetMethod instead (the same lookup the interpreter uses for obj.name(...)) and call it directly via _PyObject_VectorcallPrepend, skipping the bound-method allocation. Behaviour is unchanged: same attribute semantics, same "attribute of type ... is not callable" error, and the historical PyObject_CallMethod(o, m, "O", tuple) -> o.m(*tuple) unpacking. The shared helper callmethod() and _PyObject_CallMethodFormat() are no longer needed: their only caller (traceback.c) now uses PyObject_CallFunction, so both are removed. Co-Authored-By: Claude Opus 4.8 (1M context) * Use _PyObject_GetMethodStackRef so method calls scale in free-threading Resolve the method in callmethod_va() via _PyObject_GetMethodStackRef() instead of _PyObject_GetMethod(). The StackRef variant returns the method as a deferred reference, avoiding the per-call atomic refcount on the shared method object that otherwise serializes threads in the free-threaded build. Co-Authored-By: Claude Opus 4.8 (1M context) * rename to callmethod * review comments * review comments --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Kumar Aditya Co-authored-by: Inada Naoki --- Include/internal/pycore_call.h | 6 -- Objects/call.c | 113 +++++++++++++++++++++------------ Python/traceback.c | 5 +- 3 files changed, 75 insertions(+), 49 deletions(-) diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 477b3c20e53f6e..824bb7f9499a9a 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -57,12 +57,6 @@ extern PyObject* _PyObject_Call( PyObject *args, PyObject *kwargs); -extern PyObject * _PyObject_CallMethodFormat( - PyThreadState *tstate, - PyObject *callable, - const char *format, - ...); - // Export for 'array' shared extension PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *obj, diff --git a/Objects/call.c b/Objects/call.c index 9718642473103c..a79f167aacc8c0 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -612,18 +612,70 @@ _PyObject_CallFunction_SizeT(PyObject *callable, const char *format, ...) } -static PyObject* -callmethod(PyThreadState *tstate, PyObject* callable, const char *format, va_list va) +static PyObject * +callmethod(PyThreadState *tstate, PyObject *obj, PyObject *name, + const char *format, va_list va) { - assert(callable != NULL); + PyObject *result = NULL; + + _PyCStackRef self, method; + _PyThreadState_PushCStackRef(tstate, &self); + _PyThreadState_PushCStackRef(tstate, &method); + self.ref = PyStackRef_FromPyObjectBorrow(obj); + + int res = _PyObject_GetMethodStackRef(tstate, &self.ref, name, &method.ref); + if (res < 0) { + goto exit; + } + + PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); + PyObject *self_obj = PyStackRef_AsPyObjectBorrow(self.ref); + if (!PyCallable_Check(callable)) { _PyErr_Format(tstate, PyExc_TypeError, "attribute of type '%.200s' is not callable", Py_TYPE(callable)->tp_name); - return NULL; + goto exit; } - return _PyObject_CallFunctionVa(tstate, callable, format, va); + /* Build the positional arguments from the format string. */ + PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; + Py_ssize_t nargs = 0; + PyObject **built = NULL; + if (format != NULL && *format != '\0') { + built = _Py_VaBuildStack(small_stack, _PY_FASTCALL_SMALL_STACK, + format, va, &nargs); + if (built == NULL) { + goto exit; + } + } + + /* Backward compat: a single tuple from "O" is unpacked. */ + PyObject *const *args = built; + Py_ssize_t n = nargs; + if (nargs == 1 && PyTuple_Check(built[0])) { + args = _PyTuple_ITEMS(built[0]); + n = PyTuple_GET_SIZE(built[0]); + } + + if (self_obj != NULL) { + result = _PyObject_VectorcallPrepend(tstate, callable, self_obj, args, n, NULL); + } + else { + result = _PyObject_VectorcallTstate(tstate, callable, args, n, NULL); + } + + for (Py_ssize_t i = 0; i < nargs; i++) { + Py_DECREF(built[i]); + } + if (built != small_stack) { + PyMem_Free(built); + } + +exit: + _PyThreadState_PopCStackRef(tstate, &method); + _PyThreadState_PopCStackRef(tstate, &self); + return result; } PyObject * @@ -635,17 +687,17 @@ PyObject_CallMethod(PyObject *obj, const char *name, const char *format, ...) return null_error(tstate); } - PyObject *callable = PyObject_GetAttrString(obj, name); - if (callable == NULL) { + PyObject *name_obj = PyUnicode_FromString(name); + if (name_obj == NULL) { return NULL; } va_list va; va_start(va, format); - PyObject *retval = callmethod(tstate, callable, format, va); + PyObject *retval = callmethod(tstate, obj, name_obj, format, va); va_end(va); - Py_DECREF(callable); + Py_DECREF(name_obj); return retval; } @@ -660,17 +712,17 @@ PyEval_CallMethod(PyObject *obj, const char *name, const char *format, ...) return null_error(tstate); } - PyObject *callable = PyObject_GetAttrString(obj, name); - if (callable == NULL) { + PyObject *name_obj = PyUnicode_FromString(name); + if (name_obj == NULL) { return NULL; } va_list va; va_start(va, format); - PyObject *retval = callmethod(tstate, callable, format, va); + PyObject *retval = callmethod(tstate, obj, name_obj, format, va); va_end(va); - Py_DECREF(callable); + Py_DECREF(name_obj); return retval; } @@ -684,17 +736,11 @@ _PyObject_CallMethod(PyObject *obj, PyObject *name, return null_error(tstate); } - PyObject *callable = PyObject_GetAttr(obj, name); - if (callable == NULL) { - return NULL; - } - va_list va; va_start(va, format); - PyObject *retval = callmethod(tstate, callable, format, va); + PyObject *retval = callmethod(tstate, obj, name, format, va); va_end(va); - Py_DECREF(callable); return retval; } @@ -710,30 +756,17 @@ _PyObject_CallMethodId(PyObject *obj, _Py_Identifier *name, _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS - PyObject *callable = _PyObject_GetAttrId(obj, name); + PyObject *name_obj = _PyUnicode_FromId(name); /* borrowed */ _Py_COMP_DIAG_POP - if (callable == NULL) { + if (name_obj == NULL) { return NULL; } va_list va; va_start(va, format); - PyObject *retval = callmethod(tstate, callable, format, va); + PyObject *retval = callmethod(tstate, obj, name_obj, format, va); va_end(va); - Py_DECREF(callable); - return retval; -} - - -PyObject * _PyObject_CallMethodFormat(PyThreadState *tstate, PyObject *callable, - const char *format, ...) -{ - assert(callable != NULL); - va_list va; - va_start(va, format); - PyObject *retval = callmethod(tstate, callable, format, va); - va_end(va); return retval; } @@ -749,17 +782,17 @@ _PyObject_CallMethod_SizeT(PyObject *obj, const char *name, return null_error(tstate); } - PyObject *callable = PyObject_GetAttrString(obj, name); - if (callable == NULL) { + PyObject *name_obj = PyUnicode_FromString(name); + if (name_obj == NULL) { return NULL; } va_list va; va_start(va, format); - PyObject *retval = callmethod(tstate, callable, format, va); + PyObject *retval = callmethod(tstate, obj, name_obj, format, va); va_end(va); - Py_DECREF(callable); + Py_DECREF(name_obj); return retval; } diff --git a/Python/traceback.c b/Python/traceback.c index ca94a735db5b8f..7dc72d48274567 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -2,7 +2,7 @@ /* Traceback implementation */ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallMethodFormat() +#include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_frame.h" // PyFrameObject #include "pycore_interp.h" // PyInterpreterState.gc @@ -397,7 +397,6 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * tail++; taillen = strlen(tail); - PyThreadState *tstate = _PyThreadState_GET(); if (PySys_GetOptionalAttr(&_Py_ID(path), &syspath) < 0) { PyErr_Clear(); goto error; @@ -437,7 +436,7 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * namebuf[len++] = SEP; strcpy(namebuf+len, tail); - binary = _PyObject_CallMethodFormat(tstate, open, "ss", namebuf, "rb"); + binary = PyObject_CallFunction(open, "ss", namebuf, "rb"); if (binary != NULL) { result = binary; goto finally; From dc0add3a65fc5de8f0ec8a687dccba435d52d97a Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 5 Oct 2026 20:09:36 +0100 Subject: [PATCH 093/240] gh-158868: Reduce ASLR in the MSan CI job (#158872) --- .github/workflows/reusable-san.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index da6306a50cf7bc..f63c2b89134aca 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -72,6 +72,8 @@ jobs: - name: MSan option setup if: inputs.sanitizer == 'MSan' run: | + sudo sysctl -w vm.mmap_rnd_bits=28 # Reduce ASLR to avoid MSan re-executing + echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV" # MSan reports false positives for memory initialized by libraries # that are not built with MSan, so disable modules that use them. From 114de198c0a05d8169cdbbd8f1e0976dc1319284 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 5 Oct 2026 20:56:24 +0100 Subject: [PATCH 094/240] gh-156545, gh-158539: Fix deep flamegraph export on small C stacks and macOS runtime lookup (#158874) --- Lib/profiling/sampling/stack_collector.py | 11 ++- .../test_sampling_profiler/test_collectors.py | 3 +- ...-10-05-18-00-00.gh-issue-156545.z1mQp7.rst | 2 + ...-10-05-18-00-01.gh-issue-158539.k8Lv2a.rst | 2 + Python/remote_debug.h | 69 +++---------------- 5 files changed, 25 insertions(+), 62 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-05-18-00-00.gh-issue-156545.z1mQp7.rst create mode 100644 Misc/NEWS.d/next/Library/2026-10-05-18-00-01.gh-issue-158539.k8Lv2a.rst diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 72a8cae8807890..60061b4a2ecf11 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -68,8 +68,8 @@ def export(self, filename): return True -# Bounded by the unwinder's maximum captured stack depth (MAX_FRAMES). -_FLAMEGRAPH_RECURSION_MARGIN = 2000 +# Allow for tree conversion and the dict/list frames in the Python JSON encoder. +_FLAMEGRAPH_RECURSION_MARGIN = 6000 class FlamegraphCollector(StackTraceCollector): @@ -498,7 +498,12 @@ def _get_source_lines(self, func): return None def _create_flamegraph_html(self, data): - data_json = json.dumps(data) + try: + data_json = json.dumps(data) + except RecursionError: + # The C encoder can exhaust the C stack independently of the + # Python recursion limit. iterencode() uses the Python encoder. + data_json = "".join(json.JSONEncoder().iterencode(data)) template_dir = importlib.resources.files(__package__) vendor_dir = template_dir / "_vendor" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 36713ff1314ff9..d4ba2df5044559 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -8,7 +8,7 @@ import tempfile import unittest -from test.support import is_emscripten, set_recursion_limit +from test.support import is_emscripten, run_with_limited_c_stack, set_recursion_limit try: import _remote_debugging # noqa: F401 @@ -611,6 +611,7 @@ def test_flamegraph_collector_empty_export_fails(self): self.assertFalse(export_ok) self.assertEqual(os.path.getsize(flamegraph_out.name), 0) + @run_with_limited_c_stack(size=1024 * 1024) def test_flamegraph_deep_stack_export(self): flamegraph_out = tempfile.NamedTemporaryFile( suffix=".html", delete=False diff --git a/Misc/NEWS.d/next/Library/2026-10-05-18-00-00.gh-issue-156545.z1mQp7.rst b/Misc/NEWS.d/next/Library/2026-10-05-18-00-00.gh-issue-156545.z1mQp7.rst new file mode 100644 index 00000000000000..dd67abce09da80 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-05-18-00-00.gh-issue-156545.z1mQp7.rst @@ -0,0 +1,2 @@ +Fix deep flamegraph exports on platforms with small C stacks by falling back +to the Python JSON encoder when the C encoder runs out of stack space. diff --git a/Misc/NEWS.d/next/Library/2026-10-05-18-00-01.gh-issue-158539.k8Lv2a.rst b/Misc/NEWS.d/next/Library/2026-10-05-18-00-01.gh-issue-158539.k8Lv2a.rst new file mode 100644 index 00000000000000..aadac290075abf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-05-18-00-01.gh-issue-158539.k8Lv2a.rst @@ -0,0 +1,2 @@ +Fix macOS runtime section lookup for remote debugging when virtual memory +region sizes differ from Mach-O segment file sizes. diff --git a/Python/remote_debug.h b/Python/remote_debug.h index a279cd9dfd0305..a4a3a010926a7b 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -297,7 +297,6 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { static uintptr_t return_section_address64( const char* section, - mach_port_t proc_ref, uintptr_t base, void* map ) { @@ -307,11 +306,6 @@ return_section_address64( int cmd_cnt = 0; struct segment_command_64* cmd = map + sizeof(struct mach_header_64); - mach_vm_size_t size = 0; - mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); - mach_vm_address_t address = (mach_vm_address_t)base; - vm_region_basic_info_data_64_t r_info; - mach_port_t object_name; uintptr_t vmaddr = 0; for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { @@ -319,25 +313,8 @@ return_section_address64( vmaddr = cmd->vmaddr; } if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { - while (cmd->filesize != size) { - address += size; - kern_return_t ret = mach_vm_region( - proc_ref, - &address, - &size, - VM_REGION_BASIC_INFO_64, - (vm_region_info_t)&r_info, // cppcheck-suppress [uninitvar] - &count, - &object_name - ); - if (ret != KERN_SUCCESS) { - PyErr_Format(PyExc_RuntimeError, - "mach_vm_region failed while parsing 64-bit Mach-O binary " - "at base address 0x%lx (kern_return_t: %d)", - base, ret); - return 0; - } - } + // The section address only needs the image's ASLR slide. + // VM regions need not match the segment's on-disk size. int nsects = cmd->nsects; struct section_64* sec = (struct section_64*)( @@ -360,7 +337,6 @@ return_section_address64( static uintptr_t return_section_address32( const char* section, - mach_port_t proc_ref, uintptr_t base, void* map ) { @@ -370,11 +346,6 @@ return_section_address32( int cmd_cnt = 0; struct segment_command* cmd = map + sizeof(struct mach_header); - mach_vm_size_t size = 0; - mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_t); - mach_vm_address_t address = (mach_vm_address_t)base; - vm_region_basic_info_data_t r_info; - mach_port_t object_name; uintptr_t vmaddr = 0; for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { @@ -382,25 +353,8 @@ return_section_address32( vmaddr = cmd->vmaddr; } if (cmd->cmd == LC_SEGMENT && strcmp(cmd->segname, "__DATA") == 0) { - while (cmd->filesize != size) { - address += size; - kern_return_t ret = mach_vm_region( - proc_ref, - &address, - &size, - VM_REGION_BASIC_INFO, - (vm_region_info_t)&r_info, // cppcheck-suppress [uninitvar] - &count, - &object_name - ); - if (ret != KERN_SUCCESS) { - PyErr_Format(PyExc_RuntimeError, - "mach_vm_region failed while parsing 32-bit Mach-O binary " - "at base address 0x%lx (kern_return_t: %d)", - base, ret); - return 0; - } - } + // The section address only needs the image's ASLR slide. + // VM regions need not match the segment's on-disk size. int nsects = cmd->nsects; struct section* sec = (struct section*)( @@ -423,7 +377,6 @@ return_section_address32( static uintptr_t return_section_address_fat( const char* section, - mach_port_t proc_ref, uintptr_t base, void* map ) { @@ -473,11 +426,11 @@ return_section_address_fat( switch (hdr->magic) { case MH_MAGIC: case MH_CIGAM: - return return_section_address32(section, proc_ref, base, (void*)hdr); + return return_section_address32(section, base, (void*)hdr); case MH_MAGIC_64: case MH_CIGAM_64: - return return_section_address64(section, proc_ref, base, (void*)hdr); + return return_section_address64(section, base, (void*)hdr); default: PyErr_Format(PyExc_RuntimeError, @@ -496,7 +449,7 @@ return_section_address_fat( } static uintptr_t -search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_size_t size, mach_port_t proc_ref) +search_section_in_file(const char* secname, char* path, uintptr_t base) { int fd = open(path, O_RDONLY); if (fd == -1) { @@ -533,15 +486,15 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ switch (magic) { case MH_MAGIC: case MH_CIGAM: - result = return_section_address32(secname, proc_ref, base, map); + result = return_section_address32(secname, base, map); break; case MH_MAGIC_64: case MH_CIGAM_64: - result = return_section_address64(secname, proc_ref, base, map); + result = return_section_address64(secname, base, map); break; case FAT_MAGIC: case FAT_CIGAM: - result = return_section_address_fat(secname, proc_ref, base, map); + result = return_section_address_fat(secname, base, map); break; default: PyErr_Format(PyExc_RuntimeError, @@ -644,7 +597,7 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s if (strncmp(filename, substr, strlen(substr)) == 0) { PyErr_Clear(); uintptr_t result = search_section_in_file( - secname, map_filename, address, size, proc_ref); + secname, map_filename, address); if (result != 0) { if (validator == NULL || validator(handle, result)) { return result; From eab2d16b0b5497ed008286e510e219d6758bfa14 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 5 Oct 2026 16:09:17 -0700 Subject: [PATCH 095/240] GH-158878: Stop creating `Modules/Setup.local` as part of building for WASI (GH-158879) This makes it so the build process no longer needs to modify the source checkout itself. --- ...-10-05-12-15-57.gh-issue-158878.KWLqLy.rst | 2 ++ Platforms/WASI/_build.py | 24 ------------------- Platforms/WASI/_shared.py | 4 ---- 3 files changed, 2 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-10-05-12-15-57.gh-issue-158878.KWLqLy.rst diff --git a/Misc/NEWS.d/next/Build/2026-10-05-12-15-57.gh-issue-158878.KWLqLy.rst b/Misc/NEWS.d/next/Build/2026-10-05-12-15-57.gh-issue-158878.KWLqLy.rst new file mode 100644 index 00000000000000..961e4db00efd0c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-10-05-12-15-57.gh-issue-158878.KWLqLy.rst @@ -0,0 +1,2 @@ +Stop creating an empty Modules/Setup.local file as it no longer seems to be +required to statically link the extension modules. diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index fff7e3f9f8cdad..f73f112bdf9eb1 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -25,11 +25,6 @@ import _shared -LOCAL_SETUP_MARKER = ( - b"# Generated by Platforms/WASI .\n" - b"# Required to statically build extension modules." -) - WASMTIME_VAR_NAME = "WASMTIME" WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" @@ -147,18 +142,6 @@ def call(command, *, context=None, quiet=False, **kwargs): @subdir("build_python_path", clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" - if context.setup_local_path.exists(): - if context.setup_local_path.read_bytes() == LOCAL_SETUP_MARKER: - _shared.log("👍", f"{context.setup_local_path} exists ...") - else: - _shared.log( - "⚠️", - f"{context.setup_local_path} exists, but has unexpected contents", - ) - else: - _shared.log("📝", f"Creating {context.setup_local_path} ...") - context.setup_local_path.write_bytes(LOCAL_SETUP_MARKER) - configure = [os.path.relpath(context.checkout / "configure", working_dir)] if context.args: configure.extend(context.args) @@ -300,13 +283,6 @@ def clean_contents(context): _shared.log("🧹", f"Deleting {context.cross_build_path} ...") shutil.rmtree(context.cross_build_path) - if context.setup_local_path.exists(): - if context.setup_local_path.read_bytes() == LOCAL_SETUP_MARKER: - _shared.log( - "🧹", f"Deleting generated {context.setup_local_path} ..." - ) - context.setup_local_path.unlink() - @subdir("build_python_path") def pythoninfo_build_python(context, working_dir): diff --git a/Platforms/WASI/_shared.py b/Platforms/WASI/_shared.py index ee1a65859a1a4d..6c4cb156739d51 100644 --- a/Platforms/WASI/_shared.py +++ b/Platforms/WASI/_shared.py @@ -40,10 +40,6 @@ def _pybuilddir(self, build_path): relative_dir = (build_path / "pybuilddir.txt").read_text().strip() return build_path / relative_dir - @functools.cached_property - def setup_local_path(self): - return self.checkout / "Modules" / "Setup.local" - @functools.cached_property def host_triple(self): if self._host_triple: From 047157b8915604c3d3f9914cbba7e458b2abfe7b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 6 Oct 2026 01:56:03 +0200 Subject: [PATCH 096/240] Use PyDict_GetItemRef() in _elementtree.c (#158809) Replace PyDict_GetItemWithError() with PyDict_GetItemRef(). expat_default_handler() now holds a strong reference to value. --- Modules/_elementtree.c | 102 ++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 50b7121b63ea85..8d7010c7bb90c8 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1497,11 +1497,13 @@ _elementtree_Element_get_impl(ElementObject *self, PyObject *key, { if (self->extra && self->extra->attrib) { PyObject *attrib = Py_NewRef(self->extra->attrib); - PyObject *value = Py_XNewRef(PyDict_GetItemWithError(attrib, key)); - Py_DECREF(attrib); - if (value != NULL || PyErr_Occurred()) { + PyObject *value; + if (PyDict_GetItemRef(attrib, key, &value) != 0) { + Py_DECREF(attrib); + // The key exists or an error occurred return value; } + Py_DECREF(attrib); } return Py_NewRef(default_value); @@ -3177,56 +3179,58 @@ makeuniversal(XMLParserObject* self, const char* string) if (!key) return NULL; - value = Py_XNewRef(PyDict_GetItemWithError(self->names, key)); - - if (value == NULL && !PyErr_Occurred()) { - /* new name. convert to universal name, and decode as - necessary */ - - PyObject* tag; - Py_ssize_t i; + if (PyDict_GetItemRef(self->names, key, &value) != 0) { + // The key exists or an error occurred + Py_DECREF(key); + return value; + } - /* look for namespace separator */ - for (i = 0; i < size; i++) - if (string[i] == '}') - break; - if (i != size) { - /* convert to universal name */ - PyBytesWriter *writer = PyBytesWriter_Create(1 + size); - if (writer == NULL) { - Py_DECREF(key); - return NULL; - } - char *p = PyBytesWriter_GetData(writer); - p[0] = '{'; - memcpy(p+1, string, size); - size++; + /* new name. convert to universal name, and decode as + necessary */ - tag = PyBytesWriter_Finish(writer); - if (tag == NULL) { - Py_DECREF(key); - return NULL; - } - } else { - /* plain name; use key as tag */ - tag = Py_NewRef(key); - } + PyObject* tag; + Py_ssize_t i; - /* decode universal name */ - const char *p = PyBytes_AS_STRING(tag); - value = PyUnicode_DecodeUTF8(p, size, "strict"); - Py_DECREF(tag); - if (!value) { + /* look for namespace separator */ + for (i = 0; i < size; i++) + if (string[i] == '}') + break; + if (i != size) { + /* convert to universal name */ + PyBytesWriter *writer = PyBytesWriter_Create(1 + size); + if (writer == NULL) { Py_DECREF(key); return NULL; } + char *p = PyBytesWriter_GetData(writer); + p[0] = '{'; + memcpy(p+1, string, size); + size++; - /* add to names dictionary */ - if (PyDict_SetItem(self->names, key, value) < 0) { + tag = PyBytesWriter_Finish(writer); + if (tag == NULL) { Py_DECREF(key); - Py_DECREF(value); return NULL; } + } else { + /* plain name; use key as tag */ + tag = Py_NewRef(key); + } + + /* decode universal name */ + const char *p = PyBytes_AS_STRING(tag); + value = PyUnicode_DecodeUTF8(p, size, "strict"); + Py_DECREF(tag); + if (!value) { + Py_DECREF(key); + return NULL; + } + + /* add to names dictionary */ + if (PyDict_SetItem(self->names, key, value) < 0) { + Py_DECREF(key); + Py_DECREF(value); + return NULL; } Py_DECREF(key); @@ -3304,7 +3308,11 @@ expat_default_handler(void *op, const XML_Char *data_in, int data_len) if (!key) return; - value = PyDict_GetItemWithError(self->entity, key); + if (PyDict_GetItemRef(self->entity, key, &value) < 0) { + Py_DECREF(key); + return; + } + Py_DECREF(key); elementtreestate *st = self->state; if (value) { @@ -3316,8 +3324,10 @@ expat_default_handler(void *op, const XML_Char *data_in, int data_len) res = PyObject_CallOneArg(self->handle_data, value); else res = NULL; + Py_DECREF(value); Py_XDECREF(res); - } else if (!PyErr_Occurred()) { + } + else { /* Report the first error, not the last */ char message[128] = "undefined entity "; strncat(message, data_in, data_len < 100?data_len:100); @@ -3329,8 +3339,6 @@ expat_default_handler(void *op, const XML_Char *data_in, int data_len) message ); } - - Py_DECREF(key); } /* Append (action, node) to the list of events of the pull parser. */ From 85c03d47a240247d41799c2075f85f690558534c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 6 Oct 2026 02:06:09 +0200 Subject: [PATCH 097/240] gh-158810: Use _Py_TYPEOF in Py_ARRAY_LENGTH() (#158838) Fix the compiler warning "extension used" when building test_cext with clang. * test_cext no longer overrides the CC environment variable if it's already defined. * test_cext now also logs __STRICT_ANSI__ macro. --- Include/pymacro.h | 6 +++--- Lib/test/test_cext/extension.c | 5 +++++ Lib/test/test_cext/runtests.py | 7 ++++++- Lib/test/test_cext/setup.py | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h index 979e6563742b98..122d26315b51b9 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -214,12 +214,12 @@ // // gh-158810: Do not use __builtin_types_compatible_p() in strict C ANSI mode // and on C++. -#if (defined(__GNUC__) && __GNUC__ >= 4 \ +#if (defined(__GNUC__) && __GNUC__ >= 4 && defined(_Py_TYPEOF) \ && !defined(__STRICT_ANSI__) && !defined(__cplusplus)) #define Py_ARRAY_LENGTH(array) \ (sizeof(array) / sizeof((array)[0]) \ - + Py_BUILD_ASSERT_EXPR(!__builtin_types_compatible_p(typeof(array), \ - typeof(&(array)[0])))) + + Py_BUILD_ASSERT_EXPR(!__builtin_types_compatible_p(_Py_TYPEOF(array), \ + _Py_TYPEOF(&(array)[0])))) #else #define Py_ARRAY_LENGTH(array) \ (sizeof(array) / sizeof((array)[0])) diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index e392b66f19cc4a..f1528b4df7776f 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -388,6 +388,11 @@ module_exec(PyObject *module) return -1; } #endif +#ifdef __STRICT_ANSI__ + if (PyModule_AddIntConstant(module, "__STRICT_ANSI__", 1) < 0) { + return -1; + } +#endif #ifdef __cplusplus if (PyModule_AddIntMacro(module, __cplusplus) < 0) { return -1; diff --git a/Lib/test/test_cext/runtests.py b/Lib/test/test_cext/runtests.py index 86aab671200493..e8288ee5e11e44 100644 --- a/Lib/test/test_cext/runtests.py +++ b/Lib/test/test_cext/runtests.py @@ -28,7 +28,12 @@ def main(): testmod = importlib.import_module(module_name) newline = False - for name in ('__STDC_VERSION__', '__cplusplus', '_MSVC_LANG'): + for name in ( + '__STDC_VERSION__', + '__STRICT_ANSI__', + '__cplusplus', + '_MSVC_LANG', + ): try: value = getattr(testmod, name) except AttributeError: diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 58bee3e255a4b3..c5706d23748e53 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -135,7 +135,7 @@ def main(): # option emits a C++ compiler warning. Remove "-std11" option from the # CC command. cmd = (sysconfig.get_config_var('CC') or '') - if cmd is not None: + if cmd is not None and 'CC' not in os.environ: if support.MS_WINDOWS: std_prefix = '/std' else: From 9028df36bb9e2942b19164f6708f9cf8a55c5ce9 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:36:02 +0200 Subject: [PATCH 098/240] gh-157789: Use compiler option ``-Og`` in case of clang-cl debug builds (#157791) Use compiler option ``-Og`` in case of clang-cl debug builds on Windows to reduce stack usage. --- .../next/Windows/2026-09-19-13-18-48.gh-issue-157789.o6-rmo.rst | 2 ++ PCbuild/pyproject-clangcl.props | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2026-09-19-13-18-48.gh-issue-157789.o6-rmo.rst diff --git a/Misc/NEWS.d/next/Windows/2026-09-19-13-18-48.gh-issue-157789.o6-rmo.rst b/Misc/NEWS.d/next/Windows/2026-09-19-13-18-48.gh-issue-157789.o6-rmo.rst new file mode 100644 index 00000000000000..f7f998b9eced36 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-09-19-13-18-48.gh-issue-157789.o6-rmo.rst @@ -0,0 +1,2 @@ +Use compiler option ``-Og`` in case of clang-cl debug builds on Windows to +reduce stack usage. Patch by Chris Eibl. diff --git a/PCbuild/pyproject-clangcl.props b/PCbuild/pyproject-clangcl.props index 70a81ca7da306c..14843f18823ca9 100644 --- a/PCbuild/pyproject-clangcl.props +++ b/PCbuild/pyproject-clangcl.props @@ -50,6 +50,7 @@ -m32 %(AdditionalOptions) -m64 %(AdditionalOptions) -flto=thin %(AdditionalOptions) + %(AdditionalOptions) -Xclang -Og -fprofile-instr-generate=$(_CLANG_PROFILE_PATH)$(TargetName)_%m.profraw %(AdditionalOptions) -fprofile-instr-use=$(OutDir)instrumented\profdata.profdata -Wno-profile-instr-unprofiled %(AdditionalOptions) From 82c62abc4538e371c976658bc4444c8834c01dcc Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 6 Oct 2026 12:55:46 +0200 Subject: [PATCH 099/240] gh-158863: Drop redundant incref/decref pair in dict_dict_merge() (#158864) Drop redundant incref/decref pair in dict_dict_merge() On the override path insertdict() receives its own references to key and value via Py_NewRef(), and nothing in the loop body uses them afterwards, so the outer Py_INCREF/Py_DECREF pair was pure overhead. Keep it only on the non-override path, where _PyDict_Contains_KnownHash() can run arbitrary __eq__ code and *dupkey needs a live reference. Co-authored-by: Claude Fable 5.1 --- Objects/dictobject.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index bab983c2509c2c..eed9bd92c55bf2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4269,12 +4269,12 @@ dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject ** while (_PyDict_Next((PyObject*)other, &pos, &key, &value, &hash)) { int err = 0; - Py_INCREF(key); - Py_INCREF(value); if (override == 1) { err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); } else { + Py_INCREF(key); + Py_INCREF(value); err = _PyDict_Contains_KnownHash((PyObject *)mp, key, hash); if (err == 0) { err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); @@ -4287,9 +4287,9 @@ dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject ** } err = 0; } + Py_DECREF(value); + Py_DECREF(key); } - Py_DECREF(value); - Py_DECREF(key); if (err != 0) return -1; From ea0ee925ad755999d6095ba61de2652ec9275f04 Mon Sep 17 00:00:00 2001 From: Christian Aurich Zanettini Martins Date: Tue, 6 Oct 2026 12:12:15 -0300 Subject: [PATCH 100/240] gh-74112: Fix race in test_interrupt_blocking_call (GH-158914) On a loaded machine the KeyboardInterrupt could be raised before the main thread reached time.sleep(), escape the test and stop the whole regrtest run. --- Lib/idlelib/idle_test/test_run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py index d7e69f70fd87d8..b106fd7be78b0b 100644 --- a/Lib/idlelib/idle_test/test_run.py +++ b/Lib/idlelib/idle_test/test_run.py @@ -546,9 +546,11 @@ def test_interrupt_blocking_call(self): # gh-74112: interrupt the main thread blocked in time.sleep(). timer = threading.Timer(0.1, self.ex.interrupt_the_server) self.addCleanup(timer.join) - timer.start() start = time.monotonic() with self.assertRaises(KeyboardInterrupt): + # On a loaded machine the signal can arrive before the main + # thread reaches time.sleep(), so start the timer in the block. + timer.start() time.sleep(support.SHORT_TIMEOUT) self.assertLess(time.monotonic() - start, support.SHORT_TIMEOUT / 2) From 197607e8656a3c5c7a3c77926b52db409c8bae7d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Tue, 6 Oct 2026 20:34:53 +0200 Subject: [PATCH 101/240] gh-156049: Add support for building with HWAsan (#156721) Co-authored-by: Florian Mayer Co-authored-by: Anna Co-authored-by: Victor Stinner --- Doc/using/configure.rst | 13 +++ Doc/whatsnew/3.16.rst | 5 ++ Include/pyport.h | 9 ++ Lib/test/libregrtest/utils.py | 4 +- Lib/test/support/__init__.py | 17 ++-- Lib/test/test_capi/test_mem.py | 13 +++ ...-08-31-15-50-04.gh-issue-156049.FCf6hP.rst | 2 + configure | 83 +++++++++++++++++++ configure.ac | 28 +++++++ 9 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-08-31-15-50-04.gh-issue-156049.FCf6hP.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index d6f21712b103ba..ab4ec42a575369 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -1022,6 +1022,19 @@ Debug options .. versionadded:: 3.6 +.. option:: --with-hwaddress-sanitizer + + Enable HWAddressSanitizer memory error detector, ``hwasan`` (default is no). + Note that on x86-64 this uses `page aliasing + `_, + which only tags heap allocations and is unsafe for programs that ``fork()``, + including much of the test suite. + See the `LLVM HWASan design documentation + `_ + for more information. + + .. versionadded:: next + .. option:: --with-memory-sanitizer Enable MemorySanitizer allocation error detector, ``msan`` (default is no). diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 65db9bd6bf70ba..524252cab87936 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1171,6 +1171,11 @@ Build changes (Contributed by Stan Ulbrych in :gh:`139314`.) +* Add the :option:`--with-hwaddress-sanitizer` :program:`configure` option to + build with `HWAddressSanitizer `_. + + (Contributed by Stan Ulbrych, Florian Mayer and AnnaAr321 in :gh:`156049`.) + C API changes ============= diff --git a/Include/pyport.h b/Include/pyport.h index 2206beaa77221c..1af2b61eeacd28 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -567,6 +567,12 @@ extern "C" { # define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) # endif # endif +# if __has_feature(hwaddress_sanitizer) +# if !defined(_Py_ADDRESS_SANITIZER) +# define _Py_ADDRESS_SANITIZER +# define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize("hwaddress"))) +# endif +# endif # if __has_feature(thread_sanitizer) # if !defined(_Py_THREAD_SANITIZER) # define _Py_THREAD_SANITIZER @@ -577,6 +583,9 @@ extern "C" { # if defined(__SANITIZE_ADDRESS__) # define _Py_ADDRESS_SANITIZER # define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +# elif defined(__SANITIZE_HWADDRESS__) +# define _Py_ADDRESS_SANITIZER +# define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize("hwaddress"))) # endif # if defined(__SANITIZE_THREAD__) # define _Py_THREAD_SANITIZER diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 83e0575619c99a..b34f106a83ca85 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -397,7 +397,9 @@ def get_build_info(): # --with-address-sanitizer sanitizers = [] - if support.check_sanitizer(address=True): + if support.check_sanitizer(hwaddress=True): + sanitizers.append("HWASAN") + elif support.check_sanitizer(address=True): sanitizers.append("ASAN") # --with-memory-sanitizer if support.check_sanitizer(memory=True): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e9fc4cca229a03..5331615b1b1587 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -448,11 +448,12 @@ def skip_if_buildbot(reason=None): isbuildbot = False return unittest.skipIf(isbuildbot, reason) -def check_sanitizer(*, address=False, memory=False, ub=False, thread=False, - function=True): +def check_sanitizer(*, address=False, hwaddress=False, memory=False, ub=False, + thread=False, function=True): """Returns True if Python is compiled with sanitizer support""" - if not (address or memory or ub or thread): - raise ValueError('At least one of address, memory, ub or thread must be True') + if not (address or hwaddress or memory or ub or thread): + raise ValueError('At least one of address, hwaddress, memory, ub or ' + 'thread must be True') cflags = sysconfig.get_config_var('CFLAGS') or '' @@ -461,9 +462,14 @@ def check_sanitizer(*, address=False, memory=False, ub=False, thread=False, '-fsanitize=memory' in cflags or '--with-memory-sanitizer' in config_args ) + hwaddress_sanitizer = ( + '-fsanitize=hwaddress' in cflags or + '--with-hwaddress-sanitizer' in config_args + ) address_sanitizer = ( '-fsanitize=address' in cflags or - '--with-address-sanitizer' in config_args + '--with-address-sanitizer' in config_args or + hwaddress_sanitizer ) ub_sanitizer = ( '-fsanitize=undefined' in cflags or @@ -479,6 +485,7 @@ def check_sanitizer(*, address=False, memory=False, ub=False, thread=False, return ( (memory and memory_sanitizer) or (address and address_sanitizer) or + (hwaddress and hwaddress_sanitizer) or (ub and ub_sanitizer) or (thread and thread_sanitizer) or (function and function_sanitizer) diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index e6389b4e0c1e92..63e71501162452 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -182,5 +182,18 @@ class PyMemDefaultTests(PyMemDebugTests): PYTHONMALLOC = '' +@requires_subprocess() +@unittest.skipUnless(support.check_sanitizer(address=True), + 'need address sanitizer') +class AddressSanitizerTests(unittest.TestCase): + def test_buffer_overflow(self): + with support.SuppressCrashReport(): + out = assert_python_failure( + '-c', 'import _testcapi; _testcapi.pymem_buffer_overflow()', + PYTHONMALLOC='malloc', + ) + self.assertIn(b'AddressSanitizer', out.err) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Build/2026-08-31-15-50-04.gh-issue-156049.FCf6hP.rst b/Misc/NEWS.d/next/Build/2026-08-31-15-50-04.gh-issue-156049.FCf6hP.rst new file mode 100644 index 00000000000000..622d282f9e8873 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-08-31-15-50-04.gh-issue-156049.FCf6hP.rst @@ -0,0 +1,2 @@ +Add :option:`--with-hwaddress-sanitizer` to build with `HWAddressSanitizer +`_. diff --git a/configure b/configure index e5f33f0f4e5ac9..deb01864e73178 100755 --- a/configure +++ b/configure @@ -1122,6 +1122,7 @@ with_frame_pointers enable_experimental_jit with_dsymutil with_address_sanitizer +with_hwaddress_sanitizer with_memory_sanitizer with_undefined_behavior_sanitizer with_thread_sanitizer @@ -1930,6 +1931,9 @@ Optional Packages: --with-address-sanitizer enable AddressSanitizer memory error detector, 'asan' (default is no) + --with-hwaddress-sanitizer + enable HWAddressSanitizer memory error detector, + 'hwasan' (default is no) --with-memory-sanitizer enable MemorySanitizer allocation error detector, 'msan' (default is no) --with-undefined-behavior-sanitizer @@ -14157,6 +14161,85 @@ esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-hwaddress-sanitizer" >&5 +printf %s "checking for --with-hwaddress-sanitizer... " >&6; } + +# Check whether --with-hwaddress_sanitizer was given. +if test ${with_hwaddress_sanitizer+y} +then : + withval=$with_hwaddress_sanitizer; +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +printf "%s\n" "$withval" >&6; } +hwasan_flags="-fsanitize=hwaddress" +# x86-64 lacks address tagging, so HWASan needs the page aliasing mode there. +# See gh-156049 and +# https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html#supported-architectures +case $host_cpu in #( + x86_64|amd64) : + hwasan_flags="$hwasan_flags -fsanitize-hwaddress-experimental-aliasing" + ;; #( + *) : + ;; +esac +as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags__$hwasan_flags" | sed "$as_sed_sh"` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts $hwasan_flags" >&5 +printf %s "checking whether C compiler accepts $hwasan_flags... " >&6; } +if eval test \${$as_CACHEVAR+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS $hwasan_flags" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + eval "$as_CACHEVAR=yes" +else case e in #( + e) eval "$as_CACHEVAR=no" ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +eval ac_res=\$$as_CACHEVAR + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } +if eval test \"x\$"$as_CACHEVAR"\" = x"yes" +then : + +BASECFLAGS="$hwasan_flags -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="$hwasan_flags $LDFLAGS" + +else case e in #( + e) as_fn_error $? "The selected compiler doesn't support hardware address sanitizer" "$LINENO" 5 ;; +esac +fi + +# HWASan works by controlling memory allocation, our own malloc interferes, +# so disable it by default, but allow --with-pymalloc to override. +if test -z "$with_pymalloc"; then + with_pymalloc="no" +fi + +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5 printf %s "checking for --with-memory-sanitizer... " >&6; } diff --git a/configure.ac b/configure.ac index 86227335731371..a1802f303e95eb 100644 --- a/configure.ac +++ b/configure.ac @@ -3531,6 +3531,34 @@ with_pymalloc="no" ], [AC_MSG_RESULT([no])]) +AC_MSG_CHECKING([for --with-hwaddress-sanitizer]) +AC_ARG_WITH( + [hwaddress_sanitizer], + [AS_HELP_STRING( + [--with-hwaddress-sanitizer], + [enable HWAddressSanitizer memory error detector, 'hwasan' (default is no)] + )], +[ +AC_MSG_RESULT([$withval]) +hwasan_flags="-fsanitize=hwaddress" +# x86-64 lacks address tagging, so HWASan needs the page aliasing mode there. +# See gh-156049 and +# https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html#supported-architectures +AS_CASE([$host_cpu], + [x86_64|amd64], [hwasan_flags="$hwasan_flags -fsanitize-hwaddress-experimental-aliasing"] +) +AX_CHECK_COMPILE_FLAG([$hwasan_flags],[ +BASECFLAGS="$hwasan_flags -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="$hwasan_flags $LDFLAGS" +],[AC_MSG_ERROR([The selected compiler doesn't support hardware address sanitizer])]) +# HWASan works by controlling memory allocation, our own malloc interferes, +# so disable it by default, but allow --with-pymalloc to override. +if test -z "$with_pymalloc"; then + with_pymalloc="no" +fi +], +[AC_MSG_RESULT([no])]) + AC_MSG_CHECKING([for --with-memory-sanitizer]) AC_ARG_WITH( [memory_sanitizer], From 2db299ddecb8f4911446a1d1ebf1829db81e989d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 6 Oct 2026 11:53:07 -0700 Subject: [PATCH 102/240] gh-126273: Fix make clinic warnings (#158892) --- Modules/_elementtree.c | 3 +-- Modules/pyexpat.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 8d7010c7bb90c8..cee564c2c1c1f0 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -3775,7 +3775,6 @@ ignore_attribute_error(PyObject *value) } /*[clinic input] -@permit_long_summary _elementtree.XMLParser.__init__ * @@ -3793,7 +3792,7 @@ file: http://www.iana.org/assignments/character-sets static int _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target, const char *encoding) -/*[clinic end generated code: output=3ae45ec6cdf344e4 input=43dcd316382c80a2]*/ +/*[clinic end generated code: output=3ae45ec6cdf344e4 input=a77eb075d276d487]*/ { self->entity = PyDict_New(); if (!self->entity) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 4ab076224cdd91..8dddca42bbdf67 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1079,7 +1079,6 @@ pyexpat_xmlparser_GetSpecifiedAttributeCount_impl(xmlparseobject *self) } /*[clinic input] -@permit_long_summary pyexpat.xmlparser.GetInputContext Return the input which generated the current event as bytes. @@ -1093,7 +1092,7 @@ available. static PyObject * pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self) -/*[clinic end generated code: output=a88026d683fc22cc input=13840373d8320ab6]*/ +/*[clinic end generated code: output=a88026d683fc22cc input=2d605aaa9edbb415]*/ { if (self->in_callback) { int offset, size; From ec14f81e74272424778002d5d26002338796e09e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 6 Oct 2026 21:13:49 +0200 Subject: [PATCH 103/240] gh-158842: Add Py_SETREF() and Py_XSETREF() to the limited C API (#158843) Test these macros with the limited C API in _testlimitedcapi and test_cext. Move also other tests (refcount, Py_CLEAR, etc.) to _testlimitedcapi. --- Doc/data/stable_abi.dat | 2 + Doc/whatsnew/3.16.rst | 4 + Include/cpython/object.h | 81 ------ Include/refcount.h | 81 ++++++ Lib/test/test_cext/extension.c | 2 - ...-10-05-15-24-22.gh-issue-158842.g7eLtE.rst | 2 + Misc/stable_abi.toml | 4 + Modules/_testcapi/object.c | 248 ----------------- Modules/_testlimitedcapi/object.c | 249 ++++++++++++++++++ 9 files changed, 342 insertions(+), 331 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-10-05-15-24-22.gh-issue-158842.g7eLtE.rst diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 82447cc98cd8d0..622a079ee6b4f2 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -986,6 +986,7 @@ func,Py_REFCNT,3.14,, macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, +macro,Py_SETREF,3.16,, func,Py_SET_SIZE,3.15,, func,Py_SIZE,3.15,, func,Py_SetRecursionLimit,3.2,, @@ -1020,6 +1021,7 @@ macro,Py_UNBLOCK_THREADS,3.2,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, +macro,Py_XSETREF,3.16,, macro,Py_am_aiter,3.5,, macro,Py_am_anext,3.5,, macro,Py_am_await,3.5,, diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 524252cab87936..759651d6135981 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1186,6 +1186,10 @@ New features * Add :c:func:`Py_HashBuffer` to the limited C API. (Contributed by Victor Stinner in :gh:`148233`.) +* Add :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros to the limited C + API. + (Contributed by Nathan Goldbaum and Victor Stinner in :gh:`158842`.) + Porting to Python 3.16 ---------------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 5d35f983d2833c..12d2e614623e30 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -326,87 +326,6 @@ _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *, PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); -/* Safely decref `dst` and set `dst` to `src`. - * - * As in case of Py_CLEAR "the obvious" code can be deadly: - * - * Py_DECREF(dst); - * dst = src; - * - * The safe way is: - * - * Py_SETREF(dst, src); - * - * That arranges to set `dst` to `src` _before_ decref'ing, so that any code - * triggered as a side-effect of `dst` getting torn down no longer believes - * `dst` points to a valid object. - * - * Temporary variables are used to only evaluate macro arguments once and so - * avoid the duplication of side effects. _Py_TYPEOF(), C++ auto, or memcpy() - * is used to avoid a miscompilation caused by type punning. See Py_CLEAR() - * comment for implementation details about type punning. - * - * The memcpy() implementation does not emit a compiler warning if 'src' has - * not the same type than 'src': any pointer type is accepted for 'src'. - */ -#ifdef _Py_TYPEOF -#define Py_SETREF(dst, src) \ - do { \ - _Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \ - _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \ - *_tmp_dst_ptr = (src); \ - Py_DECREF(_tmp_old_dst); \ - } while (0) -#elif defined(__cplusplus) && (__cplusplus >= 201103L || _MSVC_LANG >= 201103L) -#define Py_SETREF(dst, src) \ - do { \ - auto _tmp_dst_ptr = &(dst); \ - auto _tmp_old_dst = (*_tmp_dst_ptr); \ - *_tmp_dst_ptr = (src); \ - Py_DECREF(_tmp_old_dst); \ - } while (0) -#else -#define Py_SETREF(dst, src) \ - do { \ - PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ - PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \ - PyObject *_tmp_src = _PyObject_CAST(src); \ - memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \ - Py_DECREF(_tmp_old_dst); \ - } while (0) -#endif - -/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of - * Py_DECREF(). - */ -#ifdef _Py_TYPEOF -#define Py_XSETREF(dst, src) \ - do { \ - _Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \ - _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \ - *_tmp_dst_ptr = (src); \ - Py_XDECREF(_tmp_old_dst); \ - } while (0) -#elif defined(__cplusplus) && (__cplusplus >= 201103L || _MSVC_LANG >= 201103L) -#define Py_XSETREF(dst, src) \ - do { \ - auto _tmp_dst_ptr = &(dst); \ - auto _tmp_old_dst = (*_tmp_dst_ptr); \ - *_tmp_dst_ptr = (src); \ - Py_XDECREF(_tmp_old_dst); \ - } while (0) -#else -#define Py_XSETREF(dst, src) \ - do { \ - PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ - PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \ - PyObject *_tmp_src = _PyObject_CAST(src); \ - memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \ - Py_XDECREF(_tmp_old_dst); \ - } while (0) -#endif - - /* Define a pair of assertion macros: _PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT(). diff --git a/Include/refcount.h b/Include/refcount.h index 39661680e85a18..afbecd48ab31a1 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -516,6 +516,87 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) #endif +/* Safely decref `dst` and set `dst` to `src`. + * + * As in case of Py_CLEAR "the obvious" code can be deadly: + * + * Py_DECREF(dst); + * dst = src; + * + * The safe way is: + * + * Py_SETREF(dst, src); + * + * That arranges to set `dst` to `src` _before_ decref'ing, so that any code + * triggered as a side-effect of `dst` getting torn down no longer believes + * `dst` points to a valid object. + * + * Temporary variables are used to only evaluate macro arguments once and so + * avoid the duplication of side effects. _Py_TYPEOF(), C++ auto, or memcpy() + * is used to avoid a miscompilation caused by type punning. See Py_CLEAR() + * comment for implementation details about type punning. + * + * The memcpy() implementation does not emit a compiler warning if 'src' has + * not the same type than 'src': any pointer type is accepted for 'src'. + */ +#ifdef _Py_TYPEOF +#define Py_SETREF(dst, src) \ + do { \ + _Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \ + _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + Py_DECREF(_tmp_old_dst); \ + } while (0) +#elif defined(__cplusplus) && (__cplusplus >= 201103L || _MSVC_LANG >= 201103L) +#define Py_SETREF(dst, src) \ + do { \ + auto _tmp_dst_ptr = &(dst); \ + auto _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + Py_DECREF(_tmp_old_dst); \ + } while (0) +#else +#define Py_SETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \ + PyObject *_tmp_src = _PyObject_CAST(src); \ + memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_dst); \ + } while (0) +#endif + +/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of + * Py_DECREF(). + */ +#ifdef _Py_TYPEOF +#define Py_XSETREF(dst, src) \ + do { \ + _Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \ + _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + Py_XDECREF(_tmp_old_dst); \ + } while (0) +#elif defined(__cplusplus) && (__cplusplus >= 201103L || _MSVC_LANG >= 201103L) +#define Py_XSETREF(dst, src) \ + do { \ + auto _tmp_dst_ptr = &(dst); \ + auto _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + Py_XDECREF(_tmp_old_dst); \ + } while (0) +#else +#define Py_XSETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \ + PyObject *_tmp_src = _PyObject_CAST(src); \ + memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \ + Py_XDECREF(_tmp_old_dst); \ + } while (0) +#endif + + /* Function to use in case the object pointer can be NULL: */ static inline void Py_XINCREF(PyObject *op) { diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index f1528b4df7776f..779ea9facca8c7 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -104,7 +104,6 @@ test_macros(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_CLEAR(slots[0]); assert(slots[0] == _Py_NULL); -#ifndef Py_LIMITED_API // Test Py_SETREF(): use typeof()/__typeof__() if available, or memcpy() obj = Py_None; Py_SETREF(obj, _Py_NULL); @@ -124,7 +123,6 @@ test_macros(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) slots[0] = Py_None; Py_XSETREF(slots[0], _Py_NULL); assert(slots[0] == _Py_NULL); -#endif // Test that Py_BEGIN_CRITICAL_SECTION is available dict = PyDict_New(); diff --git a/Misc/NEWS.d/next/C_API/2026-10-05-15-24-22.gh-issue-158842.g7eLtE.rst b/Misc/NEWS.d/next/C_API/2026-10-05-15-24-22.gh-issue-158842.g7eLtE.rst new file mode 100644 index 00000000000000..222d5279a3fa3a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-10-05-15-24-22.gh-issue-158842.g7eLtE.rst @@ -0,0 +1,2 @@ +Add :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros to the limited C +API. Patch by Nathan Goldbaum and Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 0425fba42cbe2c..751c186674c7c9 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2884,3 +2884,7 @@ [function.Py_HashBuffer] added = '3.16' +[macro.Py_SETREF] + added = '3.16' +[macro.Py_XSETREF] + added = '3.16' diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 425ec540b62dcc..8fcb86e33e55a8 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -246,63 +246,6 @@ test_py_set_immortal(PyObject *self, PyObject *unused) Py_RETURN_NONE; } -static PyObject * -_test_incref(PyObject *ob) -{ - return Py_NewRef(ob); -} - -static PyObject * -test_xincref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored)) -{ - PyObject *obj = PyLong_FromLong(0); - Py_XINCREF(_test_incref(obj)); - Py_DECREF(obj); - Py_DECREF(obj); - Py_DECREF(obj); - Py_RETURN_NONE; -} - - -static PyObject * -test_incref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored)) -{ - PyObject *obj = PyLong_FromLong(0); - Py_INCREF(_test_incref(obj)); - Py_DECREF(obj); - Py_DECREF(obj); - Py_DECREF(obj); - Py_RETURN_NONE; -} - - -static PyObject * -test_xdecref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored)) -{ - Py_XDECREF(PyLong_FromLong(0)); - Py_RETURN_NONE; -} - - -static PyObject * -test_decref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored)) -{ - Py_DECREF(PyLong_FromLong(0)); - Py_RETURN_NONE; -} - - -static PyObject * -test_incref_decref_API(PyObject *ob, PyObject *Py_UNUSED(ignored)) -{ - PyObject *obj = PyLong_FromLong(0); - Py_IncRef(obj); - Py_DecRef(obj); - Py_DecRef(obj); - Py_RETURN_NONE; -} - - #ifdef Py_REF_DEBUG static PyObject * negative_refcount(PyObject *self, PyObject *Py_UNUSED(args)) @@ -343,186 +286,6 @@ decref_freed_object(PyObject *self, PyObject *Py_UNUSED(args)) #endif -// Test Py_CLEAR() macro -static PyObject* -test_py_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - // simple case with a variable - PyObject *obj = PyList_New(0); - if (obj == NULL) { - return NULL; - } - Py_CLEAR(obj); - assert(obj == NULL); - - // gh-98724: complex case, Py_CLEAR() argument has a side effect - PyObject* array[1]; - array[0] = PyList_New(0); - if (array[0] == NULL) { - return NULL; - } - - PyObject **p = array; - Py_CLEAR(*p++); - assert(array[0] == NULL); - assert(p == array + 1); - - Py_RETURN_NONE; -} - - -// Test Py_SETREF() and Py_XSETREF() macros, similar to test_py_clear() -static PyObject* -test_py_setref(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - // Py_SETREF() simple case with a variable - PyObject *obj = PyList_New(0); - if (obj == NULL) { - return NULL; - } - Py_SETREF(obj, NULL); - assert(obj == NULL); - - // Py_XSETREF() simple case with a variable - PyObject *obj2 = PyList_New(0); - if (obj2 == NULL) { - return NULL; - } - Py_XSETREF(obj2, NULL); - assert(obj2 == NULL); - // test Py_XSETREF() when the argument is NULL - Py_XSETREF(obj2, NULL); - assert(obj2 == NULL); - - // gh-98724: complex case, Py_SETREF() argument has a side effect - PyObject* array[1]; - array[0] = PyList_New(0); - if (array[0] == NULL) { - return NULL; - } - - PyObject **p = array; - Py_SETREF(*p++, NULL); - assert(array[0] == NULL); - assert(p == array + 1); - - // gh-98724: complex case, Py_XSETREF() argument has a side effect - PyObject* array2[1]; - array2[0] = PyList_New(0); - if (array2[0] == NULL) { - return NULL; - } - - PyObject **p2 = array2; - Py_XSETREF(*p2++, NULL); - assert(array2[0] == NULL); - assert(p2 == array2 + 1); - - // test Py_XSETREF() when the argument is NULL - p2 = array2; - Py_XSETREF(*p2++, NULL); - assert(array2[0] == NULL); - assert(p2 == array2 + 1); - - Py_RETURN_NONE; -} - - -#define TEST_REFCOUNT() \ - do { \ - PyObject *obj = PyList_New(0); \ - if (obj == NULL) { \ - return NULL; \ - } \ - assert(Py_REFCNT(obj) == 1); \ - \ - /* test Py_NewRef() */ \ - PyObject *ref = Py_NewRef(obj); \ - assert(ref == obj); \ - assert(Py_REFCNT(obj) == 2); \ - Py_DECREF(ref); \ - \ - /* test Py_XNewRef() */ \ - PyObject *xref = Py_XNewRef(obj); \ - assert(xref == obj); \ - assert(Py_REFCNT(obj) == 2); \ - Py_DECREF(xref); \ - \ - assert(Py_XNewRef(NULL) == NULL); \ - \ - Py_DECREF(obj); \ - Py_RETURN_NONE; \ - } while (0) - - -// Test Py_NewRef() and Py_XNewRef() macros -static PyObject* -test_refcount_macros(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - TEST_REFCOUNT(); -} - -#undef Py_NewRef -#undef Py_XNewRef - -// Test Py_NewRef() and Py_XNewRef() functions, after undefining macros. -static PyObject* -test_refcount_funcs(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - TEST_REFCOUNT(); -} - - -// Test Py_Is() function -#define TEST_PY_IS() \ - do { \ - PyObject *o_none = Py_None; \ - PyObject *o_true = Py_True; \ - PyObject *o_false = Py_False; \ - PyObject *obj = PyList_New(0); \ - if (obj == NULL) { \ - return NULL; \ - } \ - \ - /* test Py_Is() */ \ - assert(Py_Is(obj, obj)); \ - assert(!Py_Is(obj, o_none)); \ - \ - /* test Py_None */ \ - assert(Py_Is(o_none, o_none)); \ - assert(!Py_Is(obj, o_none)); \ - \ - /* test Py_True */ \ - assert(Py_Is(o_true, o_true)); \ - assert(!Py_Is(o_false, o_true)); \ - assert(!Py_Is(obj, o_true)); \ - \ - /* test Py_False */ \ - assert(Py_Is(o_false, o_false)); \ - assert(!Py_Is(o_true, o_false)); \ - assert(!Py_Is(obj, o_false)); \ - \ - Py_DECREF(obj); \ - Py_RETURN_NONE; \ - } while (0) - -// Test Py_Is() macro -static PyObject* -test_py_is_macros(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - TEST_PY_IS(); -} - -#undef Py_Is - -// Test Py_Is() function, after undefining its macro. -static PyObject* -test_py_is_funcs(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - TEST_PY_IS(); -} - - static PyObject * clear_managed_dict(PyObject *self, PyObject *obj) { @@ -597,21 +360,10 @@ static PyMethodDef test_methods[] = { {"pyobject_is_unique_temporary_new_object", pyobject_is_unique_temporary_new_object, METH_NOARGS}, {"test_py_try_inc_ref", test_py_try_inc_ref, METH_NOARGS}, {"test_py_set_immortal", test_py_set_immortal, METH_NOARGS}, - {"test_xincref_doesnt_leak",test_xincref_doesnt_leak, METH_NOARGS}, - {"test_incref_doesnt_leak", test_incref_doesnt_leak, METH_NOARGS}, - {"test_xdecref_doesnt_leak",test_xdecref_doesnt_leak, METH_NOARGS}, - {"test_decref_doesnt_leak", test_decref_doesnt_leak, METH_NOARGS}, - {"test_incref_decref_API", test_incref_decref_API, METH_NOARGS}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, {"decref_freed_object", decref_freed_object, METH_NOARGS}, #endif - {"test_py_clear", test_py_clear, METH_NOARGS}, - {"test_py_setref", test_py_setref, METH_NOARGS}, - {"test_refcount_macros", test_refcount_macros, METH_NOARGS}, - {"test_refcount_funcs", test_refcount_funcs, METH_NOARGS}, - {"test_py_is_macros", test_py_is_macros, METH_NOARGS}, - {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, {"is_uniquely_referenced", is_uniquely_referenced, METH_O}, {"pyobject_dump", pyobject_dump, METH_VARARGS}, diff --git a/Modules/_testlimitedcapi/object.c b/Modules/_testlimitedcapi/object.c index 594ba60f9abf20..03debb56b33c4b 100644 --- a/Modules/_testlimitedcapi/object.c +++ b/Modules/_testlimitedcapi/object.c @@ -64,10 +64,259 @@ test_constants(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } + +static PyObject * +_test_incref(PyObject *ob) +{ + return Py_NewRef(ob); +} + +static PyObject * +test_xincref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + PyObject *obj = PyLong_FromLong(0); + Py_XINCREF(_test_incref(obj)); + Py_DECREF(obj); + Py_DECREF(obj); + Py_DECREF(obj); + Py_RETURN_NONE; +} + + +static PyObject * +test_incref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + PyObject *obj = PyLong_FromLong(0); + Py_INCREF(_test_incref(obj)); + Py_DECREF(obj); + Py_DECREF(obj); + Py_DECREF(obj); + Py_RETURN_NONE; +} + + +static PyObject * +test_xdecref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + Py_XDECREF(PyLong_FromLong(0)); + Py_RETURN_NONE; +} + + +static PyObject * +test_decref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + Py_DECREF(PyLong_FromLong(0)); + Py_RETURN_NONE; +} + + +static PyObject * +test_incref_decref_API(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + PyObject *obj = PyLong_FromLong(0); + Py_IncRef(obj); + Py_DecRef(obj); + Py_DecRef(obj); + Py_RETURN_NONE; +} + + +// Test Py_CLEAR() macro +static PyObject* +test_py_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // simple case with a variable + PyObject *obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + Py_CLEAR(obj); + assert(obj == NULL); + + // gh-98724: complex case, Py_CLEAR() argument has a side effect + PyObject* array[1]; + array[0] = PyList_New(0); + if (array[0] == NULL) { + return NULL; + } + + PyObject **p = array; + Py_CLEAR(*p++); + assert(array[0] == NULL); + assert(p == array + 1); + + Py_RETURN_NONE; +} + + +// Test Py_SETREF() and Py_XSETREF() macros, similar to test_py_clear() +static PyObject* +test_py_setref(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // Py_SETREF() simple case with a variable + PyObject *obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + Py_SETREF(obj, NULL); + assert(obj == NULL); + + // Py_XSETREF() simple case with a variable + PyObject *obj2 = PyList_New(0); + if (obj2 == NULL) { + return NULL; + } + Py_XSETREF(obj2, NULL); + assert(obj2 == NULL); + // test Py_XSETREF() when the argument is NULL + Py_XSETREF(obj2, NULL); + assert(obj2 == NULL); + + // gh-98724: complex case, Py_SETREF() argument has a side effect + PyObject* array[1]; + array[0] = PyList_New(0); + if (array[0] == NULL) { + return NULL; + } + + PyObject **p = array; + Py_SETREF(*p++, NULL); + assert(array[0] == NULL); + assert(p == array + 1); + + // gh-98724: complex case, Py_XSETREF() argument has a side effect + PyObject* array2[1]; + array2[0] = PyList_New(0); + if (array2[0] == NULL) { + return NULL; + } + + PyObject **p2 = array2; + Py_XSETREF(*p2++, NULL); + assert(array2[0] == NULL); + assert(p2 == array2 + 1); + + // test Py_XSETREF() when the argument is NULL + p2 = array2; + Py_XSETREF(*p2++, NULL); + assert(array2[0] == NULL); + assert(p2 == array2 + 1); + + Py_RETURN_NONE; +} + + +#define TEST_REFCOUNT() \ + do { \ + PyObject *obj = PyList_New(0); \ + if (obj == NULL) { \ + return NULL; \ + } \ + assert(Py_REFCNT(obj) == 1); \ + \ + /* test Py_NewRef() */ \ + PyObject *ref = Py_NewRef(obj); \ + assert(ref == obj); \ + assert(Py_REFCNT(obj) == 2); \ + Py_DECREF(ref); \ + \ + /* test Py_XNewRef() */ \ + PyObject *xref = Py_XNewRef(obj); \ + assert(xref == obj); \ + assert(Py_REFCNT(obj) == 2); \ + Py_DECREF(xref); \ + \ + assert(Py_XNewRef(NULL) == NULL); \ + \ + Py_DECREF(obj); \ + Py_RETURN_NONE; \ + } while (0) + + +// Test Py_NewRef() and Py_XNewRef() macros +static PyObject* +test_refcount_macros(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + TEST_REFCOUNT(); +} + +#undef Py_NewRef +#undef Py_XNewRef + +// Test Py_NewRef() and Py_XNewRef() functions, after undefining macros. +static PyObject* +test_refcount_funcs(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + TEST_REFCOUNT(); +} + + +// Test Py_Is() function +#define TEST_PY_IS() \ + do { \ + PyObject *o_none = Py_None; \ + PyObject *o_true = Py_True; \ + PyObject *o_false = Py_False; \ + PyObject *obj = PyList_New(0); \ + if (obj == NULL) { \ + return NULL; \ + } \ + \ + /* test Py_Is() */ \ + assert(Py_Is(obj, obj)); \ + assert(!Py_Is(obj, o_none)); \ + \ + /* test Py_None */ \ + assert(Py_Is(o_none, o_none)); \ + assert(!Py_Is(obj, o_none)); \ + \ + /* test Py_True */ \ + assert(Py_Is(o_true, o_true)); \ + assert(!Py_Is(o_false, o_true)); \ + assert(!Py_Is(obj, o_true)); \ + \ + /* test Py_False */ \ + assert(Py_Is(o_false, o_false)); \ + assert(!Py_Is(o_true, o_false)); \ + assert(!Py_Is(obj, o_false)); \ + \ + Py_DECREF(obj); \ + Py_RETURN_NONE; \ + } while (0) + +// Test Py_Is() macro +static PyObject* +test_py_is_macros(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + TEST_PY_IS(); +} + +#undef Py_Is + +// Test Py_Is() function, after undefining its macro. +static PyObject* +test_py_is_funcs(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + TEST_PY_IS(); +} + + static PyMethodDef test_methods[] = { {"get_constant", get_constant, METH_VARARGS}, {"get_constant_borrowed", get_constant_borrowed, METH_VARARGS}, {"test_constants", test_constants, METH_NOARGS}, + {"test_xincref_doesnt_leak",test_xincref_doesnt_leak, METH_NOARGS}, + {"test_incref_doesnt_leak", test_incref_doesnt_leak, METH_NOARGS}, + {"test_xdecref_doesnt_leak",test_xdecref_doesnt_leak, METH_NOARGS}, + {"test_decref_doesnt_leak", test_decref_doesnt_leak, METH_NOARGS}, + {"test_incref_decref_API", test_incref_decref_API, METH_NOARGS}, + {"test_py_clear", test_py_clear, METH_NOARGS}, + {"test_py_setref", test_py_setref, METH_NOARGS}, + {"test_refcount_macros", test_refcount_macros, METH_NOARGS}, + {"test_refcount_funcs", test_refcount_funcs, METH_NOARGS}, + {"test_py_is_macros", test_py_is_macros, METH_NOARGS}, + {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {NULL}, }; From 978a8befa5ec9b28b161a56b16d326330aef4733 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 6 Oct 2026 21:15:14 +0200 Subject: [PATCH 104/240] gh-157695: Parse also _Py_HAVE_xxx variables in sysconfig (#158926) Add parse_config_h() tests to test_sysconfig. --- Lib/sysconfig/__init__.py | 9 ++--- Lib/test/test_sysconfig.py | 68 +++++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index fe55f48647e9a8..0657e458806e9b 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -437,9 +437,10 @@ def parse_config_h(fp, vars=None): if vars is None: vars = {} import re - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - quoted_re = re.compile('^"(.*)"$') + name_rx = '(?:[A-Z]|_Py_)[A-Za-z0-9_]+' + define_rx = re.compile(fr"#define ({name_rx}) (.*)\n") + undef_rx = re.compile(fr"/[*] #undef ({name_rx}) [*]/\n") + quoted_re = re.compile(r'"(.*)"') while True: line = fp.readline() @@ -448,7 +449,7 @@ def parse_config_h(fp, vars=None): m = define_rx.match(line) if m: n, v = m.group(1, 2) - if mq := quoted_re.match(v): + if mq := quoted_re.fullmatch(v): v = mq.group(1) try: if n in _ALWAYS_STR: diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 126e314db14916..6b90f2960672b6 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -29,7 +29,7 @@ get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, _expand_vars, _get_preferred_schemes, - is_python_build, _PROJECT_BASE) + is_python_build, _PROJECT_BASE, parse_config_h) from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name import _imp import _osx_support @@ -767,6 +767,72 @@ def test_sysconfig_config_vars_no_prefix_cache(self): self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) self.assertEqual(config_vars['platbase'], sys.exec_prefix) + def test_parse_config_h(self): + config = textwrap.dedent(''' + #ifndef Py_PYCONFIG_H + #define Py_PYCONFIG_H + + /* C comment */ + + #define ALIGNOF_LONG 8 + #define HAVE_ACCEPT 1 + #define _Py_HAVE_COSPI 1 + #define INVALID_NUMBER abc + #define ALT_SOABI "cpython-316t-x86_64-linux-gnu" + + // Undef macros must be written as "/* #undef NAME */": + // name must be valid and there is not value. + /* #undef ANDROID_API_LEVEL */ + #undef IGNORE_UNDEF + /* #undef IGNORE_VALUE 1 */ + + # _ALWAYS_STR: don't convert values to an integer, + # but quotes are removed + #define IPHONEOS_DEPLOYMENT_TARGET "13.0" + #define MACOSX_DEPLOYMENT_TARGET 10 + + // Spaces are tolerated after the name, not before + #define SPACES_AFTER 1 + #define IGNORED_SPACES_BEFORE 1 + + // Ignore macro without value + #define IGNORE_NO_VALUE + + // Ignore macros with an invalid name + #define _PRIVATE_IGNORED 1 + #define aLOWER_IGNORED 1 + #define 123IGNORED 1 + #define INVALID-NAME 1 + #define INVALID#NAME 1 + #define NONASCII_NAME_é 1 + + // Ignore single letter names + #define A 1 + /* #undef A */ + + #endif /*Py_PYCONFIG_H*/ + ''') + + filename = TESTFN + self.addCleanup(unlink, filename) + with open(filename, "w", encoding="utf-8") as fp: + fp.write(config) + vars = {} + with open(filename, encoding="utf-8") as fp: + parse_config_h(fp, vars) + expected = { + 'ALIGNOF_LONG': 8, + 'HAVE_ACCEPT': 1, + '_Py_HAVE_COSPI': 1, + 'INVALID_NUMBER': 'abc', + 'ALT_SOABI': 'cpython-316t-x86_64-linux-gnu', + 'ANDROID_API_LEVEL': 0, + 'IPHONEOS_DEPLOYMENT_TARGET': '13.0', + 'MACOSX_DEPLOYMENT_TARGET': '10', # str, not int + 'SPACES_AFTER': 1, + } + self.assertEqual(vars, expected) + class MakefileTests(unittest.TestCase): From 6318730aeebdf7282fa9a18aca52af4e87c0c39d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 6 Oct 2026 23:02:13 +0200 Subject: [PATCH 105/240] gh-158893: Make os.strerror() thread-safe (#158927) Make os.strerror() thread-safe: use the reentrant strerror_r() function if available. * The configure script now checks if strerror_r() is supported. * Add a stress test to test_free_threading.test_os (new module). * Add a comment on decode_current_locale() assertion which fails if the input string is mutated. * Add an assertion to _Py_DecodeLocale() to detect if the input string was mutated during the function call. --- Lib/test/test_free_threading/test_os.py | 36 +++++++++ ...-10-06-20-03-01.gh-issue-158893.B6A53q.rst | 2 + Modules/posixmodule.c | 73 +++++++++++++++++++ Python/fileutils.c | 14 +++- configure | 44 +++++++++++ configure.ac | 3 + pyconfig.h.in | 3 + 7 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_free_threading/test_os.py create mode 100644 Misc/NEWS.d/next/Library/2026-10-06-20-03-01.gh-issue-158893.B6A53q.rst diff --git a/Lib/test/test_free_threading/test_os.py b/Lib/test/test_free_threading/test_os.py new file mode 100644 index 00000000000000..72582da29015e9 --- /dev/null +++ b/Lib/test/test_free_threading/test_os.py @@ -0,0 +1,36 @@ +import errno +import os +import sysconfig +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestOs(unittest.TestCase): + @unittest.skipUnless(sysconfig.get_config_var('_Py_HAVE_STRERROR_R'), + 'need _Py_HAVE_STRERROR_R macro') + def test_strerror(self): + # gh-158893: os.strerror() is implemented with strerror_r() which is + # thread safe. Well, check if it's actually the case. + last_error = max([getattr(errno, name) for name in dir(errno) + if name.startswith('E')]) + test_errors = tuple(range(1, last_error + 1)) + loops = 20 + + def worker(): + for _ in range(loops): + for i in test_errors: + os.strerror(i) + + run_concurrently( + worker_func=worker, nthreads=NTHREADS + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-10-06-20-03-01.gh-issue-158893.B6A53q.rst b/Misc/NEWS.d/next/Library/2026-10-06-20-03-01.gh-issue-158893.B6A53q.rst new file mode 100644 index 00000000000000..9345b46da0f476 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-06-20-03-01.gh-issue-158893.B6A53q.rst @@ -0,0 +1,2 @@ +Make :func:`os.strerror` thread-safe: use the reentrant ``strerror_r()`` +function if available. Patch by Victor Stinner. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index eacf6556c1ffa4..7eb17164113756 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13923,6 +13923,78 @@ static PyObject * os_strerror_impl(PyObject *module, int code) /*[clinic end generated code: output=baebf09fa02a78f2 input=75a8673d97915a91]*/ { +#ifdef _Py_HAVE_STRERROR_R + // Check which strerror_r() API is used +# if defined(__GLIBC__) && !((_POSIX_C_SOURCE >= 200112L) && !defined(_GNU_SOURCE)) +# define Py_STRERROR_R_GNU +# elif defined(__ANDROID__) && defined(_GNU_SOURCE) +# define Py_STRERROR_R_GNU +# endif +#endif + +#ifdef Py_STRERROR_R_GNU + // Implementation for the GNU flavor of strerror_r() + + // On Linux, the longest translated strerror() message is 86 bytes + // (including the NUL byte). + char buffer[100]; + char *message = strerror_r(code, buffer, Py_ARRAY_LENGTH(buffer)); + // The strerror_r() GNU flavor doesn't provide a way to check if the error + // message was truncated or not. + // + // When the buffer is used, a trailing NUL byte is always written. + assert(message != buffer || memchr(buffer, 0, Py_ARRAY_LENGTH(buffer)) != NULL); + return PyUnicode_DecodeLocale(message, "surrogateescape"); + +#elif defined(_Py_HAVE_STRERROR_R) + // Implementation for the XSI-compliant flavor of strerror_r() + + // On Linux and FreeBSD, the longest translated strerror() message is 86 + // bytes (including the NUL byte). + char small_buffer[100]; + size_t buflen = Py_ARRAY_LENGTH(small_buffer); + char *buffer = NULL; +#ifndef NDEBUG + // Make sure that strerror_r() writes a trailing null byte + small_buffer[buflen - 1] = '#'; +#endif + int len = strerror_r(code, small_buffer, buflen); + if (len == ERANGE) { + while (len == ERANGE) { + if (buflen > (size_t)PY_SSIZE_T_MAX / 2) { + PyMem_Free(buffer); + PyErr_NoMemory(); + return NULL; + } + buflen = buflen * 2; + + char *new_buffer = PyMem_Realloc(buffer, buflen); + if (new_buffer == NULL) { + PyMem_Free(buffer); + PyErr_NoMemory(); + return NULL; + } + buffer = new_buffer; +#ifndef NDEBUG + buffer[buflen - 1] = '#'; +#endif + len = strerror_r(code, buffer, buflen); + } + } + else { + buffer = small_buffer; + } + + // strerror_r() always writes a trailing NUL byte + assert(memchr(buffer, 0, buflen) != NULL); + PyObject *result = PyUnicode_DecodeLocale(buffer, "surrogateescape"); + if (buffer != small_buffer) { + PyMem_Free(buffer); + } + return result; + +#else + // strerror() implementation char *message = strerror(code); if (message == NULL) { PyErr_SetString(PyExc_ValueError, @@ -13930,6 +14002,7 @@ os_strerror_impl(PyObject *module, int code) return NULL; } return PyUnicode_DecodeLocale(message, "surrogateescape"); +#endif } diff --git a/Python/fileutils.c b/Python/fileutils.c index 8ed88047b35e4f..9deb474820b55f 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -538,8 +538,14 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, // +1 to write also the trailing NUL character size_t count = _Py_mbstowcs(res, arg, argsize + 1); if (count != DECODE_ERROR) { - // Success + // String decoded successfully. + + // gh-158893: This assertion can fail if the input string was + // mutated during this function call. For example, the assertion + // fails on decoding strerror() result if another thread mutated + // the string in-place by calling strerror() in parallel. assert(count == argsize); + *wstr = res; if (wlen != NULL) { *wlen = count; @@ -694,6 +700,7 @@ _Py_DecodeLocale(const char* arg, wchar_t **wstr, size_t *wlen, assert(wstr != NULL); #ifdef Py_DEBUG + size_t arglen = strlen(arg); size_t wlen_canary = (size_t)-2; if (wlen) { *wlen = wlen_canary; @@ -719,6 +726,11 @@ _Py_DecodeLocale(const char* arg, wchar_t **wstr, size_t *wlen, // Success assert(*wstr != NULL); #ifdef Py_DEBUG + // gh-158893: Detect if the input string was mutated during the + // function call. For example, the assertion fails on decoding + // strerror() result if another thread mutated the string in-place by + // calling strerror() in parallel. + assert(strlen(arg) == arglen); if (wlen != NULL) { assert(*wlen == wcslen(*wstr)); } diff --git a/configure b/configure index deb01864e73178..c0c0e5b193c206 100755 --- a/configure +++ b/configure @@ -21730,6 +21730,50 @@ then : fi + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for strerror_r" >&5 +printf %s "checking for strerror_r... " >&6; } +if test ${ac_cv_func_strerror_r+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +void *x=strerror_r + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_strerror_r=yes +else case e in #( + e) ac_cv_func_strerror_r=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_strerror_r" >&5 +printf "%s\n" "$ac_cv_func_strerror_r" >&6; } + if test "x$ac_cv_func_strerror_r" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_STRERROR_R 1" >>confdefs.h + +fi + + + + + + # os.statx uses Linux's statx function. AIX also has a function named statx, # but it's unrelated. Check only on Linux (including Android). case $ac_sys_system in #( diff --git a/configure.ac b/configure.ac index a1802f303e95eb..9e83af6c2d95ca 100644 --- a/configure.ac +++ b/configure.ac @@ -5587,6 +5587,9 @@ AC_CHECK_FUNCS([ \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) +PY_CHECK_FUNC_PRIVATE([strerror_r], [@%:@include ]) + + # os.statx uses Linux's statx function. AIX also has a function named statx, # but it's unrelated. Check only on Linux (including Android). AS_CASE([$ac_sys_system], diff --git a/pyconfig.h.in b/pyconfig.h.in index d1de60ad757f4d..08306a1dc8ba13 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -2214,6 +2214,9 @@ /* Define if you have the 'sinpi' function. */ #undef _Py_HAVE_SINPI +/* Define if you have the 'strerror_r' function. */ +#undef _Py_HAVE_STRERROR_R + /* Define if you have the 'tanpi' function. */ #undef _Py_HAVE_TANPI From 1818fba7e737320b4241fdde095146a4c52c0e63 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 6 Oct 2026 23:03:46 +0200 Subject: [PATCH 106/240] Use _testlimitedcapi in test_free_threading.test_capi (#158935) Replace ctypes with _testlimitedcapi. ctypes is not always available, and _testlimitedcapi is a thin wrapper to the C API. Skip the test if _testlimitedcapi is missing. --- Lib/test/test_free_threading/test_capi.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_free_threading/test_capi.py b/Lib/test/test_free_threading/test_capi.py index b0be94629af009..36021eb33d5900 100644 --- a/Lib/test/test_free_threading/test_capi.py +++ b/Lib/test/test_free_threading/test_capi.py @@ -1,14 +1,12 @@ -import ctypes.util import sys import unittest +from test.support import import_helper from test.support import threading_helper from test.support.threading_helper import run_concurrently - -@ctypes.util.wrap_dll_function(ctypes.pythonapi) -def PyImport_AddModuleRef(name: ctypes.c_char_p) -> ctypes.py_object: - pass +# Skip the test if the extension is missing +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') @threading_helper.requires_working_threading() @@ -20,6 +18,7 @@ def test_pyimport_addmoduleref_thread_safe(self): NUM_ITERS = 10 NTHREADS = 4 + PyImport_AddModuleRef = _testlimitedcapi.PyImport_AddModuleRef module_name = f"test_free_threading_addmoduleref_{id(self)}" module_name_bytes = module_name.encode() sys.modules.pop(module_name, None) From 0f1d4a79f61d6a2e848b6b9c42f0f5f14033832d Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Tue, 6 Oct 2026 22:19:55 -0400 Subject: [PATCH 107/240] gh-158936: Resolve lazy imports consistently for dataclass annotations (#158939) With string annotations, dataclasses looks up the first name of each annotation to detect ClassVar, InitVar and KW_ONLY. If that name was a lazy import that could not be resolved, the lookup raised and the class could not be created, while the same class works with non-string annotations. This resolves lazy imports the way annotationlib does: try to resolve them, and treat a failure like a missing name. --- Lib/dataclasses.py | 6 +++++- Lib/test/test_dataclasses/__init__.py | 5 +++++ Lib/test/test_dataclasses/dataclass_lazy_broken.py | 1 + Lib/test/test_dataclasses/dataclass_lazy_str.py | 12 ++++++++++++ .../2026-10-06-20-33-52.gh-issue-158936.flCm_m.rst | 3 +++ 5 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_dataclasses/dataclass_lazy_broken.py create mode 100644 Lib/test/test_dataclasses/dataclass_lazy_str.py create mode 100644 Misc/NEWS.d/next/Library/2026-10-06-20-33-52.gh-issue-158936.flCm_m.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index bd317aa9b0f2f3..5ec91cddaa3e10 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -806,7 +806,11 @@ def _get_type_from_annotation(annotation, cls): path = match[1].split(".") root = sys.modules.get(cls.__module__) for path_item in path: - root = getattr(root, path_item.strip(), None) + try: + root = getattr(root, path_item.strip(), None) + except Exception: + # Like annotationlib, ignore lazy imports that cannot be resolved. + return None if root is None: return None diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index a89999bb97938c..2b4defe35f8a33 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -4525,6 +4525,11 @@ def test_classvar_module_level_import(self): # won't exist on the instance. self.assertNotIn('not_iv4', c.__dict__) + def test_lazy_import_annotations(self): + from test.test_dataclasses import dataclass_lazy_str as m + self.assertEqual([f.name for f in fields(m.C)], ['x', 'y']) + self.assertNotIsInstance(vars(m)['t'], types.LazyImportType) + def test_text_annotations(self): from test.test_dataclasses import dataclass_textanno diff --git a/Lib/test/test_dataclasses/dataclass_lazy_broken.py b/Lib/test/test_dataclasses/dataclass_lazy_broken.py new file mode 100644 index 00000000000000..d9d29834fe1276 --- /dev/null +++ b/Lib/test/test_dataclasses/dataclass_lazy_broken.py @@ -0,0 +1 @@ +raise RuntimeError("module fails to import") diff --git a/Lib/test/test_dataclasses/dataclass_lazy_str.py b/Lib/test/test_dataclasses/dataclass_lazy_str.py new file mode 100644 index 00000000000000..cd600fb6f38071 --- /dev/null +++ b/Lib/test/test_dataclasses/dataclass_lazy_str.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import dataclasses +lazy import typing as t +lazy from _dataclass_test_missing_module import Missing +lazy from test.test_dataclasses.dataclass_lazy_broken import Broken + +@dataclasses.dataclass +class C: + x: Missing | None = None + y: Broken | None = None + cv: t.ClassVar[int] = 0 diff --git a/Misc/NEWS.d/next/Library/2026-10-06-20-33-52.gh-issue-158936.flCm_m.rst b/Misc/NEWS.d/next/Library/2026-10-06-20-33-52.gh-issue-158936.flCm_m.rst new file mode 100644 index 00000000000000..5b023af8b1b91b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-06-20-33-52.gh-issue-158936.flCm_m.rst @@ -0,0 +1,3 @@ +:func:`~dataclasses.dataclass` no longer fails when a string annotation names a +:ref:`lazy import ` that cannot be resolved, matching the +behavior for non-string annotations. From 7d3fca4e047f5b0a9beccc447c03cd6b26259b3b Mon Sep 17 00:00:00 2001 From: Bhuvansh Date: Wed, 7 Oct 2026 08:04:54 +0530 Subject: [PATCH 108/240] gh-153354: Fix bool conversion in generated __annotate__ (#153360) --- Lib/test/test_annotationlib.py | 31 +++++++++++++++++++ ...-07-09-13-28-27.gh-issue-153354.It4Jj_.rst | 2 ++ Python/codegen.c | 1 + 3 files changed, 34 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-09-13-28-27.gh-issue-153354.It4Jj_.rst diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index a7d3bca7402bf2..b5388f3f30aee0 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1720,6 +1720,37 @@ def annotate(format, /): with self.assertRaises(DemoException): annotationlib.call_annotate_function(annotate, format=fmt) + def test_generated_annotate_non_bool_compare_result(self): + class NonBool(int): + def __gt__(self, other): + return None + + # Function __annotate__ + def func(x: int): + pass + + self.assertEqual( + func.__annotate__(NonBool(1)), + {"x": int}, + ) + + # Class __annotate__ + class C: + y: int + + self.assertEqual( + C.__annotate__(NonBool(1)), + {"y": int}, + ) + + # Module __annotate__ + ns = {} + exec("z: int", ns) + + self.assertEqual( + ns["__annotate__"](NonBool(1)), + {"z": int}, + ) class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-09-13-28-27.gh-issue-153354.It4Jj_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-09-13-28-27.gh-issue-153354.It4Jj_.rst new file mode 100644 index 00000000000000..4cfd4302b3c34d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-09-13-28-27.gh-issue-153354.It4Jj_.rst @@ -0,0 +1,2 @@ +Fix a crash in compiler-generated ``__annotate__`` functions when the +annotation format comparison returned a non-boolean result. diff --git a/Python/codegen.c b/Python/codegen.c index 7143d9abc4255e..840dd3103a81c5 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -729,6 +729,7 @@ codegen_emit_annotations_prologue(compiler *c, location loc) ADDOP_I(c, loc, LOAD_FAST, 0); ADDOP_LOAD_CONST_NEW(c, loc, value_with_fake_globals); ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); + ADDOP(c, loc, TO_BOOL); NEW_JUMP_TARGET_LABEL(c, body); ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); From 2639fd65ff8e0c1949c480a8e670fe9c2467a1f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Wed, 7 Oct 2026 04:52:58 +0200 Subject: [PATCH 109/240] Fix the `sys.remote_exec` row in the audit events table (#158940) --- Doc/library/sys.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index d5c38d6c05cc3b..b447e4253a8561 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -2113,7 +2113,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only See :ref:`remote-debugging` for more information about the remote debugging mechanism. - .. audit-event:: sys.remote_exec pid script_path + .. audit-event:: sys.remote_exec pid,script_path When the code is executed in the remote process, an :ref:`auditing event ` ``sys.remote_exec`` is raised with From 6c25c98e08c250ed063e051b99135b8080d2d0d5 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Wed, 7 Oct 2026 12:44:30 +0200 Subject: [PATCH 110/240] gh-156049: Skip `AddressSanitizerTests` on NoGIL builds (GH-158932) --- Lib/test/test_capi/test_mem.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 63e71501162452..21b51d133447da 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -183,6 +183,7 @@ class PyMemDefaultTests(PyMemDebugTests): @requires_subprocess() +@support.requires_gil_enabled("PYTHONMALLOC='malloc' needs GIL build") @unittest.skipUnless(support.check_sanitizer(address=True), 'need address sanitizer') class AddressSanitizerTests(unittest.TestCase): From 37cc8dd0b9a86c1fd87c4b08360ef3dc6a2c306b Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 7 Oct 2026 11:51:10 +0100 Subject: [PATCH 111/240] Doc: update abi3t build tool support (GH-158171) Co-authored-by: Petr Viktorin Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/howto/abi3t-migration.rst | 25 ++++++++++++++++++++----- Doc/whatsnew/3.15.rst | 11 +++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/Doc/howto/abi3t-migration.rst b/Doc/howto/abi3t-migration.rst index 501106679b02bd..3aa11a0dbf653d 100644 --- a/Doc/howto/abi3t-migration.rst +++ b/Doc/howto/abi3t-migration.rst @@ -169,14 +169,22 @@ or :c:member:`PyTypeObject.tp_itemsize`), it cannot be ported to ``abi3t`` 3.15. +.. _abi3t-migration-build: + Setting up the build ==================== -If you use a build tool (such as setuptools, meson-python, scikit-build-core), -search its documentation for a way to select ``abi3t``. -At the time of writing, not all of them have this; but if your tool does, -use it. -You may want to verify that it set the right flag by temporarily adding the +If you use a build tool, search its documentation for "``abi3t``", and follow +any instructions to select the ABI. +For reference, here are direct links for several popular build tools: + +- `meson-python + `__ +- `scikit-build-core + `__ +- `Maturin `__ + +You may want to verify that the tool set the right flag by temporarily adding the following just after ``#include ``:: #if Py_TARGET_ABI3T+0 <= 0x30f0000 @@ -185,6 +193,13 @@ following just after ``#include ``:: This should result in a different error than "``abi3t`` define is not set". +.. seealso:: + + `Building and distributing abi3t extensions + `__: + Build configuration and wheel testing examples in the community-maintained + Python Free-Threading Guide. + .. note:: If your build tool doesn't support ``abi3t`` yet, set the following macro diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ca0f67e7c76ace..ea7e7a68c08f9f 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -572,16 +572,19 @@ the existing Stable ABI (``abi3``) and the version-specific ABI for free-threading (``cp315t``) separately. Stable ABI for Free-Threaded Builds should typically -be selected in a build tool (such as, for example, Setuptools, meson-python, -scikit-build-core, or Maturin). -At the time of writing, these tools do **not** support ``abi3t``. -If this is the case for your tool, compile for ``cp315t`` separately. +be selected in a build tool. Tools with ``abi3t`` support include +meson-python, scikit-build-core, and Maturin; see +:ref:`abi3t-migration-build` for links to their documentation on ``abi3t`` builds. +If your build tool does not support ``abi3t``, compile for ``cp315t`` separately. If not using a build tool -- or when writing such a tool -- you can select ``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed in :ref:`abi3-compiling`. A practical :ref:`migration guide ` for switching to ``abi3t`` is available. +The community-maintained `Python Free-Threading Guide +`__ provides examples of building +and testing ``abi3t`` wheels. .. seealso:: :pep:`803` for further details. From ba9d96ff8d7c572a0c90f072102d54b1fc3044dc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Oct 2026 14:12:32 +0200 Subject: [PATCH 112/240] gh-149044: Avoid C++ warnings in PySlot_END macro (GH-158866) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Initialize all fields in the macro. Otherwise, g++ -Wall -Wextra complains that some fields are not initialized [-Werror=missing-field-initializers]: error: missing initializer for member ‘PySlot::sl_flags’ error: missing initializer for member ‘PySlot::’ error: missing initializer for member ‘PySlot::’ --- Include/slots.h | 2 +- .../next/C_API/2026-10-05-17-09-35.gh-issue-149044.1HUiag.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-10-05-17-09-35.gh-issue-149044.1HUiag.rst diff --git a/Include/slots.h b/Include/slots.h index 4bf7bda0208a8d..4afdfd3dcc3107 100644 --- a/Include/slots.h +++ b/Include/slots.h @@ -42,7 +42,7 @@ struct PySlot { #define PySlot_STATIC_DATA(NAME, VALUE) \ {.sl_id=(NAME), .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} -#define PySlot_END {0} +#define PySlot_END {0, 0, {0}, {_Py_NULL}} // Macros without designated initializers (for C++11 and below): diff --git a/Misc/NEWS.d/next/C_API/2026-10-05-17-09-35.gh-issue-149044.1HUiag.rst b/Misc/NEWS.d/next/C_API/2026-10-05-17-09-35.gh-issue-149044.1HUiag.rst new file mode 100644 index 00000000000000..3861d23e27f8b3 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-10-05-17-09-35.gh-issue-149044.1HUiag.rst @@ -0,0 +1,2 @@ +Fix the :c:macro:`PySlot_END` macro for C++: initialize all members to avoid +compiler warnings. Patch by Victor Stinner. From c120cae880759160ac9bc042e4c615aeeb26d7a9 Mon Sep 17 00:00:00 2001 From: he_tao <53343436+hetaozdh@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:21:00 -0500 Subject: [PATCH 113/240] gh-158790: Improve performance for list.insert() and del list[i] free-threaded builds (gh-158791) --- ...-10-04-01-40-31.gh-issue-158790.F3kQ7z.rst | 1 + Objects/listobject.c | 44 ++++++++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-01-40-31.gh-issue-158790.F3kQ7z.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-01-40-31.gh-issue-158790.F3kQ7z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-01-40-31.gh-issue-158790.F3kQ7z.rst new file mode 100644 index 00000000000000..b509071e16eea2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-01-40-31.gh-issue-158790.F3kQ7z.rst @@ -0,0 +1 @@ +Speed up :meth:`list.insert` and item deletion via :keyword:`del` in free-threaded builds by using :manpage:`memmove(3)`. diff --git a/Objects/listobject.c b/Objects/listobject.c index 962ed70aed6b8d..87a669f203589d 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -478,10 +478,43 @@ end:; return ret; } +static void ptr_wise_atomic_memmove(PyListObject *a, PyObject **dest, + PyObject **src, Py_ssize_t n); + +static inline void +list_shift_items_right_lock_held(PyListObject *self, Py_ssize_t first, + Py_ssize_t last) +{ +#ifdef Py_GIL_DISABLED + ptr_wise_atomic_memmove(self, &self->ob_item[first + 1], + &self->ob_item[first], last - first); +#else + PyObject **items = self->ob_item; + for (Py_ssize_t i = last; --i >= first; ) { + items[i + 1] = items[i]; + } +#endif +} + +static inline void +list_shift_items_left_lock_held(PyListObject *self, Py_ssize_t first, + Py_ssize_t last) +{ +#ifdef Py_GIL_DISABLED + ptr_wise_atomic_memmove(self, &self->ob_item[first], + &self->ob_item[first + 1], last - first); +#else + PyObject **items = self->ob_item; + for (Py_ssize_t i = first; i < last; i++) { + items[i] = items[i + 1]; + } +#endif +} + static int ins1(PyListObject *self, Py_ssize_t where, PyObject *v) { - Py_ssize_t i, n = Py_SIZE(self); + Py_ssize_t n = Py_SIZE(self); PyObject **items; if (v == NULL) { PyErr_BadInternalCall(); @@ -500,8 +533,9 @@ ins1(PyListObject *self, Py_ssize_t where, PyObject *v) if (where > n) where = n; items = self->ob_item; - for (i = n; --i >= where; ) - FT_ATOMIC_STORE_PTR_RELEASE(items[i+1], items[i]); + if (where < n) { + list_shift_items_right_lock_held(self, where, n); + } FT_ATOMIC_STORE_PTR_RELEASE(items[where], Py_NewRef(v)); return 0; } @@ -1145,8 +1179,8 @@ list_ass_item_lock_held(PyListObject *a, Py_ssize_t i, PyObject *v) PyObject *tmp = a->ob_item[i]; if (v == NULL) { Py_ssize_t size = Py_SIZE(a); - for (Py_ssize_t idx = i; idx < size - 1; idx++) { - FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item[idx], a->ob_item[idx + 1]); + if (i < size - 1) { + list_shift_items_left_lock_held(a, i, size - 1); } Py_SET_SIZE(a, size - 1); } From 16108074b1496b1dd60dfdafc762d9e26d6f149c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Oct 2026 15:18:10 +0200 Subject: [PATCH 114/240] gh-158897: Optimize io.BufferedReader.readline() by calling memchr() (#158944) Optimize io.BufferedReader.readline() when lines are close to the buffer size (128 kB by default) or longer than the buffer size. Replace the C loop searching for the newline byte in the buffer with a memchr() call which is more efficient. In the good case, it makes readline() up to 4x faster! --- ...026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst | 5 +++++ Modules/_io/bufferedio.c | 16 +++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst diff --git a/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst new file mode 100644 index 00000000000000..4524077daefe9a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst @@ -0,0 +1,5 @@ +Optimize :meth:`!io.BufferedReader.readline` when lines are close to the +buffer size (128 kB by default) or longer than the buffer size. Replace the +C loop searching for the newline byte in the buffer with a :c:func:`!memchr` +call which is more efficient. In the good case, it makes ``readline()`` up to +4x faster! Patch by Victor Stinner. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index d6318af1263d9c..44c3f56caee18b 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1280,16 +1280,14 @@ _buffered_readline(buffered *self, Py_ssize_t limit) n = limit; } start = self->buffer; - const char *end = start + n; - s = start; - while (s < end) { - if (*s++ == '\n') { - if (PyBytesWriter_WriteBytes(writer, start, s - start) < 0) { - goto error; - } - self->pos = s - start; - goto found; + const char *newline = memchr(start, '\n', n); + if (newline != NULL) { + n = newline - start + 1; + if (PyBytesWriter_WriteBytes(writer, start, n) < 0) { + goto error; } + self->pos = n; + goto found; } if (PyBytesWriter_WriteBytes(writer, start, n) < 0) { From acfa77671ce0f342585d6f557f526c3d0a1b0a5b Mon Sep 17 00:00:00 2001 From: Shamil Date: Wed, 7 Oct 2026 18:01:09 +0300 Subject: [PATCH 115/240] gh-157176: Make time.struct_time type immutable (#157179) Co-authored-by: Victor Stinner --- Lib/test/test_structseq.py | 10 +++++++--- .../2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst | 1 + Modules/timemodule.c | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index 74506fc54de50e..04b6497265809f 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -6,6 +6,7 @@ import textwrap import time import unittest +from test import support from test.support import script_helper @@ -351,9 +352,9 @@ def test_reference_cycle(self): # was cleared first, the structseq instance would crash in the # destructor. script_helper.assert_python_ok("-c", textwrap.dedent(r""" - import time - t = time.gmtime() - type(t).refcyle = t + import os + t = os.stat('.') + type(t).refcycle = t """)) def test_replace_gc_tracked(self): @@ -365,5 +366,8 @@ def test_replace_gc_tracked(self): self.assertTrue(gc.is_tracked(replaced_struct)) + def test_struct_time_type_immutable(self): + support.check_immutable_type(self, type(time.gmtime())) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst new file mode 100644 index 00000000000000..ac2bcf09a1f2fd --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst @@ -0,0 +1 @@ +Make the :class:`time.struct_time` type immutable to prevent reference cycles between the type and its instances. Patch by Shamil Abdulaev. diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 98db74c2222f20..e5162fd3b67855 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_structseq.h" // _PyStructSequence_NewType() #include "pycore_time.h" // _PyTimeFraction #include // clock() @@ -2164,7 +2165,8 @@ time_exec(PyObject *module) } // struct_time type - state->struct_time_type = PyStructSequence_NewType(&struct_time_type_desc); + state->struct_time_type = _PyStructSequence_NewType( + &struct_time_type_desc, Py_TPFLAGS_IMMUTABLETYPE); if (state->struct_time_type == NULL) { return -1; } From 2791c35cae1fa274198bb2d3b3dccb378dfe40f3 Mon Sep 17 00:00:00 2001 From: Vasiliy Kiryanov Date: Wed, 7 Oct 2026 12:05:37 -0400 Subject: [PATCH 116/240] gh-136872: Build pymalloc in sanitizer builds, but disable it by default (GH-157934) --with-*address-sanitizer and --with-memory-sanitizer unconditionally disabled pymalloc (even when --with-pymalloc was given explicitly). Enable it, so that it's available with `PYTHONMALLOC=pymalloc`. Disable it by default on the affected sanitizer builds. --- Doc/c-api/memory.rst | 14 +++++++++++--- Doc/using/configure.rst | 12 +++++++++--- Include/internal/pycore_pymem_init.h | 4 ++++ Lib/test/test_cmd_line.py | 14 +++++++++++--- Lib/test/test_sys.py | 4 +++- .../2026-09-21-19-20-42.gh-issue-136872.a2tSc_.rst | 4 ++++ Objects/obmalloc.c | 6 ++++++ configure | 9 --------- configure.ac | 9 --------- 9 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-09-21-19-20-42.gh-issue-136872.a2tSc_.rst diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 73310670ac371c..94be7dde89dbf3 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -437,6 +437,8 @@ Release build ``"pymalloc"`` ``malloc`` Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +Release build, with ASan or MSan ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, with ASan or MSan ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug Free-threaded build ``"mimalloc"`` ``mimalloc`` ``mimalloc`` ``mimalloc`` Free-threaded debug build ``"mimalloc_debug"`` ``mimalloc`` + debug ``mimalloc`` + debug ``mimalloc`` + debug =================================== ======================= ==================== ====================== ====================== @@ -451,6 +453,10 @@ Legend: * "+ debug": with :ref:`debug hooks on the Python memory allocators `. * "Debug build": :ref:`Python build in debug mode `. +* "with ASan or MSan": sanitizer build as configured using the + :option:`--with-address-sanitizer`, + :option:`--with-hwaddress-sanitizer`, and/or + :option:`--with-memory-sanitizer` option. .. _customize-memory-allocators: @@ -705,9 +711,11 @@ This allocator is disabled if Python is configured with the :option:`--without-pymalloc` option. It can also be disabled at runtime using the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). -Typically, it makes sense to disable the pymalloc allocator when building -Python with AddressSanitizer (:option:`--with-address-sanitizer`) which helps -uncover low level bugs within the C code. +The pymalloc allocator is disabled by default when Python is built with +a sanitizer which does not track pymalloc allocations +(:option:`--with-address-sanitizer`, :option:`--with-hwaddress-sanitizer`, +:option:`--with-memory-sanitizer`). +Use :envvar:`PYTHONMALLOC=pymalloc ` to enable pymalloc. Customize pymalloc Arena Allocator ---------------------------------- diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index ab4ec42a575369..8a510a2dc0ba61 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -1016,9 +1016,9 @@ Debug options .. option:: --with-address-sanitizer Enable AddressSanitizer memory error detector, ``asan`` (default is no). - To improve ASan detection capabilities you may also want to combine this - with :option:`--without-pymalloc` to disable the specialized small-object - allocator whose allocations are not tracked by ASan. + + When built with ``asan``, Python uses ``malloc`` instead of :ref:`pymalloc ` by default. + Set :envvar:`PYTHONMALLOC=pymalloc ` to use pymalloc. .. versionadded:: 3.6 @@ -1033,6 +1033,9 @@ Debug options `_ for more information. + When built with ``hwasan``, Python uses ``malloc`` instead of :ref:`pymalloc ` by default. + Set :envvar:`PYTHONMALLOC=pymalloc ` to use pymalloc. + .. versionadded:: next .. option:: --with-memory-sanitizer @@ -1043,6 +1046,9 @@ Debug options not built with MSan, so either build all dependencies with MSan or disable the extension modules that use them in :file:`Modules/Setup.local`. + When built with ``msan``, Python uses ``malloc`` instead of :ref:`pymalloc ` by default. + Set :envvar:`PYTHONMALLOC=pymalloc ` to use pymalloc. + .. versionadded:: 3.6 .. option:: --with-undefined-behavior-sanitizer diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 2a0e0817dcc7f8..f89eb8bfc4964c 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -36,6 +36,10 @@ extern void _PyMem_MiRawFree(void *, void *); extern void* _PyMem_MiRawRealloc(void *, void *, size_t); # undef PYRAW_ALLOC # define PYRAW_ALLOC {NULL, _PyMem_MiRawMalloc, _PyMem_MiRawCalloc, _PyMem_MiRawRealloc, _PyMem_MiRawFree} +#elif defined(_Py_ADDRESS_SANITIZER) || defined(_Py_MEMORY_SANITIZER) +// Keep in sync with the default allocators in Objects/obmalloc.c. +# define PYOBJ_ALLOC PYRAW_ALLOC +# define PYMEM_ALLOC PYOBJ_ALLOC #elif defined(WITH_PYMALLOC) extern void* _PyObject_Malloc(void *, size_t); extern void* _PyObject_Calloc(void *, size_t, size_t); diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1e303dc3bf6770..0af0debd8150e3 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -845,10 +845,13 @@ def test_xdev(self): code = "import _testinternalcapi; print(_testinternalcapi.pymem_getallocatorsname())" with support.SuppressCrashReport(): out = self.run_xdev("-c", code, check_exitcode=False) - if support.with_pymalloc(): - alloc_name = "pymalloc_debug" - elif support.Py_GIL_DISABLED: + if support.Py_GIL_DISABLED: alloc_name = "mimalloc_debug" + elif support.check_sanitizer(address=True, memory=True): + # ASan and MSan builds default to malloc, even with pymalloc. + alloc_name = "malloc_debug" + elif support.with_pymalloc(): + alloc_name = "pymalloc_debug" else: alloc_name = "malloc_debug" self.assertEqual(out, alloc_name) @@ -927,10 +930,15 @@ def test_pythonmalloc(self): # Test the PYTHONMALLOC environment variable malloc = not support.Py_GIL_DISABLED pymalloc = support.with_pymalloc() + sanitizer = support.check_sanitizer(address=True, memory=True) mimalloc = support.with_mimalloc() if support.Py_GIL_DISABLED: default_name = 'mimalloc_debug' if support.Py_DEBUG else 'mimalloc' default_name_debug = 'mimalloc_debug' + elif sanitizer: + # ASan and MSan builds default to malloc, even with pymalloc. + default_name = 'malloc_debug' if support.Py_DEBUG else 'malloc' + default_name_debug = 'malloc_debug' elif pymalloc: default_name = 'pymalloc_debug' if support.Py_DEBUG else 'pymalloc' default_name_debug = 'pymalloc_debug' diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1ccdcf5a187bbb..c74ca05ca66c9c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1194,8 +1194,10 @@ def test_debugmallocstats(self): # The sysconfig vars are not available on Windows. if sys.platform != "win32": with_pymalloc = sysconfig.get_config_var("WITH_PYMALLOC") + with_sanitizer = support.check_sanitizer(address=True, memory=True) self.assertIn(b"free PyDictObjects", err) - if with_pymalloc: + # ASan and MSan builds default to malloc, even with pymalloc. + if with_pymalloc and not with_sanitizer: self.assertIn(b'Small block threshold', err) # The function has no parameter diff --git a/Misc/NEWS.d/next/Build/2026-09-21-19-20-42.gh-issue-136872.a2tSc_.rst b/Misc/NEWS.d/next/Build/2026-09-21-19-20-42.gh-issue-136872.a2tSc_.rst new file mode 100644 index 00000000000000..7e51844695d0e4 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-09-21-19-20-42.gh-issue-136872.a2tSc_.rst @@ -0,0 +1,4 @@ +Python built with :option:`--with-address-sanitizer` or +:option:`--with-memory-sanitizer` now includes pymalloc, but uses ``malloc`` +as the default memory allocator. Set +:envvar:`PYTHONMALLOC=pymalloc ` to use pymalloc. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 9a41a422467223..841fb796e1ced1 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -433,6 +433,12 @@ void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); # define PYRAW_ALLOC MIMALLOC_RAWALLOC # define PYMEM_ALLOC MIMALLOC_ALLOC # define PYOBJ_ALLOC MIMALLOC_OBJALLOC +#elif defined(_Py_ADDRESS_SANITIZER) || defined(_Py_MEMORY_SANITIZER) +// ASan and MSan do not track pymalloc blocks, so use malloc by default. +// pymalloc can still be selected at runtime. +# define PYRAW_ALLOC MALLOC_ALLOC +# define PYMEM_ALLOC MALLOC_ALLOC +# define PYOBJ_ALLOC MALLOC_ALLOC #elif defined(WITH_PYMALLOC) # define PYRAW_ALLOC MALLOC_ALLOC # define PYMEM_ALLOC PYMALLOC_ALLOC diff --git a/configure b/configure index c0c0e5b193c206..d6f5319beb7037 100755 --- a/configure +++ b/configure @@ -14151,8 +14151,6 @@ then : printf "%s\n" "$withval" >&6; } BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=address $LDFLAGS" -# ASan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" else case e in #( e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 @@ -14227,11 +14225,6 @@ else case e in #( esac fi -# HWASan works by controlling memory allocation, our own malloc interferes, -# so disable it by default, but allow --with-pymalloc to override. -if test -z "$with_pymalloc"; then - with_pymalloc="no" -fi else case e in #( e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 @@ -14293,8 +14286,6 @@ else case e in #( esac fi -# MSan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" else case e in #( e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 diff --git a/configure.ac b/configure.ac index 9e83af6c2d95ca..9ed32b686f033a 100644 --- a/configure.ac +++ b/configure.ac @@ -3526,8 +3526,6 @@ AC_ARG_WITH([address_sanitizer], AC_MSG_RESULT([$withval]) BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=address $LDFLAGS" -# ASan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" ], [AC_MSG_RESULT([no])]) @@ -3551,11 +3549,6 @@ AX_CHECK_COMPILE_FLAG([$hwasan_flags],[ BASECFLAGS="$hwasan_flags -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="$hwasan_flags $LDFLAGS" ],[AC_MSG_ERROR([The selected compiler doesn't support hardware address sanitizer])]) -# HWASan works by controlling memory allocation, our own malloc interferes, -# so disable it by default, but allow --with-pymalloc to override. -if test -z "$with_pymalloc"; then - with_pymalloc="no" -fi ], [AC_MSG_RESULT([no])]) @@ -3572,8 +3565,6 @@ AX_CHECK_COMPILE_FLAG([-fsanitize=memory],[ BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" ],[AC_MSG_ERROR([The selected compiler doesn't support memory sanitizer])]) -# MSan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" ], [AC_MSG_RESULT([no])]) From e19dc470f4c2afd7ac944896232a3a6ca0204399 Mon Sep 17 00:00:00 2001 From: Shamil Date: Wed, 7 Oct 2026 21:08:54 +0300 Subject: [PATCH 117/240] gh-157364: Fix use-after-free in io.TextIOWrapper during reentrant detach (#157370) Co-authored-by: Victor Stinner --- Lib/test/test_io/test_textio.py | 30 +++++++++++++++++++ ...-09-12-18-03-15.gh-issue-157364.Qd7mKs.rst | 2 ++ Modules/_io/textio.c | 28 +++++++++++------ 3 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-12-18-03-15.gh-issue-157364.Qd7mKs.rst diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py index f9fb84101d4a2e..65b0812f071e21 100644 --- a/Lib/test/test_io/test_textio.py +++ b/Lib/test/test_io/test_textio.py @@ -1633,6 +1633,36 @@ def make_text(buffer): wrapper.write('x') self.assertRaisesRegex(ValueError, "detached", wrapper.read) + def test_reentrant_detach_during_read(self): + # gh-157363, gh-157364: The buffer must stay alive until its active + # read operation returns. + wrapper = None + + class DetachOnRead(self.RawIOBase): + detached = False + + def readable(self): + return True + + def readinto(self, b): + if self.detached: + return 0 + self.detached = True + wrapper.detach() + b[:3] = b"ab\n" + return 3 + + for method_name in ("read", "readline"): + with self.subTest(method_name): + raw = DetachOnRead() + wrapper = self.TextIOWrapper( + self.BufferedReader(raw), encoding="utf-8") + method = getattr(wrapper, method_name) + self.assertEqual(method(), "ab\n") + with self.assertRaisesRegex(ValueError, + "underlying buffer has been detached"): + getattr(wrapper, 'buffer') + def test_reentrant_seek_during_tell(self): # gh-153539: reading short of _CHUNK_SIZE leaves residual bytes in the # snapshot, so tell() re-decodes and calls the decoder's getstate(); a diff --git a/Misc/NEWS.d/next/Library/2026-09-12-18-03-15.gh-issue-157364.Qd7mKs.rst b/Misc/NEWS.d/next/Library/2026-09-12-18-03-15.gh-issue-157364.Qd7mKs.rst new file mode 100644 index 00000000000000..3ca44d25b57910 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-12-18-03-15.gh-issue-157364.Qd7mKs.rst @@ -0,0 +1,2 @@ +Fix a use-after-free in :class:`io.TextIOWrapper` when a call to the +underlying buffer reentrantly detaches it. Patched by Shamil Abdulaev. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 8f98f7d80095b0..3736179807f45d 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -758,10 +758,8 @@ buffer_access_safe(textio *self) return NULL; } - /* Returning a borrowed reference is safe since TextIOWrapper methods are - protected by critical sections. */ _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - return self->buffer; + return Py_NewRef(self->buffer); } static PyObject * @@ -772,7 +770,9 @@ buffer_getattr(textio *self, PyObject *attr_name) return NULL; } - return PyObject_GetAttr(buffer, attr_name); + PyObject *res = PyObject_GetAttr(buffer, attr_name); + Py_DECREF(buffer); + return res; } static PyObject * @@ -783,7 +783,9 @@ buffer_callmethod_noargs(textio *self, PyObject *name) return NULL; } - return PyObject_CallMethodNoArgs(buffer, name); + PyObject *res = PyObject_CallMethodNoArgs(buffer, name); + Py_DECREF(buffer); + return res; } static PyObject * @@ -794,7 +796,9 @@ buffer_callmethod_onearg(textio *self, PyObject *name, PyObject *arg) return NULL; } - return PyObject_CallMethodOneArg(buffer, name, arg); + PyObject *res = PyObject_CallMethodOneArg(buffer, name, arg); + Py_DECREF(buffer); + return res; } static void @@ -1644,7 +1648,7 @@ _io_TextIOWrapper_detach_impl(textio *self) if (buffer == NULL) { return NULL; } - self->buffer = NULL; + Py_CLEAR(self->buffer); self->detached = 1; return buffer; } @@ -1868,7 +1872,12 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) if (needflush) { PyObject *buffer = buffer_access_safe(self); - if (buffer == NULL || _PyFile_Flush(buffer) < 0) { + if (buffer == NULL) { + return NULL; + } + int res = _PyFile_Flush(buffer); + Py_DECREF(buffer); + if (res < 0) { return NULL; } } @@ -2687,6 +2696,7 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) goto fail; } res = _PyObject_CallMethod(buf, &_Py_ID(seek), "ii", 0, 2); + Py_DECREF(buf); Py_CLEAR(cookieObj); if (res == NULL) goto fail; @@ -3441,7 +3451,7 @@ static PyObject * _io_TextIOWrapper_buffer_get_impl(textio *self) /*[clinic end generated code: output=d265a34555aa5d4b input=5951cfa148f7350a]*/ { - return Py_XNewRef(buffer_access_safe(self)); + return buffer_access_safe(self); } static PyMethodDef incrementalnewlinedecoder_methods[] = { From f9949a9728938fecad4b77c2d42be48e1a377a4f Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Wed, 7 Oct 2026 22:18:46 +0200 Subject: [PATCH 118/240] gh-157176: Move blurb for gh-157176 to correct section (#158983) --- .../2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/{Core_and_Builtins => Library}/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst (100%) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst b/Misc/NEWS.d/next/Library/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst similarity index 100% rename from Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst rename to Misc/NEWS.d/next/Library/2026-09-08-15-55-00.gh-issue-157176.yZ8a1B.rst From 15dd7356f300733b9c3fcc35f0d75e61a6ef3e05 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Oct 2026 22:31:27 +0200 Subject: [PATCH 119/240] gh-158893: Add internal _Py_strerror() function (#158982) Add a new internal _Py_strerror() function to Python/fileutils.c. It uses strerror_r() if available, or use strerror() otherwise. Replace all strerror(code) calls with _Py_strerror(code). --- Include/internal/pycore_fileutils.h | 3 + Modules/_remote_debugging/subprocess.c | 21 ++++-- Modules/_remote_debugging/threads.c | 45 ++++++++---- Modules/posixmodule.c | 81 +--------------------- Python/errors.c | 3 +- Python/fileutils.c | 94 ++++++++++++++++++++++++++ 6 files changed, 147 insertions(+), 100 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index a765eb5fe2d322..a5fe5206477d05 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -321,6 +321,9 @@ extern int _Py_GetTicksPerSecond(long *ticks_per_second); // Export for '_testcapi' shared extension PyAPI_FUNC(int) _Py_IsValidFD(int fd); +// Export for '_remote_debugging' shared extension +PyAPI_FUNC(PyObject*) _Py_strerror(int code); + #ifdef __cplusplus } #endif diff --git a/Modules/_remote_debugging/subprocess.c b/Modules/_remote_debugging/subprocess.c index 1388a75a95c47b..eeb9693cef9222 100644 --- a/Modules/_remote_debugging/subprocess.c +++ b/Modules/_remote_debugging/subprocess.c @@ -6,6 +6,7 @@ ******************************************************************************/ #include "_remote_debugging.h" +#include "pycore_fileutils.h" // _Py_strerror() #ifndef MS_WINDOWS #include @@ -229,9 +230,13 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) if (entry == NULL) { if (errno != 0) { int err = errno; - _set_debug_oserror_from_errno_with_filename(err, "/proc", - "Failed to read process directory '/proc': %s", - strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno_with_filename(err, "/proc", + "Failed to read process directory '/proc': %S", + message); + Py_DECREF(message); + } goto done; } break; @@ -259,9 +264,13 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) if (closedir(proc_dir) != 0) { int err = errno; proc_dir = NULL; - _set_debug_oserror_from_errno_with_filename(err, "/proc", - "Failed to close process directory '/proc': %s", - strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno_with_filename(err, "/proc", + "Failed to close process directory '/proc': %S", + message); + Py_DECREF(message); + } goto done; } proc_dir = NULL; diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 198134fe6cfbea..120065022ddc8e 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -6,6 +6,7 @@ ******************************************************************************/ #include "_remote_debugging.h" +#include "pycore_fileutils.h" // _Py_strerror() #ifndef MS_WINDOWS #include @@ -716,9 +717,13 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st int err = errno; closedir(dir); _Py_RemoteDebug_InitThreadsState(unwinder, st); - _set_debug_oserror_from_errno_with_filename(err, task_path, - "Failed to read process task directory '%s': %s", - task_path, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno_with_filename(err, task_path, + "Failed to read process task directory '%s': %S", + task_path, message); + Py_DECREF(message); + } return -1; } break; @@ -749,9 +754,13 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st if (closedir(dir) != 0) { int err = errno; _Py_RemoteDebug_InitThreadsState(unwinder, st); - _set_debug_oserror_from_errno_with_filename(err, task_path, - "Failed to close process task directory '%s': %s", - task_path, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno_with_filename(err, task_path, + "Failed to close process task directory '%s': %S", + task_path, message); + Py_DECREF(message); + } return -1; } st->tids = unwinder->thread_tids; @@ -816,8 +825,12 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T } if (ret < 0) { detach_threads(st, seized); - _set_debug_oserror_from_errno(err, - "Failed to seize thread %d: %s", tid, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno(err, + "Failed to seize thread %d: %S", tid, message); + Py_DECREF(message); + } _Py_RemoteDebug_InitThreadsState(unwinder, st); return -1; } @@ -827,8 +840,12 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T err = errno; if (err != ESRCH) { detach_threads(st, seized); - _set_debug_oserror_from_errno(err, - "Failed to interrupt thread %d: %s", tid, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno(err, + "Failed to interrupt thread %d: %S", tid, message); + Py_DECREF(message); + } _Py_RemoteDebug_InitThreadsState(unwinder, st); return -1; } @@ -839,8 +856,12 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T err = errno; if (err != ECHILD && err != ESRCH) { detach_threads(st, seized); - _set_debug_oserror_from_errno(err, - "waitpid failed for thread %d: %s", tid, strerror(err)); + PyObject *message = _Py_strerror(err); + if (message != NULL) { + _set_debug_oserror_from_errno(err, + "waitpid failed for thread %d: %S", tid, message); + Py_DECREF(message); + } _Py_RemoteDebug_InitThreadsState(unwinder, st); return -1; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7eb17164113756..028270e7b841fd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13923,86 +13923,7 @@ static PyObject * os_strerror_impl(PyObject *module, int code) /*[clinic end generated code: output=baebf09fa02a78f2 input=75a8673d97915a91]*/ { -#ifdef _Py_HAVE_STRERROR_R - // Check which strerror_r() API is used -# if defined(__GLIBC__) && !((_POSIX_C_SOURCE >= 200112L) && !defined(_GNU_SOURCE)) -# define Py_STRERROR_R_GNU -# elif defined(__ANDROID__) && defined(_GNU_SOURCE) -# define Py_STRERROR_R_GNU -# endif -#endif - -#ifdef Py_STRERROR_R_GNU - // Implementation for the GNU flavor of strerror_r() - - // On Linux, the longest translated strerror() message is 86 bytes - // (including the NUL byte). - char buffer[100]; - char *message = strerror_r(code, buffer, Py_ARRAY_LENGTH(buffer)); - // The strerror_r() GNU flavor doesn't provide a way to check if the error - // message was truncated or not. - // - // When the buffer is used, a trailing NUL byte is always written. - assert(message != buffer || memchr(buffer, 0, Py_ARRAY_LENGTH(buffer)) != NULL); - return PyUnicode_DecodeLocale(message, "surrogateescape"); - -#elif defined(_Py_HAVE_STRERROR_R) - // Implementation for the XSI-compliant flavor of strerror_r() - - // On Linux and FreeBSD, the longest translated strerror() message is 86 - // bytes (including the NUL byte). - char small_buffer[100]; - size_t buflen = Py_ARRAY_LENGTH(small_buffer); - char *buffer = NULL; -#ifndef NDEBUG - // Make sure that strerror_r() writes a trailing null byte - small_buffer[buflen - 1] = '#'; -#endif - int len = strerror_r(code, small_buffer, buflen); - if (len == ERANGE) { - while (len == ERANGE) { - if (buflen > (size_t)PY_SSIZE_T_MAX / 2) { - PyMem_Free(buffer); - PyErr_NoMemory(); - return NULL; - } - buflen = buflen * 2; - - char *new_buffer = PyMem_Realloc(buffer, buflen); - if (new_buffer == NULL) { - PyMem_Free(buffer); - PyErr_NoMemory(); - return NULL; - } - buffer = new_buffer; -#ifndef NDEBUG - buffer[buflen - 1] = '#'; -#endif - len = strerror_r(code, buffer, buflen); - } - } - else { - buffer = small_buffer; - } - - // strerror_r() always writes a trailing NUL byte - assert(memchr(buffer, 0, buflen) != NULL); - PyObject *result = PyUnicode_DecodeLocale(buffer, "surrogateescape"); - if (buffer != small_buffer) { - PyMem_Free(buffer); - } - return result; - -#else - // strerror() implementation - char *message = strerror(code); - if (message == NULL) { - PyErr_SetString(PyExc_ValueError, - "strerror() argument out of range"); - return NULL; - } - return PyUnicode_DecodeLocale(message, "surrogateescape"); -#endif + return _Py_strerror(code); } diff --git a/Python/errors.c b/Python/errors.c index edb1557e23f63b..0645c688a9af7e 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -842,8 +842,7 @@ PyErr_SetFromErrnoWithFilenameObjects(PyObject *exc, PyObject *filenameObject, P #ifndef MS_WINDOWS if (i != 0) { - const char *s = strerror(i); - message = PyUnicode_DecodeLocale(s, "surrogateescape"); + message = _Py_strerror(i); } else { /* Sometimes errno didn't get set */ diff --git a/Python/fileutils.c b/Python/fileutils.c index 9deb474820b55f..3ea0f59f69ad1e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -3302,3 +3302,97 @@ _Py_IsValidFD(int fd) return (fstat(fd, &st) == 0); #endif } + + +// Call strerror_r(code) if available, or use strerror() otherwise. Decode the +// result from the locale encoding using surrogateescape error handler. +// +// On success, return a Unicode string. On error, set an exception and return +// NULL. +PyObject* +_Py_strerror(int code) +/*[clinic end generated code: output=baebf09fa02a78f2 input=75a8673d97915a91]*/ +{ + const char *errors = "surrogateescape"; + +#ifdef _Py_HAVE_STRERROR_R + // Check which strerror_r() API is used +# if defined(__GLIBC__) && !((_POSIX_C_SOURCE >= 200112L) && !defined(_GNU_SOURCE)) +# define Py_STRERROR_R_GNU +# elif defined(__ANDROID__) && defined(_GNU_SOURCE) +# define Py_STRERROR_R_GNU +# endif +#endif + +#ifdef Py_STRERROR_R_GNU + // Implementation for the GNU flavor of strerror_r() + + // On Linux, the longest translated strerror() message is 86 bytes + // (including the NUL byte). + char buffer[100]; + char *message = strerror_r(code, buffer, Py_ARRAY_LENGTH(buffer)); + // The strerror_r() GNU flavor doesn't provide a way to check if the error + // message was truncated or not. + // + // When the buffer is used, a trailing NUL byte is always written. + assert(message != buffer || memchr(buffer, 0, Py_ARRAY_LENGTH(buffer)) != NULL); + return PyUnicode_DecodeLocale(message, errors); + +#elif defined(_Py_HAVE_STRERROR_R) + // Implementation for the XSI-compliant flavor of strerror_r() + + // On Linux and FreeBSD, the longest translated strerror() message is 86 + // bytes (including the NUL byte). + char small_buffer[100]; + size_t buflen = Py_ARRAY_LENGTH(small_buffer); + char *buffer = NULL; +#ifndef NDEBUG + // Make sure that strerror_r() writes a trailing null byte + small_buffer[buflen - 1] = '#'; +#endif + int len = strerror_r(code, small_buffer, buflen); + if (len == ERANGE) { + while (len == ERANGE) { + if (buflen > (size_t)PY_SSIZE_T_MAX / 2) { + PyMem_Free(buffer); + PyErr_NoMemory(); + return NULL; + } + buflen = buflen * 2; + + char *new_buffer = PyMem_Realloc(buffer, buflen); + if (new_buffer == NULL) { + PyMem_Free(buffer); + PyErr_NoMemory(); + return NULL; + } + buffer = new_buffer; +#ifndef NDEBUG + buffer[buflen - 1] = '#'; +#endif + len = strerror_r(code, buffer, buflen); + } + } + else { + buffer = small_buffer; + } + + // strerror_r() always writes a trailing NUL byte + assert(memchr(buffer, 0, buflen) != NULL); + PyObject *result = PyUnicode_DecodeLocale(buffer, errors); + if (buffer != small_buffer) { + PyMem_Free(buffer); + } + return result; + +#else + // strerror() implementation (usually not thread-safe) + char *message = strerror(code); + if (message == NULL) { + PyErr_SetString(PyExc_ValueError, + "strerror() argument out of range"); + return NULL; + } + return PyUnicode_DecodeLocale(message, errors); +#endif +} From b51d4b3561df7b228c1cdc2ffda4c97456d86942 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 7 Oct 2026 16:37:31 -0700 Subject: [PATCH 120/240] Document the commands provided by `Platforms/WASI/__main__.py` (GH-158976) Along the way, fix some bugs and add a `path` command to make is easier to programmatically figure out where things are. The README should also be self-contained enough that one could point an LLM at it for working on WASI. --- Platforms/WASI/README.md | 233 ++++++++++++++++++++++++++++++++++++- Platforms/WASI/__main__.py | 24 ++++ Platforms/WASI/_build.py | 8 +- Platforms/WASI/_package.py | 22 +--- Platforms/WASI/_shared.py | 24 +++- 5 files changed, 288 insertions(+), 23 deletions(-) diff --git a/Platforms/WASI/README.md b/Platforms/WASI/README.md index 62c82924d437b1..a62efbda536654 100644 --- a/Platforms/WASI/README.md +++ b/Platforms/WASI/README.md @@ -9,9 +9,237 @@ use WASM runtimes such as [wasmtime](https://wasmtime.dev/). **NOTE**: If you are looking for general information about WebAssembly that is not directly related to CPython, please see https://github.com/psf/webassembly. -## Build -See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). +## Working on the WASI build + +This directory provides a CLI for building and packaging WASI builds for +distribution. + +Run all commands below from the root of the CPython source checkout. + +To see all the available commands, run: + +```shell +python3 Platforms/WASI --help +``` + +The `python3` interpreter used to run this CLI must be a Python version that +is receiving bugfixes. This requirement does not apply to the build Python, +which the CLI builds from the source checkout. + + +### Prerequisites + +There are some tools that must be available to successfully build. + +1. C compiler +2. `make` +3. WASI SDK +4. Wasmtime (or some other WASI runtime configured via `--host-runner`) + +The default runner requires Wasmtime be on `PATH`. + +The WASI SDK must be the same version as specified in config.toml. The search +for the WASI SDK is done via: + +1. `--wasi-sdk` CLI option +2. `WASI_SDK_PATH` environment variable +3. `/opt` where the WASI SDK has been unpacked from its tarball + +Note that all prerequisites are included and configured appropriately in the +[WASI dev container image](https://github.com/python/cpython-devcontainers/pkgs/container/wasicontainer). +You can download it via: + +```shell +podman pull ghcr.io/python/wasicontainer:latest +``` + +The `latest` image contains the WASI SDK versions required by all supported +CPython branches. + +### Development loop + +The common way to get started is to first do a full build: + +```shell +python3 Platforms/WASI build --quiet --logdir cross-build/logs -- --with-pydebug --config-cache +``` + +In the end, you will end up with a "build Python" which is a local build of +Python used for cross-builds. You will also have the WASI build. + +Once you have the build you can run the test you want. + +Bash: +```bash +"$(python3 Platforms/WASI path)/python.sh" -m test test_os +``` + +Fish: +```fish +set -l wasi_dir (python3 Platforms/WASI path); "$wasi_dir/python.sh" -m test test_os +``` + +If you are working on C code and need a rebuild: + +```shell +python3 Platforms/WASI make-host +``` + + +### Building + +In general, +[the devguide covers how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi), +but we will cover some of the details here. + +The simplest way to get a pydebug WASI build is: + +```shell +python3 Platforms/WASI build -- --with-pydebug +``` + +This builds the build Python and the WASI build with `--with-pydebug` passed to +`configure` (as is anything that comes after `--`). The builds are placed in +the `cross-build/` directory of the source checkout, each in a subdirectory +matching the compiler triple for the build. + +You can do the two builds separately if you want: + +```shell +python3 Platforms/WASI build-python -- --with-pydebug +python3 Platforms/WASI build-host +``` + +This can be broken down even more to the separate `configure` and `make` steps: + +```shell +python3 Platforms/WASI configure-build-python -- --with-pydebug +python3 Platforms/WASI make-build-python +python3 Platforms/WASI configure-host +python3 Platforms/WASI make-host +``` + +Note that `configure-host` figures out to do a pydebug build by looking at the +build Python. + +There is a `--quiet` flag to redirect output from the underlying commands to a +directory. The `--logdir` flag controls where the log files go (which defaults +to `/tmp`). + +```shell +python3 Platforms/WASI build --quiet --logdir cross-build/logs +``` + + +### Packaging + +The `package` command is used to gather all the files necessary to make a +release and place them in an archive: + +```shell +python3 Platforms/WASI package +``` + +The files are gathered into a versioned directory inside `dist/` in the source +checkout. An archive containing that directory is placed alongside it in +`dist/`. The gathered directory includes a `bin/python3.wasmtime` file to ease +launching the interpreter. + +If you just need to gather the files for a release, you can use the `gather` +command: + +```shell +python3 Platforms/WASI gather +``` + +Both `package` and `gather` require a completed build and delete the entire +existing `dist/` directory before gathering files. + +There is no command just to archive the gathered files. + + +### Paths + +The `path` command prints the location of the build Python, WASI build, or +gathered distribution files: + +```shell +python3 Platforms/WASI path build-python +python3 Platforms/WASI path wasi +python3 Platforms/WASI path dist +``` + +With no location argument, `path` defaults to `wasi`. The `dist` location +returns the versioned distribution directory inside `dist/`, not the top-level +`dist/` directory. The `build-python` and `wasi` locations can be queried before +building; the `dist` location requires WASI build metadata to determine the +versioned directory name. + + +### Cleanup + +The `clean` command deletes the entire `cross-build/` and `dist/` directories, +including all build files, gathered distribution files, and archives: + +```shell +python3 Platforms/WASI clean +``` + + +### Testing + +To find out where the WASI build directory is, you can run: + +```shell +python3 Platforms/WASI path +``` + +From there, you can run the test suite. + +For Bash-like shells: + +```bash +make buildbottest -C "$(python3 Platforms/WASI path)" +``` + +For fish: + +```fish +make buildbottest -C (python3 Platforms/WASI path) +``` + +There is a `python.sh` file in the WASI build directory, so you can also run +tests that way. + +Bash: + +```bash +"$(python3 Platforms/WASI path)/python.sh" -m test +``` + +Fish: + +```fish +set -l wasi_dir (python3 Platforms/WASI path); "$wasi_dir/python.sh" -m test +``` + +If you want to test the files meant for distribution, the directory containing +the files can be found via `python3 Platforms/WASI path dist` and there is a +`bin/python3.wasmtime` shell script. + +Bash: + +```bash +"$(python3 Platforms/WASI path dist)/bin/python3.wasmtime" -m test +``` + +Fish: + +```fish +set -l dist_dir (python3 Platforms/WASI path dist); "$dist_dir/bin/python3.wasmtime" -m test +``` + ## Detecting WASI builds @@ -48,6 +276,7 @@ posix.uname_result( 'wasi' ``` + ### C code WASI SDK defines several built-in macros. You can dump a full list of built-ins diff --git a/Platforms/WASI/__main__.py b/Platforms/WASI/__main__.py index 58cbc44834dd28..dc1df16f829efe 100644 --- a/Platforms/WASI/__main__.py +++ b/Platforms/WASI/__main__.py @@ -68,6 +68,19 @@ def main(): package = subcommands.add_parser( "package", help="Package the host/WASI Python into an archive" ) + gather = subcommands.add_parser( + "gather", help="Gather all the files for distribution" + ) + path = subcommands.add_parser( + "path", help="Print the path to a build or distribution directory" + ) + path.add_argument( + "location", + nargs="?", + choices=("build-python", "wasi", "dist"), + default="wasi", + help="Directory whose path to print", + ) subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) @@ -144,7 +157,9 @@ def main(): make_host, build_host, pythoninfo_host, + gather, package, + path, ): subcommand.add_argument( "--host-triple", @@ -191,9 +206,18 @@ def main(): _build.pythoninfo_wasi_python(context) case "clean": _build.clean_contents(context) + case "gather": + _package.gather(context) case "package": _package.gather(context) _package.archive(context) + case "path": + paths = { + "build-python": "build_python_path", + "wasi": "wasi_build_path", + "dist": "archive_dir", + } + print(getattr(context, paths[context.location])) case None: parser.print_help() case _: diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index f73f112bdf9eb1..8d9cc3c1bf281f 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -113,6 +113,7 @@ def call(command, *, context=None, quiet=False, **kwargs): else: if (log_path := getattr(context, "log_path", None)) is None: log_path = pathlib.Path(tempfile.gettempdir()) + log_path.mkdir(parents=True, exist_ok=True) stdout = tempfile.NamedTemporaryFile( "w", encoding="utf-8", @@ -279,9 +280,10 @@ def make_wasi_python(context, working_dir): def clean_contents(context): """Delete all files created by this script.""" context.clean = True - if context.cross_build_path.exists(): - _shared.log("🧹", f"Deleting {context.cross_build_path} ...") - shutil.rmtree(context.cross_build_path) + for path in [context.cross_build_path, context.dist_path]: + if path.exists(): + _shared.log("🧹", f"Deleting {path} ...") + shutil.rmtree(path) @subdir("build_python_path") diff --git a/Platforms/WASI/_package.py b/Platforms/WASI/_package.py index d1d43e5da2843c..862ffb30130d04 100644 --- a/Platforms/WASI/_package.py +++ b/Platforms/WASI/_package.py @@ -264,18 +264,6 @@ def config_symlink(config_path, context): return [(symlink, config_path) for symlink in symlinks] -def filename_stem(context): - """Calculate the stem of the archive file name.""" - version_info = context.wasi_build_details["language"]["version_info"] - version = f"python-{version_info['major']}.{version_info['minor']}.{version_info['micro']}" - if version_info["releaselevel"] != "final": - version += version_info["releaselevel"][0] + str( - version_info["serial"] - ) - - return f"{version}-{context.host_triple}" - - def copy_files(files, base): for dest, src in files: target = base / dest @@ -296,13 +284,13 @@ def gather(context): py_version = python_version(context) py_d_version = python_version(context, debug_ok=True) - dist = context.checkout / "dist" + dist = context.dist_path if dist.exists(): _shared.log("🧹", f"Deleting {dist} ...") shutil.rmtree(dist) indent = " " - base = dist / filename_stem(context) + base = context.archive_dir _shared.log("📝", f"Copying files to {base} ...") _shared.log("📁", "bin/", spacing=indent * 2) @@ -365,12 +353,12 @@ def gather(context): def archive(context): - file_name = f"{filename_stem(context)}.tar.xz" - file_path = context.checkout / "dist" / file_name + file_name = f"{context.archive_stem}.tar.xz" + file_path = context.dist_path / file_name if file_path.exists(): _shared.log("🧹", f"Deleting {file_path} ...") file_path.unlink() - to_compress = context.checkout / "dist" / filename_stem(context) + to_compress = context.archive_dir _shared.log("🗜️", f"Archiving to {file_path} ...") mtime_format = "%Y-%m-%dT%H:%M:%SZ" if source_date_epoch := os.environ.get("SOURCE_DATE_EPOCH"): diff --git a/Platforms/WASI/_shared.py b/Platforms/WASI/_shared.py index 6c4cb156739d51..306aabd1f1c097 100644 --- a/Platforms/WASI/_shared.py +++ b/Platforms/WASI/_shared.py @@ -24,6 +24,7 @@ class Context: def __init__(self): self.here = pathlib.Path(__file__).parent + self.orig_cwd = pathlib.Path.cwd() @functools.cached_property def checkout(self): @@ -180,10 +181,31 @@ def wasi_sdk_path(self): @functools.cached_property def log_path(self): if self._log_path is not None: - return self._log_path + if not (path := self._log_path).is_absolute(): + path = (self.orig_cwd / self._log_path).resolve() + return path return pathlib.Path(tempfile.gettempdir()) + @functools.cached_property + def dist_path(self): + return self.checkout / "dist" + + @functools.cached_property + def archive_stem(self): + version_info = self.wasi_build_details["language"]["version_info"] + version = f"python-{version_info['major']}.{version_info['minor']}.{version_info['micro']}" + if version_info["releaselevel"] != "final": + version += version_info["releaselevel"][0] + str( + version_info["serial"] + ) + + return f"{version}-{self.host_triple}" + + @functools.cached_property + def archive_dir(self): + return self.dist_path / self.archive_stem + def log(emoji, message, *, spacing=None): """Print a notification with an emoji. From 12fbc90e063b118f489241ae9f806a9d694a5682 Mon Sep 17 00:00:00 2001 From: Ilya Egorov <0x42005e1f@gmail.com> Date: Thu, 8 Oct 2026 03:56:38 +0400 Subject: [PATCH 121/240] gh-158592: Reallocate the list's array when deleting by index (gh-158602) --- .../2026-10-02-16-19-35.gh-issue-158592.V6W21J.rst | 1 + Objects/listobject.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-02-16-19-35.gh-issue-158592.V6W21J.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-02-16-19-35.gh-issue-158592.V6W21J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-02-16-19-35.gh-issue-158592.V6W21J.rst new file mode 100644 index 00000000000000..1112ca8e63d314 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-02-16-19-35.gh-issue-158592.V6W21J.rst @@ -0,0 +1 @@ +``del list[index]`` now triggers shrinking the list's internal array. diff --git a/Objects/listobject.c b/Objects/listobject.c index 87a669f203589d..a4ad1094ba51ce 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1182,7 +1182,7 @@ list_ass_item_lock_held(PyListObject *a, Py_ssize_t i, PyObject *v) if (i < size - 1) { list_shift_items_left_lock_held(a, i, size - 1); } - Py_SET_SIZE(a, size - 1); + list_resize(a, size - 1); // NB: shrinking a list can't fail } else { FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item[i], Py_NewRef(v)); From cdfdf7bb73a5751ecec7d2780e3f34ec60699fb1 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Thu, 8 Oct 2026 08:58:23 +0900 Subject: [PATCH 122/240] gh-158592: Skip reallocation when shrinking a small list (gh-158787) --- ...-10-04-11-59-33.gh-issue-158592._BpoeY.rst | 2 ++ Objects/listobject.c | 27 ++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-11-59-33.gh-issue-158592._BpoeY.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-11-59-33.gh-issue-158592._BpoeY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-11-59-33.gh-issue-158592._BpoeY.rst new file mode 100644 index 00000000000000..d937a81cc4630a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-11-59-33.gh-issue-158592._BpoeY.rst @@ -0,0 +1,2 @@ +Skip the array reallocation when shrinking a small :class:`list`. Patch by +Donghee Na. diff --git a/Objects/listobject.c b/Objects/listobject.c index a4ad1094ba51ce..f9488cc5c384ba 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -91,6 +91,10 @@ ensure_shared_on_resize(PyListObject *self) #endif } +#define LIST_SMALL_ALLOCATED 32 + +static int py_list_resize(PyListObject *self, Py_ssize_t newsize); + /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -99,22 +103,37 @@ ensure_shared_on_resize(PyListObject *self) * Failure is impossible if newsize <= self.allocated on entry. * Note that self->ob_item may change, and even if newsize is less * than ob_size on entry. + * + * Always inlining list_resize() makes the fast path a few instructions + * in each caller instead of a function call. */ -static int +static inline Py_ALWAYS_INLINE int list_resize(PyListObject *self, Py_ssize_t newsize) { - size_t new_allocated, target_bytes; Py_ssize_t allocated = self->allocated; /* Bypass realloc() when a previous overallocation is large enough to accommodate the newsize. If the newsize falls lower than half the allocated size, then proceed with the realloc() to shrink the list. + gh-158592: do not shrink a small list, the realloc() cost is bigger + than the memory we get back. */ - if (allocated >= newsize && newsize >= (allocated >> 1)) { + if (allocated >= newsize + && (newsize >= (allocated >> 1) || allocated <= LIST_SMALL_ALLOCATED)) + { assert(self->ob_item != NULL || newsize == 0); Py_SET_SIZE(self, newsize); return 0; } + return py_list_resize(self, newsize); +} + +/* Slow path of list_resize(): allocate or reallocate ob_item. */ +static int +py_list_resize(PyListObject *self, Py_ssize_t newsize) +{ + size_t new_allocated, target_bytes; + Py_ssize_t allocated = self->allocated; /* This over-allocates proportional to the list size, making room * for additional growth. The over-allocation is mild, but is @@ -136,6 +155,8 @@ list_resize(PyListObject *self, Py_ssize_t newsize) if (newsize == 0) new_allocated = 0; + assert(newsize > allocated || new_allocated < (size_t)allocated); + ensure_shared_on_resize(self); #ifdef Py_GIL_DISABLED From b22a8c9fd44855d7cc013eb49f689dfd396639a7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 8 Oct 2026 03:26:19 +0200 Subject: [PATCH 123/240] gh-149044: Build test_cext with -Wall -Wextra (#158867) --- Lib/test/test_cext/setup.py | 38 ++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index c5706d23748e53..10a3887931932a 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -21,6 +21,8 @@ if not MSVC: # C compiler flags for GCC and clang BASE_CFLAGS = [ + '-Wall', + '-Wextra', # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', @@ -47,7 +49,11 @@ # with the -Werror=declaration-after-statement compiler flag. '-Werror=declaration-after-statement', ) - INTERNAL_CFLAGS = [*BASE_CFLAGS] + INTERNAL_CFLAGS = [ + *BASE_CFLAGS, + # Do not complain about unused parameteres in the internal C API + '-Wno-unused-parameter', + ] else: # MSVC compiler flags BASE_CFLAGS = [ @@ -70,14 +76,16 @@ if not MSVC: # C++ compiler flags for GCC and clang - CPPFLAGS = [ + CXXFLAGS = [ + '-Wall', + '-Wextra', # gh-91321: The purpose of _testcppext extension is to check that building # a C++ extension using the Python C API does not emit C++ compiler # warnings '-Werror', ] - CPPFLAGS_PEDANTIC = [ + PEDANTIC_CXXFLAGS = [ # Ask for strict(er) compliance with the standard. # We cannot do this for c++03 unlimited API, since several headers in # Include/cpython/ use commas at end of `enum` declarations, a C++11 @@ -87,15 +95,20 @@ # We also use `long long`, a C++11 feature we can enable individually. '-Wno-long-long', ] + INTERNAL_CXXFLAGS = [ + # Do not complain about unused parameteres in the internal C API + '-Wno-unused-parameter', + ] else: # MSVC compiler flags - CPPFLAGS = [ + CXXFLAGS = [ # Display warnings level 1 to 4 '/W4', # Treat all compiler warnings as compiler errors '/WX', ] - CPPFLAGS_PEDANTIC = [] + PEDANTIC_CXXFLAGS = [] + INTERNAL_CXXFLAGS = [] def main(): @@ -109,10 +122,13 @@ def main(): libdirs = os.environ.get("CPYTHON_TEST_EXTRA_LIBDIRS", "") extra_cflags = os.environ.get("CPYTHON_TEST_EXTRA_CFLAGS", "") + if language not in ('C', 'C++'): + raise ValueError(f"invalid language: {language}") + source = SOURCE[language] if language == 'C++': - flags = list(CPPFLAGS) + flags = list(CXXFLAGS) else: if not internal: flags = list(PUBLIC_CFLAGS) @@ -127,9 +143,13 @@ def main(): else: flags.append(f'-std={std}') - if language == 'C++' and (limited or (std != 'c++03') and not internal): - # See CPPFLAGS_PEDANTIC docstring - flags.extend(CPPFLAGS_PEDANTIC) + if language == 'C++': + if limited or (std != 'c++03') and not internal: + # See PEDANTIC_CXXFLAGS comment + flags.extend(PEDANTIC_CXXFLAGS) + elif internal: + flags.extend(INTERNAL_CXXFLAGS) + # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 # option emits a C++ compiler warning. Remove "-std11" option from the From 20cb7fc7077c20f4595fa65031f7047700a7bab3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 8 Oct 2026 14:12:57 +0200 Subject: [PATCH 124/240] gh-158886: Add INCREF/DECREF macros to stable_abi.toml (GH-158941) * Add 7 INCREF/DECREF macros * distinguish between 'const' and 'macro' in stable_abi.dat The file now has the 'kind' from the toml, not the ReST role. * Fix kind of PY_VECTORCALL_ARGUMENTS_OFFSET * Make it explicit that macros aren't part of the ABI * Add is_static_inline_function to the TOML file (these are treated as macros in the docs) Co-authored-by: Petr Viktorin --- Doc/data/stable_abi.dat | 2039 +++++++++++++------------ Doc/tools/extensions/c_annotations.py | 48 +- Misc/stable_abi.toml | 29 +- Tools/build/stable_abi.py | 34 +- 4 files changed, 1104 insertions(+), 1046 deletions(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 622a079ee6b4f2..3a6832c1ff72e1 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,247 +1,247 @@ -role,name,added,ifdef_note,struct_abi_kind -macro,METH_CLASS,3.2,, -macro,METH_COEXIST,3.2,, -macro,METH_FASTCALL,3.10,, -macro,METH_METHOD,3.7,, -macro,METH_NOARGS,3.2,, -macro,METH_O,3.2,, -macro,METH_STATIC,3.2,, -macro,METH_VARARGS,3.2,, -macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, -type,PyABIInfo,3.15,,full-abi -func,PyABIInfo_Check,3.15,, -macro,PyABIInfo_DEFAULT_ABI_VERSION,3.15,, -macro,PyABIInfo_DEFAULT_FLAGS,3.15,, -macro,PyABIInfo_FREETHREADED,3.15,, -macro,PyABIInfo_FREETHREADING_AGNOSTIC,3.15,, -macro,PyABIInfo_GIL,3.15,, -macro,PyABIInfo_STABLE,3.15,, +kind,name,added,ifdef_note,struct_abi_kind +const,METH_CLASS,3.2,, +const,METH_COEXIST,3.2,, +const,METH_FASTCALL,3.10,, +const,METH_METHOD,3.7,, +const,METH_NOARGS,3.2,, +const,METH_O,3.2,, +const,METH_STATIC,3.2,, +const,METH_VARARGS,3.2,, +const,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, +struct,PyABIInfo,3.15,,full-abi +function,PyABIInfo_Check,3.15,, +const,PyABIInfo_DEFAULT_ABI_VERSION,3.15,, +const,PyABIInfo_DEFAULT_FLAGS,3.15,, +const,PyABIInfo_FREETHREADED,3.15,, +const,PyABIInfo_FREETHREADING_AGNOSTIC,3.15,, +const,PyABIInfo_GIL,3.15,, +const,PyABIInfo_STABLE,3.15,, macro,PyABIInfo_VAR,3.15,, -func,PyAIter_Check,3.10,, -func,PyArg_Parse,3.2,, -func,PyArg_ParseTuple,3.2,, -func,PyArg_ParseTupleAndKeywords,3.2,, -func,PyArg_UnpackTuple,3.2,, -func,PyArg_VaParse,3.2,, -func,PyArg_VaParseTupleAndKeywords,3.2,, -func,PyArg_ValidateKeywordArguments,3.2,, -macro,PyBUF_ANY_CONTIGUOUS,3.11,, -macro,PyBUF_CONTIG,3.11,, -macro,PyBUF_CONTIG_RO,3.11,, -macro,PyBUF_C_CONTIGUOUS,3.11,, -macro,PyBUF_FORMAT,3.11,, -macro,PyBUF_FULL,3.11,, -macro,PyBUF_FULL_RO,3.11,, -macro,PyBUF_F_CONTIGUOUS,3.11,, -macro,PyBUF_INDIRECT,3.11,, -macro,PyBUF_MAX_NDIM,3.11,, -macro,PyBUF_ND,3.11,, -macro,PyBUF_READ,3.11,, -macro,PyBUF_RECORDS,3.11,, -macro,PyBUF_RECORDS_RO,3.11,, -macro,PyBUF_SIMPLE,3.11,, -macro,PyBUF_STRIDED,3.11,, -macro,PyBUF_STRIDED_RO,3.11,, -macro,PyBUF_STRIDES,3.11,, -macro,PyBUF_WRITABLE,3.11,, -macro,PyBUF_WRITE,3.11,, +function,PyAIter_Check,3.10,, +function,PyArg_Parse,3.2,, +function,PyArg_ParseTuple,3.2,, +function,PyArg_ParseTupleAndKeywords,3.2,, +function,PyArg_UnpackTuple,3.2,, +function,PyArg_VaParse,3.2,, +function,PyArg_VaParseTupleAndKeywords,3.2,, +function,PyArg_ValidateKeywordArguments,3.2,, +const,PyBUF_ANY_CONTIGUOUS,3.11,, +const,PyBUF_CONTIG,3.11,, +const,PyBUF_CONTIG_RO,3.11,, +const,PyBUF_C_CONTIGUOUS,3.11,, +const,PyBUF_FORMAT,3.11,, +const,PyBUF_FULL,3.11,, +const,PyBUF_FULL_RO,3.11,, +const,PyBUF_F_CONTIGUOUS,3.11,, +const,PyBUF_INDIRECT,3.11,, +const,PyBUF_MAX_NDIM,3.11,, +const,PyBUF_ND,3.11,, +const,PyBUF_READ,3.11,, +const,PyBUF_RECORDS,3.11,, +const,PyBUF_RECORDS_RO,3.11,, +const,PyBUF_SIMPLE,3.11,, +const,PyBUF_STRIDED,3.11,, +const,PyBUF_STRIDED_RO,3.11,, +const,PyBUF_STRIDES,3.11,, +const,PyBUF_WRITABLE,3.11,, +const,PyBUF_WRITE,3.11,, data,PyBaseObject_Type,3.2,, -func,PyBool_FromLong,3.2,, +function,PyBool_FromLong,3.2,, data,PyBool_Type,3.2,, -func,PyBuffer_FillContiguousStrides,3.11,, -func,PyBuffer_FillInfo,3.11,, -func,PyBuffer_FromContiguous,3.11,, -func,PyBuffer_GetPointer,3.11,, -func,PyBuffer_IsContiguous,3.11,, -func,PyBuffer_Release,3.11,, -func,PyBuffer_SizeFromFormat,3.11,, -func,PyBuffer_ToContiguous,3.11,, +function,PyBuffer_FillContiguousStrides,3.11,, +function,PyBuffer_FillInfo,3.11,, +function,PyBuffer_FromContiguous,3.11,, +function,PyBuffer_GetPointer,3.11,, +function,PyBuffer_IsContiguous,3.11,, +function,PyBuffer_Release,3.11,, +function,PyBuffer_SizeFromFormat,3.11,, +function,PyBuffer_ToContiguous,3.11,, data,PyByteArrayIter_Type,3.2,, -func,PyByteArray_AsString,3.2,, -func,PyByteArray_Concat,3.2,, -func,PyByteArray_FromObject,3.2,, -func,PyByteArray_FromStringAndSize,3.2,, -func,PyByteArray_Resize,3.2,, -func,PyByteArray_Size,3.2,, +function,PyByteArray_AsString,3.2,, +function,PyByteArray_Concat,3.2,, +function,PyByteArray_FromObject,3.2,, +function,PyByteArray_FromStringAndSize,3.2,, +function,PyByteArray_Resize,3.2,, +function,PyByteArray_Size,3.2,, data,PyByteArray_Type,3.2,, data,PyBytesIter_Type,3.2,, -func,PyBytes_AsString,3.2,, -func,PyBytes_AsStringAndSize,3.2,, -func,PyBytes_Concat,3.2,, -func,PyBytes_ConcatAndDel,3.2,, -func,PyBytes_DecodeEscape,3.2,, -func,PyBytes_FromFormat,3.2,, -func,PyBytes_FromFormatV,3.2,, -func,PyBytes_FromObject,3.2,, -func,PyBytes_FromString,3.2,, -func,PyBytes_FromStringAndSize,3.2,, -func,PyBytes_Repr,3.2,, -func,PyBytes_Size,3.2,, +function,PyBytes_AsString,3.2,, +function,PyBytes_AsStringAndSize,3.2,, +function,PyBytes_Concat,3.2,, +function,PyBytes_ConcatAndDel,3.2,, +function,PyBytes_DecodeEscape,3.2,, +function,PyBytes_FromFormat,3.2,, +function,PyBytes_FromFormatV,3.2,, +function,PyBytes_FromObject,3.2,, +function,PyBytes_FromString,3.2,, +function,PyBytes_FromStringAndSize,3.2,, +function,PyBytes_Repr,3.2,, +function,PyBytes_Size,3.2,, data,PyBytes_Type,3.2,, -type,PyCFunction,3.2,, -type,PyCFunctionFast,3.13,, -type,PyCFunctionFastWithKeywords,3.13,, -type,PyCFunctionWithKeywords,3.2,, -func,PyCFunction_GetFlags,3.2,, -func,PyCFunction_GetFunction,3.2,, -func,PyCFunction_GetSelf,3.2,, -func,PyCFunction_New,3.4,, -func,PyCFunction_NewEx,3.2,, +typedef,PyCFunction,3.2,, +typedef,PyCFunctionFast,3.13,, +typedef,PyCFunctionFastWithKeywords,3.13,, +typedef,PyCFunctionWithKeywords,3.2,, +function,PyCFunction_GetFlags,3.2,, +function,PyCFunction_GetFunction,3.2,, +function,PyCFunction_GetSelf,3.2,, +function,PyCFunction_New,3.4,, +function,PyCFunction_NewEx,3.2,, data,PyCFunction_Type,3.2,, -func,PyCMethod_New,3.9,, -func,PyCallIter_New,3.2,, +function,PyCMethod_New,3.9,, +function,PyCallIter_New,3.2,, data,PyCallIter_Type,3.2,, -func,PyCallable_Check,3.2,, -type,PyCapsule_Destructor,3.2,, -func,PyCapsule_GetContext,3.2,, -func,PyCapsule_GetDestructor,3.2,, -func,PyCapsule_GetName,3.2,, -func,PyCapsule_GetPointer,3.2,, -func,PyCapsule_Import,3.2,, -func,PyCapsule_IsValid,3.2,, -func,PyCapsule_New,3.2,, -func,PyCapsule_SetContext,3.2,, -func,PyCapsule_SetDestructor,3.2,, -func,PyCapsule_SetName,3.2,, -func,PyCapsule_SetPointer,3.2,, +function,PyCallable_Check,3.2,, +typedef,PyCapsule_Destructor,3.2,, +function,PyCapsule_GetContext,3.2,, +function,PyCapsule_GetDestructor,3.2,, +function,PyCapsule_GetName,3.2,, +function,PyCapsule_GetPointer,3.2,, +function,PyCapsule_Import,3.2,, +function,PyCapsule_IsValid,3.2,, +function,PyCapsule_New,3.2,, +function,PyCapsule_SetContext,3.2,, +function,PyCapsule_SetDestructor,3.2,, +function,PyCapsule_SetName,3.2,, +function,PyCapsule_SetPointer,3.2,, data,PyCapsule_Type,3.2,, data,PyClassMethodDescr_Type,3.2,, -func,PyCodec_BackslashReplaceErrors,3.2,, -func,PyCodec_Decode,3.2,, -func,PyCodec_Decoder,3.2,, -func,PyCodec_Encode,3.2,, -func,PyCodec_Encoder,3.2,, -func,PyCodec_IgnoreErrors,3.2,, -func,PyCodec_IncrementalDecoder,3.2,, -func,PyCodec_IncrementalEncoder,3.2,, -func,PyCodec_KnownEncoding,3.2,, -func,PyCodec_LookupError,3.2,, -func,PyCodec_NameReplaceErrors,3.7,, -func,PyCodec_Register,3.2,, -func,PyCodec_RegisterError,3.2,, -func,PyCodec_ReplaceErrors,3.2,, -func,PyCodec_StreamReader,3.2,, -func,PyCodec_StreamWriter,3.2,, -func,PyCodec_StrictErrors,3.2,, -func,PyCodec_Unregister,3.10,, -func,PyCodec_XMLCharRefReplaceErrors,3.2,, -func,PyComplex_FromDoubles,3.2,, -func,PyComplex_ImagAsDouble,3.2,, -func,PyComplex_RealAsDouble,3.2,, +function,PyCodec_BackslashReplaceErrors,3.2,, +function,PyCodec_Decode,3.2,, +function,PyCodec_Decoder,3.2,, +function,PyCodec_Encode,3.2,, +function,PyCodec_Encoder,3.2,, +function,PyCodec_IgnoreErrors,3.2,, +function,PyCodec_IncrementalDecoder,3.2,, +function,PyCodec_IncrementalEncoder,3.2,, +function,PyCodec_KnownEncoding,3.2,, +function,PyCodec_LookupError,3.2,, +function,PyCodec_NameReplaceErrors,3.7,, +function,PyCodec_Register,3.2,, +function,PyCodec_RegisterError,3.2,, +function,PyCodec_ReplaceErrors,3.2,, +function,PyCodec_StreamReader,3.2,, +function,PyCodec_StreamWriter,3.2,, +function,PyCodec_StrictErrors,3.2,, +function,PyCodec_Unregister,3.10,, +function,PyCodec_XMLCharRefReplaceErrors,3.2,, +function,PyComplex_FromDoubles,3.2,, +function,PyComplex_ImagAsDouble,3.2,, +function,PyComplex_RealAsDouble,3.2,, data,PyComplex_Type,3.2,, -type,PyCriticalSection,3.15,,full-abi -type,PyCriticalSection2,3.15,,full-abi -func,PyCriticalSection2_Begin,3.15,, -func,PyCriticalSection2_End,3.15,, -func,PyCriticalSection_Begin,3.15,, -func,PyCriticalSection_End,3.15,, -func,PyDescr_NewClassMethod,3.2,, -func,PyDescr_NewGetSet,3.2,, -func,PyDescr_NewMember,3.2,, -func,PyDescr_NewMethod,3.2,, +struct,PyCriticalSection,3.15,,full-abi +struct,PyCriticalSection2,3.15,,full-abi +function,PyCriticalSection2_Begin,3.15,, +function,PyCriticalSection2_End,3.15,, +function,PyCriticalSection_Begin,3.15,, +function,PyCriticalSection_End,3.15,, +function,PyDescr_NewClassMethod,3.2,, +function,PyDescr_NewGetSet,3.2,, +function,PyDescr_NewMember,3.2,, +function,PyDescr_NewMethod,3.2,, data,PyDictItems_Type,3.2,, data,PyDictIterItem_Type,3.2,, data,PyDictIterKey_Type,3.2,, data,PyDictIterValue_Type,3.2,, data,PyDictKeys_Type,3.2,, -func,PyDictProxy_New,3.2,, +function,PyDictProxy_New,3.2,, data,PyDictProxy_Type,3.2,, data,PyDictRevIterItem_Type,3.8,, data,PyDictRevIterKey_Type,3.8,, data,PyDictRevIterValue_Type,3.8,, data,PyDictValues_Type,3.2,, -func,PyDict_Clear,3.2,, -func,PyDict_Contains,3.2,, -func,PyDict_Copy,3.2,, -func,PyDict_DelItem,3.2,, -func,PyDict_DelItemString,3.2,, -func,PyDict_GetItem,3.2,, -func,PyDict_GetItemRef,3.13,, -func,PyDict_GetItemString,3.2,, -func,PyDict_GetItemStringRef,3.13,, -func,PyDict_GetItemWithError,3.2,, -func,PyDict_Items,3.2,, -func,PyDict_Keys,3.2,, -func,PyDict_Merge,3.2,, -func,PyDict_MergeFromSeq2,3.2,, -func,PyDict_New,3.2,, -func,PyDict_Next,3.2,, -func,PyDict_SetDefaultRef,3.15,, -func,PyDict_SetItem,3.2,, -func,PyDict_SetItemString,3.2,, -func,PyDict_Size,3.2,, +function,PyDict_Clear,3.2,, +function,PyDict_Contains,3.2,, +function,PyDict_Copy,3.2,, +function,PyDict_DelItem,3.2,, +function,PyDict_DelItemString,3.2,, +function,PyDict_GetItem,3.2,, +function,PyDict_GetItemRef,3.13,, +function,PyDict_GetItemString,3.2,, +function,PyDict_GetItemStringRef,3.13,, +function,PyDict_GetItemWithError,3.2,, +function,PyDict_Items,3.2,, +function,PyDict_Keys,3.2,, +function,PyDict_Merge,3.2,, +function,PyDict_MergeFromSeq2,3.2,, +function,PyDict_New,3.2,, +function,PyDict_Next,3.2,, +function,PyDict_SetDefaultRef,3.15,, +function,PyDict_SetItem,3.2,, +function,PyDict_SetItemString,3.2,, +function,PyDict_Size,3.2,, data,PyDict_Type,3.2,, -func,PyDict_Update,3.2,, -func,PyDict_Values,3.2,, +function,PyDict_Update,3.2,, +function,PyDict_Values,3.2,, data,PyEllipsis_Type,3.2,, data,PyEnum_Type,3.2,, -func,PyErr_BadArgument,3.2,, -func,PyErr_BadInternalCall,3.2,, -func,PyErr_CheckSignals,3.2,, -func,PyErr_Clear,3.2,, -func,PyErr_Display,3.2,, -func,PyErr_DisplayException,3.12,, -func,PyErr_ExceptionMatches,3.2,, -func,PyErr_Fetch,3.2,, -func,PyErr_Format,3.2,, -func,PyErr_FormatV,3.5,, -func,PyErr_GetExcInfo,3.7,, -func,PyErr_GetHandledException,3.11,, -func,PyErr_GetRaisedException,3.12,, -func,PyErr_GivenExceptionMatches,3.2,, -func,PyErr_NewException,3.2,, -func,PyErr_NewExceptionWithDoc,3.2,, -func,PyErr_NoMemory,3.2,, -func,PyErr_NormalizeException,3.2,, -func,PyErr_Occurred,3.2,, -func,PyErr_Print,3.2,, -func,PyErr_PrintEx,3.2,, -func,PyErr_ProgramText,3.2,, -func,PyErr_ResourceWarning,3.6,, -func,PyErr_Restore,3.2,, -func,PyErr_SetExcFromWindowsErr,3.7,on Windows, -func,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, -func,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, -func,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, -func,PyErr_SetExcInfo,3.7,, -func,PyErr_SetFromErrno,3.2,, -func,PyErr_SetFromErrnoWithFilename,3.2,, -func,PyErr_SetFromErrnoWithFilenameObject,3.2,, -func,PyErr_SetFromErrnoWithFilenameObjects,3.7,, -func,PyErr_SetFromWindowsErr,3.7,on Windows, -func,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, -func,PyErr_SetHandledException,3.11,, -func,PyErr_SetImportError,3.7,, -func,PyErr_SetImportErrorSubclass,3.6,, -func,PyErr_SetInterrupt,3.2,, -func,PyErr_SetInterruptEx,3.10,, -func,PyErr_SetNone,3.2,, -func,PyErr_SetObject,3.2,, -func,PyErr_SetRaisedException,3.12,, -func,PyErr_SetString,3.2,, -func,PyErr_SyntaxLocation,3.2,, -func,PyErr_SyntaxLocationEx,3.7,, -func,PyErr_WarnEx,3.2,, -func,PyErr_WarnExplicit,3.2,, -func,PyErr_WarnFormat,3.2,, -func,PyErr_WriteUnraisable,3.2,, -func,PyEval_AcquireThread,3.2,, -func,PyEval_EvalCode,3.2,, -func,PyEval_EvalCodeEx,3.2,, -func,PyEval_EvalFrame,3.2,, -func,PyEval_EvalFrameEx,3.2,, -func,PyEval_GetBuiltins,3.2,, -func,PyEval_GetFrame,3.2,, -func,PyEval_GetFrameBuiltins,3.13,, -func,PyEval_GetFrameGlobals,3.13,, -func,PyEval_GetFrameLocals,3.13,, -func,PyEval_GetFuncDesc,3.2,, -func,PyEval_GetFuncName,3.2,, -func,PyEval_GetGlobals,3.2,, -func,PyEval_GetLocals,3.2,, -func,PyEval_ReleaseThread,3.2,, -func,PyEval_RestoreThread,3.2,, -func,PyEval_SaveThread,3.2,, +function,PyErr_BadArgument,3.2,, +function,PyErr_BadInternalCall,3.2,, +function,PyErr_CheckSignals,3.2,, +function,PyErr_Clear,3.2,, +function,PyErr_Display,3.2,, +function,PyErr_DisplayException,3.12,, +function,PyErr_ExceptionMatches,3.2,, +function,PyErr_Fetch,3.2,, +function,PyErr_Format,3.2,, +function,PyErr_FormatV,3.5,, +function,PyErr_GetExcInfo,3.7,, +function,PyErr_GetHandledException,3.11,, +function,PyErr_GetRaisedException,3.12,, +function,PyErr_GivenExceptionMatches,3.2,, +function,PyErr_NewException,3.2,, +function,PyErr_NewExceptionWithDoc,3.2,, +function,PyErr_NoMemory,3.2,, +function,PyErr_NormalizeException,3.2,, +function,PyErr_Occurred,3.2,, +function,PyErr_Print,3.2,, +function,PyErr_PrintEx,3.2,, +function,PyErr_ProgramText,3.2,, +function,PyErr_ResourceWarning,3.6,, +function,PyErr_Restore,3.2,, +function,PyErr_SetExcFromWindowsErr,3.7,on Windows, +function,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, +function,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, +function,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, +function,PyErr_SetExcInfo,3.7,, +function,PyErr_SetFromErrno,3.2,, +function,PyErr_SetFromErrnoWithFilename,3.2,, +function,PyErr_SetFromErrnoWithFilenameObject,3.2,, +function,PyErr_SetFromErrnoWithFilenameObjects,3.7,, +function,PyErr_SetFromWindowsErr,3.7,on Windows, +function,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, +function,PyErr_SetHandledException,3.11,, +function,PyErr_SetImportError,3.7,, +function,PyErr_SetImportErrorSubclass,3.6,, +function,PyErr_SetInterrupt,3.2,, +function,PyErr_SetInterruptEx,3.10,, +function,PyErr_SetNone,3.2,, +function,PyErr_SetObject,3.2,, +function,PyErr_SetRaisedException,3.12,, +function,PyErr_SetString,3.2,, +function,PyErr_SyntaxLocation,3.2,, +function,PyErr_SyntaxLocationEx,3.7,, +function,PyErr_WarnEx,3.2,, +function,PyErr_WarnExplicit,3.2,, +function,PyErr_WarnFormat,3.2,, +function,PyErr_WriteUnraisable,3.2,, +function,PyEval_AcquireThread,3.2,, +function,PyEval_EvalCode,3.2,, +function,PyEval_EvalCodeEx,3.2,, +function,PyEval_EvalFrame,3.2,, +function,PyEval_EvalFrameEx,3.2,, +function,PyEval_GetBuiltins,3.2,, +function,PyEval_GetFrame,3.2,, +function,PyEval_GetFrameBuiltins,3.13,, +function,PyEval_GetFrameGlobals,3.13,, +function,PyEval_GetFrameLocals,3.13,, +function,PyEval_GetFuncDesc,3.2,, +function,PyEval_GetFuncName,3.2,, +function,PyEval_GetGlobals,3.2,, +function,PyEval_GetLocals,3.2,, +function,PyEval_ReleaseThread,3.2,, +function,PyEval_RestoreThread,3.2,, +function,PyEval_SaveThread,3.2,, data,PyExc_ArithmeticError,3.2,, data,PyExc_AssertionError,3.2,, data,PyExc_AttributeError,3.2,, @@ -311,860 +311,867 @@ data,PyExc_ValueError,3.2,, data,PyExc_Warning,3.2,, data,PyExc_WindowsError,3.7,on Windows, data,PyExc_ZeroDivisionError,3.2,, -func,PyExceptionClass_Name,3.8,, -func,PyException_GetArgs,3.12,, -func,PyException_GetCause,3.2,, -func,PyException_GetContext,3.2,, -func,PyException_GetTraceback,3.2,, -func,PyException_SetArgs,3.12,, -func,PyException_SetCause,3.2,, -func,PyException_SetContext,3.2,, -func,PyException_SetTraceback,3.2,, -func,PyFile_FromFd,3.2,, -func,PyFile_GetLine,3.2,, -func,PyFile_WriteObject,3.2,, -func,PyFile_WriteString,3.2,, +function,PyExceptionClass_Name,3.8,, +function,PyException_GetArgs,3.12,, +function,PyException_GetCause,3.2,, +function,PyException_GetContext,3.2,, +function,PyException_GetTraceback,3.2,, +function,PyException_SetArgs,3.12,, +function,PyException_SetCause,3.2,, +function,PyException_SetContext,3.2,, +function,PyException_SetTraceback,3.2,, +function,PyFile_FromFd,3.2,, +function,PyFile_GetLine,3.2,, +function,PyFile_WriteObject,3.2,, +function,PyFile_WriteString,3.2,, data,PyFilter_Type,3.2,, -func,PyFloat_AsDouble,3.2,, -func,PyFloat_FromDouble,3.2,, -func,PyFloat_FromString,3.2,, -func,PyFloat_GetInfo,3.2,, -func,PyFloat_GetMax,3.2,, -func,PyFloat_GetMin,3.2,, +function,PyFloat_AsDouble,3.2,, +function,PyFloat_FromDouble,3.2,, +function,PyFloat_FromString,3.2,, +function,PyFloat_GetInfo,3.2,, +function,PyFloat_GetMax,3.2,, +function,PyFloat_GetMin,3.2,, data,PyFloat_Type,3.2,, -type,PyFrameObject,3.2,,opaque -func,PyFrame_GetCode,3.10,, -func,PyFrame_GetLineNumber,3.10,, -func,PyFrozenSet_New,3.2,, +struct,PyFrameObject,3.2,,opaque +function,PyFrame_GetCode,3.10,, +function,PyFrame_GetLineNumber,3.10,, +function,PyFrozenSet_New,3.2,, data,PyFrozenSet_Type,3.2,, -func,PyGC_Collect,3.2,, -func,PyGC_Disable,3.10,, -func,PyGC_Enable,3.10,, -func,PyGC_IsEnabled,3.10,, -func,PyGILState_Ensure,3.2,, -func,PyGILState_GetThisThreadState,3.2,, -func,PyGILState_Release,3.2,, -type,PyGILState_STATE,3.2,, -type,PyGetSetDef,3.2,,full-abi +function,PyGC_Collect,3.2,, +function,PyGC_Disable,3.10,, +function,PyGC_Enable,3.10,, +function,PyGC_IsEnabled,3.10,, +function,PyGILState_Ensure,3.2,, +function,PyGILState_GetThisThreadState,3.2,, +function,PyGILState_Release,3.2,, +typedef,PyGILState_STATE,3.2,, +struct,PyGetSetDef,3.2,,full-abi data,PyGetSetDescr_Type,3.2,, -func,PyImport_AddModule,3.2,, -func,PyImport_AddModuleObject,3.7,, -func,PyImport_AddModuleRef,3.13,, -func,PyImport_AppendInittab,3.2,, -func,PyImport_ExecCodeModule,3.2,, -func,PyImport_ExecCodeModuleEx,3.2,, -func,PyImport_ExecCodeModuleObject,3.7,, -func,PyImport_ExecCodeModuleWithPathnames,3.2,, -func,PyImport_GetImporter,3.2,, -func,PyImport_GetMagicNumber,3.2,, -func,PyImport_GetMagicTag,3.2,, -func,PyImport_GetModule,3.8,, -func,PyImport_GetModuleDict,3.2,, -func,PyImport_Import,3.2,, -func,PyImport_ImportFrozenModule,3.2,, -func,PyImport_ImportFrozenModuleObject,3.7,, -func,PyImport_ImportModule,3.2,, -func,PyImport_ImportModuleLevel,3.2,, -func,PyImport_ImportModuleLevelObject,3.7,, -func,PyImport_ReloadModule,3.2,, -func,PyIndex_Check,3.8,, -type,PyInterpreterGuard,3.15,,opaque -func,PyInterpreterGuard_Close,3.15,, -func,PyInterpreterGuard_FromCurrent,3.15,, -func,PyInterpreterGuard_FromView,3.15,, -type,PyInterpreterState,3.2,,opaque -func,PyInterpreterState_Clear,3.2,, -func,PyInterpreterState_Delete,3.2,, -func,PyInterpreterState_Get,3.9,, -func,PyInterpreterState_GetDict,3.8,, -func,PyInterpreterState_GetID,3.7,, -func,PyInterpreterState_New,3.2,, -type,PyInterpreterView,3.15,,opaque -func,PyInterpreterView_Close,3.15,, -func,PyInterpreterView_FromCurrent,3.15,, -func,PyInterpreterView_FromMain,3.15,, -func,PyIter_Check,3.8,, -func,PyIter_Next,3.2,, -func,PyIter_NextItem,3.14,, -func,PyIter_Send,3.10,, +function,PyImport_AddModule,3.2,, +function,PyImport_AddModuleObject,3.7,, +function,PyImport_AddModuleRef,3.13,, +function,PyImport_AppendInittab,3.2,, +function,PyImport_ExecCodeModule,3.2,, +function,PyImport_ExecCodeModuleEx,3.2,, +function,PyImport_ExecCodeModuleObject,3.7,, +function,PyImport_ExecCodeModuleWithPathnames,3.2,, +function,PyImport_GetImporter,3.2,, +function,PyImport_GetMagicNumber,3.2,, +function,PyImport_GetMagicTag,3.2,, +function,PyImport_GetModule,3.8,, +function,PyImport_GetModuleDict,3.2,, +function,PyImport_Import,3.2,, +function,PyImport_ImportFrozenModule,3.2,, +function,PyImport_ImportFrozenModuleObject,3.7,, +function,PyImport_ImportModule,3.2,, +function,PyImport_ImportModuleLevel,3.2,, +function,PyImport_ImportModuleLevelObject,3.7,, +function,PyImport_ReloadModule,3.2,, +function,PyIndex_Check,3.8,, +struct,PyInterpreterGuard,3.15,,opaque +function,PyInterpreterGuard_Close,3.15,, +function,PyInterpreterGuard_FromCurrent,3.15,, +function,PyInterpreterGuard_FromView,3.15,, +struct,PyInterpreterState,3.2,,opaque +function,PyInterpreterState_Clear,3.2,, +function,PyInterpreterState_Delete,3.2,, +function,PyInterpreterState_Get,3.9,, +function,PyInterpreterState_GetDict,3.8,, +function,PyInterpreterState_GetID,3.7,, +function,PyInterpreterState_New,3.2,, +struct,PyInterpreterView,3.15,,opaque +function,PyInterpreterView_Close,3.15,, +function,PyInterpreterView_FromCurrent,3.15,, +function,PyInterpreterView_FromMain,3.15,, +function,PyIter_Check,3.8,, +function,PyIter_Next,3.2,, +function,PyIter_NextItem,3.14,, +function,PyIter_Send,3.10,, data,PyListIter_Type,3.2,, data,PyListRevIter_Type,3.2,, -func,PyList_Append,3.2,, -func,PyList_AsTuple,3.2,, -func,PyList_GetItem,3.2,, -func,PyList_GetItemRef,3.13,, -func,PyList_GetSlice,3.2,, -func,PyList_Insert,3.2,, -func,PyList_New,3.2,, -func,PyList_Reverse,3.2,, -func,PyList_SetItem,3.2,, -func,PyList_SetSlice,3.2,, -func,PyList_Size,3.2,, -func,PyList_Sort,3.2,, +function,PyList_Append,3.2,, +function,PyList_AsTuple,3.2,, +function,PyList_GetItem,3.2,, +function,PyList_GetItemRef,3.13,, +function,PyList_GetSlice,3.2,, +function,PyList_Insert,3.2,, +function,PyList_New,3.2,, +function,PyList_Reverse,3.2,, +function,PyList_SetItem,3.2,, +function,PyList_SetSlice,3.2,, +function,PyList_Size,3.2,, +function,PyList_Sort,3.2,, data,PyList_Type,3.2,, -type,PyLongExport,3.15,,full-abi -type,PyLongLayout,3.15,,full-abi -type,PyLongObject,3.2,,opaque +struct,PyLongExport,3.15,,full-abi +struct,PyLongLayout,3.15,,full-abi +struct,PyLongObject,3.2,,opaque data,PyLongRangeIter_Type,3.2,, -type,PyLongWriter,3.15,,opaque -func,PyLongWriter_Create,3.15,, -func,PyLongWriter_Discard,3.15,, -func,PyLongWriter_Finish,3.15,, -func,PyLong_AsDouble,3.2,, -func,PyLong_AsInt,3.13,, -func,PyLong_AsInt32,3.14,, -func,PyLong_AsInt64,3.14,, -func,PyLong_AsLong,3.2,, -func,PyLong_AsLongAndOverflow,3.2,, -func,PyLong_AsLongLong,3.2,, -func,PyLong_AsLongLongAndOverflow,3.2,, -func,PyLong_AsNativeBytes,3.14,, -func,PyLong_AsSize_t,3.2,, -func,PyLong_AsSsize_t,3.2,, -func,PyLong_AsUInt32,3.14,, -func,PyLong_AsUInt64,3.14,, -func,PyLong_AsUnsignedLong,3.2,, -func,PyLong_AsUnsignedLongLong,3.2,, -func,PyLong_AsUnsignedLongLongMask,3.2,, -func,PyLong_AsUnsignedLongMask,3.2,, -func,PyLong_AsVoidPtr,3.2,, -func,PyLong_Export,3.15,, -func,PyLong_FreeExport,3.15,, -func,PyLong_FromDouble,3.2,, -func,PyLong_FromInt32,3.14,, -func,PyLong_FromInt64,3.14,, -func,PyLong_FromLong,3.2,, -func,PyLong_FromLongLong,3.2,, -func,PyLong_FromNativeBytes,3.14,, -func,PyLong_FromSize_t,3.2,, -func,PyLong_FromSsize_t,3.2,, -func,PyLong_FromString,3.2,, -func,PyLong_FromUInt32,3.14,, -func,PyLong_FromUInt64,3.14,, -func,PyLong_FromUnsignedLong,3.2,, -func,PyLong_FromUnsignedLongLong,3.2,, -func,PyLong_FromUnsignedNativeBytes,3.14,, -func,PyLong_FromVoidPtr,3.2,, -func,PyLong_GetInfo,3.2,, -func,PyLong_GetNativeLayout,3.15,, +struct,PyLongWriter,3.15,,opaque +function,PyLongWriter_Create,3.15,, +function,PyLongWriter_Discard,3.15,, +function,PyLongWriter_Finish,3.15,, +function,PyLong_AsDouble,3.2,, +function,PyLong_AsInt,3.13,, +function,PyLong_AsInt32,3.14,, +function,PyLong_AsInt64,3.14,, +function,PyLong_AsLong,3.2,, +function,PyLong_AsLongAndOverflow,3.2,, +function,PyLong_AsLongLong,3.2,, +function,PyLong_AsLongLongAndOverflow,3.2,, +function,PyLong_AsNativeBytes,3.14,, +function,PyLong_AsSize_t,3.2,, +function,PyLong_AsSsize_t,3.2,, +function,PyLong_AsUInt32,3.14,, +function,PyLong_AsUInt64,3.14,, +function,PyLong_AsUnsignedLong,3.2,, +function,PyLong_AsUnsignedLongLong,3.2,, +function,PyLong_AsUnsignedLongLongMask,3.2,, +function,PyLong_AsUnsignedLongMask,3.2,, +function,PyLong_AsVoidPtr,3.2,, +function,PyLong_Export,3.15,, +function,PyLong_FreeExport,3.15,, +function,PyLong_FromDouble,3.2,, +function,PyLong_FromInt32,3.14,, +function,PyLong_FromInt64,3.14,, +function,PyLong_FromLong,3.2,, +function,PyLong_FromLongLong,3.2,, +function,PyLong_FromNativeBytes,3.14,, +function,PyLong_FromSize_t,3.2,, +function,PyLong_FromSsize_t,3.2,, +function,PyLong_FromString,3.2,, +function,PyLong_FromUInt32,3.14,, +function,PyLong_FromUInt64,3.14,, +function,PyLong_FromUnsignedLong,3.2,, +function,PyLong_FromUnsignedLongLong,3.2,, +function,PyLong_FromUnsignedNativeBytes,3.14,, +function,PyLong_FromVoidPtr,3.2,, +function,PyLong_GetInfo,3.2,, +function,PyLong_GetNativeLayout,3.15,, data,PyLong_Type,3.2,, macro,PyMODEXPORT_FUNC,3.15,, data,PyMap_Type,3.2,, -func,PyMapping_Check,3.2,, -func,PyMapping_GetItemString,3.2,, -func,PyMapping_GetOptionalItem,3.13,, -func,PyMapping_GetOptionalItemString,3.13,, -func,PyMapping_HasKey,3.2,, -func,PyMapping_HasKeyString,3.2,, -func,PyMapping_HasKeyStringWithError,3.13,, -func,PyMapping_HasKeyWithError,3.13,, -func,PyMapping_Items,3.2,, -func,PyMapping_Keys,3.2,, -func,PyMapping_Length,3.2,, -func,PyMapping_SetItemString,3.2,, -func,PyMapping_Size,3.2,, -func,PyMapping_Values,3.2,, -func,PyMem_Calloc,3.7,, -func,PyMem_Free,3.2,, -func,PyMem_Malloc,3.2,, -func,PyMem_RawCalloc,3.13,, -func,PyMem_RawFree,3.13,, -func,PyMem_RawMalloc,3.13,, -func,PyMem_RawRealloc,3.13,, -func,PyMem_Realloc,3.2,, -type,PyMemberDef,3.2,,full-abi +function,PyMapping_Check,3.2,, +function,PyMapping_GetItemString,3.2,, +function,PyMapping_GetOptionalItem,3.13,, +function,PyMapping_GetOptionalItemString,3.13,, +function,PyMapping_HasKey,3.2,, +function,PyMapping_HasKeyString,3.2,, +function,PyMapping_HasKeyStringWithError,3.13,, +function,PyMapping_HasKeyWithError,3.13,, +function,PyMapping_Items,3.2,, +function,PyMapping_Keys,3.2,, +function,PyMapping_Length,3.2,, +function,PyMapping_SetItemString,3.2,, +function,PyMapping_Size,3.2,, +function,PyMapping_Values,3.2,, +function,PyMem_Calloc,3.7,, +function,PyMem_Free,3.2,, +function,PyMem_Malloc,3.2,, +function,PyMem_RawCalloc,3.13,, +function,PyMem_RawFree,3.13,, +function,PyMem_RawMalloc,3.13,, +function,PyMem_RawRealloc,3.13,, +function,PyMem_Realloc,3.2,, +struct,PyMemberDef,3.2,,full-abi data,PyMemberDescr_Type,3.2,, -func,PyMember_GetOne,3.2,, -func,PyMember_SetOne,3.2,, -func,PyMemoryView_FromBuffer,3.11,, -func,PyMemoryView_FromMemory,3.7,, -func,PyMemoryView_FromObject,3.2,, -func,PyMemoryView_GetContiguous,3.2,, +function,PyMember_GetOne,3.2,, +function,PyMember_SetOne,3.2,, +function,PyMemoryView_FromBuffer,3.11,, +function,PyMemoryView_FromMemory,3.7,, +function,PyMemoryView_FromObject,3.2,, +function,PyMemoryView_GetContiguous,3.2,, data,PyMemoryView_Type,3.2,, -type,PyMethodDef,3.2,,full-abi +struct,PyMethodDef,3.2,,full-abi data,PyMethodDescr_Type,3.2,, -type,PyModuleDef,3.2,,abi3t-opaque -type,PyModuleDef_Base,3.2,,abi3t-opaque -func,PyModuleDef_Init,3.5,, -type,PyModuleDef_Slot,3.5,,full-abi +struct,PyModuleDef,3.2,,abi3t-opaque +struct,PyModuleDef_Base,3.2,,abi3t-opaque +function,PyModuleDef_Init,3.5,, +struct,PyModuleDef_Slot,3.5,,full-abi data,PyModuleDef_Type,3.5,, -func,PyModule_Add,3.13,, -func,PyModule_AddFunctions,3.7,, -func,PyModule_AddIntConstant,3.2,, -func,PyModule_AddObject,3.2,, -func,PyModule_AddObjectRef,3.10,, -func,PyModule_AddStringConstant,3.2,, -func,PyModule_AddType,3.10,, -func,PyModule_Create2,3.2,, -func,PyModule_Exec,3.15,, -func,PyModule_ExecDef,3.7,, -func,PyModule_FromDefAndSpec2,3.7,, -func,PyModule_FromSlotsAndSpec,3.15,, -func,PyModule_GetDef,3.2,, -func,PyModule_GetDict,3.2,, -func,PyModule_GetFilename,3.2,, -func,PyModule_GetFilenameObject,3.2,, -func,PyModule_GetName,3.2,, -func,PyModule_GetNameObject,3.7,, -func,PyModule_GetState,3.2,, -func,PyModule_GetStateSize,3.15,, -func,PyModule_GetState_DuringGC,3.15,, -func,PyModule_GetToken,3.15,, -func,PyModule_GetToken_DuringGC,3.15,, -func,PyModule_New,3.2,, -func,PyModule_NewObject,3.7,, -func,PyModule_SetDocString,3.7,, +function,PyModule_Add,3.13,, +function,PyModule_AddFunctions,3.7,, +function,PyModule_AddIntConstant,3.2,, +function,PyModule_AddObject,3.2,, +function,PyModule_AddObjectRef,3.10,, +function,PyModule_AddStringConstant,3.2,, +function,PyModule_AddType,3.10,, +function,PyModule_Create2,3.2,, +function,PyModule_Exec,3.15,, +function,PyModule_ExecDef,3.7,, +function,PyModule_FromDefAndSpec2,3.7,, +function,PyModule_FromSlotsAndSpec,3.15,, +function,PyModule_GetDef,3.2,, +function,PyModule_GetDict,3.2,, +function,PyModule_GetFilename,3.2,, +function,PyModule_GetFilenameObject,3.2,, +function,PyModule_GetName,3.2,, +function,PyModule_GetNameObject,3.7,, +function,PyModule_GetState,3.2,, +function,PyModule_GetStateSize,3.15,, +function,PyModule_GetState_DuringGC,3.15,, +function,PyModule_GetToken,3.15,, +function,PyModule_GetToken_DuringGC,3.15,, +function,PyModule_New,3.2,, +function,PyModule_NewObject,3.7,, +function,PyModule_SetDocString,3.7,, data,PyModule_Type,3.2,, -func,PyNumber_Absolute,3.2,, -func,PyNumber_Add,3.2,, -func,PyNumber_And,3.2,, -func,PyNumber_AsSsize_t,3.2,, -func,PyNumber_Check,3.2,, -func,PyNumber_Divmod,3.2,, -func,PyNumber_Float,3.2,, -func,PyNumber_FloorDivide,3.2,, -func,PyNumber_InPlaceAdd,3.2,, -func,PyNumber_InPlaceAnd,3.2,, -func,PyNumber_InPlaceFloorDivide,3.2,, -func,PyNumber_InPlaceLshift,3.2,, -func,PyNumber_InPlaceMatrixMultiply,3.7,, -func,PyNumber_InPlaceMultiply,3.2,, -func,PyNumber_InPlaceOr,3.2,, -func,PyNumber_InPlacePower,3.2,, -func,PyNumber_InPlaceRemainder,3.2,, -func,PyNumber_InPlaceRshift,3.2,, -func,PyNumber_InPlaceSubtract,3.2,, -func,PyNumber_InPlaceTrueDivide,3.2,, -func,PyNumber_InPlaceXor,3.2,, -func,PyNumber_Index,3.2,, -func,PyNumber_Invert,3.2,, -func,PyNumber_Long,3.2,, -func,PyNumber_Lshift,3.2,, -func,PyNumber_MatrixMultiply,3.7,, -func,PyNumber_Multiply,3.2,, -func,PyNumber_Negative,3.2,, -func,PyNumber_Or,3.2,, -func,PyNumber_Positive,3.2,, -func,PyNumber_Power,3.2,, -func,PyNumber_Remainder,3.2,, -func,PyNumber_Rshift,3.2,, -func,PyNumber_Subtract,3.2,, -func,PyNumber_ToBase,3.2,, -func,PyNumber_TrueDivide,3.2,, -func,PyNumber_Xor,3.2,, -func,PyOS_AfterFork,3.2,on platforms with fork(), -func,PyOS_AfterFork_Child,3.7,on platforms with fork(), -func,PyOS_AfterFork_Parent,3.7,on platforms with fork(), -func,PyOS_BeforeFork,3.7,on platforms with fork(), -func,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, -func,PyOS_FSPath,3.6,, +function,PyNumber_Absolute,3.2,, +function,PyNumber_Add,3.2,, +function,PyNumber_And,3.2,, +function,PyNumber_AsSsize_t,3.2,, +function,PyNumber_Check,3.2,, +function,PyNumber_Divmod,3.2,, +function,PyNumber_Float,3.2,, +function,PyNumber_FloorDivide,3.2,, +function,PyNumber_InPlaceAdd,3.2,, +function,PyNumber_InPlaceAnd,3.2,, +function,PyNumber_InPlaceFloorDivide,3.2,, +function,PyNumber_InPlaceLshift,3.2,, +function,PyNumber_InPlaceMatrixMultiply,3.7,, +function,PyNumber_InPlaceMultiply,3.2,, +function,PyNumber_InPlaceOr,3.2,, +function,PyNumber_InPlacePower,3.2,, +function,PyNumber_InPlaceRemainder,3.2,, +function,PyNumber_InPlaceRshift,3.2,, +function,PyNumber_InPlaceSubtract,3.2,, +function,PyNumber_InPlaceTrueDivide,3.2,, +function,PyNumber_InPlaceXor,3.2,, +function,PyNumber_Index,3.2,, +function,PyNumber_Invert,3.2,, +function,PyNumber_Long,3.2,, +function,PyNumber_Lshift,3.2,, +function,PyNumber_MatrixMultiply,3.7,, +function,PyNumber_Multiply,3.2,, +function,PyNumber_Negative,3.2,, +function,PyNumber_Or,3.2,, +function,PyNumber_Positive,3.2,, +function,PyNumber_Power,3.2,, +function,PyNumber_Remainder,3.2,, +function,PyNumber_Rshift,3.2,, +function,PyNumber_Subtract,3.2,, +function,PyNumber_ToBase,3.2,, +function,PyNumber_TrueDivide,3.2,, +function,PyNumber_Xor,3.2,, +function,PyOS_AfterFork,3.2,on platforms with fork(), +function,PyOS_AfterFork_Child,3.7,on platforms with fork(), +function,PyOS_AfterFork_Parent,3.7,on platforms with fork(), +function,PyOS_BeforeFork,3.7,on platforms with fork(), +function,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, +function,PyOS_FSPath,3.6,, data,PyOS_InputHook,3.2,, -func,PyOS_InterruptOccurred,3.2,, -func,PyOS_double_to_string,3.2,, -func,PyOS_getsig,3.2,, -func,PyOS_mystricmp,3.2,, -func,PyOS_mystrnicmp,3.2,, -func,PyOS_setsig,3.2,, -type,PyOS_sighandler_t,3.2,, -func,PyOS_snprintf,3.2,, -func,PyOS_string_to_double,3.2,, -func,PyOS_strtol,3.2,, -func,PyOS_strtoul,3.2,, -func,PyOS_vsnprintf,3.2,, -type,PyObject,3.2,,members +function,PyOS_InterruptOccurred,3.2,, +function,PyOS_double_to_string,3.2,, +function,PyOS_getsig,3.2,, +function,PyOS_mystricmp,3.2,, +function,PyOS_mystrnicmp,3.2,, +function,PyOS_setsig,3.2,, +typedef,PyOS_sighandler_t,3.2,, +function,PyOS_snprintf,3.2,, +function,PyOS_string_to_double,3.2,, +function,PyOS_strtol,3.2,, +function,PyOS_strtoul,3.2,, +function,PyOS_vsnprintf,3.2,, +struct,PyObject,3.2,,members member,PyObject.ob_refcnt,3.2,, member,PyObject.ob_type,3.2,, -func,PyObject_ASCII,3.2,, -func,PyObject_AsFileDescriptor,3.2,, -func,PyObject_Bytes,3.2,, -func,PyObject_Call,3.2,, -func,PyObject_CallFinalizerFromDealloc,3.15,, -func,PyObject_CallFunction,3.2,, -func,PyObject_CallFunctionObjArgs,3.2,, -func,PyObject_CallMethod,3.2,, -func,PyObject_CallMethodObjArgs,3.2,, -func,PyObject_CallNoArgs,3.10,, -func,PyObject_CallObject,3.2,, -func,PyObject_Calloc,3.7,, -func,PyObject_CheckBuffer,3.11,, -func,PyObject_ClearWeakRefs,3.2,, -func,PyObject_CopyData,3.11,, -func,PyObject_DelAttr,3.13,, -func,PyObject_DelAttrString,3.13,, -func,PyObject_DelItem,3.2,, -func,PyObject_DelItemString,3.2,, -func,PyObject_Dir,3.2,, -func,PyObject_Format,3.2,, -func,PyObject_Free,3.2,, -func,PyObject_GC_Del,3.2,, -func,PyObject_GC_IsFinalized,3.9,, -func,PyObject_GC_IsTracked,3.9,, -func,PyObject_GC_Track,3.2,, -func,PyObject_GC_UnTrack,3.2,, -func,PyObject_GenericGetAttr,3.2,, -func,PyObject_GenericGetDict,3.10,, -func,PyObject_GenericSetAttr,3.2,, -func,PyObject_GenericSetDict,3.7,, -func,PyObject_GetAIter,3.10,, -func,PyObject_GetAttr,3.2,, -func,PyObject_GetAttrString,3.2,, -func,PyObject_GetBuffer,3.11,, -func,PyObject_GetItem,3.2,, -func,PyObject_GetIter,3.2,, -func,PyObject_GetOptionalAttr,3.13,, -func,PyObject_GetOptionalAttrString,3.13,, -func,PyObject_GetTypeData,3.12,, -func,PyObject_GetTypeData_DuringGC,3.15,, -func,PyObject_HasAttr,3.2,, -func,PyObject_HasAttrString,3.2,, -func,PyObject_HasAttrStringWithError,3.13,, -func,PyObject_HasAttrWithError,3.13,, -func,PyObject_Hash,3.2,, -func,PyObject_HashNotImplemented,3.2,, -func,PyObject_Init,3.2,, -func,PyObject_InitVar,3.2,, -func,PyObject_IsInstance,3.2,, -func,PyObject_IsSubclass,3.2,, -func,PyObject_IsTrue,3.2,, -func,PyObject_Length,3.2,, -func,PyObject_Malloc,3.2,, -func,PyObject_Not,3.2,, -func,PyObject_Realloc,3.2,, -func,PyObject_Repr,3.2,, -func,PyObject_RichCompare,3.2,, -func,PyObject_RichCompareBool,3.2,, -func,PyObject_SelfIter,3.2,, -func,PyObject_SetAttr,3.2,, -func,PyObject_SetAttrString,3.2,, -func,PyObject_SetItem,3.2,, -func,PyObject_Size,3.2,, -func,PyObject_Str,3.2,, -func,PyObject_Type,3.2,, -func,PyObject_Vectorcall,3.12,, -func,PyObject_VectorcallMethod,3.12,, +function,PyObject_ASCII,3.2,, +function,PyObject_AsFileDescriptor,3.2,, +function,PyObject_Bytes,3.2,, +function,PyObject_Call,3.2,, +function,PyObject_CallFinalizerFromDealloc,3.15,, +function,PyObject_CallFunction,3.2,, +function,PyObject_CallFunctionObjArgs,3.2,, +function,PyObject_CallMethod,3.2,, +function,PyObject_CallMethodObjArgs,3.2,, +function,PyObject_CallNoArgs,3.10,, +function,PyObject_CallObject,3.2,, +function,PyObject_Calloc,3.7,, +function,PyObject_CheckBuffer,3.11,, +function,PyObject_ClearWeakRefs,3.2,, +function,PyObject_CopyData,3.11,, +function,PyObject_DelAttr,3.13,, +function,PyObject_DelAttrString,3.13,, +function,PyObject_DelItem,3.2,, +function,PyObject_DelItemString,3.2,, +function,PyObject_Dir,3.2,, +function,PyObject_Format,3.2,, +function,PyObject_Free,3.2,, +function,PyObject_GC_Del,3.2,, +function,PyObject_GC_IsFinalized,3.9,, +function,PyObject_GC_IsTracked,3.9,, +function,PyObject_GC_Track,3.2,, +function,PyObject_GC_UnTrack,3.2,, +function,PyObject_GenericGetAttr,3.2,, +function,PyObject_GenericGetDict,3.10,, +function,PyObject_GenericSetAttr,3.2,, +function,PyObject_GenericSetDict,3.7,, +function,PyObject_GetAIter,3.10,, +function,PyObject_GetAttr,3.2,, +function,PyObject_GetAttrString,3.2,, +function,PyObject_GetBuffer,3.11,, +function,PyObject_GetItem,3.2,, +function,PyObject_GetIter,3.2,, +function,PyObject_GetOptionalAttr,3.13,, +function,PyObject_GetOptionalAttrString,3.13,, +function,PyObject_GetTypeData,3.12,, +function,PyObject_GetTypeData_DuringGC,3.15,, +function,PyObject_HasAttr,3.2,, +function,PyObject_HasAttrString,3.2,, +function,PyObject_HasAttrStringWithError,3.13,, +function,PyObject_HasAttrWithError,3.13,, +function,PyObject_Hash,3.2,, +function,PyObject_HashNotImplemented,3.2,, +function,PyObject_Init,3.2,, +function,PyObject_InitVar,3.2,, +function,PyObject_IsInstance,3.2,, +function,PyObject_IsSubclass,3.2,, +function,PyObject_IsTrue,3.2,, +function,PyObject_Length,3.2,, +function,PyObject_Malloc,3.2,, +function,PyObject_Not,3.2,, +function,PyObject_Realloc,3.2,, +function,PyObject_Repr,3.2,, +function,PyObject_RichCompare,3.2,, +function,PyObject_RichCompareBool,3.2,, +function,PyObject_SelfIter,3.2,, +function,PyObject_SetAttr,3.2,, +function,PyObject_SetAttrString,3.2,, +function,PyObject_SetItem,3.2,, +function,PyObject_Size,3.2,, +function,PyObject_Str,3.2,, +function,PyObject_Type,3.2,, +function,PyObject_Vectorcall,3.12,, +function,PyObject_VectorcallMethod,3.12,, data,PyProperty_Type,3.2,, data,PyRangeIter_Type,3.2,, data,PyRange_Type,3.2,, data,PyReversed_Type,3.2,, -func,PySeqIter_New,3.2,, +function,PySeqIter_New,3.2,, data,PySeqIter_Type,3.2,, -func,PySequence_Check,3.2,, -func,PySequence_Concat,3.2,, -func,PySequence_Contains,3.2,, -func,PySequence_Count,3.2,, -func,PySequence_DelItem,3.2,, -func,PySequence_DelSlice,3.2,, -func,PySequence_Fast,3.2,, -func,PySequence_GetItem,3.2,, -func,PySequence_GetSlice,3.2,, -func,PySequence_In,3.2,, -func,PySequence_InPlaceConcat,3.2,, -func,PySequence_InPlaceRepeat,3.2,, -func,PySequence_Index,3.2,, -func,PySequence_Length,3.2,, -func,PySequence_List,3.2,, -func,PySequence_Repeat,3.2,, -func,PySequence_SetItem,3.2,, -func,PySequence_SetSlice,3.2,, -func,PySequence_Size,3.2,, -func,PySequence_Tuple,3.2,, +function,PySequence_Check,3.2,, +function,PySequence_Concat,3.2,, +function,PySequence_Contains,3.2,, +function,PySequence_Count,3.2,, +function,PySequence_DelItem,3.2,, +function,PySequence_DelSlice,3.2,, +function,PySequence_Fast,3.2,, +function,PySequence_GetItem,3.2,, +function,PySequence_GetSlice,3.2,, +function,PySequence_In,3.2,, +function,PySequence_InPlaceConcat,3.2,, +function,PySequence_InPlaceRepeat,3.2,, +function,PySequence_Index,3.2,, +function,PySequence_Length,3.2,, +function,PySequence_List,3.2,, +function,PySequence_Repeat,3.2,, +function,PySequence_SetItem,3.2,, +function,PySequence_SetSlice,3.2,, +function,PySequence_Size,3.2,, +function,PySequence_Tuple,3.2,, data,PySetIter_Type,3.2,, -func,PySet_Add,3.2,, -func,PySet_Clear,3.2,, -func,PySet_Contains,3.2,, -func,PySet_Discard,3.2,, -func,PySet_New,3.2,, -func,PySet_Pop,3.2,, -func,PySet_Size,3.2,, +function,PySet_Add,3.2,, +function,PySet_Clear,3.2,, +function,PySet_Contains,3.2,, +function,PySet_Discard,3.2,, +function,PySet_New,3.2,, +function,PySet_Pop,3.2,, +function,PySet_Size,3.2,, data,PySet_Type,3.2,, -func,PySlice_AdjustIndices,3.7,, -func,PySlice_GetIndices,3.2,, -func,PySlice_GetIndicesEx,3.2,, -func,PySlice_New,3.2,, +function,PySlice_AdjustIndices,3.7,, +function,PySlice_GetIndices,3.2,, +function,PySlice_GetIndicesEx,3.2,, +function,PySlice_New,3.2,, data,PySlice_Type,3.2,, -func,PySlice_Unpack,3.7,, -type,PySlot,3.15,,full-abi +function,PySlice_Unpack,3.7,, +struct,PySlot,3.15,,full-abi macro,PySlot_DATA,3.15,, macro,PySlot_END,3.15,, macro,PySlot_FUNC,3.15,, macro,PySlot_INT64,3.15,, -macro,PySlot_INTPTR,3.15,, -macro,PySlot_OPTIONAL,3.15,, +const,PySlot_INTPTR,3.15,, +const,PySlot_OPTIONAL,3.15,, macro,PySlot_PTR,3.15,, macro,PySlot_PTR_STATIC,3.15,, macro,PySlot_SIZE,3.15,, -macro,PySlot_STATIC,3.15,, +const,PySlot_STATIC,3.15,, macro,PySlot_STATIC_DATA,3.15,, macro,PySlot_UINT64,3.15,, -func,PyState_AddModule,3.3,, -func,PyState_FindModule,3.2,, -func,PyState_RemoveModule,3.3,, -type,PyStructSequence_Desc,3.2,,full-abi -type,PyStructSequence_Field,3.2,,full-abi -func,PyStructSequence_GetItem,3.2,, -func,PyStructSequence_New,3.2,, -func,PyStructSequence_NewType,3.2,, -func,PyStructSequence_SetItem,3.2,, +function,PyState_AddModule,3.3,, +function,PyState_FindModule,3.2,, +function,PyState_RemoveModule,3.3,, +struct,PyStructSequence_Desc,3.2,,full-abi +struct,PyStructSequence_Field,3.2,,full-abi +function,PyStructSequence_GetItem,3.2,, +function,PyStructSequence_New,3.2,, +function,PyStructSequence_NewType,3.2,, +function,PyStructSequence_SetItem,3.2,, data,PyStructSequence_UnnamedField,3.11,, data,PySuper_Type,3.2,, -func,PySys_Audit,3.13,, -func,PySys_AuditTuple,3.13,, -func,PySys_FormatStderr,3.2,, -func,PySys_FormatStdout,3.2,, -func,PySys_GetAttr,3.15,, -func,PySys_GetAttrString,3.15,, -func,PySys_GetObject,3.2,, -func,PySys_GetOptionalAttr,3.15,, -func,PySys_GetOptionalAttrString,3.15,, -func,PySys_GetXOptions,3.7,, -func,PySys_SetObject,3.2,, -func,PySys_WriteStderr,3.2,, -func,PySys_WriteStdout,3.2,, -type,PyThreadState,3.2,,opaque -type,PyThreadStateToken,3.15,,opaque -func,PyThreadState_Clear,3.2,, -func,PyThreadState_Delete,3.2,, -func,PyThreadState_Ensure,3.15,, -func,PyThreadState_EnsureFromView,3.15,, -func,PyThreadState_Get,3.2,, -func,PyThreadState_GetDict,3.2,, -func,PyThreadState_GetFrame,3.10,, -func,PyThreadState_GetID,3.10,, -func,PyThreadState_GetInterpreter,3.10,, -func,PyThreadState_New,3.2,, -func,PyThreadState_Release,3.15,, -func,PyThreadState_SetAsyncExc,3.2,, -func,PyThreadState_Swap,3.2,, -func,PyThread_GetInfo,3.3,, -func,PyThread_ReInitTLS,3.2,, -func,PyThread_acquire_lock,3.2,, -func,PyThread_acquire_lock_timed,3.2,, -func,PyThread_allocate_lock,3.2,, -func,PyThread_create_key,3.2,, -func,PyThread_delete_key,3.2,, -func,PyThread_delete_key_value,3.2,, -func,PyThread_exit_thread,3.2,, -func,PyThread_free_lock,3.2,, -func,PyThread_get_key_value,3.2,, -func,PyThread_get_stacksize,3.2,, -func,PyThread_get_thread_ident,3.2,, -func,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, -func,PyThread_init_thread,3.2,, -func,PyThread_release_lock,3.2,, -func,PyThread_set_key_value,3.2,, -func,PyThread_set_stacksize,3.2,, -func,PyThread_start_new_thread,3.2,, -func,PyThread_tss_alloc,3.7,, -func,PyThread_tss_create,3.7,, -func,PyThread_tss_delete,3.7,, -func,PyThread_tss_free,3.7,, -func,PyThread_tss_get,3.7,, -func,PyThread_tss_is_created,3.7,, -func,PyThread_tss_set,3.7,, -func,PyTraceBack_Here,3.2,, -func,PyTraceBack_Print,3.2,, +function,PySys_Audit,3.13,, +function,PySys_AuditTuple,3.13,, +function,PySys_FormatStderr,3.2,, +function,PySys_FormatStdout,3.2,, +function,PySys_GetAttr,3.15,, +function,PySys_GetAttrString,3.15,, +function,PySys_GetObject,3.2,, +function,PySys_GetOptionalAttr,3.15,, +function,PySys_GetOptionalAttrString,3.15,, +function,PySys_GetXOptions,3.7,, +function,PySys_SetObject,3.2,, +function,PySys_WriteStderr,3.2,, +function,PySys_WriteStdout,3.2,, +struct,PyThreadState,3.2,,opaque +struct,PyThreadStateToken,3.15,,opaque +function,PyThreadState_Clear,3.2,, +function,PyThreadState_Delete,3.2,, +function,PyThreadState_Ensure,3.15,, +function,PyThreadState_EnsureFromView,3.15,, +function,PyThreadState_Get,3.2,, +function,PyThreadState_GetDict,3.2,, +function,PyThreadState_GetFrame,3.10,, +function,PyThreadState_GetID,3.10,, +function,PyThreadState_GetInterpreter,3.10,, +function,PyThreadState_New,3.2,, +function,PyThreadState_Release,3.15,, +function,PyThreadState_SetAsyncExc,3.2,, +function,PyThreadState_Swap,3.2,, +function,PyThread_GetInfo,3.3,, +function,PyThread_ReInitTLS,3.2,, +function,PyThread_acquire_lock,3.2,, +function,PyThread_acquire_lock_timed,3.2,, +function,PyThread_allocate_lock,3.2,, +function,PyThread_create_key,3.2,, +function,PyThread_delete_key,3.2,, +function,PyThread_delete_key_value,3.2,, +function,PyThread_exit_thread,3.2,, +function,PyThread_free_lock,3.2,, +function,PyThread_get_key_value,3.2,, +function,PyThread_get_stacksize,3.2,, +function,PyThread_get_thread_ident,3.2,, +function,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, +function,PyThread_init_thread,3.2,, +function,PyThread_release_lock,3.2,, +function,PyThread_set_key_value,3.2,, +function,PyThread_set_stacksize,3.2,, +function,PyThread_start_new_thread,3.2,, +function,PyThread_tss_alloc,3.7,, +function,PyThread_tss_create,3.7,, +function,PyThread_tss_delete,3.7,, +function,PyThread_tss_free,3.7,, +function,PyThread_tss_get,3.7,, +function,PyThread_tss_is_created,3.7,, +function,PyThread_tss_set,3.7,, +function,PyTraceBack_Here,3.2,, +function,PyTraceBack_Print,3.2,, data,PyTraceBack_Type,3.2,, data,PyTupleIter_Type,3.2,, -func,PyTuple_GetItem,3.2,, -func,PyTuple_GetSlice,3.2,, -func,PyTuple_New,3.2,, -func,PyTuple_Pack,3.2,, -func,PyTuple_SetItem,3.2,, -func,PyTuple_Size,3.2,, +function,PyTuple_GetItem,3.2,, +function,PyTuple_GetSlice,3.2,, +function,PyTuple_New,3.2,, +function,PyTuple_Pack,3.2,, +function,PyTuple_SetItem,3.2,, +function,PyTuple_Size,3.2,, data,PyTuple_Type,3.2,, -type,PyTypeObject,3.2,,opaque -func,PyType_ClearCache,3.2,, -func,PyType_Freeze,3.14,, -func,PyType_FromMetaclass,3.12,, -func,PyType_FromModuleAndSpec,3.10,, -func,PyType_FromSlots,3.15,, -func,PyType_FromSpec,3.2,, -func,PyType_FromSpecWithBases,3.3,, -func,PyType_GenericAlloc,3.2,, -func,PyType_GenericNew,3.2,, -func,PyType_GetBaseByToken,3.14,, -func,PyType_GetBaseByToken_DuringGC,3.15,, -func,PyType_GetFlags,3.2,, -func,PyType_GetFullyQualifiedName,3.13,, -func,PyType_GetModule,3.10,, -func,PyType_GetModuleByDef,3.13,, -func,PyType_GetModuleByToken,3.15,, -func,PyType_GetModuleByToken_DuringGC,3.15,, -func,PyType_GetModuleName,3.13,, -func,PyType_GetModuleState,3.10,, -func,PyType_GetModuleState_DuringGC,3.15,, -func,PyType_GetModule_DuringGC,3.15,, -func,PyType_GetName,3.11,, -func,PyType_GetQualName,3.11,, -func,PyType_GetSlot,3.4,, -func,PyType_GetTypeDataSize,3.12,, -func,PyType_IsSubtype,3.2,, -func,PyType_Modified,3.2,, -func,PyType_Ready,3.2,, -type,PyType_Slot,3.2,,full-abi -type,PyType_Spec,3.2,,full-abi +struct,PyTypeObject,3.2,,opaque +function,PyType_ClearCache,3.2,, +function,PyType_Freeze,3.14,, +function,PyType_FromMetaclass,3.12,, +function,PyType_FromModuleAndSpec,3.10,, +function,PyType_FromSlots,3.15,, +function,PyType_FromSpec,3.2,, +function,PyType_FromSpecWithBases,3.3,, +function,PyType_GenericAlloc,3.2,, +function,PyType_GenericNew,3.2,, +function,PyType_GetBaseByToken,3.14,, +function,PyType_GetBaseByToken_DuringGC,3.15,, +function,PyType_GetFlags,3.2,, +function,PyType_GetFullyQualifiedName,3.13,, +function,PyType_GetModule,3.10,, +function,PyType_GetModuleByDef,3.13,, +function,PyType_GetModuleByToken,3.15,, +function,PyType_GetModuleByToken_DuringGC,3.15,, +function,PyType_GetModuleName,3.13,, +function,PyType_GetModuleState,3.10,, +function,PyType_GetModuleState_DuringGC,3.15,, +function,PyType_GetModule_DuringGC,3.15,, +function,PyType_GetName,3.11,, +function,PyType_GetQualName,3.11,, +function,PyType_GetSlot,3.4,, +function,PyType_GetTypeDataSize,3.12,, +function,PyType_IsSubtype,3.2,, +function,PyType_Modified,3.2,, +function,PyType_Ready,3.2,, +struct,PyType_Slot,3.2,,full-abi +struct,PyType_Spec,3.2,,full-abi data,PyType_Type,3.2,, -func,PyUnicodeDecodeError_Create,3.2,, -func,PyUnicodeDecodeError_GetEncoding,3.2,, -func,PyUnicodeDecodeError_GetEnd,3.2,, -func,PyUnicodeDecodeError_GetObject,3.2,, -func,PyUnicodeDecodeError_GetReason,3.2,, -func,PyUnicodeDecodeError_GetStart,3.2,, -func,PyUnicodeDecodeError_SetEnd,3.2,, -func,PyUnicodeDecodeError_SetReason,3.2,, -func,PyUnicodeDecodeError_SetStart,3.2,, -func,PyUnicodeEncodeError_GetEncoding,3.2,, -func,PyUnicodeEncodeError_GetEnd,3.2,, -func,PyUnicodeEncodeError_GetObject,3.2,, -func,PyUnicodeEncodeError_GetReason,3.2,, -func,PyUnicodeEncodeError_GetStart,3.2,, -func,PyUnicodeEncodeError_SetEnd,3.2,, -func,PyUnicodeEncodeError_SetReason,3.2,, -func,PyUnicodeEncodeError_SetStart,3.2,, +function,PyUnicodeDecodeError_Create,3.2,, +function,PyUnicodeDecodeError_GetEncoding,3.2,, +function,PyUnicodeDecodeError_GetEnd,3.2,, +function,PyUnicodeDecodeError_GetObject,3.2,, +function,PyUnicodeDecodeError_GetReason,3.2,, +function,PyUnicodeDecodeError_GetStart,3.2,, +function,PyUnicodeDecodeError_SetEnd,3.2,, +function,PyUnicodeDecodeError_SetReason,3.2,, +function,PyUnicodeDecodeError_SetStart,3.2,, +function,PyUnicodeEncodeError_GetEncoding,3.2,, +function,PyUnicodeEncodeError_GetEnd,3.2,, +function,PyUnicodeEncodeError_GetObject,3.2,, +function,PyUnicodeEncodeError_GetReason,3.2,, +function,PyUnicodeEncodeError_GetStart,3.2,, +function,PyUnicodeEncodeError_SetEnd,3.2,, +function,PyUnicodeEncodeError_SetReason,3.2,, +function,PyUnicodeEncodeError_SetStart,3.2,, data,PyUnicodeIter_Type,3.2,, -func,PyUnicodeTranslateError_GetEnd,3.2,, -func,PyUnicodeTranslateError_GetObject,3.2,, -func,PyUnicodeTranslateError_GetReason,3.2,, -func,PyUnicodeTranslateError_GetStart,3.2,, -func,PyUnicodeTranslateError_SetEnd,3.2,, -func,PyUnicodeTranslateError_SetReason,3.2,, -func,PyUnicodeTranslateError_SetStart,3.2,, -func,PyUnicode_Append,3.2,, -func,PyUnicode_AppendAndDel,3.2,, -func,PyUnicode_AsASCIIString,3.2,, -func,PyUnicode_AsCharmapString,3.2,, -func,PyUnicode_AsEncodedString,3.2,, -func,PyUnicode_AsLatin1String,3.2,, -func,PyUnicode_AsMBCSString,3.7,on Windows, -func,PyUnicode_AsRawUnicodeEscapeString,3.2,, -func,PyUnicode_AsUCS4,3.7,, -func,PyUnicode_AsUCS4Copy,3.7,, -func,PyUnicode_AsUTF16String,3.2,, -func,PyUnicode_AsUTF32String,3.2,, -func,PyUnicode_AsUTF8AndSize,3.10,, -func,PyUnicode_AsUTF8String,3.2,, -func,PyUnicode_AsUnicodeEscapeString,3.2,, -func,PyUnicode_AsWideChar,3.2,, -func,PyUnicode_AsWideCharString,3.7,, -func,PyUnicode_BuildEncodingMap,3.2,, -func,PyUnicode_Compare,3.2,, -func,PyUnicode_CompareWithASCIIString,3.2,, -func,PyUnicode_Concat,3.2,, -func,PyUnicode_Contains,3.2,, -func,PyUnicode_Count,3.2,, -func,PyUnicode_Decode,3.2,, -func,PyUnicode_DecodeASCII,3.2,, -func,PyUnicode_DecodeCharmap,3.2,, -func,PyUnicode_DecodeCodePageStateful,3.7,on Windows, -func,PyUnicode_DecodeFSDefault,3.2,, -func,PyUnicode_DecodeFSDefaultAndSize,3.2,, -func,PyUnicode_DecodeLatin1,3.2,, -func,PyUnicode_DecodeLocale,3.7,, -func,PyUnicode_DecodeLocaleAndSize,3.7,, -func,PyUnicode_DecodeMBCS,3.7,on Windows, -func,PyUnicode_DecodeMBCSStateful,3.7,on Windows, -func,PyUnicode_DecodeRawUnicodeEscape,3.2,, -func,PyUnicode_DecodeUTF16,3.2,, -func,PyUnicode_DecodeUTF16Stateful,3.2,, -func,PyUnicode_DecodeUTF32,3.2,, -func,PyUnicode_DecodeUTF32Stateful,3.2,, -func,PyUnicode_DecodeUTF7,3.2,, -func,PyUnicode_DecodeUTF7Stateful,3.2,, -func,PyUnicode_DecodeUTF8,3.2,, -func,PyUnicode_DecodeUTF8Stateful,3.2,, -func,PyUnicode_DecodeUnicodeEscape,3.2,, -func,PyUnicode_EncodeCodePage,3.7,on Windows, -func,PyUnicode_EncodeFSDefault,3.2,, -func,PyUnicode_EncodeLocale,3.7,, -func,PyUnicode_Equal,3.14,, -func,PyUnicode_EqualToUTF8,3.13,, -func,PyUnicode_EqualToUTF8AndSize,3.13,, -func,PyUnicode_FSConverter,3.2,, -func,PyUnicode_FSDecoder,3.2,, -func,PyUnicode_Find,3.2,, -func,PyUnicode_FindChar,3.7,, -func,PyUnicode_Format,3.2,, -func,PyUnicode_FromEncodedObject,3.2,, -func,PyUnicode_FromFormat,3.2,, -func,PyUnicode_FromFormatV,3.2,, -func,PyUnicode_FromObject,3.2,, -func,PyUnicode_FromOrdinal,3.2,, -func,PyUnicode_FromString,3.2,, -func,PyUnicode_FromStringAndSize,3.2,, -func,PyUnicode_FromWideChar,3.2,, -func,PyUnicode_GetDefaultEncoding,3.2,, -func,PyUnicode_GetLength,3.7,, -func,PyUnicode_InternFromString,3.2,, -func,PyUnicode_InternInPlace,3.2,, -func,PyUnicode_IsIdentifier,3.2,, -func,PyUnicode_Join,3.2,, -func,PyUnicode_Partition,3.2,, -func,PyUnicode_RPartition,3.2,, -func,PyUnicode_RSplit,3.2,, -func,PyUnicode_ReadChar,3.7,, -func,PyUnicode_Replace,3.2,, -func,PyUnicode_Resize,3.2,, -func,PyUnicode_RichCompare,3.2,, -func,PyUnicode_Split,3.2,, -func,PyUnicode_Splitlines,3.2,, -func,PyUnicode_Substring,3.7,, -func,PyUnicode_Tailmatch,3.2,, -func,PyUnicode_Translate,3.2,, +function,PyUnicodeTranslateError_GetEnd,3.2,, +function,PyUnicodeTranslateError_GetObject,3.2,, +function,PyUnicodeTranslateError_GetReason,3.2,, +function,PyUnicodeTranslateError_GetStart,3.2,, +function,PyUnicodeTranslateError_SetEnd,3.2,, +function,PyUnicodeTranslateError_SetReason,3.2,, +function,PyUnicodeTranslateError_SetStart,3.2,, +function,PyUnicode_Append,3.2,, +function,PyUnicode_AppendAndDel,3.2,, +function,PyUnicode_AsASCIIString,3.2,, +function,PyUnicode_AsCharmapString,3.2,, +function,PyUnicode_AsEncodedString,3.2,, +function,PyUnicode_AsLatin1String,3.2,, +function,PyUnicode_AsMBCSString,3.7,on Windows, +function,PyUnicode_AsRawUnicodeEscapeString,3.2,, +function,PyUnicode_AsUCS4,3.7,, +function,PyUnicode_AsUCS4Copy,3.7,, +function,PyUnicode_AsUTF16String,3.2,, +function,PyUnicode_AsUTF32String,3.2,, +function,PyUnicode_AsUTF8AndSize,3.10,, +function,PyUnicode_AsUTF8String,3.2,, +function,PyUnicode_AsUnicodeEscapeString,3.2,, +function,PyUnicode_AsWideChar,3.2,, +function,PyUnicode_AsWideCharString,3.7,, +function,PyUnicode_BuildEncodingMap,3.2,, +function,PyUnicode_Compare,3.2,, +function,PyUnicode_CompareWithASCIIString,3.2,, +function,PyUnicode_Concat,3.2,, +function,PyUnicode_Contains,3.2,, +function,PyUnicode_Count,3.2,, +function,PyUnicode_Decode,3.2,, +function,PyUnicode_DecodeASCII,3.2,, +function,PyUnicode_DecodeCharmap,3.2,, +function,PyUnicode_DecodeCodePageStateful,3.7,on Windows, +function,PyUnicode_DecodeFSDefault,3.2,, +function,PyUnicode_DecodeFSDefaultAndSize,3.2,, +function,PyUnicode_DecodeLatin1,3.2,, +function,PyUnicode_DecodeLocale,3.7,, +function,PyUnicode_DecodeLocaleAndSize,3.7,, +function,PyUnicode_DecodeMBCS,3.7,on Windows, +function,PyUnicode_DecodeMBCSStateful,3.7,on Windows, +function,PyUnicode_DecodeRawUnicodeEscape,3.2,, +function,PyUnicode_DecodeUTF16,3.2,, +function,PyUnicode_DecodeUTF16Stateful,3.2,, +function,PyUnicode_DecodeUTF32,3.2,, +function,PyUnicode_DecodeUTF32Stateful,3.2,, +function,PyUnicode_DecodeUTF7,3.2,, +function,PyUnicode_DecodeUTF7Stateful,3.2,, +function,PyUnicode_DecodeUTF8,3.2,, +function,PyUnicode_DecodeUTF8Stateful,3.2,, +function,PyUnicode_DecodeUnicodeEscape,3.2,, +function,PyUnicode_EncodeCodePage,3.7,on Windows, +function,PyUnicode_EncodeFSDefault,3.2,, +function,PyUnicode_EncodeLocale,3.7,, +function,PyUnicode_Equal,3.14,, +function,PyUnicode_EqualToUTF8,3.13,, +function,PyUnicode_EqualToUTF8AndSize,3.13,, +function,PyUnicode_FSConverter,3.2,, +function,PyUnicode_FSDecoder,3.2,, +function,PyUnicode_Find,3.2,, +function,PyUnicode_FindChar,3.7,, +function,PyUnicode_Format,3.2,, +function,PyUnicode_FromEncodedObject,3.2,, +function,PyUnicode_FromFormat,3.2,, +function,PyUnicode_FromFormatV,3.2,, +function,PyUnicode_FromObject,3.2,, +function,PyUnicode_FromOrdinal,3.2,, +function,PyUnicode_FromString,3.2,, +function,PyUnicode_FromStringAndSize,3.2,, +function,PyUnicode_FromWideChar,3.2,, +function,PyUnicode_GetDefaultEncoding,3.2,, +function,PyUnicode_GetLength,3.7,, +function,PyUnicode_InternFromString,3.2,, +function,PyUnicode_InternInPlace,3.2,, +function,PyUnicode_IsIdentifier,3.2,, +function,PyUnicode_Join,3.2,, +function,PyUnicode_Partition,3.2,, +function,PyUnicode_RPartition,3.2,, +function,PyUnicode_RSplit,3.2,, +function,PyUnicode_ReadChar,3.7,, +function,PyUnicode_Replace,3.2,, +function,PyUnicode_Resize,3.2,, +function,PyUnicode_RichCompare,3.2,, +function,PyUnicode_Split,3.2,, +function,PyUnicode_Splitlines,3.2,, +function,PyUnicode_Substring,3.7,, +function,PyUnicode_Tailmatch,3.2,, +function,PyUnicode_Translate,3.2,, data,PyUnicode_Type,3.2,, -func,PyUnicode_WriteChar,3.7,, -type,PyVarObject,3.2,,members +function,PyUnicode_WriteChar,3.7,, +struct,PyVarObject,3.2,,members member,PyVarObject.ob_base,3.2,, member,PyVarObject.ob_size,3.2,, -func,PyVectorcall_Call,3.12,, -func,PyVectorcall_NARGS,3.12,, -type,PyWeakReference,3.2,,opaque -func,PyWeakref_GetRef,3.13,, -func,PyWeakref_NewProxy,3.2,, -func,PyWeakref_NewRef,3.2,, +function,PyVectorcall_Call,3.12,, +function,PyVectorcall_NARGS,3.12,, +struct,PyWeakReference,3.2,,opaque +function,PyWeakref_GetRef,3.13,, +function,PyWeakref_NewProxy,3.2,, +function,PyWeakref_NewRef,3.2,, data,PyWrapperDescr_Type,3.2,, -func,PyWrapper_New,3.2,, +function,PyWrapper_New,3.2,, data,PyZip_Type,3.2,, -macro,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, -macro,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, -macro,Py_ASNATIVEBYTES_DEFAULTS,3.14,, -macro,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, -macro,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, -macro,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, -macro,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, -macro,Py_AUDIT_READ,3.12,, -func,Py_AddPendingCall,3.2,, -func,Py_AtExit,3.2,, +const,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, +const,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, +const,Py_ASNATIVEBYTES_DEFAULTS,3.14,, +const,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, +const,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, +const,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, +const,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, +const,Py_AUDIT_READ,3.12,, +function,Py_AddPendingCall,3.2,, +function,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, macro,Py_BEGIN_CRITICAL_SECTION,3.15,, macro,Py_BEGIN_CRITICAL_SECTION2,3.15,, macro,Py_BLOCK_THREADS,3.2,, -func,Py_BuildValue,3.2,, -func,Py_BytesMain,3.8,, -func,Py_CompileString,3.2,, -func,Py_DecRef,3.2,, -func,Py_DecodeLocale,3.7,, +function,Py_BuildValue,3.2,, +function,Py_BytesMain,3.8,, +macro,Py_CLEAR,3.2,, +function,Py_CompileString,3.2,, +macro,Py_DECREF,3.2,, +function,Py_DecRef,3.2,, +function,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, macro,Py_END_CRITICAL_SECTION,3.15,, macro,Py_END_CRITICAL_SECTION2,3.15,, -func,Py_EncodeLocale,3.7,, -func,Py_EndInterpreter,3.2,, -func,Py_EnterRecursiveCall,3.9,, -func,Py_Exit,3.2,, -func,Py_FatalError,3.2,, -func,Py_Finalize,3.2,, -func,Py_FinalizeEx,3.6,, -func,Py_GenericAlias,3.9,, +function,Py_EncodeLocale,3.7,, +function,Py_EndInterpreter,3.2,, +function,Py_EnterRecursiveCall,3.9,, +function,Py_Exit,3.2,, +function,Py_FatalError,3.2,, +function,Py_Finalize,3.2,, +function,Py_FinalizeEx,3.6,, +function,Py_GenericAlias,3.9,, data,Py_GenericAliasType,3.9,, -func,Py_GetBuildInfo,3.2,, -func,Py_GetCompiler,3.2,, -func,Py_GetConstant,3.13,, -func,Py_GetConstantBorrowed,3.13,, -func,Py_GetCopyright,3.2,, -func,Py_GetPlatform,3.2,, -func,Py_GetRecursionLimit,3.2,, -func,Py_GetVersion,3.2,, -func,Py_HashBuffer,3.16,, -func,Py_IS_TYPE,3.15,, -func,Py_IncRef,3.2,, -func,Py_Initialize,3.2,, -func,Py_InitializeEx,3.2,, -func,Py_Is,3.10,, -func,Py_IsFalse,3.10,, -func,Py_IsFinalizing,3.13,, -func,Py_IsInitialized,3.2,, -func,Py_IsNone,3.10,, -func,Py_IsTrue,3.10,, -func,Py_LeaveRecursiveCall,3.9,, -func,Py_Main,3.2,, -func,Py_MakePendingCalls,3.2,, -func,Py_NewInterpreter,3.2,, -func,Py_NewRef,3.10,, -func,Py_PACK_FULL_VERSION,3.14,, -func,Py_PACK_VERSION,3.14,, -macro,Py_READONLY,3.12,, -func,Py_REFCNT,3.14,, -macro,Py_RELATIVE_OFFSET,3.12,, -func,Py_ReprEnter,3.2,, -func,Py_ReprLeave,3.2,, +function,Py_GetBuildInfo,3.2,, +function,Py_GetCompiler,3.2,, +function,Py_GetConstant,3.13,, +function,Py_GetConstantBorrowed,3.13,, +function,Py_GetCopyright,3.2,, +function,Py_GetPlatform,3.2,, +function,Py_GetRecursionLimit,3.2,, +function,Py_GetVersion,3.2,, +function,Py_HashBuffer,3.16,, +macro,Py_INCREF,3.2,, +function,Py_IS_TYPE,3.15,, +function,Py_IncRef,3.2,, +function,Py_Initialize,3.2,, +function,Py_InitializeEx,3.2,, +function,Py_Is,3.10,, +function,Py_IsFalse,3.10,, +function,Py_IsFinalizing,3.13,, +function,Py_IsInitialized,3.2,, +function,Py_IsNone,3.10,, +function,Py_IsTrue,3.10,, +function,Py_LeaveRecursiveCall,3.9,, +function,Py_Main,3.2,, +function,Py_MakePendingCalls,3.2,, +function,Py_NewInterpreter,3.2,, +function,Py_NewRef,3.10,, +function,Py_PACK_FULL_VERSION,3.14,, +function,Py_PACK_VERSION,3.14,, +const,Py_READONLY,3.12,, +function,Py_REFCNT,3.14,, +const,Py_RELATIVE_OFFSET,3.12,, +function,Py_ReprEnter,3.2,, +function,Py_ReprLeave,3.2,, macro,Py_SETREF,3.16,, -func,Py_SET_SIZE,3.15,, -func,Py_SIZE,3.15,, -func,Py_SetRecursionLimit,3.2,, -macro,Py_TPFLAGS_BASETYPE,3.2,, -macro,Py_TPFLAGS_DEFAULT,3.2,, -macro,Py_TPFLAGS_HAVE_GC,3.2,, -macro,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, -macro,Py_TPFLAGS_ITEMS_AT_END,3.12,, -macro,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, -macro,Py_TP_USE_SPEC,3.14,, -func,Py_TYPE,3.14,, -macro,Py_T_BOOL,3.12,, -macro,Py_T_BYTE,3.12,, -macro,Py_T_CHAR,3.12,, -macro,Py_T_DOUBLE,3.12,, -macro,Py_T_FLOAT,3.12,, -macro,Py_T_INT,3.12,, -macro,Py_T_LONG,3.12,, -macro,Py_T_LONGLONG,3.12,, -macro,Py_T_OBJECT_EX,3.12,, -macro,Py_T_PYSSIZET,3.12,, -macro,Py_T_SHORT,3.12,, -macro,Py_T_STRING,3.12,, -macro,Py_T_STRING_INPLACE,3.12,, -macro,Py_T_UBYTE,3.12,, -macro,Py_T_UINT,3.12,, -macro,Py_T_ULONG,3.12,, -macro,Py_T_ULONGLONG,3.12,, -macro,Py_T_USHORT,3.12,, -type,Py_UCS4,3.2,, +macro,Py_SET_REFCNT,3.9,, +function,Py_SET_SIZE,3.15,, +macro,Py_SET_TYPE,3.9,, +function,Py_SIZE,3.15,, +function,Py_SetRecursionLimit,3.2,, +const,Py_TPFLAGS_BASETYPE,3.2,, +const,Py_TPFLAGS_DEFAULT,3.2,, +const,Py_TPFLAGS_HAVE_GC,3.2,, +const,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, +const,Py_TPFLAGS_ITEMS_AT_END,3.12,, +const,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, +const,Py_TP_USE_SPEC,3.14,, +function,Py_TYPE,3.14,, +const,Py_T_BOOL,3.12,, +const,Py_T_BYTE,3.12,, +const,Py_T_CHAR,3.12,, +const,Py_T_DOUBLE,3.12,, +const,Py_T_FLOAT,3.12,, +const,Py_T_INT,3.12,, +const,Py_T_LONG,3.12,, +const,Py_T_LONGLONG,3.12,, +const,Py_T_OBJECT_EX,3.12,, +const,Py_T_PYSSIZET,3.12,, +const,Py_T_SHORT,3.12,, +const,Py_T_STRING,3.12,, +const,Py_T_STRING_INPLACE,3.12,, +const,Py_T_UBYTE,3.12,, +const,Py_T_UINT,3.12,, +const,Py_T_ULONG,3.12,, +const,Py_T_ULONGLONG,3.12,, +const,Py_T_USHORT,3.12,, +typedef,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, -func,Py_VaBuildValue,3.2,, +function,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, -func,Py_XNewRef,3.10,, +macro,Py_XDECREF,3.2,, +macro,Py_XINCREF,3.2,, +function,Py_XNewRef,3.10,, macro,Py_XSETREF,3.16,, -macro,Py_am_aiter,3.5,, -macro,Py_am_anext,3.5,, -macro,Py_am_await,3.5,, -macro,Py_am_send,3.10,, -macro,Py_bf_getbuffer,3.11,, -macro,Py_bf_releasebuffer,3.11,, -type,Py_buffer,3.11,,full-abi -type,Py_intptr_t,3.2,, -macro,Py_mod_abi,3.15,, -macro,Py_mod_create,3.5,, -macro,Py_mod_doc,3.15,, -macro,Py_mod_exec,3.5,, -macro,Py_mod_gil,3.13,, -macro,Py_mod_methods,3.15,, -macro,Py_mod_multiple_interpreters,3.12,, -macro,Py_mod_name,3.15,, -macro,Py_mod_slots,3.15,, -macro,Py_mod_state_clear,3.15,, -macro,Py_mod_state_free,3.15,, -macro,Py_mod_state_size,3.15,, -macro,Py_mod_state_traverse,3.15,, -macro,Py_mod_token,3.15,, -macro,Py_mp_ass_subscript,3.2,, -macro,Py_mp_length,3.2,, -macro,Py_mp_subscript,3.2,, -macro,Py_nb_absolute,3.2,, -macro,Py_nb_add,3.2,, -macro,Py_nb_and,3.2,, -macro,Py_nb_bool,3.2,, -macro,Py_nb_divmod,3.2,, -macro,Py_nb_float,3.2,, -macro,Py_nb_floor_divide,3.2,, -macro,Py_nb_index,3.2,, -macro,Py_nb_inplace_add,3.2,, -macro,Py_nb_inplace_and,3.2,, -macro,Py_nb_inplace_floor_divide,3.2,, -macro,Py_nb_inplace_lshift,3.2,, -macro,Py_nb_inplace_matrix_multiply,3.5,, -macro,Py_nb_inplace_multiply,3.2,, -macro,Py_nb_inplace_or,3.2,, -macro,Py_nb_inplace_power,3.2,, -macro,Py_nb_inplace_remainder,3.2,, -macro,Py_nb_inplace_rshift,3.2,, -macro,Py_nb_inplace_subtract,3.2,, -macro,Py_nb_inplace_true_divide,3.2,, -macro,Py_nb_inplace_xor,3.2,, -macro,Py_nb_int,3.2,, -macro,Py_nb_invert,3.2,, -macro,Py_nb_lshift,3.2,, -macro,Py_nb_matrix_multiply,3.5,, -macro,Py_nb_multiply,3.2,, -macro,Py_nb_negative,3.2,, -macro,Py_nb_or,3.2,, -macro,Py_nb_positive,3.2,, -macro,Py_nb_power,3.2,, -macro,Py_nb_remainder,3.2,, -macro,Py_nb_rshift,3.2,, -macro,Py_nb_subtract,3.2,, -macro,Py_nb_true_divide,3.2,, -macro,Py_nb_xor,3.2,, -macro,Py_slot_end,3.15,, -macro,Py_slot_invalid,3.15,, -macro,Py_slot_subslots,3.15,, -macro,Py_sq_ass_item,3.2,, -macro,Py_sq_concat,3.2,, -macro,Py_sq_contains,3.2,, -macro,Py_sq_inplace_concat,3.2,, -macro,Py_sq_inplace_repeat,3.2,, -macro,Py_sq_item,3.2,, -macro,Py_sq_length,3.2,, -macro,Py_sq_repeat,3.2,, -type,Py_ssize_t,3.2,, -macro,Py_tp_alloc,3.2,, -macro,Py_tp_base,3.2,, -macro,Py_tp_bases,3.2,, -macro,Py_tp_basicsize,3.15,, -macro,Py_tp_call,3.2,, -macro,Py_tp_clear,3.2,, -macro,Py_tp_dealloc,3.2,, -macro,Py_tp_del,3.2,, -macro,Py_tp_descr_get,3.2,, -macro,Py_tp_descr_set,3.2,, -macro,Py_tp_doc,3.2,, -macro,Py_tp_extra_basicsize,3.15,, -macro,Py_tp_finalize,3.5,, -macro,Py_tp_flags,3.15,, -macro,Py_tp_free,3.2,, -macro,Py_tp_getattr,3.2,, -macro,Py_tp_getattro,3.2,, -macro,Py_tp_getset,3.2,, -macro,Py_tp_hash,3.2,, -macro,Py_tp_init,3.2,, -macro,Py_tp_is_gc,3.2,, -macro,Py_tp_itemsize,3.15,, -macro,Py_tp_iter,3.2,, -macro,Py_tp_iternext,3.2,, -macro,Py_tp_members,3.2,, -macro,Py_tp_metaclass,3.15,, -macro,Py_tp_methods,3.2,, -macro,Py_tp_module,3.15,, -macro,Py_tp_name,3.15,, -macro,Py_tp_new,3.2,, -macro,Py_tp_repr,3.2,, -macro,Py_tp_richcompare,3.2,, -macro,Py_tp_setattr,3.2,, -macro,Py_tp_setattro,3.2,, -macro,Py_tp_slots,3.15,, -macro,Py_tp_str,3.2,, -macro,Py_tp_token,3.14,, -macro,Py_tp_traverse,3.2,, -macro,Py_tp_vectorcall,3.14,, -type,Py_uintptr_t,3.2,, -type,allocfunc,3.2,, -type,binaryfunc,3.2,, -type,descrgetfunc,3.2,, -type,descrsetfunc,3.2,, -type,destructor,3.2,, -type,getattrfunc,3.2,, -type,getattrofunc,3.2,, -type,getbufferproc,3.12,, -type,getiterfunc,3.2,, -type,getter,3.2,, -type,hashfunc,3.2,, -type,initproc,3.2,, -type,inquiry,3.2,, -type,iternextfunc,3.2,, -type,lenfunc,3.2,, -type,newfunc,3.2,, -type,objobjargproc,3.2,, -type,objobjproc,3.2,, -type,releasebufferproc,3.12,, -type,reprfunc,3.2,, -type,richcmpfunc,3.2,, -type,setattrfunc,3.2,, -type,setattrofunc,3.2,, -type,setter,3.2,, -type,ssizeargfunc,3.2,, -type,ssizeobjargproc,3.2,, -type,ssizessizeargfunc,3.2,, -type,ssizessizeobjargproc,3.2,, -type,symtable,3.2,,opaque -type,ternaryfunc,3.2,, -type,traverseproc,3.2,, -type,unaryfunc,3.2,, -type,vectorcallfunc,3.12,, -type,visitproc,3.2,, +const,Py_am_aiter,3.5,, +const,Py_am_anext,3.5,, +const,Py_am_await,3.5,, +const,Py_am_send,3.10,, +const,Py_bf_getbuffer,3.11,, +const,Py_bf_releasebuffer,3.11,, +struct,Py_buffer,3.11,,full-abi +typedef,Py_intptr_t,3.2,, +const,Py_mod_abi,3.15,, +const,Py_mod_create,3.5,, +const,Py_mod_doc,3.15,, +const,Py_mod_exec,3.5,, +const,Py_mod_gil,3.13,, +const,Py_mod_methods,3.15,, +const,Py_mod_multiple_interpreters,3.12,, +const,Py_mod_name,3.15,, +const,Py_mod_slots,3.15,, +const,Py_mod_state_clear,3.15,, +const,Py_mod_state_free,3.15,, +const,Py_mod_state_size,3.15,, +const,Py_mod_state_traverse,3.15,, +const,Py_mod_token,3.15,, +const,Py_mp_ass_subscript,3.2,, +const,Py_mp_length,3.2,, +const,Py_mp_subscript,3.2,, +const,Py_nb_absolute,3.2,, +const,Py_nb_add,3.2,, +const,Py_nb_and,3.2,, +const,Py_nb_bool,3.2,, +const,Py_nb_divmod,3.2,, +const,Py_nb_float,3.2,, +const,Py_nb_floor_divide,3.2,, +const,Py_nb_index,3.2,, +const,Py_nb_inplace_add,3.2,, +const,Py_nb_inplace_and,3.2,, +const,Py_nb_inplace_floor_divide,3.2,, +const,Py_nb_inplace_lshift,3.2,, +const,Py_nb_inplace_matrix_multiply,3.5,, +const,Py_nb_inplace_multiply,3.2,, +const,Py_nb_inplace_or,3.2,, +const,Py_nb_inplace_power,3.2,, +const,Py_nb_inplace_remainder,3.2,, +const,Py_nb_inplace_rshift,3.2,, +const,Py_nb_inplace_subtract,3.2,, +const,Py_nb_inplace_true_divide,3.2,, +const,Py_nb_inplace_xor,3.2,, +const,Py_nb_int,3.2,, +const,Py_nb_invert,3.2,, +const,Py_nb_lshift,3.2,, +const,Py_nb_matrix_multiply,3.5,, +const,Py_nb_multiply,3.2,, +const,Py_nb_negative,3.2,, +const,Py_nb_or,3.2,, +const,Py_nb_positive,3.2,, +const,Py_nb_power,3.2,, +const,Py_nb_remainder,3.2,, +const,Py_nb_rshift,3.2,, +const,Py_nb_subtract,3.2,, +const,Py_nb_true_divide,3.2,, +const,Py_nb_xor,3.2,, +const,Py_slot_end,3.15,, +const,Py_slot_invalid,3.15,, +const,Py_slot_subslots,3.15,, +const,Py_sq_ass_item,3.2,, +const,Py_sq_concat,3.2,, +const,Py_sq_contains,3.2,, +const,Py_sq_inplace_concat,3.2,, +const,Py_sq_inplace_repeat,3.2,, +const,Py_sq_item,3.2,, +const,Py_sq_length,3.2,, +const,Py_sq_repeat,3.2,, +typedef,Py_ssize_t,3.2,, +const,Py_tp_alloc,3.2,, +const,Py_tp_base,3.2,, +const,Py_tp_bases,3.2,, +const,Py_tp_basicsize,3.15,, +const,Py_tp_call,3.2,, +const,Py_tp_clear,3.2,, +const,Py_tp_dealloc,3.2,, +const,Py_tp_del,3.2,, +const,Py_tp_descr_get,3.2,, +const,Py_tp_descr_set,3.2,, +const,Py_tp_doc,3.2,, +const,Py_tp_extra_basicsize,3.15,, +const,Py_tp_finalize,3.5,, +const,Py_tp_flags,3.15,, +const,Py_tp_free,3.2,, +const,Py_tp_getattr,3.2,, +const,Py_tp_getattro,3.2,, +const,Py_tp_getset,3.2,, +const,Py_tp_hash,3.2,, +const,Py_tp_init,3.2,, +const,Py_tp_is_gc,3.2,, +const,Py_tp_itemsize,3.15,, +const,Py_tp_iter,3.2,, +const,Py_tp_iternext,3.2,, +const,Py_tp_members,3.2,, +const,Py_tp_metaclass,3.15,, +const,Py_tp_methods,3.2,, +const,Py_tp_module,3.15,, +const,Py_tp_name,3.15,, +const,Py_tp_new,3.2,, +const,Py_tp_repr,3.2,, +const,Py_tp_richcompare,3.2,, +const,Py_tp_setattr,3.2,, +const,Py_tp_setattro,3.2,, +const,Py_tp_slots,3.15,, +const,Py_tp_str,3.2,, +const,Py_tp_token,3.14,, +const,Py_tp_traverse,3.2,, +const,Py_tp_vectorcall,3.14,, +typedef,Py_uintptr_t,3.2,, +typedef,allocfunc,3.2,, +typedef,binaryfunc,3.2,, +typedef,descrgetfunc,3.2,, +typedef,descrsetfunc,3.2,, +typedef,destructor,3.2,, +typedef,getattrfunc,3.2,, +typedef,getattrofunc,3.2,, +typedef,getbufferproc,3.12,, +typedef,getiterfunc,3.2,, +typedef,getter,3.2,, +typedef,hashfunc,3.2,, +typedef,initproc,3.2,, +typedef,inquiry,3.2,, +typedef,iternextfunc,3.2,, +typedef,lenfunc,3.2,, +typedef,newfunc,3.2,, +typedef,objobjargproc,3.2,, +typedef,objobjproc,3.2,, +typedef,releasebufferproc,3.12,, +typedef,reprfunc,3.2,, +typedef,richcmpfunc,3.2,, +typedef,setattrfunc,3.2,, +typedef,setattrofunc,3.2,, +typedef,setter,3.2,, +typedef,ssizeargfunc,3.2,, +typedef,ssizeobjargproc,3.2,, +typedef,ssizessizeargfunc,3.2,, +typedef,ssizessizeobjargproc,3.2,, +struct,symtable,3.2,,opaque +typedef,ternaryfunc,3.2,, +typedef,traverseproc,3.2,, +typedef,unaryfunc,3.2,, +typedef,vectorcallfunc,3.12,, +typedef,visitproc,3.2,, diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 0e762042979c2b..fbbf01fa7da02b 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -28,12 +28,25 @@ from sphinx.application import Sphinx from sphinx.util.typing import ExtensionMetadata -ROLE_TO_OBJECT_TYPE = { - "func": "function", - "macro": "macro", - "member": "member", - "type": "type", - "data": "var", +ABI_KIND_TO_OBJECT_TYPES = { + "function": ["function"], + "member": ["member"], + "struct": ["type"], + "data": ["var"], + "const": ["macro"], + "typedef": ["type"], + # Some macros are documented as functions + "macro": ["macro", "function"], +} + +ABI_KIND_TO_ROLE = { + 'function': 'func', + "member": 'member', + 'data': 'data', + 'struct': 'type', + 'macro': 'macro', + 'const': 'macro', + 'typedef': 'type', } @@ -61,11 +74,11 @@ class ThreadSafetyEntry: @dataclasses.dataclass(frozen=True, slots=True) class StableABIEntry: - # Role of the object. - # Source: Each [item_kind] in stable_abi.toml is mapped to a C Domain role. - role: str + # Kind of the object. + # Source: [.] in stable_abi.toml. + kind: str # Name of the object. - # Source: [.*] in stable_abi.toml. + # Source: [.] in stable_abi.toml. name: str # Version when the object was added to the stable ABI. # (Source: [.*.added] in stable_abi.toml. @@ -177,10 +190,11 @@ def add_annotations(app: Sphinx, doctree: nodes.document) -> None: # Stable ABI annotation. if record := stable_abi_data.get(name): - if ROLE_TO_OBJECT_TYPE[record.role] != objtype: + record_object_types = ABI_KIND_TO_OBJECT_TYPES[record.kind] + if objtype not in record_object_types: msg = ( f"Object type mismatch in limited API annotation for {name}: " - f"{ROLE_TO_OBJECT_TYPE[record.role]!r} != {objtype!r}" + f"{objtype!r} not in {record_object_types!r}" ) raise ValueError(msg) annotation = _stable_abi_annotation(record) @@ -238,10 +252,14 @@ def _stable_abi_annotation( emph_node += nodes.Text(" ") emph_node += nodes.literal(record.name, record.name) message = sphinx_gettext("is part of the") - emph_node += nodes.Text(" " + message + " ") + elif record.kind == "macro": + # Macros are not part of the ABI + message = sphinx_gettext( + "This macro is available when compiling for the" + ) else: message = sphinx_gettext("Part of the") - emph_node += nodes.Text(" " + message + " ") + emph_node += nodes.Text(" " + message + " ") ref_node = addnodes.pending_xref( "Stable ABI", refdomain="std", @@ -365,7 +383,7 @@ class LimitedAPIList(SphinxDirective): def run(self) -> list[nodes.Node]: state = self.env.domaindata["c_annotations"] content = [ - f"* :c:{record.role}:`{record.name}`" + f"* :c:{ABI_KIND_TO_ROLE[record.kind]}:`{record.name}`" for record in state["stable_abi_data"].values() ] node = nodes.paragraph() diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 751c186674c7c9..e79fa33319a39b 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -64,6 +64,8 @@ # on all platforms.) # - abi3t_opaque: In abi3t, this struct is opaque (as if `struct_abi_kind` # was 'opaque' and `members` was missing). +# - is_static_inline_function: for `macro`: actually implemented as a static +# inline function, rather than a macro. # Removing items from this file is generally not allowed, and additions should # be considered with that in mind. See the devguide for exact rules: @@ -1872,6 +1874,22 @@ [function.PyThread_start_new_thread] added = '3.2' +# Python 3.2 macros + +[macro.Py_CLEAR] + added = '3.2' +[macro.Py_DECREF] + added = '3.2' +[macro.Py_INCREF] + added = '3.2' + is_static_inline_function = true +[macro.Py_XDECREF] + added = '3.2' + is_static_inline_function = true +[macro.Py_XINCREF] + added = '3.2' + is_static_inline_function = true + ### Python 3.3 ############################################################## @@ -2225,6 +2243,15 @@ [function.PyObject_GC_IsTracked] added = '3.9' +# Python 3.9 macros + +[macro.Py_SET_REFCNT] + added = '3.9' + is_static_inline_function = true +[macro.Py_SET_TYPE] + added = '3.9' + is_static_inline_function = true + ### Python 3.10 ############################################################# @@ -2451,7 +2478,7 @@ added = '3.12' [function.PyObject_VectorcallMethod] added = '3.12' -[macro.PY_VECTORCALL_ARGUMENTS_OFFSET] +[const.PY_VECTORCALL_ARGUMENTS_OFFSET] added = '3.12' [typedef.getbufferproc] added = '3.12' diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 39115b331ba642..5f70a706d139b6 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -116,7 +116,6 @@ def decorator(cls): return decorator @itemclass('function') -@itemclass('macro') @itemclass('data') @itemclass('const') @itemclass('typedef') @@ -144,6 +143,11 @@ class Struct(ABIItem): struct_abi_kind: str members: list = None +@itemclass('macro') +@dataclasses.dataclass +class Macro(ABIItem): + is_static_inline_function: bool = False + def parse_manifest(file): """Parse the given file (iterable of lines) to a Manifest""" @@ -227,13 +231,13 @@ def sort_key(item): key=sort_key): write(f'EXPORT_DATA({item.name})') -ITEM_KIND_TO_DOC_ROLE = { - 'function': 'func', - 'data': 'data', - 'struct': 'type', - 'macro': 'macro', - 'const': 'macro', - 'typedef': 'type', +DOC_KINDS = { + 'function', + 'data', + 'struct', + 'macro', + 'const', + 'typedef', } @generator("doc_list", 'Doc/data/stable_abi.dat') @@ -245,17 +249,16 @@ def gen_doc_annotations(manifest, args, outfile): """ writer = csv.DictWriter( outfile, - ['role', 'name', 'added', 'ifdef_note', 'struct_abi_kind'], + ['kind', 'name', 'added', 'ifdef_note', 'struct_abi_kind'], lineterminator='\n') writer.writeheader() - kinds = set(ITEM_KIND_TO_DOC_ROLE) - for item in manifest.select(kinds, include_abi_only=False): + for item in manifest.select(DOC_KINDS, include_abi_only=False): if item.ifdef: ifdef_note = manifest.contents[item.ifdef].doc else: ifdef_note = None row = { - 'role': ITEM_KIND_TO_DOC_ROLE[item.kind], + 'kind': item.kind, 'name': item.name, 'added': item.added, 'ifdef_note': ifdef_note, @@ -265,7 +268,7 @@ def gen_doc_annotations(manifest, args, outfile): row['struct_abi_kind'] = item.struct_abi_kind for member_name in item.members or (): rows.append({ - 'role': 'member', + 'kind': 'member', 'name': f'{item.name}.{member_name}', 'added': item.added, }) @@ -408,7 +411,10 @@ def do_unixy_check(manifest, args): feature_macros &= present_macros # Check that we have all needed macros - expected_macros = {item.name for item in manifest.select({'macro'})} + expected_macros = { + item.name for item in manifest.select({'macro'}) + if not item.is_static_inline_function + } missing_macros = expected_macros - present_macros okay &= _report_unexpected_items( missing_macros, From 2eb2af805c156ca52b316bb4ced5e0ab3971d0ae Mon Sep 17 00:00:00 2001 From: metsw24-max Date: Thu, 8 Oct 2026 18:10:51 +0530 Subject: [PATCH 125/240] gh-150751: validate http.client Content-Length and chunk-size (GH-150752) RFC 9112 defines Content-Length as 1*DIGIT and chunk-size as 1*HEXDIG, but int() also accepts a sign, underscores, surrounding whitespace and an 0x prefix, so malformed framing values were parsed instead of rejected. --- Lib/http/client.py | 18 +++++++------ Lib/test/test_httplib.py | 27 +++++++++++++++++++ ...-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst | 5 ++++ 3 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 7ef99e7201c005..4b6fc549a9b517 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -165,6 +165,11 @@ # to prevent http header injection. _contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') +# RFC 9112: Content-Length = 1*DIGIT and chunk-size = 1*HEXDIG. +# int() is more permissive, so we match against the grammar before calling it. +_is_legal_content_length = re.compile(r'[0-9]+').fullmatch +_is_legal_chunk_size = re.compile(rb'[0-9a-fA-F]+').fullmatch + # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} @@ -392,14 +397,8 @@ def begin(self, *, _max_headers=None): # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked" self.length = None length = self.headers.get("content-length") - if length and not self.chunked: - try: - self.length = int(length) - except ValueError: - self.length = None - else: - if self.length < 0: # ignore nonsensical negative lengths - self.length = None + if length and not self.chunked and _is_legal_content_length(length): + self.length = int(length) else: self.length = None @@ -566,7 +565,10 @@ def _read_next_chunk_size(self): i = line.find(b";") if i >= 0: line = line[:i] # strip chunk-extensions + line = line.rstrip() try: + if not _is_legal_chunk_size(line): + raise ValueError("invalid chunk size") return int(line, 16) except ValueError: # close the connection as protocol synchronisation is diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 5b1d6e0aa52079..ad28e921978e98 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1348,6 +1348,33 @@ def test_negative_content_length(self): self.assertEqual(resp.read(), b'Hello\r\n') self.assertTrue(resp.isclosed()) + def test_malformed_content_length(self): + # RFC 9112: Content-Length = 1*DIGIT. Values that int() accepts but + # the grammar forbids must not be used to frame the body. + for value in ('+5', '5_0'): + with self.subTest(value=value): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\nHello\r\n' % value) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertIsNone(resp.length) + self.assertEqual(resp.read(), b'Hello\r\n') + resp.close() + + def test_malformed_chunk_size(self): + # RFC 9112: chunk-size = 1*HEXDIG. Reject sizes that int(_, 16) accepts + # but the grammar forbids (a sign, an "0x" prefix, underscores or + # leading whitespace). + start = 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n' + for size in ('-5', '+5', '0x5', '1_f', ' 5'): + with self.subTest(size=size): + sock = FakeSocket(start + '%s\r\nHELLO\r\n0\r\n\r\n' % size) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertRaises(client.IncompleteRead, resp.read) + self.assertTrue(resp.isclosed()) + resp.close() + def test_incomplete_read(self): sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') resp = client.HTTPResponse(sock, method="GET") diff --git a/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst b/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst new file mode 100644 index 00000000000000..670d677aff79ff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst @@ -0,0 +1,5 @@ +:mod:`http.client` now validates the ``Content-Length`` header and the +chunked ``chunk-size`` against the RFC 9112 grammar (``1*DIGIT`` and +``1*HEXDIG``) before parsing them, rejecting values such as ``+5``, ``5_0`` +or a ``0x``-prefixed or negative chunk size that :func:`int` would otherwise +accept. This avoids framing a response differently from a strict peer. From 56f9837db25fd65fa6d10862e572270574ab2b0d Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Thu, 8 Oct 2026 08:52:30 -0400 Subject: [PATCH 126/240] gh-158824: Fix `__getattr__` blocking reification of lazy imports (GH-158825) When a module attribute was a lazy import placeholder, module attribute lookup called the module's __getattr__ before resolving it. This was needed while lazy from-imports published placeholders on the source module. That no longer happens, so the check only affected the module's own lazy imports: a catch-all __getattr__ hid them, unlike eager imports. Resolve the placeholder directly. --- Lib/test/test_lazy_import/__init__.py | 13 +++++++++++ .../module_with_lazy_import_and_getattr.py | 5 ++++ ...-10-05-03-25-30.gh-issue-158824.h-hJj7.rst | 2 ++ Objects/moduleobject.c | 23 ------------------- 4 files changed, 20 insertions(+), 23 deletions(-) create mode 100644 Lib/test/test_lazy_import/data/module_with_lazy_import_and_getattr.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-05-03-25-30.gh-issue-158824.h-hJj7.rst diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index e0c5552fe11c53..8a90ffa662e6e2 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -168,6 +168,19 @@ def test_from_import_with_imported_module_getattr(self): """) assert_python_ok("-c", code) + @support.requires_subprocess() + def test_module_getattr_does_not_shadow_own_lazy_import(self): + """Module __getattr__ should not shadow the module's own lazy imports.""" + code = textwrap.dedent(""" + import types + import test.test_lazy_import.data.module_with_lazy_import_and_getattr as mod + assert mod.basic2.__name__ == "test.test_lazy_import.data.basic2" + assert mod.f is mod.basic2.f + assert not isinstance(vars(mod)["basic2"], types.LazyImportType) + assert mod.missing == "from_getattr:missing" + """) + assert_python_ok("-c", code) + class GlobalLazyImportModeTests(LazyImportTestCase): """Tests for sys.set_lazy_imports() global mode control.""" diff --git a/Lib/test/test_lazy_import/data/module_with_lazy_import_and_getattr.py b/Lib/test/test_lazy_import/data/module_with_lazy_import_and_getattr.py new file mode 100644 index 00000000000000..e9a1f1a88e909f --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_with_lazy_import_and_getattr.py @@ -0,0 +1,5 @@ +lazy import test.test_lazy_import.data.basic2 as basic2 +lazy from test.test_lazy_import.data.basic2 import f + +def __getattr__(name): + return f"from_getattr:{name}" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-05-03-25-30.gh-issue-158824.h-hJj7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-05-03-25-30.gh-issue-158824.h-hJj7.rst new file mode 100644 index 00000000000000..b7bcc999ac68d7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-05-03-25-30.gh-issue-158824.h-hJj7.rst @@ -0,0 +1,2 @@ +Fix module-level :meth:`~module.__getattr__` being called before a module's +own lazy imports were resolved, which allowed it to shadow them. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d13b513ab468e4..164f24e69c69ba 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1362,29 +1362,6 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) } if (attr) { if (PyLazyImport_CheckExact(attr)) { - // gh-144957: Module __getattr__ should get a chance to provide - // the attribute before resolving a lazy import placeholder. - // Resolving __getattr__ itself must not invoke the hook. - getattr = NULL; - if (!_PyUnicode_EqualToASCIIString(name, "__getattr__") && - module_get_resolved_dict_item( - m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { - Py_DECREF(attr); - return NULL; - } - if (getattr) { - PyObject *result = PyObject_CallOneArg(getattr, name); - Py_DECREF(getattr); - if (result != NULL) { - Py_DECREF(attr); - return result; - } - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - Py_DECREF(attr); - return NULL; - } - PyErr_Clear(); - } PyThreadState *tstate = PyThreadState_GET(); if (suppress && _PyLazyImport_IsResolving(tstate, attr)) { // Only direct reentry means this attribute is unavailable. From eb77b4b2e4180b779e3d4c7d42c6a47ba965f74b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Oct 2026 15:45:24 +0200 Subject: [PATCH 127/240] Add os.supports_dir_fd to test.pythoninfo (GH-158605) Split the attribute lists so the 'supports_*' names don't need to be repeated twice. Add 'supports_dir_fd'. --- Lib/test/pythoninfo.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 100e61a20211c2..bc6735d0749f52 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -247,21 +247,23 @@ def collect_urandom(info_add): def collect_os(info_add): import os - def format_attr(attr, value): - if attr in ('supports_follow_symlinks', 'supports_fd', - 'supports_effective_ids'): - return str(sorted(func.__name__ for func in value)) - else: - return value - - attributes = ( + plain_attributes = ( 'name', 'supports_bytes_environ', + ) + copy_attributes(info_add, os, 'os.%s', plain_attributes) + + def format_set_of_functions(attr, value): + return str(sorted(func.__name__ for func in value)) + + supports_attributes = ( + 'supports_dir_fd', 'supports_effective_ids', 'supports_fd', 'supports_follow_symlinks', ) - copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr) + copy_attributes(info_add, os, 'os.%s', supports_attributes, + formatter=format_set_of_functions) for func in ( 'cpu_count', From 05d80ccfe77a781667d1b9e25247c193559cd7cc Mon Sep 17 00:00:00 2001 From: Christian Aurich Zanettini Martins Date: Thu, 8 Oct 2026 10:58:00 -0300 Subject: [PATCH 128/240] gh-158803: Fix crash in bytes.join() on a concurrently mutated list (#158910) In the free-threaded build, bytes.join() and bytearray.join() read items from the list with borrowed references and without holding its lock, so another thread could replace and free an item before it was increfed. Run the join under Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST, as PyUnicode_Join() already does. --- .../test_free_threading/test_bytes_object.py | 24 ++++++++++++- ...-10-06-05-12-21.gh-issue-158803.iYTpRQ.rst | 3 ++ Objects/stringlib/join.h | 34 ++++++++++++------- 3 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-06-05-12-21.gh-issue-158803.iYTpRQ.rst diff --git a/Lib/test/test_free_threading/test_bytes_object.py b/Lib/test/test_free_threading/test_bytes_object.py index a371e3d533a2cb..a9017bac6de029 100644 --- a/Lib/test/test_free_threading/test_bytes_object.py +++ b/Lib/test/test_free_threading/test_bytes_object.py @@ -1,5 +1,5 @@ import unittest -from threading import Thread, Barrier +from threading import Thread, Barrier, Event from test.support import threading_helper threading_helper.requires_working_threading(module=True) @@ -32,6 +32,28 @@ def work(ii): barrier.reset() + def test_racing_join_replace(self): + # gh-158803: join() must not use a list item that another thread + # replaces (and frees) concurrently. + lst = [bytes(10) for _ in range(100)] + done = Event() + + def writer(): + try: + for _ in range(100): + for i in range(len(lst)): + lst[i] = bytearray(10) if i % 2 else bytes(10) + finally: + done.set() + + def reader(): + while not done.is_set(): + b''.join(lst) + b'-'.join(lst) + bytearray().join(lst) + + threading_helper.run_concurrently([writer] + [reader] * 4) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-06-05-12-21.gh-issue-158803.iYTpRQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-06-05-12-21.gh-issue-158803.iYTpRQ.rst new file mode 100644 index 00000000000000..8c95d4882f7863 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-06-05-12-21.gh-issue-158803.iYTpRQ.rst @@ -0,0 +1,3 @@ +Fix a crash in :meth:`bytes.join` and :meth:`bytearray.join` in the +:term:`free-threaded build` when another thread concurrently mutates the +list being joined. Patch by Christian Aurich Zanettini Martins. diff --git a/Objects/stringlib/join.h b/Objects/stringlib/join.h index 5fd2ca70f98dd3..4764baac181b2a 100644 --- a/Objects/stringlib/join.h +++ b/Objects/stringlib/join.h @@ -5,7 +5,7 @@ #endif Py_LOCAL_INLINE(PyObject *) -STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) +STRINGLIB(bytes_join_lock_held)(PyObject *sep, PyObject *seq) { const char *sepstr = STRINGLIB_STR(sep); Py_ssize_t seplen = STRINGLIB_LEN(sep); @@ -14,7 +14,7 @@ STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) Py_ssize_t seqlen = 0; Py_ssize_t sz = 0; Py_ssize_t i, nbufs; - PyObject *seq, *item; + PyObject *item; Py_buffer *buffers = NULL; #define NB_STATIC_BUFFERS 10 Py_buffer static_buffers[NB_STATIC_BUFFERS]; @@ -22,30 +22,21 @@ STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) int drop_gil = 1; PyThreadState *save = NULL; - seq = PySequence_Fast(iterable, "can only join an iterable"); - if (seq == NULL) { - return NULL; - } - seqlen = PySequence_Fast_GET_SIZE(seq); if (seqlen == 0) { - Py_DECREF(seq); return STRINGLIB_NEW(NULL, 0); } #if !STRINGLIB_MUTABLE if (seqlen == 1) { item = PySequence_Fast_GET_ITEM(seq, 0); if (STRINGLIB_CHECK_EXACT(item)) { - Py_INCREF(item); - Py_DECREF(seq); - return item; + return Py_NewRef(item); } } #endif if (seqlen > NB_STATIC_BUFFERS) { buffers = PyMem_NEW(Py_buffer, seqlen); if (buffers == NULL) { - Py_DECREF(seq); PyErr_NoMemory(); return NULL; } @@ -157,7 +148,6 @@ STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) error: res = NULL; done: - Py_DECREF(seq); for (i = 0; i < nbufs; i++) PyBuffer_Release(&buffers[i]); if (buffers != static_buffers) @@ -165,5 +155,23 @@ STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) return res; } +Py_LOCAL_INLINE(PyObject *) +STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) +{ + PyObject *seq, *res; + + seq = PySequence_Fast(iterable, "can only join an iterable"); + if (seq == NULL) { + return NULL; + } + + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(iterable); + res = STRINGLIB(bytes_join_lock_held)(sep, seq); + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); + + Py_DECREF(seq); + return res; +} + #undef NB_STATIC_BUFFERS #undef GIL_THRESHOLD From 2423814eb64730c148600d0704717e01d5b59aa8 Mon Sep 17 00:00:00 2001 From: Lazizbek Ergashev Date: Thu, 8 Oct 2026 20:12:09 +0500 Subject: [PATCH 129/240] gh-156895: Document CXX in ./configure --help output (#157046) --- configure | 2 ++ configure.ac | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/configure b/configure index d6f5319beb7037..35035e45b21a19 100755 --- a/configure +++ b/configure @@ -1175,6 +1175,7 @@ LDFLAGS LIBS CPPFLAGS CPP +CXX PROFILE_TASK BOLT_COMMON_FLAGS BOLT_INSTRUMENT_FLAGS @@ -2033,6 +2034,7 @@ Some influential environment variables: CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor + CXX C++ compiler command PROFILE_TASK Python args for PGO generation task BOLT_COMMON_FLAGS diff --git a/configure.ac b/configure.ac index 9ed32b686f033a..6037e79801b450 100644 --- a/configure.ac +++ b/configure.ac @@ -1146,7 +1146,7 @@ AC_CACHE_CHECK([for GCC compatible compiler], [ac_cv_gcc_compat=yes], [ac_cv_gcc_compat=no])]) -AC_SUBST([CXX]) +AC_ARG_VAR([CXX], [C++ compiler command]) preset_cxx="$CXX" if test -z "$CXX" From e2c3c7e4dd8624315b1b263cfc50c1b3201c249a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 8 Oct 2026 19:48:43 +0200 Subject: [PATCH 130/240] gh-158942: Use more unique variable names in Py_MIN/MAX/ABS() (#158969) If Py_MIN/MAX/ABS() macros are called on an existing "_x" or "_y" variable name, the external variable is used instead of macro local variable. So use more unique names in the macros to avoid the issue. Add test_min_max_abs_macros() to _testlimitedcapi. --- Include/pymacro.h | 16 ++++----- Modules/_testlimitedcapi/object.c | 58 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h index 122d26315b51b9..41dfd9155b9b24 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -118,20 +118,20 @@ /* Minimum value between x and y */ # define Py_MIN(x, y) \ __extension__ \ - ({ _Py_TYPEOF (x) _x = (x); \ - _Py_TYPEOF (y) _y = (y); \ - _x < _y ? _x : _y; }) + ({ _Py_TYPEOF (x) _PyMIN_x = (x); \ + _Py_TYPEOF (y) _PyMIN_y = (y); \ + _PyMIN_x < _PyMIN_y ? _PyMIN_x : _PyMIN_y; }) /* Maximum value between x and y */ # define Py_MAX(x, y) \ __extension__ \ - ({ _Py_TYPEOF (x) _x = (x); \ - _Py_TYPEOF (y) _y = (y); \ - _x > _y ? _x : _y; }) + ({ _Py_TYPEOF (x) _PyMAX_x = (x); \ + _Py_TYPEOF (y) _PyMAX_y = (y); \ + _PyMAX_x > _PyMAX_y ? _PyMAX_x : _PyMAX_y; }) /* Absolute value of the number x */ # define Py_ABS(x) \ __extension__ \ - ({ _Py_TYPEOF (x) _x = (x); \ - _x < 0 ? -_x : _x; }) + ({ _Py_TYPEOF (x) _PyABS_x = (x); \ + _PyABS_x < 0 ? -_PyABS_x : _PyABS_x; }) #else /* Minimum value between x and y */ # define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) diff --git a/Modules/_testlimitedcapi/object.c b/Modules/_testlimitedcapi/object.c index 03debb56b33c4b..5eee2a9a5210e7 100644 --- a/Modules/_testlimitedcapi/object.c +++ b/Modules/_testlimitedcapi/object.c @@ -241,6 +241,63 @@ test_refcount_macros(PyObject *self, PyObject *Py_UNUSED(ignored)) TEST_REFCOUNT(); } + +// Test Py_MIN(), Py_MAX() and Py_ABS() macros. +// On GCC/clang, they are implemented as a statement expression to only +// evaluate each argument only once. +static PyObject* +test_min_max_abs_macros(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int x = 5, y = 7, z = -11; + + // Simple usage + assert(Py_MIN(x, y) == 5); + assert(Py_MAX(x, y) == 7); + assert(Py_ABS(z) == 11); + + // Combined macros + assert(Py_MIN(x, Py_MIN(y, z)) == -11); + assert(Py_MAX(x, Py_MAX(y, z)) == 7); + assert(Py_MIN(x, Py_MAX(y, z)) == 5); + assert(Py_MAX(x, Py_MIN(y, z)) == 5); + assert(Py_ABS(Py_ABS(z)) == 11); + + // A few more tests + assert(Py_MIN(2, Py_MIN(6, 12)) == 2); + assert(Py_MAX(2, Py_MAX(6, 12)) == 12); + assert(Py_MIN(2, Py_MAX(6, 12)) == 2); + assert(Py_MAX(2, Py_MIN(6, 12)) == 6); + assert(Py_MIN(Py_MIN(2, 6), 12) == 2); + assert(Py_MAX(Py_MAX(2, 6), 12) == 12); + assert(Py_MIN(Py_MAX(2, 6), 12) == 6); + assert(Py_MAX(Py_MIN(2, 6), 12) == 12); + assert(Py_ABS(Py_ABS(12)) == 12); + assert(Py_ABS(Py_ABS(12)) == 12); + + // Integer limits + assert(Py_MIN(123, INT_MIN) == INT_MIN); + assert(Py_MAX(123, INT_MAX) == INT_MAX); + assert(Py_ABS(INT_MAX) == INT_MAX); + // Do not tests Py_ABS(INT_MIN), since the behavior is documented. + // This limitation is documented in Py_ABS() documentation. + +#if ((defined(__GNUC__) || defined(__clang__)) \ + && defined(_Py_TYPEOF) && !defined(__cplusplus)) + // Check that arguments are only evaluated once + int a = 5, b = 7, c = -11; + assert(Py_MIN(++a, ++b) == 6); + assert(a == 6); + assert(b == 8); + assert(Py_MAX(++a, ++b) == 9); + assert(a == 7); + assert(b == 9); + assert(Py_ABS(--c) == 12); + assert(c == -12); +#endif + + Py_RETURN_NONE; +} + #undef Py_NewRef #undef Py_XNewRef @@ -315,6 +372,7 @@ static PyMethodDef test_methods[] = { {"test_py_setref", test_py_setref, METH_NOARGS}, {"test_refcount_macros", test_refcount_macros, METH_NOARGS}, {"test_refcount_funcs", test_refcount_funcs, METH_NOARGS}, + {"test_min_max_abs_macros", test_min_max_abs_macros, METH_NOARGS}, {"test_py_is_macros", test_py_is_macros, METH_NOARGS}, {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {NULL}, From 2cd747b89bd5e0ad06edc506bb62a7c5e40e02cc Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Thu, 8 Oct 2026 22:03:51 +0200 Subject: [PATCH 131/240] Add a template for GHSAs (#158612) Co-authored-by: Ezio Melotti --- .github/VULNERABILITY_REPORT.yml | 73 ++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/VULNERABILITY_REPORT.yml diff --git a/.github/VULNERABILITY_REPORT.yml b/.github/VULNERABILITY_REPORT.yml new file mode 100644 index 00000000000000..2887c7092d19fe --- /dev/null +++ b/.github/VULNERABILITY_REPORT.yml @@ -0,0 +1,73 @@ +name: Vulnerability report +description: Privately report a potential security vulnerability in CPython +body: + - type: markdown + attributes: + value: | + **Not all bugs are vulnerabilities.** Before submitting, read the [Python security policy](https://devguide.python.org/security/policy/) to understand which issues are vulnerabilities and what versions of Python accept reports. + + Keep the report short and in plain text: no headers, tables, PDFs, binaries, or severity and CVSS information. + + Reports that do not contain a potential security vulnerability will be discarded without a reply. + + To report vulnerabilities that affect other projects (such as pip or python.org), or if you are not sure where to send your report, email [security@python.org](mailto:security@python.org). + - type: textarea + id: summary + attributes: + label: Summary + description: A few sentences describing the vulnerability. + validations: + required: true + - type: textarea + id: threat_model + attributes: + label: Threat model + description: > + What does the attacker control, and what do they gain? Describe the code, configuration, or deployment that may exist in the real world and is exploitable. Where possible, cite the relevant part of the [security policy](https://devguide.python.org/security/policy/#what-types-of-bugs-are-vulnerabilities). + validations: + required: true + - type: textarea + id: proof_of_concept + attributes: + label: Proof of concept + description: > + A script that reproduces the issue and clearly indicates whether the vulnerability is present, such as exiting with `1` if vulnerable and `0` if not. If it depends on a specially constructed binary file, include a script to construct the file rather than the file itself. + + Wrap scripts longer than a few lines in a [collapsed section](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections) using `
...
`. + validations: + required: true + - type: dropdown + id: versions + attributes: + label: "CPython versions tested on:" + description: > + If any tested version was not vulnerable, say which in the summary. Only [supported versions](https://devguide.python.org/versions/) accept reports. + multiple: true + options: + - "3.11" + - "3.12" + - "3.13" + - "3.14" + - "3.15" + - "3.16" + - "CPython main branch" + validations: + required: true + - type: textarea + id: patch + attributes: + label: Suggested fix + description: Ideally, a minimal patch with the mitigation. + validations: + required: false + - type: checkboxes + id: checklist + attributes: + label: Before submitting + options: + - label: I have read the security policy and evaluated this report against it. + required: true + - label: I have checked that this issue is not already resolved on the `main` branch. + required: true + - label: I have verified the factual validity of everything in this report, including any content produced by an LLM. + required: true From 560738126e42208e71fa372c2826f9ccda3e4068 Mon Sep 17 00:00:00 2001 From: Alex Malyshev Date: Thu, 8 Oct 2026 17:32:33 -0400 Subject: [PATCH 132/240] gh-156491: Avoid KeyErrors in os._Environ.get() and __contains__() (GH-156492) Implement get() and __contains__() on os._Environ directly, which cuts out the cost of raising and catching a KeyError when the keys do not exist. --- Lib/os.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/os.py b/Lib/os.py index 87547e369db817..98d476f98419b2 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -720,6 +720,11 @@ def get_exec_path(env=None): # Change environ to automatically call putenv() and unsetenv() from _collections_abc import MutableMapping, Mapping +# Sentinel used for seeing if a value is found within the internal _Environ +# dictionary. +_MISSING = sentinel("MISSING") + + class _Environ(MutableMapping): def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue): self.encodekey = encodekey @@ -728,6 +733,9 @@ def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue): self.decodevalue = decodevalue self._data = data + def __contains__(self, key): + return self.encodekey(key) in self._data + def __getitem__(self, key): try: value = self._data[self.encodekey(key)] @@ -770,6 +778,10 @@ def __repr__(self): def copy(self): return dict(self) + def get(self, key, default=None): + val = self._data.get(self.encodekey(key), _MISSING) + return default if val is _MISSING else self.decodevalue(val) + def setdefault(self, key, value): if key not in self: self[key] = value From bc12b7285f87f3ca9118cb4b61c28a84e7984ba3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 8 Oct 2026 19:45:03 -0400 Subject: [PATCH 133/240] gh-158572: Add an external flag to interpreter frames (#158573) Adds an `external` flag to a bit-packed byte on the _PyInterpreterFrame struct in order to provide external frame evaluators a way to distinguish VM-pushed frames vs externally- pushed frames. --- Include/internal/pycore_interpframe_structs.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 560fb66a97fc89..0d6a37cc2c1a17 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -42,12 +42,11 @@ struct _PyInterpreterFrame { #endif uint16_t return_offset; /* Only relevant during a function call */ char owner; -#ifdef Py_DEBUG uint8_t visited:1; + uint8_t external:1; /* Set when the frame was pushed by something other than the VM */ +#ifdef Py_DEBUG uint8_t stackpointer_valid:1; - uint8_t lltrace:6; -#else - uint8_t visited; + uint8_t lltrace:5; #endif /* Locals and stack */ _PyStackRef localsplus[1]; From c524b30b8e80738127cc6fedf54abf61c1a57974 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:42:39 +0200 Subject: [PATCH 134/240] gh-142927: Update Tachyon examples (#159050) --- Doc/_static/tachyon-example-flamegraph.html | 1013 ++++++++++++++----- Doc/_static/tachyon-example-heatmap.html | 410 ++++++-- 2 files changed, 1045 insertions(+), 378 deletions(-) diff --git a/Doc/_static/tachyon-example-flamegraph.html b/Doc/_static/tachyon-example-flamegraph.html index 701c959bda3ccc..0563e7ce4acfe6 100644 --- a/Doc/_static/tachyon-example-flamegraph.html +++ b/Doc/_static/tachyon-example-flamegraph.html @@ -1,5 +1,5 @@ - + @@ -166,6 +166,14 @@ /* Heatmap span highlighting colors */ --span-hot-base: 255, 100, 50; --span-cold-base: 150, 150, 150; + + /* Summary card colors - optimized for 4.5:1 contrast on light bg */ + --card-blue: 44, 102, 149; + --card-green: 26, 116, 49; + --card-yellow: 134, 100, 4; + --card-purple: 102, 57, 166; + --card-red: 180, 40, 50; + --card-orange: 166, 90, 0; } /* Dark theme */ @@ -190,15 +198,15 @@ --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); - /* Dark mode heat palette - muted colors that provide sufficient contrast with light text */ - --heat-1: rgba(74, 123, 167, 0.35); - --heat-2: rgba(90, 159, 168, 0.38); - --heat-3: rgba(106, 181, 181, 0.40); - --heat-4: rgba(126, 196, 136, 0.42); - --heat-5: rgba(160, 216, 120, 0.45); - --heat-6: rgba(196, 222, 106, 0.48); - --heat-7: rgba(244, 212, 77, 0.50); - --heat-8: rgba(255, 107, 53, 0.55); + /* Dark mode heat palette - cool to warm gradient for visualization */ + --heat-1: rgba(90, 123, 167, 1); + --heat-2: rgba(106, 148, 168, 1); + --heat-3: rgba(122, 172, 172, 1); + --heat-4: rgba(142, 196, 152, 1); + --heat-5: rgba(168, 216, 136, 1); + --heat-6: rgba(200, 222, 122, 1); + --heat-7: rgba(244, 212, 93, 1); + --heat-8: rgba(255, 122, 69, 1); /* Code view specific - dark mode */ --code-bg: #0d1117; @@ -228,6 +236,14 @@ /* Heatmap span highlighting colors - dark theme */ --span-hot-base: 255, 107, 53; --span-cold-base: 189, 189, 189; + + /* Summary card colors - optimized for 4.5:1 contrast on dark bg */ + --card-blue: 88, 166, 255; + --card-green: 63, 185, 80; + --card-yellow: 210, 153, 34; + --card-purple: 163, 113, 247; + --card-red: 248, 113, 113; + --card-orange: 251, 146, 60; } /* -------------------------------------------------------------------------- @@ -571,6 +587,46 @@ This file extends the shared foundation with flamegraph-specific styles. ========================================================================== */ +/* -------------------------------------------------------------------------- + Differential Flamegraph + -------------------------------------------------------------------------- */ + +:root { + /* Regression colors */ + --diff-regression-deep: #d32f2f; + --diff-regression-medium: #e57373; + --diff-regression-light: #ef9a9a; + --diff-regression-verylight: #ffcdd2; + + /* Improvement colors */ + --diff-improvement-deep: #1976d2; + --diff-improvement-medium: #42a5f5; + --diff-improvement-light: #64b5f6; + --diff-improvement-verylight: #90caf9; + + /* Other differential colors */ + --diff-neutral: #bdbdbd; + --diff-new: #9575cd; + --diff-elided: #b39ddb; +} + +/* Dark mode differential colors - adjusted for contrast against dark backgrounds */ +[data-theme="dark"] { + --diff-regression-deep: #ef5350; + --diff-regression-medium: #e57373; + --diff-regression-light: #ef9a9a; + --diff-regression-verylight: #ffcdd2; + + --diff-improvement-deep: #42a5f5; + --diff-improvement-medium: #64b5f6; + --diff-improvement-light: #90caf9; + --diff-improvement-verylight: #bbdefb; + + --diff-neutral: #757575; + --diff-new: #b39ddb; + --diff-elided: #ce93d8; +} + /* -------------------------------------------------------------------------- Layout Overrides (Flamegraph-specific) -------------------------------------------------------------------------- */ @@ -841,9 +897,17 @@ } /* View Mode Section */ +.view-mode-section { + display: flex; + flex-direction: column; + gap: 8px; +} + .view-mode-section .section-content { display: flex; - justify-content: center; + flex-direction: column; + gap: 10px; + align-items: center; } /* Collapsible sections */ @@ -912,10 +976,10 @@ position: relative; } -.summary-card:nth-child(1) { --i: 0; --card-color: 55, 118, 171; } -.summary-card:nth-child(2) { --i: 1; --card-color: 40, 167, 69; } -.summary-card:nth-child(3) { --i: 2; --card-color: 255, 193, 7; } -.summary-card:nth-child(4) { --i: 3; --card-color: 111, 66, 193; } +.summary-card:nth-child(1) { --i: 0; --card-color: var(--card-blue); } +.summary-card:nth-child(2) { --i: 1; --card-color: var(--card-green); } +.summary-card:nth-child(3) { --i: 2; --card-color: var(--card-yellow); } +.summary-card:nth-child(4) { --i: 3; --card-color: var(--card-purple); } .summary-card:hover { border-color: rgba(var(--card-color), 0.6); @@ -971,21 +1035,18 @@ text-overflow: ellipsis; } -/* Efficiency Bar */ -.efficiency-section { - margin-top: 10px; - padding-top: 10px; - border-top: 1px solid var(--border); -} +/* -------------------------------------------------------------------------- + Progress Bars + -------------------------------------------------------------------------- */ -.efficiency-header { +.bar-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 5px; } -.efficiency-label { +.bar-label { font-size: 9px; font-weight: 600; color: var(--text-secondary); @@ -993,21 +1054,21 @@ letter-spacing: 0.2px; } -.efficiency-value { +.bar-value { font-family: var(--font-mono); font-size: 11px; font-weight: 700; color: var(--accent); } -.efficiency-bar { +.bar { height: 6px; background: var(--bg-tertiary); border-radius: 3px; overflow: hidden; } -.efficiency-fill { +.bar-fill { height: 100%; background: linear-gradient(90deg, #28a745 0%, #20c997 50%, #17a2b8 100%); border-radius: 3px; @@ -1016,7 +1077,7 @@ overflow: hidden; } -.efficiency-fill::after { +.bar-fill::after { content: ''; position: absolute; top: 0; @@ -1033,68 +1094,56 @@ } /* -------------------------------------------------------------------------- - Thread Stats Grid (in Sidebar) + Efficiency Section Container + -------------------------------------------------------------------------- */ + +.efficiency-section { + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid var(--border); +} + +/* -------------------------------------------------------------------------- + Thread Stats Progress Bars (in Sidebar) -------------------------------------------------------------------------- */ .thread-stats-section { display: block; } -.stats-grid { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 8px; +.stats-container { + display: flex; + flex-direction: column; + gap: 10px; } -.stat-tile { - background: var(--bg-primary); - border-radius: 8px; - padding: 10px; - text-align: center; - border: 2px solid var(--border); - transition: all var(--transition-fast); +.stat-item { animation: fadeIn 0.4s ease-out backwards; animation-delay: calc(var(--i, 0) * 0.05s); } -.stat-tile:nth-child(1) { --i: 0; } -.stat-tile:nth-child(2) { --i: 1; } -.stat-tile:nth-child(3) { --i: 2; } -.stat-tile:nth-child(4) { --i: 3; } +.stat-item:nth-child(1) { --i: 0; } +.stat-item:nth-child(2) { --i: 1; } +.stat-item:nth-child(3) { --i: 2; } +.stat-item:nth-child(4) { --i: 3; } +.stat-item:nth-child(5) { --i: 4; } -.stat-tile:hover { - transform: translateY(-2px); - box-shadow: var(--shadow-sm); +/* Color variants for bar-fill */ +.bar-fill--green { + background: linear-gradient(90deg, #28a745 0%, #20c997 100%); } -.stat-tile-value { - font-family: var(--font-mono); - font-size: 16px; - font-weight: 700; - color: var(--text-primary); - line-height: 1.2; +.bar-fill--yellow { + background: linear-gradient(90deg, #ffc107 0%, #ffdb4d 100%); } -.stat-tile-label { - font-size: 9px; - font-weight: 600; - text-transform: uppercase; - letter-spacing: 0.3px; - color: var(--text-muted); - margin-top: 2px; +.bar-fill--purple { + background: linear-gradient(90deg, #6f42c1 0%, #9b6dd6 100%); } -/* Stat tile color variants */ -.stat-tile--green { --tile-color: 40, 167, 69; --tile-text: #28a745; } -.stat-tile--red { --tile-color: 220, 53, 69; --tile-text: #dc3545; } -.stat-tile--yellow { --tile-color: 255, 193, 7; --tile-text: #d39e00; } -.stat-tile--purple { --tile-color: 111, 66, 193; --tile-text: #6f42c1; } - -.stat-tile[class*="--"] { - border-color: rgba(var(--tile-color), 0.4); - background: linear-gradient(135deg, rgba(var(--tile-color), 0.08) 0%, var(--bg-primary) 100%); +.bar-fill--red { + background: linear-gradient(90deg, #dc3545 0%, #ff6b7a 100%); } -.stat-tile[class*="--"] .stat-tile-value { color: var(--tile-text); } /* -------------------------------------------------------------------------- Hotspot Cards @@ -1396,6 +1445,41 @@ color: var(--accent); } +.tooltip-diff { + margin-top: 12px; + padding-top: 12px; + border-top: 1px solid var(--border); +} + +.tooltip-diff-title { + font-size: 11px; + font-weight: 600; + color: var(--accent); + margin-bottom: 8px; +} + +.tooltip-diff-row { + display: grid; + grid-template-columns: auto 1fr; + gap: 4px 14px; + font-size: 12px; + margin-bottom: 4px; +} + +.tooltip-diff-row.regression .tooltip-stat-value { + color: var(--diff-regression-deep); + font-weight: 700; +} + +.tooltip-diff-row.improvement .tooltip-stat-value { + color: var(--diff-improvement-deep); + font-weight: 700; +} + +.tooltip-diff-row.neutral .tooltip-stat-value { + color: var(--text-secondary); +} + .tooltip-source { margin-top: 10px; padding-top: 10px; @@ -1551,10 +1635,6 @@ .brand-info { display: none; } - - .stats-grid { - grid-template-columns: 1fr; - } } /* -------------------------------------------------------------------------- @@ -1574,7 +1654,9 @@ Flamegraph-Specific Toggle Override -------------------------------------------------------------------------- */ -#toggle-invert .toggle-track.on { +#toggle-invert .toggle-track.on, +#toggle-elided .toggle-track.on, +#toggle-path-display .toggle-track.on { background: #8e44ad; border-color: #8e44ad; box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); @@ -1674,7 +1756,7 @@