From f8d2bdc06f984227e5736a76d219ebb52250ed72 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 May 2026 19:05:45 +0200 Subject: [PATCH] gh-150107: Fix asyncio sendfile fallback ignoring non-zero offset The fallback paths in BaseEventLoop._sock_sendfile_fallback and _sendfile_fallback only seeked the file when offset was truthy, so an offset of 0 was respected but later non-zero offsets were dropped when the file lacked seek tracking. Seek whenever the file supports seek(). Also seek the CRT file pointer on Windows TransmitFile, which ignores OVERLAPPED.Offset for handles not opened with FILE_FLAG_OVERLAPPED. Co-Authored-By: Claude Opus 4.7 --- Lib/asyncio/base_events.py | 5 ++-- Lib/asyncio/windows_events.py | 3 +++ Lib/test/libregrtest/logger.py | 16 ++++++++++-- Lib/test/libregrtest/run_workers.py | 22 +++++++++++++++- Lib/test/libregrtest/utils.py | 23 ++++++++++++++++ Lib/test/test_asyncio/test_sendfile.py | 26 +++++++++++++++++++ Lib/test/test_regrtest.py | 6 ++++- ...-05-22-17-09-28.gh-issue-150107.GD72-D.rst | 3 +++ ...-05-22-16-41-45.gh-issue-150114.UdMikH.rst | 2 ++ 9 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst create mode 100644 Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 1fedb066f94c539..3732294c5848f02 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -969,7 +969,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): f"and file {file!r} combination") async def _sock_sendfile_fallback(self, sock, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = ( min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) @@ -1286,7 +1286,6 @@ async def sendfile(self, transport, file, offset=0, count=None, raise RuntimeError( f"fallback is disabled and native sendfile is not " f"supported for transport {transport!r}") - return await self._sendfile_fallback(transport, file, offset, count) @@ -1295,7 +1294,7 @@ async def _sendfile_native(self, transp, file, offset, count): "sendfile syscall is not supported") async def _sendfile_fallback(self, transp, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = min(count, 16384) if count else 16384 buf = bytearray(blocksize) diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 5f75b17d8ca649b..0bf7732136f1f8e 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -610,6 +610,9 @@ def sendfile(self, sock, file, offset, count): ov = _overlapped.Overlapped(NULL) offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff + # TransmitFile ignores OVERLAPPED.Offset for handles not opened with + # FILE_FLAG_OVERLAPPED, so seek the CRT file pointer to match. + file.seek(offset) ov.TransmitFile(sock.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index fa1d4d575c8fd4f..4e011ef06f8a913 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -1,5 +1,6 @@ import os import time +from typing import Callable from test.support import MS_WINDOWS from .results import TestResults @@ -19,16 +20,27 @@ def __init__(self, results: TestResults, quiet: bool, pgo: bool): self._results: TestResults = results self._quiet: bool = quiet self._pgo: bool = pgo + self.get_mem_usage: Callable[[], int | None] | None = None def log(self, line: str = '') -> None: empty = not line - # add the system load prefix: "load avg: 1.80 " + # Add the memory usage: "mem: 1 GiB " + if self.get_mem_usage is not None: + mem = self.get_mem_usage() + if mem: + mib = mem / (1024*1024) + if mib >= 1024: + line = f"mem: {mib / 1024:.1f} GiB {line}" + else: + line = f"mem: {mib:.1f} MiB {line}" + + # Add the system load prefix: "load avg: 1.80 " load_avg = self.get_load_avg() if load_avg is not None: line = f"load avg: {load_avg:.2f} {line}" - # add the timestamp prefix: "0:01:05 " + # Add the timestamp prefix: "0:01:05 " log_time = time.perf_counter() - self.start_time mins, secs = divmod(int(log_time), 60) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 424085a0050eb59..befdac7ee77f107 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -22,7 +22,7 @@ from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, - format_duration, print_warning, count, plural) + format_duration, print_warning, count, plural, get_process_memory_usage) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -452,6 +452,12 @@ def wait_stopped(self, start_time: float) -> None: print_warning(f"Failed to join {self} in {format_duration(dt)}") break + def get_mem_usage(self): + popen = self._popen + if popen is None: + return + return get_process_memory_usage(popen.pid) + def get_running(workers: list[WorkerThread]) -> str | None: running: list[str] = [] @@ -473,6 +479,7 @@ def __init__(self, num_workers: int, runtests: RunTests, logger: Logger, results: TestResults) -> None: self.num_workers = num_workers self.runtests = runtests + self.logger = logger self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results @@ -598,9 +605,21 @@ def _process_result(self, item: QueueOutput) -> TestResult: return result + def get_mem_usage(self): + usage = 0 + main_mem = get_process_memory_usage(os.getpid()) + if main_mem: + usage += main_mem + for worker in self.workers: + worker_mem = worker.get_mem_usage() + if worker_mem: + usage += worker_mem + return usage + def run(self) -> None: fail_fast = self.runtests.fail_fast fail_env_changed = self.runtests.fail_env_changed + self.logger.get_mem_usage = self.get_mem_usage self.start_workers() @@ -625,3 +644,4 @@ def run(self) -> None: # worker when we exit this function self.pending.stop() self.stop_workers() + self.logger.get_mem_usage = None diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 00703d6c074855b..1b4cb96406d6f60 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -752,3 +752,26 @@ def display_title(title): print(title) print("#" * len(title)) print(flush=True) + + +def get_process_memory_usage(pid: int) -> int | None: + """ + Read the private memory in bytes from /proc/pid/smaps. + """ + try: + fp = open(f"/proc/{pid}/smaps", "rb") + except OSError: + return None + + try: + total = 0 + with fp: + for line in fp: + # Include both Private_Clean and Private_Dirty sections. + line = line.rstrip() + if line.startswith(b"Private_") and line.endswith(b'kB'): + parts = line.split() + total += int(parts[1]) * 1024 + return total + except ProcessLookupError: + return None diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index dcd963b3355ef86..ae7a2e32c526873 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -228,6 +228,32 @@ def test_sock_sendfile_zero_size(self): self.assertEqual(ret, 0) self.assertEqual(self.file.tell(), 0) + def check_sock_sendfile_offset(self, data, offset, force_fallback=False): + sock, proto = self.prepare_socksendfile() + with tempfile.TemporaryFile() as f: + f.write(data) + f.flush() + self.assertEqual(f.tell(), len(data)) + + if force_fallback: + async def _sock_sendfile_fail(sock, file, offset, count): + raise asyncio.exceptions.SendfileNotAvailableError() + with support.swap_attr(self.loop, '_sock_sendfile_native', _sock_sendfile_fail): + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + else: + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + self.assertEqual(f.tell(), len(data)) + sock.close() + self.run_loop(proto.wait_closed()) + self.assertEqual(ret, len(data) - offset) + + def test_sock_sendfile_offset(self): + data = b'abcdef' + for offset in (0, len(data) // 2, len(data)): + for force_fallback in (False, True): + with self.subTest(offset=offset, force_fallback=force_fallback): + self.check_sock_sendfile_offset(data, offset, force_fallback) + def test_sock_sendfile_mix_with_regular_send(self): buf = b"mix_regular_send" * (4 * 1024) # 64 KiB sock, proto = self.prepare_socksendfile() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 02f6e0c74b5ce84..207b144d01d9255 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -41,7 +41,11 @@ ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) -LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' +LOG_PREFIX = ( + r'[0-9]+:[0-9]+:[0-9]+ ' + r'(?:load avg: [0-9]+\.[0-9]{2} )?' + r'(?:mem: [0-9]+\.[0-9] (?:MiB|GiB) )?' +) RESULT_REGEX = ( 'passed', 'failed', diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst new file mode 100644 index 000000000000000..a13f249e48cc021 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst @@ -0,0 +1,3 @@ +:mod:`asyncio`: ``sendfile()`` and ``sock_sendfile()`` event loop methods +now call ``file.seek(offset)`` if *file* has a ``seek()`` method, +even if *offset* is ``0`` (default value). diff --git a/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst b/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst new file mode 100644 index 000000000000000..a140bf921972ed9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst @@ -0,0 +1,2 @@ +On Linux, regrtest now logs the total memory usage of all Python processes. +Read the private memory in ``/proc/pid/smaps``. Patch by Victor Stinner.