diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 1fedb066f94c539..3732294c5848f02 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -969,7 +969,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): f"and file {file!r} combination") async def _sock_sendfile_fallback(self, sock, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = ( min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) @@ -1286,7 +1286,6 @@ async def sendfile(self, transport, file, offset=0, count=None, raise RuntimeError( f"fallback is disabled and native sendfile is not " f"supported for transport {transport!r}") - return await self._sendfile_fallback(transport, file, offset, count) @@ -1295,7 +1294,7 @@ async def _sendfile_native(self, transp, file, offset, count): "sendfile syscall is not supported") async def _sendfile_fallback(self, transp, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = min(count, 16384) if count else 16384 buf = bytearray(blocksize) diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 5f75b17d8ca649b..0bf7732136f1f8e 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -610,6 +610,9 @@ def sendfile(self, sock, file, offset, count): ov = _overlapped.Overlapped(NULL) offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff + # TransmitFile ignores OVERLAPPED.Offset for handles not opened with + # FILE_FLAG_OVERLAPPED, so seek the CRT file pointer to match. + file.seek(offset) ov.TransmitFile(sock.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index fa1d4d575c8fd4f..4e011ef06f8a913 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -1,5 +1,6 @@ import os import time +from typing import Callable from test.support import MS_WINDOWS from .results import TestResults @@ -19,16 +20,27 @@ def __init__(self, results: TestResults, quiet: bool, pgo: bool): self._results: TestResults = results self._quiet: bool = quiet self._pgo: bool = pgo + self.get_mem_usage: Callable[[], int | None] | None = None def log(self, line: str = '') -> None: empty = not line - # add the system load prefix: "load avg: 1.80 " + # Add the memory usage: "mem: 1 GiB " + if self.get_mem_usage is not None: + mem = self.get_mem_usage() + if mem: + mib = mem / (1024*1024) + if mib >= 1024: + line = f"mem: {mib / 1024:.1f} GiB {line}" + else: + line = f"mem: {mib:.1f} MiB {line}" + + # Add the system load prefix: "load avg: 1.80 " load_avg = self.get_load_avg() if load_avg is not None: line = f"load avg: {load_avg:.2f} {line}" - # add the timestamp prefix: "0:01:05 " + # Add the timestamp prefix: "0:01:05 " log_time = time.perf_counter() - self.start_time mins, secs = divmod(int(log_time), 60) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 424085a0050eb59..befdac7ee77f107 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -22,7 +22,7 @@ from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, - format_duration, print_warning, count, plural) + format_duration, print_warning, count, plural, get_process_memory_usage) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -452,6 +452,12 @@ def wait_stopped(self, start_time: float) -> None: print_warning(f"Failed to join {self} in {format_duration(dt)}") break + def get_mem_usage(self): + popen = self._popen + if popen is None: + return + return get_process_memory_usage(popen.pid) + def get_running(workers: list[WorkerThread]) -> str | None: running: list[str] = [] @@ -473,6 +479,7 @@ def __init__(self, num_workers: int, runtests: RunTests, logger: Logger, results: TestResults) -> None: self.num_workers = num_workers self.runtests = runtests + self.logger = logger self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results @@ -598,9 +605,21 @@ def _process_result(self, item: QueueOutput) -> TestResult: return result + def get_mem_usage(self): + usage = 0 + main_mem = get_process_memory_usage(os.getpid()) + if main_mem: + usage += main_mem + for worker in self.workers: + worker_mem = worker.get_mem_usage() + if worker_mem: + usage += worker_mem + return usage + def run(self) -> None: fail_fast = self.runtests.fail_fast fail_env_changed = self.runtests.fail_env_changed + self.logger.get_mem_usage = self.get_mem_usage self.start_workers() @@ -625,3 +644,4 @@ def run(self) -> None: # worker when we exit this function self.pending.stop() self.stop_workers() + self.logger.get_mem_usage = None diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 00703d6c074855b..1b4cb96406d6f60 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -752,3 +752,26 @@ def display_title(title): print(title) print("#" * len(title)) print(flush=True) + + +def get_process_memory_usage(pid: int) -> int | None: + """ + Read the private memory in bytes from /proc/pid/smaps. + """ + try: + fp = open(f"/proc/{pid}/smaps", "rb") + except OSError: + return None + + try: + total = 0 + with fp: + for line in fp: + # Include both Private_Clean and Private_Dirty sections. + line = line.rstrip() + if line.startswith(b"Private_") and line.endswith(b'kB'): + parts = line.split() + total += int(parts[1]) * 1024 + return total + except ProcessLookupError: + return None diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index dcd963b3355ef86..ae7a2e32c526873 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -228,6 +228,32 @@ def test_sock_sendfile_zero_size(self): self.assertEqual(ret, 0) self.assertEqual(self.file.tell(), 0) + def check_sock_sendfile_offset(self, data, offset, force_fallback=False): + sock, proto = self.prepare_socksendfile() + with tempfile.TemporaryFile() as f: + f.write(data) + f.flush() + self.assertEqual(f.tell(), len(data)) + + if force_fallback: + async def _sock_sendfile_fail(sock, file, offset, count): + raise asyncio.exceptions.SendfileNotAvailableError() + with support.swap_attr(self.loop, '_sock_sendfile_native', _sock_sendfile_fail): + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + else: + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + self.assertEqual(f.tell(), len(data)) + sock.close() + self.run_loop(proto.wait_closed()) + self.assertEqual(ret, len(data) - offset) + + def test_sock_sendfile_offset(self): + data = b'abcdef' + for offset in (0, len(data) // 2, len(data)): + for force_fallback in (False, True): + with self.subTest(offset=offset, force_fallback=force_fallback): + self.check_sock_sendfile_offset(data, offset, force_fallback) + def test_sock_sendfile_mix_with_regular_send(self): buf = b"mix_regular_send" * (4 * 1024) # 64 KiB sock, proto = self.prepare_socksendfile() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 02f6e0c74b5ce84..207b144d01d9255 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -41,7 +41,11 @@ ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) -LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' +LOG_PREFIX = ( + r'[0-9]+:[0-9]+:[0-9]+ ' + r'(?:load avg: [0-9]+\.[0-9]{2} )?' + r'(?:mem: [0-9]+\.[0-9] (?:MiB|GiB) )?' +) RESULT_REGEX = ( 'passed', 'failed', diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst new file mode 100644 index 000000000000000..a13f249e48cc021 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst @@ -0,0 +1,3 @@ +:mod:`asyncio`: ``sendfile()`` and ``sock_sendfile()`` event loop methods +now call ``file.seek(offset)`` if *file* has a ``seek()`` method, +even if *offset* is ``0`` (default value). diff --git a/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst b/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst new file mode 100644 index 000000000000000..a140bf921972ed9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst @@ -0,0 +1,2 @@ +On Linux, regrtest now logs the total memory usage of all Python processes. +Read the private memory in ``/proc/pid/smaps``. Patch by Victor Stinner.