Skip to content

Commit 2224fff

Browse files
pablogsaldeadlovelll
pablogsal
and
deadlovelll
authored
[3.14] gh-159620: Fix remote asyncio stacks stopping at async generators (GH-159623) (GH-159655)
Follow asynchronous-generator send/throw wrappers when unwinding suspended and running asyncio tasks. Retry transient sampling errors in the existing live-process stress test, matching main and 3.15. (cherry picked from commit 2188f9b) Co-authored-by: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Co-authored-by: Pablo Galindo Salgado <pablogsal@gmail.com>
1 parent 2261a3e commit 2224fff

3 files changed

Lines changed: 179 additions & 49 deletions

File tree

‎Lib/test/test_external_inspection.py‎

Lines changed: 111 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import unittest
22
from contextlib import contextmanager
33
import asyncio
4+
import contextlib
45
import os
56
import textwrap
67
import importlib
@@ -68,6 +69,112 @@ def get_all_awaited_by(pid):
6869
class TestGetStackTrace(unittest.TestCase):
6970
maxDiff = None
7071

72+
def _extract_coroutine_funcnames(self, trace):
73+
return {
74+
task.task_name: [
75+
frame.funcname.rpartition(".")[2]
76+
for coro in task.coroutine_stack
77+
for frame in coro.call_stack
78+
]
79+
for info in trace
80+
for task in info.awaited_by
81+
}
82+
83+
@skip_if_not_supported
84+
def test_async_generator_stack_is_not_truncated(self):
85+
# gh-159620
86+
async def agen():
87+
await asyncio.sleep(10_000)
88+
yield
89+
90+
async def agen_with_cleanup():
91+
try:
92+
yield
93+
finally:
94+
await asyncio.sleep(10_000)
95+
96+
async def outer_agen():
97+
async for _ in agen():
98+
yield
99+
100+
async def consumer(gen):
101+
async for _ in gen:
102+
pass
103+
104+
async def main():
105+
g = agen_with_cleanup()
106+
await anext(g)
107+
async_for_task = asyncio.create_task(consumer(agen()), name="async for")
108+
nested_task = asyncio.create_task(
109+
consumer(outer_agen()), name="nested async for")
110+
anext_task = asyncio.create_task(anext(agen()), name="anext")
111+
aclose_task = asyncio.create_task(g.aclose(), name="aclose")
112+
await asyncio.sleep(0)
113+
return RemoteUnwinder(os.getpid()).get_all_awaited_by()
114+
115+
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
116+
self.assertEqual(stacks["async for"], ["sleep", "agen", "consumer"])
117+
self.assertEqual(stacks["nested async for"],
118+
["sleep", "agen", "outer_agen", "consumer"])
119+
self.assertEqual(stacks["anext"], ["sleep", "agen"])
120+
self.assertEqual(stacks["aclose"], ["sleep", "agen_with_cleanup"])
121+
122+
@skip_if_not_supported
123+
def test_async_generator_waiter_in_async_stack_trace(self):
124+
# gh-159620
125+
async def agen(task):
126+
await task
127+
yield
128+
129+
async def main():
130+
waiter_task = asyncio.create_task(
131+
anext(agen(asyncio.current_task())), name="anext")
132+
await asyncio.sleep(0)
133+
return RemoteUnwinder(os.getpid()).get_async_stack_trace()
134+
135+
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
136+
self.assertEqual(stacks["anext"], ["agen"])
137+
138+
@skip_if_not_supported
139+
def test_async_generator_running_task_stack(self):
140+
# gh-159620
141+
async def agen():
142+
yield RemoteUnwinder(os.getpid()).get_async_stack_trace()
143+
144+
async def main():
145+
return await asyncio.create_task(anext(agen()), name="anext")
146+
147+
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
148+
self.assertEqual(stacks["anext"], ["agen"])
149+
150+
@skip_if_not_supported
151+
def test_class_named_like_async_generator_wrapper_is_not_walked(self):
152+
# gh-159620
153+
async def decoy():
154+
await asyncio.sleep(10_000)
155+
156+
class async_generator_asend:
157+
__slots__ = ("gen",)
158+
def __init__(self, gen):
159+
self.gen = gen
160+
def __await__(self):
161+
return self
162+
def __next__(self):
163+
return None
164+
165+
async def consumer(aw):
166+
await aw
167+
168+
async def main():
169+
with contextlib.closing(decoy()) as gen:
170+
consumer_task = asyncio.create_task(
171+
consumer(async_generator_asend(gen)), name="consumer")
172+
await asyncio.sleep(0)
173+
return RemoteUnwinder(os.getpid()).get_all_awaited_by()
174+
175+
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
176+
self.assertEqual(stacks["consumer"], ["consumer"])
177+
71178
@skip_if_not_supported
72179
def test_long_task_name_is_truncated(self):
73180
# gh-157788
@@ -935,22 +1042,10 @@ async def main():
9351042
for _ in busy_retry(SHORT_TIMEOUT):
9361043
try:
9371044
all_awaited_by = get_all_awaited_by(p.pid)
938-
except RuntimeError as re:
939-
# This call reads a linked list in another process with
940-
# no synchronization. That occasionally leads to invalid
941-
# reads. Here we avoid making the test flaky.
942-
msg = str(re)
943-
if msg.startswith("Task list appears corrupted"):
944-
continue
945-
elif msg.startswith(
946-
"Invalid linked list structure reading remote memory"
947-
):
948-
continue
949-
elif msg.startswith("Unknown error reading memory"):
950-
continue
951-
elif msg.startswith("Unhandled frame owner"):
952-
continue
953-
raise # Unrecognized exception, safest not to ignore it
1045+
except RuntimeError:
1046+
# Tasks and awaited objects can change while we read
1047+
# the running process. Retry with a fresh sample.
1048+
continue
9541049
else:
9551050
break
9561051
# expected: a list of two elements: 1 thread, 1 interp

‎Misc/NEWS.d/next/Library/2026-10-17-15-22-16.gh-issue-159620.Ynsu5L.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix ``python -m asyncio ps`` and ``pstree`` stacks stopping at
2+
asynchronous generators. Patch by Timofei Ivankov.

‎Modules/_remote_debugging_module.c‎

Lines changed: 66 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1426,11 +1426,63 @@ parse_task_name(
14261426
}
14271427

14281428

1429+
// gh-159620
1430+
/* Return in *coro_addr the coroutine or async generator to walk for the
1431+
awaitable at addr, or 0 if it is neither (for example, a future). */
1432+
static int
1433+
get_coro_like_address(
1434+
RemoteUnwinderObject *unwinder,
1435+
uintptr_t addr,
1436+
uintptr_t *coro_addr
1437+
) {
1438+
*coro_addr = 0;
1439+
1440+
// Skip user classes that share a name with the generator types below
1441+
uintptr_t type_addr;
1442+
unsigned long tp_flags;
1443+
if (read_ptr(unwinder, addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type,
1444+
&type_addr) < 0
1445+
|| _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle,
1446+
type_addr + (uintptr_t)unwinder->debug_offsets.type_object.tp_flags,
1447+
sizeof(tp_flags), &tp_flags) < 0)
1448+
{
1449+
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read awaitable type");
1450+
return -1;
1451+
}
1452+
1453+
if (tp_flags & Py_TPFLAGS_HEAPTYPE) {
1454+
return 0;
1455+
}
1456+
1457+
uintptr_t name_addr;
1458+
char name[32] = {0};
1459+
if (read_ptr(unwinder, type_addr + (uintptr_t)unwinder->debug_offsets.type_object.tp_name,
1460+
&name_addr) < 0
1461+
|| _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, name_addr,
1462+
sizeof(name) - 1, name) < 0)
1463+
{
1464+
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read awaitable type name");
1465+
return -1;
1466+
}
1467+
1468+
if (strcmp(name, "coroutine") == 0) {
1469+
*coro_addr = addr;
1470+
}
1471+
else if (strcmp(name, "async_generator_asend") == 0
1472+
|| strcmp(name, "async_generator_athrow") == 0)
1473+
{
1474+
// asend and athrow are separate wrapper objects: read the pointer
1475+
// to their parent async generator, stored right after PyObject_HEAD
1476+
return read_ptr(unwinder, addr + (uintptr_t)unwinder->debug_offsets.pyobject.size,
1477+
coro_addr);
1478+
}
1479+
return 0;
1480+
}
1481+
14291482
static int
14301483
get_awaited_coro_address(
14311484
RemoteUnwinderObject *unwinder,
14321485
uintptr_t gi_iframe_addr,
1433-
uintptr_t gen_type_addr,
14341486
uintptr_t *next_coro
14351487
) {
14361488
*next_coro = 0;
@@ -1458,41 +1510,14 @@ get_awaited_coro_address(
14581510
uintptr_t stackpointer_addr = GET_MEMBER_NO_TAG(uintptr_t, iframe, unwinder->debug_offsets.interpreter_frame.stackpointer);
14591511

14601512
if ((void*)stackpointer_addr != NULL) {
1461-
uintptr_t gi_await_addr;
14621513
err = read_py_ptr(
14631514
unwinder,
14641515
stackpointer_addr - sizeof(void*),
1465-
&gi_await_addr);
1516+
next_coro);
14661517
if (err) {
14671518
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await address");
14681519
return -1;
14691520
}
1470-
1471-
if ((void*)gi_await_addr != NULL) {
1472-
uintptr_t gi_await_addr_type_addr;
1473-
err = read_ptr(
1474-
unwinder,
1475-
gi_await_addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type,
1476-
&gi_await_addr_type_addr);
1477-
if (err) {
1478-
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await type address");
1479-
return -1;
1480-
}
1481-
1482-
if (gen_type_addr == gi_await_addr_type_addr) {
1483-
/* This needs an explanation. We always start with parsing
1484-
native coroutine / generator frames. Ultimately they
1485-
are awaiting on something. That something can be
1486-
a native coroutine frame or... an iterator.
1487-
If it's the latter -- we can't continue building
1488-
our chain. So the condition to bail out of this is
1489-
to do that when the type of the current coroutine
1490-
doesn't match the type of whatever it points to
1491-
in its cr_await.
1492-
*/
1493-
*next_coro = gi_await_addr;
1494-
}
1495-
}
14961521
}
14971522

14981523
return 0;
@@ -1515,6 +1540,13 @@ parse_coro_chain(
15151540
return -1;
15161541
}
15171542

1543+
if (get_coro_like_address(unwinder, coro_address, &coro_address) < 0) {
1544+
return -1;
1545+
}
1546+
if ((void*)coro_address == NULL) {
1547+
return 0;
1548+
}
1549+
15181550
// Read the entire generator object at once
15191551
char gen_object[SIZEOF_GEN_OBJ];
15201552
int err = _Py_RemoteDebug_PagedReadRemoteMemory(
@@ -1532,8 +1564,6 @@ parse_coro_chain(
15321564
return 0;
15331565
}
15341566

1535-
uintptr_t gen_type_addr = GET_MEMBER(uintptr_t, gen_object, unwinder->debug_offsets.pyobject.ob_type);
1536-
15371567
PyObject* name = NULL;
15381568

15391569
// Parse the previous frame using the gi_iframe from local copy
@@ -1560,8 +1590,7 @@ parse_coro_chain(
15601590
return 0;
15611591
}
15621592

1563-
if (get_awaited_coro_address(unwinder, gi_iframe_addr, gen_type_addr,
1564-
&coro_address) < 0) {
1593+
if (get_awaited_coro_address(unwinder, gi_iframe_addr, &coro_address) < 0) {
15651594
return -1;
15661595
}
15671596
}
@@ -2034,6 +2063,10 @@ get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintpt
20342063
return -1;
20352064
}
20362065

2066+
if (get_coro_like_address(unwinder, running_coro_addr, &running_coro_addr) < 0) {
2067+
return -1;
2068+
}
2069+
20372070
// note: genobject's gi_iframe is an embedded struct so the address to
20382071
// the offset leads directly to its first field: f_executable
20392072
if (read_py_ptr(

0 commit comments

Comments
 (0)