Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions Lib/asyncio/futures.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ def __init__(self, *, loop=None):
loop object used by the future. If it's not provided, the future uses
the default event loop.
"""
if self._loop is not None:
raise RuntimeError(f"{self.__class__.__name__} object is already "
"initialized")

if loop is None:
self._loop = events.get_event_loop()
else:
Expand Down Expand Up @@ -351,22 +355,19 @@ def _set_concurrent_future_state(concurrent, source):
def _copy_future_state(source, dest):
"""Internal helper to copy state from another Future.

The other Future may be a concurrent.futures.Future.
The other Future must be a concurrent.futures.Future.
"""
assert source.done()
if dest.cancelled():
return
assert not dest.done()
if source.cancelled():
done, cancelled, result, exception = source._get_snapshot()

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve asyncio-to-asyncio future chaining

When both arguments to _chain_future() are ordinary asyncio Futures (an accepted combination per its checks and docstring), completing the source reaches this call and raises AttributeError: repo-wide, _get_snapshot() exists only on concurrent.futures.Future, not on either _PyFuture or native _asyncio.Future. The callback failure leaves the destination pending, so retain the prior asyncio copy path or use the snapshot path only for concurrent sources.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed by source inspection: this asyncio-to-asyncio private-helper gap is also present in exact-final CPython d220409a13b77de7a6c30abe16ea90db01528edc. This PR’s Lib/asyncio/futures.py is byte-identical to that file. _copy_future_state now explicitly requires a concurrent Future, but _chain_future still accepts two asyncio Futures; neither canonical asyncio implementation provides _get_snapshot.

The in-tree production callers avoid that combination: wrap_future chains concurrent → asyncio (and returns asyncio inputs unchanged); run_coroutine_threadsafe and the asyncio REPL chain Task → concurrent. The compatibility gap remains for direct private-helper callers. I’m retaining the canonical import rather than adding a noncanonical native method, which would also leave _PyFuture and protocol-compatible sources uncovered. No source change or runtime validation was made for this finding.

assert done
if cancelled:
dest.cancel()
elif exception is not None:
dest.set_exception(_convert_future_exc(exception))
else:
exception = source.exception()
if exception is not None:
dest.set_exception(_convert_future_exc(exception))
else:
result = source.result()
dest.set_result(result)

dest.set_result(result)

def _chain_future(source, destination):
"""Chain two futures so that when one completes, so does the other.
Expand Down
43 changes: 36 additions & 7 deletions Lib/asyncio/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,20 @@ def __init__(self, coro, *, loop=None, name=None, context=None,
self._coro = coro
if context is None:
self._context = contextvars.copy_context()
elif not isinstance(context, contextvars.Context):
# gh-157301: the passed value must be a contextvars.Context
self._log_destroy_pending = False
raise TypeError('a contextvars.Context was expected, '
f'got {type(context).__name__}')
else:
self._context = context

if eager_start and self._loop.is_running():
self.__eager_start()
try:
self.__eager_start()
except:
self._log_destroy_pending = False
raise
else:
self._loop.call_soon(self.__step, context=self._context)
_py_register_task(self)
Expand Down Expand Up @@ -440,15 +449,13 @@ def _release_waiter(waiter, *args):
async def wait_for(fut, timeout):
"""Wait for the single Future or coroutine to complete, with timeout.

Coroutine will be wrapped in Task.

Returns result of the Future or coroutine. When a timeout occurs,
it cancels the task and raises TimeoutError. To avoid the task
cancellation, wrap it in shield().
it cancels fut and raises TimeoutError. To prevent fut from being
cancelled, wrap it in shield().

If the wait is cancelled, the task is also cancelled.
If the wait is cancelled, fut is also cancelled.

If the task suppresses the cancellation and returns a value instead,
If fut suppresses the cancellation and returns a value instead,
that value is returned.

This function is a coroutine.
Expand Down Expand Up @@ -524,6 +531,7 @@ def _on_completion(f):
timeout_handle.cancel()
for f in fs:
f.remove_done_callback(_on_completion)
futures.future_discard_from_awaited_by(f, cur_task)

done, pending = set(), set()
for f in fs:
Expand All @@ -542,13 +550,18 @@ async def _cancel_and_wait(fut):
cb = functools.partial(_release_waiter, waiter)
fut.add_done_callback(cb)

# gh-157058: awaiting the waiter leaves no edge on fut, add it here
cur_task = current_task()
futures.future_add_to_awaited_by(fut, cur_task)

try:
fut.cancel()
# We cannot wait on *fut* directly to make
# sure _cancel_and_wait itself is reliably cancellable.
await waiter
finally:
fut.remove_done_callback(cb)
futures.future_discard_from_awaited_by(fut, cur_task)


class _AsCompletedIterator:
Expand All @@ -563,9 +576,11 @@ def __init__(self, aws, timeout):
self._timeout_handle = None

loop = events.get_event_loop()
self._cur_task = current_task()
todo = {ensure_future(aw, loop=loop) for aw in set(aws)}
for f in todo:
f.add_done_callback(self._handle_completion)
futures.future_add_to_awaited_by(f, self._cur_task)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Ignore missing tasks in awaited-by registration

When as_completed() is constructed from a synchronous event-loop callback, the loop is running but current_task() returns None. This unconditional call reaches _asyncio.future_add_to_awaited_by, whose native implementation stores the waiter without validating it, so a pending native Future gets _asyncio_awaited_by == {None} and asyncio.capture_call_graph(future) then recurses into None and raises TypeError; fix the native helper to ignore non-Future waiters rather than adding a Python-side workaround.

AGENTS.md reference: AGENTS.md:L193-L193

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed against exact-final CPython: both native awaited-by helpers require Future/Task objects for both arguments. RustPython checked only the child, so a synchronous as_completed callback could record None as a parent. The local fix adds waiter validation to both add and discard while retaining the canonical Python source. The existing native reentrant fixture now uses Future subclasses to preserve its intended coverage. The two bounded CPython reference checks pass; matching RustPython build/runtime validation is still pending. The fix has not been pushed yet.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 33715c9. Both awaited-by helpers now ignore non-native Future/Task waiters, and the existing native reentrant fixture uses valid Future subclasses without weakening its assertions. The matching narrow release build and strict VM/stdlib all-target Clippy passed. All 26 bounded candidate selections passed: 23 unchanged canonical methods, the two P2 oracle checks, and the reviewed existing awaited-by suffix. Both oracle checks also passed on exact-final CPython. This is selected coverage, not a full workspace/CAPI or GC/thread-stress suite; current-head hosted CI remains pending.

if todo and timeout is not None:
self._timeout_handle = (
loop.call_later(timeout, self._handle_timeout)
Expand Down Expand Up @@ -596,13 +611,15 @@ def __next__(self):
def _handle_timeout(self):
for f in self._todo:
f.remove_done_callback(self._handle_completion)
futures.future_discard_from_awaited_by(f, self._cur_task)
self._done.put_nowait(None) # Sentinel for _wait_for_one().
self._todo.clear() # Can't do todo.remove(f) in the loop.

def _handle_completion(self, f):
if not self._todo:
return # _handle_timeout() was here first.
self._todo.remove(f)
futures.future_discard_from_awaited_by(f, self._cur_task)
self._done.put_nowait(f)
if not self._todo and self._timeout_handle is not None:
self._timeout_handle.cancel()
Expand Down Expand Up @@ -767,6 +784,11 @@ def cancel(self, msg=None):
return ret


def _discard_awaited_by(children, waiter, outer):
for fut in children:
futures.future_discard_from_awaited_by(fut, waiter)


def gather(*coros_or_futures, return_exceptions=False):
"""Return a future aggregating results from the given coroutines/futures.

Expand Down Expand Up @@ -900,6 +922,10 @@ def _done_callback(fut, cur_task=cur_task):
children.append(fut)

outer = _GatheringFuture(children, loop=loop)
if cur_task is not None:
# gh-157213: a child outliving gather() must lose the awaited-by edge
outer.add_done_callback(
functools.partial(_discard_awaited_by, children, cur_task))
# Run done callbacks after GatheringFuture created so any post-processing
# can be performed at this point
# optimization: in the special case that *all* futures finished eagerly,
Expand Down Expand Up @@ -996,6 +1022,9 @@ def _outer_done_callback(outer):
# Keep only one callback to log on cancel
inner.remove_done_callback(_log_on_exception)
inner.add_done_callback(_log_on_exception)
if cur_task is not None:
inner.remove_done_callback(_clear_awaited_by_callback)
futures.future_discard_from_awaited_by(inner, cur_task)

if cur_task is not None:
inner.add_done_callback(_clear_awaited_by_callback)
Expand Down
29 changes: 28 additions & 1 deletion Lib/concurrent/futures/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,33 @@ def set_exception(self, exception):
self._condition.notify_all()
self._invoke_callbacks()

def _get_snapshot(self):
"""Get a snapshot of the future's current state.

This method atomically retrieves the state in one lock acquisition,
which is significantly faster than multiple method calls.

Returns:
Tuple of (done, cancelled, result, exception)
- done: True if the future is done (cancelled or finished)
- cancelled: True if the future was cancelled
- result: The result if available and not cancelled
- exception: The exception if available and not cancelled
"""
# Fast path: check if already finished without lock
if self._state == FINISHED:
return True, False, self._result, self._exception

# Need lock for other states since they can change
with self._condition:
# We have to check the state again after acquiring the lock
# because it may have changed in the meantime.
if self._state == FINISHED:
return True, False, self._result, self._exception
if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED}:
return True, True, None, None
return False, False, None, None

__class_getitem__ = classmethod(types.GenericAlias)

class Executor(object):
Expand Down Expand Up @@ -676,5 +703,5 @@ def __exit__(self, exc_type, exc_val, exc_tb):

class BrokenExecutor(RuntimeError):
"""
Raised when a executor has become non-functional after a severe failure.
Raised when an executor has become non-functional after a severe failure.
"""
89 changes: 58 additions & 31 deletions Lib/test/test_asyncio/test_futures.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,23 +413,23 @@ def func_repr(func):
def test_copy_state(self):
from asyncio.futures import _copy_future_state

f = self._new_future(loop=self.loop)
f = concurrent.futures.Future()

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Revert forbidden generated test rewrites

This replaces the existing asyncio Future test input with a concurrent Future, and the same commit changes test logic/data and adds or removes whole tests across all three Lib/test files. The repository's critical test policy permits only adding or removing expectedFailure markers, so restore the original test files rather than rewriting canonical tests to match the implementation.

AGENTS.md reference: AGENTS.md:L273-L279

Useful? React with 👍 / 👎.

f.set_result(10)

newf = self._new_future(loop=self.loop)
_copy_future_state(f, newf)
self.assertTrue(newf.done())
self.assertEqual(newf.result(), 10)

f_exception = self._new_future(loop=self.loop)
f_exception = concurrent.futures.Future()
f_exception.set_exception(RuntimeError())

newf_exception = self._new_future(loop=self.loop)
_copy_future_state(f_exception, newf_exception)
self.assertTrue(newf_exception.done())
self.assertRaises(RuntimeError, newf_exception.result)

f_cancelled = self._new_future(loop=self.loop)
f_cancelled = concurrent.futures.Future()
f_cancelled.cancel()

newf_cancelled = self._new_future(loop=self.loop)
Expand All @@ -441,7 +441,7 @@ def test_copy_state(self):
except BaseException as e:
f_exc = e

f_conexc = self._new_future(loop=self.loop)
f_conexc = concurrent.futures.Future()
f_conexc.set_exception(f_exc)

newf_conexc = self._new_future(loop=self.loop)
Expand All @@ -454,6 +454,56 @@ def test_copy_state(self):
newf_tb = ''.join(traceback.format_tb(newf_exc.__traceback__))
self.assertEqual(newf_tb.count('raise concurrent.futures.InvalidStateError'), 1)

def test_copy_state_from_concurrent_futures(self):
"""Test _copy_future_state from concurrent.futures.Future.

This tests the optimized path using _get_snapshot when available.
"""
from asyncio.futures import _copy_future_state

# Test with a result
f_concurrent = concurrent.futures.Future()
f_concurrent.set_result(42)
f_asyncio = self._new_future(loop=self.loop)
_copy_future_state(f_concurrent, f_asyncio)
self.assertTrue(f_asyncio.done())
self.assertEqual(f_asyncio.result(), 42)

# Test with an exception
f_concurrent_exc = concurrent.futures.Future()
f_concurrent_exc.set_exception(ValueError("test exception"))
f_asyncio_exc = self._new_future(loop=self.loop)
_copy_future_state(f_concurrent_exc, f_asyncio_exc)
self.assertTrue(f_asyncio_exc.done())
with self.assertRaises(ValueError) as cm:
f_asyncio_exc.result()
self.assertEqual(str(cm.exception), "test exception")

# Test with cancelled state
f_concurrent_cancelled = concurrent.futures.Future()
f_concurrent_cancelled.cancel()
f_asyncio_cancelled = self._new_future(loop=self.loop)
_copy_future_state(f_concurrent_cancelled, f_asyncio_cancelled)
self.assertTrue(f_asyncio_cancelled.cancelled())

# Test that destination already cancelled prevents copy
f_concurrent_result = concurrent.futures.Future()
f_concurrent_result.set_result(10)
f_asyncio_precancelled = self._new_future(loop=self.loop)
f_asyncio_precancelled.cancel()
_copy_future_state(f_concurrent_result, f_asyncio_precancelled)
self.assertTrue(f_asyncio_precancelled.cancelled())

# Test exception type conversion
f_concurrent_invalid = concurrent.futures.Future()
f_concurrent_invalid.set_exception(concurrent.futures.InvalidStateError("invalid"))
f_asyncio_invalid = self._new_future(loop=self.loop)
_copy_future_state(f_concurrent_invalid, f_asyncio_invalid)
self.assertTrue(f_asyncio_invalid.done())
with self.assertRaises(asyncio.exceptions.InvalidStateError) as cm:
f_asyncio_invalid.result()
self.assertEqual(str(cm.exception), "invalid")

def test_iter(self):
fut = self._new_future(loop=self.loop)

Expand Down Expand Up @@ -700,6 +750,10 @@ def test_future_cancelled_exception_refcycles(self):
self.assertIsNotNone(exc)
self.assertListEqual(gc.get_referrers(exc), [])

def test_future_disallow_multiple_initialization(self):
f = self._new_future(loop=self.loop)
with self.assertRaises(RuntimeError, msg="is already initialized"):
f.__init__(loop=self.loop)

@unittest.skipUnless(hasattr(futures, '_CFuture'),
'requires the C _asyncio module')
Expand Down Expand Up @@ -1041,33 +1095,6 @@ def __getattribute__(self, name):
fut.add_done_callback(fut_callback_0)
self.assertRaises(ReachableCode, fut.set_result, "boom")

def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self):
# see: https://github.com/python/cpython/issues/125984

class EvilEventLoop(SimpleEvilEventLoop):
def call_soon(self, *args, **kwargs):
super().call_soon(*args, **kwargs)
raise ReachableCode

def __getattribute__(self, name):
if name == 'call_soon':
# resets the future's event loop
fut.__init__(loop=SimpleEvilEventLoop())
return object.__getattribute__(self, name)

evil_loop = EvilEventLoop()
with mock.patch.object(self, 'loop', evil_loop):
fut = self._new_future()
self.assertIs(fut.get_loop(), evil_loop)

fut_callback_0 = mock.Mock()
fut_context_0 = mock.Mock()
fut.add_done_callback(fut_callback_0, context=fut_context_0)
del fut_context_0
del fut_callback_0
self.assertRaises(ReachableCode, fut.set_result, "boom")


@unittest.skipUnless(hasattr(futures, '_CFuture'),
'requires the C _asyncio module')
class CFutureDoneCallbackTests(BaseFutureDoneCallbackTests,
Expand Down
Loading
Loading