From 15a9cfebdf1077fc87a08129de3cb3293bf48be2 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 27 Dec 2021 19:56:56 +0530 Subject: [PATCH 01/19] bpo-45924: Fix asyncio incorrect traceback when future's exception is raised multiple times --- Lib/asyncio/futures.py | 3 ++- Lib/test/test_asyncio/test_futures2.py | 25 ++++++++++++++++++- .../2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst | 1 + Modules/_asynciomodule.c | 15 +++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 8e8cd87612588c..48a32f868385c4 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -198,7 +198,7 @@ def result(self): raise exceptions.InvalidStateError('Result is not ready.') self.__log_traceback = False if self._exception is not None: - raise self._exception + raise self._exception.with_traceback(self._exception_tb) return self._result def exception(self): @@ -274,6 +274,7 @@ def set_exception(self, exception): raise TypeError("StopIteration interacts badly with generators " "and cannot be raised into a Future") self._exception = exception + self._exception_tb = exception.__traceback__ self._state = _FINISHED self.__schedule_callbacks() self.__log_traceback = True diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 57d24190bc0bd5..da2e80ef3bc451 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -1,9 +1,32 @@ # IsolatedAsyncioTestCase based tests import asyncio import unittest - +import traceback class FutureTests(unittest.IsolatedAsyncioTestCase): + async def test_future_traceback(self): + async def raise_exc(): + raise TypeError(42) + future = asyncio.create_task(raise_exc()) + for _ in range(10): + try: + await future + except TypeError: + tb = traceback.format_exc() + expected = ( +f"""Traceback (most recent call last): + File "{__file__}", line 13, in test_future_traceback + await future + ^^^^^^^^^^^^ + File "{__file__}", line 9, in raise_exc + raise TypeError(42) + ^^^^^^^^^^^^^^^^^^^ +TypeError: 42 +""" + ) + self.assertEqual(tb, expected) + + async def test_recursive_repr_for_pending_tasks(self): # The call crashes if the guard for recursive call # in base_futures:_future_repr_info is absent diff --git a/Misc/NEWS.d/next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst b/Misc/NEWS.d/next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst new file mode 100644 index 00000000000000..fdb4d42c0621e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst @@ -0,0 +1 @@ +Fix :mod:`asyncio` incorrect traceback when future's exception is raised multiple times. Patch by Kumar Aditya. \ No newline at end of file diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 2216dd0178173a..598fe54a3db1a1 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -68,6 +68,7 @@ typedef enum { PyObject *prefix##_context0; \ PyObject *prefix##_callbacks; \ PyObject *prefix##_exception; \ + PyObject *prefix##_exception_tb; \ PyObject *prefix##_result; \ PyObject *prefix##_source_tb; \ PyObject *prefix##_cancel_msg; \ @@ -492,6 +493,7 @@ future_init(FutureObj *fut, PyObject *loop) Py_CLEAR(fut->fut_callbacks); Py_CLEAR(fut->fut_result); Py_CLEAR(fut->fut_exception); + Py_CLEAR(fut->fut_exception_tb); Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); _PyErr_ClearExcState(&fut->fut_cancelled_exc_state); @@ -598,7 +600,9 @@ future_set_exception(FutureObj *fut, PyObject *exc) } assert(!fut->fut_exception); + assert(!fut->fut_exception_tb); fut->fut_exception = exc_val; + fut->fut_exception_tb = PyException_GetTraceback(exc_val); fut->fut_state = STATE_FINISHED; if (future_schedule_callbacks(fut) == -1) { @@ -646,6 +650,15 @@ future_get_result(FutureObj *fut, PyObject **result) fut->fut_log_tb = 0; if (fut->fut_exception != NULL) { + if (fut->fut_exception_tb != NULL) { + if (PyException_SetTraceback(fut->fut_exception, fut->fut_exception_tb) < 0) + { + return -1; + } + } + else { + PyException_SetTraceback(fut->fut_exception, Py_None); + } Py_INCREF(fut->fut_exception); *result = fut->fut_exception; return 1; @@ -789,6 +802,7 @@ FutureObj_clear(FutureObj *fut) Py_CLEAR(fut->fut_callbacks); Py_CLEAR(fut->fut_result); Py_CLEAR(fut->fut_exception); + Py_CLEAR(fut->fut_exception_tb); Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); _PyErr_ClearExcState(&fut->fut_cancelled_exc_state); @@ -805,6 +819,7 @@ FutureObj_traverse(FutureObj *fut, visitproc visit, void *arg) Py_VISIT(fut->fut_callbacks); Py_VISIT(fut->fut_result); Py_VISIT(fut->fut_exception); + Py_VISIT(fut->fut_exception_tb); Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->dict); From 4e5accffe506cc49e7b4c603798a8d51d6f359e2 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 15 Feb 2022 21:39:43 +0200 Subject: [PATCH 02/19] Update Lib/test/test_asyncio/test_futures2.py --- Lib/test/test_asyncio/test_futures2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index da2e80ef3bc451..8d80b11a53e204 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -25,6 +25,8 @@ async def raise_exc(): """ ) self.assertEqual(tb, expected) + else: + self.fail('TypeError not raised') async def test_recursive_repr_for_pending_tasks(self): From 42600eb47103bb64c08a6ca0ee072717497d5e04 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 15 Feb 2022 21:41:14 +0200 Subject: [PATCH 03/19] Update test_futures2.py --- Lib/test/test_asyncio/test_futures2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 8d80b11a53e204..3231ab7b1b1068 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -25,8 +25,8 @@ async def raise_exc(): """ ) self.assertEqual(tb, expected) - else: - self.fail('TypeError not raised') + else: + self.fail('TypeError not raised') async def test_recursive_repr_for_pending_tasks(self): From d792cc4ed47d1995ea4cc02be299f2e3a9f2b875 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 15 Feb 2022 21:50:10 +0200 Subject: [PATCH 04/19] Fix test --- Lib/test/test_asyncio/test_futures2.py | 39 +++++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 3231ab7b1b1068..255db89042c436 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -3,32 +3,37 @@ import unittest import traceback +from textwrap import dedent + + class FutureTests(unittest.IsolatedAsyncioTestCase): - async def test_future_traceback(self): + maxDiff = None + + async def test_future_traceback(self): + async def raise_exc(): raise TypeError(42) + future = asyncio.create_task(raise_exc()) for _ in range(10): try: await future except TypeError: tb = traceback.format_exc() - expected = ( -f"""Traceback (most recent call last): - File "{__file__}", line 13, in test_future_traceback - await future - ^^^^^^^^^^^^ - File "{__file__}", line 9, in raise_exc - raise TypeError(42) - ^^^^^^^^^^^^^^^^^^^ -TypeError: 42 -""" - ) - self.assertEqual(tb, expected) - else: - self.fail('TypeError not raised') - - + expected = dedent(f"""\ + Traceback (most recent call last): + File "{__file__}", line 20, in test_future_traceback + await future + ^^^^^^^^^^^^ + File "{__file__}", line 15, in raise_exc + raise TypeError(42) + ^^^^^^^^^^^^^^^^^^^ + TypeError: 42 + """) + self.assertEqual(tb, expected) + else: + self.fail('TypeError not raised') + async def test_recursive_repr_for_pending_tasks(self): # The call crashes if the guard for recursive call # in base_futures:_future_repr_info is absent From 858fcaff7bdd245772935c9848688cc59e144e90 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 15 Feb 2022 21:51:25 +0200 Subject: [PATCH 05/19] Fix altered execution error --- Lib/test/test_asyncio/test_futures2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 255db89042c436..19257d031d64af 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -6,6 +6,10 @@ from textwrap import dedent +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + class FutureTests(unittest.IsolatedAsyncioTestCase): maxDiff = None From c192f03b30d7b619222df4d2f3e15b1739ba2a49 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 15 Feb 2022 21:52:24 +0200 Subject: [PATCH 06/19] Fix line numbers --- Lib/test/test_asyncio/test_futures2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 19257d031d64af..7b80fe39862453 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -26,10 +26,10 @@ async def raise_exc(): tb = traceback.format_exc() expected = dedent(f"""\ Traceback (most recent call last): - File "{__file__}", line 20, in test_future_traceback + File "{__file__}", line 24, in test_future_traceback await future ^^^^^^^^^^^^ - File "{__file__}", line 15, in raise_exc + File "{__file__}", line 19, in raise_exc raise TypeError(42) ^^^^^^^^^^^^^^^^^^^ TypeError: 42 From 72e23effa625d8a2bb4f0995c7d32817c2fb784a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 15 Feb 2022 22:47:28 +0200 Subject: [PATCH 07/19] Make test more robust --- Lib/test/test_asyncio/test_futures2.py | 67 ++++++++++++++++++-------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 7b80fe39862453..77590863ed9136 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -1,10 +1,9 @@ # IsolatedAsyncioTestCase based tests import asyncio +import inspect import unittest import traceback -from textwrap import dedent - def tearDownModule(): asyncio.set_event_loop_policy(None) @@ -14,29 +13,55 @@ class FutureTests(unittest.IsolatedAsyncioTestCase): maxDiff = None async def test_future_traceback(self): + # Note: the test is sensitive to line numbers. + # + # To make the test more flexible + # and ready for future file modifications, + # RAISE_MARKER and AWAIT_MARKER_ are + # used for finding analyzed line numbers. + + markers = {} + + def init_markers(): + source_lines, offset = inspect.getsourcelines( + FutureTests.test_future_traceback, + ) + mark = "# pos: " + for pos, line in enumerate(source_lines): + found = line.find(mark) + if found != -1: + marker = line[found + len(mark):].strip() + if marker == '"': + # skip 'mark' variable definition from above + continue + markers[marker] = offset + pos + + init_markers() + + def analyze_traceback(tb, await_marker): + summary = traceback.extract_tb(tb) + self.assertEqual(2, len(summary)) + self.assertEqual(markers[await_marker], summary[0].lineno) + self.assertEqual(markers["RAISE_MARKER"], summary[1].lineno) async def raise_exc(): - raise TypeError(42) + raise TypeError(42) # pos: RAISE_MARKER future = asyncio.create_task(raise_exc()) - for _ in range(10): - try: - await future - except TypeError: - tb = traceback.format_exc() - expected = dedent(f"""\ - Traceback (most recent call last): - File "{__file__}", line 24, in test_future_traceback - await future - ^^^^^^^^^^^^ - File "{__file__}", line 19, in raise_exc - raise TypeError(42) - ^^^^^^^^^^^^^^^^^^^ - TypeError: 42 - """) - self.assertEqual(tb, expected) - else: - self.fail('TypeError not raised') + # first await + try: + await future # pos: AWAIT_MARKER_1 + except TypeError as exc: + analyze_traceback(exc.__traceback__, "AWAIT_MARKER_1") + else: + self.fail('TypeError not raised') + # the second await replaces traceback + try: + await future # pos: AWAIT_MARKER_2 + except TypeError as exc: + analyze_traceback(exc.__traceback__, "AWAIT_MARKER_2") + else: + self.fail('TypeError not raised') async def test_recursive_repr_for_pending_tasks(self): # The call crashes if the guard for recursive call From 10b1125a37c17a1e06715cbf18d1cb1bed68c33c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 15 Feb 2022 22:59:21 +0200 Subject: [PATCH 08/19] Fix docs --- .../next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst b/Misc/NEWS.d/next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst index fdb4d42c0621e7..5cda22737adb79 100644 --- a/Misc/NEWS.d/next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst +++ b/Misc/NEWS.d/next/Library/2021-12-27-15-32-15.bpo-45924.0ZpHX2.rst @@ -1 +1 @@ -Fix :mod:`asyncio` incorrect traceback when future's exception is raised multiple times. Patch by Kumar Aditya. \ No newline at end of file +Fix :mod:`asyncio` incorrect traceback when future's exception is raised multiple times. Patch by Kumar Aditya. From 02955ae44d2609425fa8e271b51b9531004843d5 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 15 Feb 2022 23:15:07 +0200 Subject: [PATCH 09/19] Cleanup --- Lib/test/test_asyncio/test_futures2.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 2664bbab1fbf2f..77590863ed9136 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -5,10 +5,6 @@ import traceback -def tearDownModule(): - asyncio.set_event_loop_policy(None) - - def tearDownModule(): asyncio.set_event_loop_policy(None) From 3a81ce8641b86a85a187f538bfb0eea1d25cb2fc Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 28 Feb 2022 10:59:58 +0000 Subject: [PATCH 10/19] fix it with IsolatedAsyncioTestCase --- Lib/test/test_asyncio/test_tasks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index b30f8f56dfa726..522d36ed582a0b 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -3284,13 +3284,12 @@ async def coro(s): self.one_loop.call_exception_handler.assert_not_called() -class RunCoroutineThreadsafeTests(test_utils.TestCase): +class RunCoroutineThreadsafeTests(unittest.IsolatedAsyncioTestCase): """Test case for asyncio.run_coroutine_threadsafe.""" - def setUp(self): + async def asyncSetUp(self) -> None: super().setUp() - self.loop = asyncio.new_event_loop() - self.set_event_loop(self.loop) # Will cleanup properly + self.loop = asyncio.get_running_loop() async def add(self, a, b, fail=False, cancel=False): """Wait 0.05 second and return a + b.""" From 63b6fc1bc32ed4ec10733016260c38eb3d6f428a Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 21 Mar 2022 09:52:44 +0000 Subject: [PATCH 11/19] revert --- Lib/test/test_asyncio/test_tasks.py | 429 +++++++--------------------- 1 file changed, 97 insertions(+), 332 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 8d31f6c1335415..b86646ee398ae3 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -12,7 +12,6 @@ import textwrap import traceback import unittest -import weakref from unittest import mock from types import GenericAlias @@ -85,19 +84,13 @@ def __await__(self): return self -# The following value can be used as a very small timeout: -# it passes check "timeout > 0", but has almost -# no effect on the test performance -_EPSILON = 0.0001 - - class BaseTaskTests: Task = None Future = None - def new_task(self, loop, coro, name='TestTask'): - return self.__class__.Task(coro, loop=loop, name=name) + def new_task(self, loop, coro, name='TestTask', context=None): + return self.__class__.Task(coro, loop=loop, name=name, context=context) def new_future(self, loop): return self.__class__.Future(loop=loop) @@ -108,7 +101,6 @@ def setUp(self): self.loop.set_task_factory(self.new_task) self.loop.create_future = lambda: self.new_future(self.loop) - def test_generic_alias(self): task = self.__class__.Task[str] self.assertEqual(task.__args__, (str,)) @@ -972,251 +964,6 @@ async def coro(): task._log_traceback = True self.loop.run_until_complete(task) - def test_wait_for_timeout_less_then_0_or_0_future_done(self): - def gen(): - when = yield - self.assertAlmostEqual(0, when) - - loop = self.new_test_loop(gen) - - fut = self.new_future(loop) - fut.set_result('done') - - ret = loop.run_until_complete(asyncio.wait_for(fut, 0)) - - self.assertEqual(ret, 'done') - self.assertTrue(fut.done()) - self.assertAlmostEqual(0, loop.time()) - - def test_wait_for_timeout_less_then_0_or_0_coroutine_do_not_started(self): - def gen(): - when = yield - self.assertAlmostEqual(0, when) - - loop = self.new_test_loop(gen) - - foo_started = False - - async def foo(): - nonlocal foo_started - foo_started = True - - with self.assertRaises(asyncio.TimeoutError): - loop.run_until_complete(asyncio.wait_for(foo(), 0)) - - self.assertAlmostEqual(0, loop.time()) - self.assertEqual(foo_started, False) - - def test_wait_for_timeout_less_then_0_or_0(self): - def gen(): - when = yield - self.assertAlmostEqual(0.2, when) - when = yield 0 - self.assertAlmostEqual(0, when) - - for timeout in [0, -1]: - with self.subTest(timeout=timeout): - loop = self.new_test_loop(gen) - - foo_running = None - - async def foo(): - nonlocal foo_running - foo_running = True - try: - await asyncio.sleep(0.2) - finally: - foo_running = False - return 'done' - - fut = self.new_task(loop, foo()) - - with self.assertRaises(asyncio.TimeoutError): - loop.run_until_complete(asyncio.wait_for(fut, timeout)) - self.assertTrue(fut.done()) - # it should have been cancelled due to the timeout - self.assertTrue(fut.cancelled()) - self.assertAlmostEqual(0, loop.time()) - self.assertEqual(foo_running, False) - - def test_wait_for(self): - - def gen(): - when = yield - self.assertAlmostEqual(0.2, when) - when = yield 0 - self.assertAlmostEqual(0.1, when) - when = yield 0.1 - - loop = self.new_test_loop(gen) - - foo_running = None - - async def foo(): - nonlocal foo_running - foo_running = True - try: - await asyncio.sleep(0.2) - finally: - foo_running = False - return 'done' - - fut = self.new_task(loop, foo()) - - with self.assertRaises(asyncio.TimeoutError): - loop.run_until_complete(asyncio.wait_for(fut, 0.1)) - self.assertTrue(fut.done()) - # it should have been cancelled due to the timeout - self.assertTrue(fut.cancelled()) - self.assertAlmostEqual(0.1, loop.time()) - self.assertEqual(foo_running, False) - - def test_wait_for_blocking(self): - loop = self.new_test_loop() - - async def coro(): - return 'done' - - res = loop.run_until_complete(asyncio.wait_for(coro(), timeout=None)) - self.assertEqual(res, 'done') - - def test_wait_for_race_condition(self): - - def gen(): - yield 0.1 - yield 0.1 - yield 0.1 - - loop = self.new_test_loop(gen) - - fut = self.new_future(loop) - task = asyncio.wait_for(fut, timeout=0.2) - loop.call_later(0.1, fut.set_result, "ok") - res = loop.run_until_complete(task) - self.assertEqual(res, "ok") - - def test_wait_for_cancellation_race_condition(self): - async def inner(): - with contextlib.suppress(asyncio.CancelledError): - await asyncio.sleep(1) - return 1 - - async def main(): - result = await asyncio.wait_for(inner(), timeout=.01) - self.assertEqual(result, 1) - - asyncio.run(main()) - - def test_wait_for_waits_for_task_cancellation(self): - loop = asyncio.new_event_loop() - self.addCleanup(loop.close) - - task_done = False - - async def foo(): - async def inner(): - nonlocal task_done - try: - await asyncio.sleep(0.2) - except asyncio.CancelledError: - await asyncio.sleep(_EPSILON) - raise - finally: - task_done = True - - inner_task = self.new_task(loop, inner()) - - await asyncio.wait_for(inner_task, timeout=_EPSILON) - - with self.assertRaises(asyncio.TimeoutError) as cm: - loop.run_until_complete(foo()) - - self.assertTrue(task_done) - chained = cm.exception.__context__ - self.assertEqual(type(chained), asyncio.CancelledError) - - def test_wait_for_waits_for_task_cancellation_w_timeout_0(self): - loop = asyncio.new_event_loop() - self.addCleanup(loop.close) - - task_done = False - - async def foo(): - async def inner(): - nonlocal task_done - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - await asyncio.sleep(_EPSILON) - raise - finally: - task_done = True - - inner_task = self.new_task(loop, inner()) - await asyncio.sleep(_EPSILON) - await asyncio.wait_for(inner_task, timeout=0) - - with self.assertRaises(asyncio.TimeoutError) as cm: - loop.run_until_complete(foo()) - - self.assertTrue(task_done) - chained = cm.exception.__context__ - self.assertEqual(type(chained), asyncio.CancelledError) - - def test_wait_for_reraises_exception_during_cancellation(self): - loop = asyncio.new_event_loop() - self.addCleanup(loop.close) - - class FooException(Exception): - pass - - async def foo(): - async def inner(): - try: - await asyncio.sleep(0.2) - finally: - raise FooException - - inner_task = self.new_task(loop, inner()) - - await asyncio.wait_for(inner_task, timeout=_EPSILON) - - with self.assertRaises(FooException): - loop.run_until_complete(foo()) - - def test_wait_for_self_cancellation(self): - loop = asyncio.new_event_loop() - self.addCleanup(loop.close) - - async def foo(): - async def inner(): - try: - await asyncio.sleep(0.3) - except asyncio.CancelledError: - try: - await asyncio.sleep(0.3) - except asyncio.CancelledError: - await asyncio.sleep(0.3) - - return 42 - - inner_task = self.new_task(loop, inner()) - - wait = asyncio.wait_for(inner_task, timeout=0.1) - - # Test that wait_for itself is properly cancellable - # even when the initial task holds up the initial cancellation. - task = self.new_task(loop, wait) - await asyncio.sleep(0.2) - task.cancel() - - with self.assertRaises(asyncio.CancelledError): - await task - - self.assertEqual(await inner_task, 42) - - loop.run_until_complete(foo()) - def test_wait(self): def gen(): @@ -1250,13 +997,12 @@ def test_wait_duplicate_coroutines(self): async def coro(s): return s - c = coro('test') + c = self.loop.create_task(coro('test')) task = self.new_task( self.loop, - asyncio.wait([c, c, coro('spam')])) + asyncio.wait([c, c, self.loop.create_task(coro('spam'))])) - with self.assertWarns(DeprecationWarning): - done, pending = self.loop.run_until_complete(task) + done, pending = self.loop.run_until_complete(task) self.assertFalse(pending) self.assertEqual(set(f.result() for f in done), {'test', 'spam'}) @@ -1633,11 +1379,9 @@ def gen(): async def test(): futs = list(asyncio.as_completed(fs)) self.assertEqual(len(futs), 2) - waiter = asyncio.wait(futs) - # Deprecation from passing coros in futs to asyncio.wait() - with self.assertWarns(DeprecationWarning) as cm: - done, pending = await waiter - self.assertEqual(cm.warnings[0].filename, __file__) + done, pending = await asyncio.wait( + [asyncio.ensure_future(fut) for fut in futs] + ) self.assertEqual(set(f.result() for f in done), {'a', 'b'}) loop = self.new_test_loop(gen) @@ -1687,21 +1431,6 @@ async def test(): loop.run_until_complete(test()) - def test_as_completed_coroutine_use_global_loop(self): - # Deprecated in 3.10 - async def coro(): - return 42 - - loop = self.new_test_loop() - asyncio.set_event_loop(loop) - self.addCleanup(asyncio.set_event_loop, None) - futs = asyncio.as_completed([coro()]) - with self.assertWarns(DeprecationWarning) as cm: - futs = list(futs) - self.assertEqual(cm.warnings[0].filename, __file__) - self.assertEqual(len(futs), 1) - self.assertEqual(loop.run_until_complete(futs[0]), 42) - def test_sleep(self): def gen(): @@ -2004,7 +1733,7 @@ async def inner(): async def outer(): nonlocal proof with self.assertWarns(DeprecationWarning): - d, p = await asyncio.wait([inner()]) + d, p = await asyncio.wait([asyncio.create_task(inner())]) proof += 100 f = asyncio.ensure_future(outer(), loop=self.loop) @@ -2243,32 +1972,6 @@ def test_task_source_traceback(self): 'test_task_source_traceback')) self.loop.run_until_complete(task) - def _test_cancel_wait_for(self, timeout): - loop = asyncio.new_event_loop() - self.addCleanup(loop.close) - - async def blocking_coroutine(): - fut = self.new_future(loop) - # Block: fut result is never set - await fut - - task = loop.create_task(blocking_coroutine()) - - wait = loop.create_task(asyncio.wait_for(task, timeout)) - loop.call_soon(wait.cancel) - - self.assertRaises(asyncio.CancelledError, - loop.run_until_complete, wait) - - # Python issue #23219: cancelling the wait must also cancel the task - self.assertTrue(task.cancelled()) - - def test_cancel_blocking_wait_for(self): - self._test_cancel_wait_for(None) - - def test_cancel_wait_for(self): - self._test_cancel_wait_for(60.0) - def test_cancel_gather_1(self): """Ensure that a gathering future refuses to be cancelled once all children are done""" @@ -2528,6 +2231,90 @@ async def main(): self.assertEqual(cvar.get(), -1) + def test_context_4(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = self.new_task(loop, coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = self.new_task(loop, coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + + def test_context_5(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = asyncio.create_task(coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = asyncio.create_task(coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + + def test_context_6(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = loop.create_task(coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = loop.create_task(coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = loop.create_task(main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + def test_get_coro(self): loop = asyncio.new_event_loop() coro = coroutine_function() @@ -3288,12 +3075,13 @@ async def coro(s): self.one_loop.call_exception_handler.assert_not_called() -class RunCoroutineThreadsafeTests(unittest.IsolatedAsyncioTestCase): +class RunCoroutineThreadsafeTests(test_utils.TestCase): """Test case for asyncio.run_coroutine_threadsafe.""" - async def asyncSetUp(self) -> None: + def setUp(self): super().setUp() - self.loop = asyncio.get_running_loop() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) # Will cleanup properly async def add(self, a, b, fail=False, cancel=False): """Wait 0.05 second and return a + b.""" @@ -3414,29 +3202,6 @@ async def coro(): self.assertEqual(result, 11) -class WaitTests(test_utils.TestCase): - def setUp(self): - super().setUp() - self.loop = asyncio.new_event_loop() - self.set_event_loop(self.loop) - - def tearDown(self): - self.loop.close() - self.loop = None - super().tearDown() - - def test_coro_is_deprecated_in_wait(self): - # Remove test when passing coros to asyncio.wait() is removed in 3.11 - with self.assertWarns(DeprecationWarning): - self.loop.run_until_complete( - asyncio.wait([coroutine_function()])) - - task = self.loop.create_task(coroutine_function()) - with self.assertWarns(DeprecationWarning): - self.loop.run_until_complete( - asyncio.wait([task, coroutine_function()])) - - class CompatibilityTests(test_utils.TestCase): # Tests for checking a bridge between old-styled coroutines # and async/await syntax From e437cd879a352a238e471d8c91b244ba0656386d Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 21 Mar 2022 15:24:03 +0530 Subject: [PATCH 12/19] Update Modules/_asynciomodule.c Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Modules/_asynciomodule.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 01bd51c159d7ef..51d328b2bfbaf2 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -659,14 +659,12 @@ future_get_result(FutureObj *fut, PyObject **result) fut->fut_log_tb = 0; if (fut->fut_exception != NULL) { - if (fut->fut_exception_tb != NULL) { - if (PyException_SetTraceback(fut->fut_exception, fut->fut_exception_tb) < 0) - { - return -1; - } + PyObject *tb = fut->fut_exception_tb; + if (tb == NULL) { + tb = Py_None; } - else { - PyException_SetTraceback(fut->fut_exception, Py_None); + if (PyException_SetTraceback(fut->fut_exception, tb) < 0) { + return -1; } Py_INCREF(fut->fut_exception); *result = fut->fut_exception; From 07f500069a1f788600fd0e07f3a70aad0d2d8eec Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 30 Jun 2022 10:20:25 +0000 Subject: [PATCH 13/19] finally working! --- Lib/test/test_asyncio/test_futures2.py | 57 +++++--------------------- Modules/_asynciomodule.c | 1 + 2 files changed, 11 insertions(+), 47 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 77590863ed9136..634283b48c0c27 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -1,6 +1,5 @@ # IsolatedAsyncioTestCase based tests import asyncio -import inspect import unittest import traceback @@ -10,58 +9,22 @@ def tearDownModule(): class FutureTests(unittest.IsolatedAsyncioTestCase): - maxDiff = None async def test_future_traceback(self): - # Note: the test is sensitive to line numbers. - # - # To make the test more flexible - # and ready for future file modifications, - # RAISE_MARKER and AWAIT_MARKER_ are - # used for finding analyzed line numbers. - - markers = {} - - def init_markers(): - source_lines, offset = inspect.getsourcelines( - FutureTests.test_future_traceback, - ) - mark = "# pos: " - for pos, line in enumerate(source_lines): - found = line.find(mark) - if found != -1: - marker = line[found + len(mark):].strip() - if marker == '"': - # skip 'mark' variable definition from above - continue - markers[marker] = offset + pos - - init_markers() - - def analyze_traceback(tb, await_marker): - summary = traceback.extract_tb(tb) - self.assertEqual(2, len(summary)) - self.assertEqual(markers[await_marker], summary[0].lineno) - self.assertEqual(markers["RAISE_MARKER"], summary[1].lineno) async def raise_exc(): - raise TypeError(42) # pos: RAISE_MARKER + raise TypeError(42) future = asyncio.create_task(raise_exc()) - # first await - try: - await future # pos: AWAIT_MARKER_1 - except TypeError as exc: - analyze_traceback(exc.__traceback__, "AWAIT_MARKER_1") - else: - self.fail('TypeError not raised') - # the second await replaces traceback - try: - await future # pos: AWAIT_MARKER_2 - except TypeError as exc: - analyze_traceback(exc.__traceback__, "AWAIT_MARKER_2") - else: - self.fail('TypeError not raised') + + for _ in range(5): + try: + await future + except TypeError as e: + tb = ''.join(traceback.format_tb(e.__traceback__)) + self.assertEqual(tb.count("await fut"), 1) + else: + self.fail('TypeError was not raised') async def test_recursive_repr_for_pending_tasks(self): # The call crashes if the guard for recursive call diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 0c3f71b7f01dfa..46175ccb8eb3cb 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -669,6 +669,7 @@ future_get_result(FutureObj *fut, PyObject **result) } Py_INCREF(fut->fut_exception); *result = fut->fut_exception; + Py_CLEAR(fut->fut_exception_tb); return 1; } From a433abc64d16adb6ef3b21bc890da4991ab1d401 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 1 Jul 2022 08:41:44 +0000 Subject: [PATCH 14/19] rename var --- Lib/test/test_asyncio/test_futures2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 634283b48c0c27..ea7b1ee0d43d9c 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -22,7 +22,7 @@ async def raise_exc(): await future except TypeError as e: tb = ''.join(traceback.format_tb(e.__traceback__)) - self.assertEqual(tb.count("await fut"), 1) + self.assertEqual(tb.count("await future"), 1) else: self.fail('TypeError was not raised') From 9d6bae0b281e764a8f83f2e91b9dfff37588917c Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 9 Jul 2022 07:38:53 +0000 Subject: [PATCH 15/19] test both implementations --- Lib/test/test_asyncio/test_futures2.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index ea7b1ee0d43d9c..4e8435b4ae6dec 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -1,21 +1,22 @@ # IsolatedAsyncioTestCase based tests import asyncio -import unittest import traceback +import unittest +from asyncio import tasks def tearDownModule(): asyncio.set_event_loop_policy(None) -class FutureTests(unittest.IsolatedAsyncioTestCase): +class FutureTests: async def test_future_traceback(self): async def raise_exc(): raise TypeError(42) - future = asyncio.create_task(raise_exc()) + future = self.cls(raise_exc()) for _ in range(5): try: @@ -39,6 +40,13 @@ async def func(): # exact comparison for the whole string is even weaker. self.assertIn('...', repr(await asyncio.wait_for(func(), timeout=10))) +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._CTask + +class PyFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._PyTask if __name__ == '__main__': unittest.main() From 998f33f71f3391a92f001c4fe4cc2d0c5decdf03 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 10 Jul 2022 16:51:01 +0000 Subject: [PATCH 16/19] separate tests --- Lib/test/test_asyncio/test_futures2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 4e8435b4ae6dec..94817bc5ab5bbb 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -27,6 +27,8 @@ async def raise_exc(): else: self.fail('TypeError was not raised') +class FutureReprTests(unittest.IsolatedAsyncioTestCase): + async def test_recursive_repr_for_pending_tasks(self): # The call crashes if the guard for recursive call # in base_futures:_future_repr_info is absent From bf757ce40cbca602632dab587712694a5aecb12a Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 11 Jul 2022 11:10:45 +0530 Subject: [PATCH 17/19] Update Lib/test/test_asyncio/test_futures2.py Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/test/test_asyncio/test_futures2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 94817bc5ab5bbb..4f93d3c3619444 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -27,6 +27,13 @@ async def raise_exc(): else: self.fail('TypeError was not raised') +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') + class CFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._CTask + + class PyFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._PyTask class FutureReprTests(unittest.IsolatedAsyncioTestCase): async def test_recursive_repr_for_pending_tasks(self): From 0eb9594219498b553b3f9632e6cd57abc0d9acca Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 11 Jul 2022 11:10:55 +0530 Subject: [PATCH 18/19] Update Lib/test/test_asyncio/test_futures2.py Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/test/test_asyncio/test_futures2.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 4f93d3c3619444..fa8264b5710e52 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -49,13 +49,6 @@ async def func(): # exact comparison for the whole string is even weaker. self.assertIn('...', repr(await asyncio.wait_for(func(), timeout=10))) -@unittest.skipUnless(hasattr(tasks, '_CTask'), - 'requires the C _asyncio module') -class CFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): - cls = tasks._CTask - -class PyFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): - cls = tasks._PyTask if __name__ == '__main__': unittest.main() From c9a6fe7630b135f7af8252f716cf91338f20bf72 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 11 Jul 2022 06:10:16 +0000 Subject: [PATCH 19/19] fix indent --- Lib/test/test_asyncio/test_futures2.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index fa8264b5710e52..71279b69c7929e 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -29,11 +29,12 @@ async def raise_exc(): @unittest.skipUnless(hasattr(tasks, '_CTask'), 'requires the C _asyncio module') - class CFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): - cls = tasks._CTask - - class PyFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): - cls = tasks._PyTask +class CFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._CTask + +class PyFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._PyTask + class FutureReprTests(unittest.IsolatedAsyncioTestCase): async def test_recursive_repr_for_pending_tasks(self):