Skip to content

Commit 702f8f3

Browse files
serhiy-storchakaserhiy-storchaka
authored
bpo-33041: Rework compiling an "async for" loop. (python#6142)
* Added new opcode END_ASYNC_FOR. * Setting global StopAsyncIteration no longer breaks "async for" loops. * Jumping into an "async for" loop is now disabled. * Jumping out of an "async for" loop no longer corrupts the stack. * Simplify the compiler.
1 parent c65bf3f commit 702f8f3

14 files changed

Lines changed: 275 additions & 225 deletions

File tree

‎Doc/library/dis.rst‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -588,6 +588,17 @@ the original TOS1.
588588
.. versionadded:: 3.5
589589

590590

591+
.. opcode:: END_ASYNC_FOR
592+
593+
Terminates an :keyword:`async for` loop. Handles an exception raised
594+
when awaiting a next item. If TOS is :exc:`StopAsyncIteration` pop 7
595+
values from the stack and restore the exception state using the second
596+
three of them. Otherwise re-raise the exception using the three values
597+
from the stack. An exception handler block is removed from the block stack.
598+
599+
.. versionadded:: 3.8
600+
601+
591602
.. opcode:: BEFORE_ASYNC_WITH
592603

593604
Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the

‎Doc/whatsnew/3.8.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,3 +157,7 @@ CPython bytecode changes
157157

158158
(Contributed by Mark Shannon, Antoine Pitrou and Serhiy Storchaka in
159159
:issue:`17611`.)
160+
161+
* Added new opcode :opcode:`END_ASYNC_FOR` for handling exceptions raised
162+
when awaiting a next item in an :keyword:`async for` loop.
163+
(Contributed by Serhiy Storchaka in :issue:`33041`.)

‎Include/opcode.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ extern "C" {
3434
#define GET_ANEXT 51
3535
#define BEFORE_ASYNC_WITH 52
3636
#define BEGIN_FINALLY 53
37+
#define END_ASYNC_FOR 54
3738
#define INPLACE_ADD 55
3839
#define INPLACE_SUBTRACT 56
3940
#define INPLACE_MULTIPLY 57

‎Lib/importlib/_bootstrap_external.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ def _write_atomic(path, data, mode=0o666):
247247
# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650)
248248
# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550)
249249
# Python 3.8a1 3400 (move frame block handling to compiler #17611)
250+
# Python 3.8a1 3401 (add END_ASYNC_FOR #33041)
250251
#
251252
# MAGIC must change whenever the bytecode emitted by the compiler may no
252253
# longer be understood by older implementations of the eval loop (usually
@@ -255,7 +256,7 @@ def _write_atomic(path, data, mode=0o666):
255256
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
256257
# in PC/launcher.c must also be updated.
257258

258-
MAGIC_NUMBER = (3400).to_bytes(2, 'little') + b'\r\n'
259+
MAGIC_NUMBER = (3401).to_bytes(2, 'little') + b'\r\n'
259260
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
260261

261262
_PYCACHE = '__pycache__'

‎Lib/opcode.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ def jabs_op(name, op):
8888
def_op('GET_ANEXT', 51)
8989
def_op('BEFORE_ASYNC_WITH', 52)
9090
def_op('BEGIN_FINALLY', 53)
91-
91+
def_op('END_ASYNC_FOR', 54)
9292
def_op('INPLACE_ADD', 55)
9393
def_op('INPLACE_SUBTRACT', 56)
9494
def_op('INPLACE_MULTIPLY', 57)

‎Lib/test/test_coroutines.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1846,6 +1846,36 @@ async def run_gen():
18461846
run_async(run_gen()),
18471847
([], [121]))
18481848

1849+
def test_comp_4_2(self):
1850+
async def f(it):
1851+
for i in it:
1852+
yield i
1853+
1854+
async def run_list():
1855+
return [i + 10 async for i in f(range(5)) if 0 < i < 4]
1856+
self.assertEqual(
1857+
run_async(run_list()),
1858+
([], [11, 12, 13]))
1859+
1860+
async def run_set():
1861+
return {i + 10 async for i in f(range(5)) if 0 < i < 4}
1862+
self.assertEqual(
1863+
run_async(run_set()),
1864+
([], {11, 12, 13}))
1865+
1866+
async def run_dict():
1867+
return {i + 10: i + 100 async for i in f(range(5)) if 0 < i < 4}
1868+
self.assertEqual(
1869+
run_async(run_dict()),
1870+
([], {11: 101, 12: 102, 13: 103}))
1871+
1872+
async def run_gen():
1873+
gen = (i + 10 async for i in f(range(5)) if 0 < i < 4)
1874+
return [g + 100 async for g in gen]
1875+
self.assertEqual(
1876+
run_async(run_gen()),
1877+
([], [111, 112, 113]))
1878+
18491879
def test_comp_5(self):
18501880
async def f(it):
18511881
for i in it:

‎Lib/test/test_dis.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -747,8 +747,7 @@ async def async_def():
747747
1: 1
748748
Names:
749749
0: b
750-
1: StopAsyncIteration
751-
2: c
750+
1: c
752751
Variable names:
753752
0: a
754753
1: d"""

‎Lib/test/test_sys_settrace.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ async def __aenter__(self):
3333
async def __aexit__(self, *exc_info):
3434
self.output.append(-self.value)
3535

36+
async def asynciter(iterable):
37+
"""Convert an iterable to an asynchronous iterator."""
38+
for x in iterable:
39+
yield x
3640

3741

3842
# A very basic example. If this fails, we're in deep trouble.
@@ -720,6 +724,23 @@ def test_jump_out_of_block_backwards(output):
720724
output.append(6)
721725
output.append(7)
722726

727+
@async_jump_test(4, 5, [3, 5])
728+
async def test_jump_out_of_async_for_block_forwards(output):
729+
for i in [1]:
730+
async for i in asynciter([1, 2]):
731+
output.append(3)
732+
output.append(4)
733+
output.append(5)
734+
735+
@async_jump_test(5, 2, [2, 4, 2, 4, 5, 6])
736+
async def test_jump_out_of_async_for_block_backwards(output):
737+
for i in [1]:
738+
output.append(2)
739+
async for i in asynciter([1]):
740+
output.append(4)
741+
output.append(5)
742+
output.append(6)
743+
723744
@jump_test(1, 2, [3])
724745
def test_jump_to_codeless_line(output):
725746
output.append(1)
@@ -1030,6 +1051,17 @@ def test_jump_over_for_block_before_else(output):
10301051
output.append(7)
10311052
output.append(8)
10321053

1054+
@async_jump_test(1, 7, [7, 8])
1055+
async def test_jump_over_async_for_block_before_else(output):
1056+
output.append(1)
1057+
if not output: # always false
1058+
async for i in asynciter([3]):
1059+
output.append(4)
1060+
else:
1061+
output.append(6)
1062+
output.append(7)
1063+
output.append(8)
1064+
10331065
# The second set of 'jump' tests are for things that are not allowed:
10341066

10351067
@jump_test(2, 3, [1], (ValueError, 'after'))
@@ -1081,12 +1113,24 @@ def test_no_jump_forwards_into_for_block(output):
10811113
for i in 1, 2:
10821114
output.append(3)
10831115

1116+
@async_jump_test(1, 3, [], (ValueError, 'into'))
1117+
async def test_no_jump_forwards_into_async_for_block(output):
1118+
output.append(1)
1119+
async for i in asynciter([1, 2]):
1120+
output.append(3)
1121+
10841122
@jump_test(3, 2, [2, 2], (ValueError, 'into'))
10851123
def test_no_jump_backwards_into_for_block(output):
10861124
for i in 1, 2:
10871125
output.append(2)
10881126
output.append(3)
10891127

1128+
@async_jump_test(3, 2, [2, 2], (ValueError, 'into'))
1129+
async def test_no_jump_backwards_into_async_for_block(output):
1130+
async for i in asynciter([1, 2]):
1131+
output.append(2)
1132+
output.append(3)
1133+
10901134
@jump_test(1, 3, [], (ValueError, 'into'))
10911135
def test_no_jump_forwards_into_with_block(output):
10921136
output.append(1)
@@ -1220,6 +1264,17 @@ def test_no_jump_into_for_block_before_else(output):
12201264
output.append(7)
12211265
output.append(8)
12221266

1267+
@async_jump_test(7, 4, [1, 6], (ValueError, 'into'))
1268+
async def test_no_jump_into_async_for_block_before_else(output):
1269+
output.append(1)
1270+
if not output: # always false
1271+
async for i in asynciter([3]):
1272+
output.append(4)
1273+
else:
1274+
output.append(6)
1275+
output.append(7)
1276+
output.append(8)
1277+
12231278
def test_no_jump_to_non_integers(self):
12241279
self.run_test(no_jump_to_non_integers, 2, "Spam", [True])
12251280

‎Misc/NEWS.d/next/Core and Builtins/2018-03-18-13-56-14.bpo-33041.XwPhI2.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Added new opcode :opcode:`END_ASYNC_FOR` and fixes the following issues:
2+
3+
* Setting global :exc:`StopAsyncIteration` no longer breaks ``async for``
4+
loops.
5+
* Jumping into an ``async for`` loop is now disabled.
6+
* Jumping out of an ``async for`` loop no longer corrupts the stack.

‎Objects/frameobject.c‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
100100
int line = 0; /* (ditto) */
101101
int addr = 0; /* (ditto) */
102102
int delta_iblock = 0; /* Scanning the SETUPs and POPs */
103-
int for_loop_delta = 0; /* (ditto) */
104-
int delta;
103+
int delta = 0;
105104
int blockstack[CO_MAXBLOCKS]; /* Walking the 'finally' blocks */
106105
int blockstack_top = 0; /* (ditto) */
107106

@@ -256,14 +255,16 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
256255
return -1;
257256
}
258257
if (first_in && !second_in) {
259-
if (op == FOR_ITER && !delta_iblock) {
260-
for_loop_delta++;
261-
}
262-
if (op != FOR_ITER) {
258+
if (op != FOR_ITER && code[target_addr] != END_ASYNC_FOR) {
263259
delta_iblock++;
264260
}
261+
else if (!delta_iblock) {
262+
/* Pop the iterators of any 'for' and 'async for' loop
263+
* we're jumping out of. */
264+
delta++;
265+
}
265266
}
266-
if (op != FOR_ITER) {
267+
if (op != FOR_ITER && code[target_addr] != END_ASYNC_FOR) {
267268
blockstack[blockstack_top++] = target_addr;
268269
}
269270
break;
@@ -289,21 +290,17 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno)
289290
assert(blockstack_top == 0);
290291

291292
/* Pop any blocks that we're jumping out of. */
292-
delta = 0;
293293
if (delta_iblock > 0) {
294294
f->f_iblock -= delta_iblock;
295295
PyTryBlock *b = &f->f_blockstack[f->f_iblock];
296-
delta = (f->f_stacktop - f->f_valuestack) - b->b_level;
296+
delta += (f->f_stacktop - f->f_valuestack) - b->b_level;
297297
if (b->b_type == SETUP_FINALLY &&
298298
code[b->b_handler] == WITH_CLEANUP_START)
299299
{
300300
/* Pop the exit function. */
301301
delta++;
302302
}
303303
}
304-
/* Pop the iterators of any 'for' loop we're jumping out of. */
305-
delta += for_loop_delta;
306-
307304
while (delta > 0) {
308305
PyObject *v = (*--f->f_stacktop);
309306
Py_DECREF(v);

0 commit comments

Comments
 (0)