From cfd631bd1751988f392d49e18e8e038b9e79127f Mon Sep 17 00:00:00 2001
From: Yury Selivanov <yury@magic.io>
Date: Sat, 4 May 2019 17:48:00 -0400
Subject: [PATCH 01/19] WIP async repl implementation

This now works:

    import asyncio
    import types

    print('here')

    L = {
        'asyncio': asyncio,
        'a': 'something'
    }

    code = compile('''
    async with asyncio.Lock():
        a = await asyncio.sleep(1, result=42)
    ''', '', 'single', 0x2000)

    f = types.FunctionType(code, L)

    print('before', L)
    asyncio.run(f())
    print('after', L)
---
 Include/compile.h |  1 +
 Python/compile.c  | 25 +++++++++++++++++++------
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/Include/compile.h b/Include/compile.h
index 13708678675f7b..094c9f51f0685c 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -23,6 +23,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
 #define PyCF_TYPE_COMMENTS 0x1000
+#define PyCF_AWAIT 0x2000
 
 #ifndef Py_LIMITED_API
 typedef struct {
diff --git a/Python/compile.c b/Python/compile.c
index b20548c777246c..ea3ae4ff971ab0 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -4565,7 +4565,9 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
 
     assert(s->kind == AsyncWith_kind);
     if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
-        return compiler_error(c, "'async with' outside async function");
+        if (!(c->c_flags->cf_flags & 0x2000)) {
+            return compiler_error(c, "'async with' outside async function");
+        }
     }
 
     block = compiler_new_block(c);
@@ -4773,12 +4775,19 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
         ADDOP(c, YIELD_FROM);
         break;
     case Await_kind:
-        if (c->u->u_ste->ste_type != FunctionBlock)
-            return compiler_error(c, "'await' outside function");
+        if (c->u->u_ste->ste_type != FunctionBlock) {
+            if (!(c->c_flags->cf_flags & 0x2000)) {
+                return compiler_error(c, "'await' outside function");
+            }
+
+            c->u->u_ste->ste_coroutine = 1;
+        }
 
-        if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
-                c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
-            return compiler_error(c, "'await' outside async function");
+        if (!(c->c_flags->cf_flags & 0x2000)) {
+            if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
+                    c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
+                return compiler_error(c, "'await' outside async function");
+        }
 
         VISIT(c, expr, e->v.Await.value);
         ADDOP(c, GET_AWAITABLE);
@@ -5712,6 +5721,10 @@ compute_code_flags(struct compiler *c)
     /* (Only) inherit compilerflags in PyCF_MASK */
     flags |= (c->c_flags->cf_flags & PyCF_MASK);
 
+    if ((c->c_flags->cf_flags & 0x2000) && c->u->u_ste->ste_coroutine) {
+        flags |= CO_COROUTINE;
+    }
+
     return flags;
 }
 

From 1dafcdc2c2ed3037cfcd1b6fddc3f1e55ff5b988 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Sun, 5 May 2019 06:22:37 -0400
Subject: [PATCH 02/19] Add option to `compile()` to accept top-level await
 features.

In some narrow but rate case; it is useful to be allow to compile code
with top-level await features; for example when writing a REPL.
This attempt to allow that.
---
 Doc/library/functions.rst                     | 10 +++++++
 Include/compile.h                             |  2 +-
 Lib/test/test_builtin.py                      | 29 ++++++++++++++++++
 .../2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst  |  1 +
 Parser/asdl_c.py                              |  2 ++
 Python/Python-ast.c                           |  2 ++
 Python/compile.c                              | 30 ++++++++++---------
 7 files changed, 61 insertions(+), 15 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 613e4f74ac4176..e0b5a47030abd1 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -257,6 +257,12 @@ are always available.  They are listed here in alphabetical order.
    can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on
    the :class:`~__future__._Feature` instance in the :mod:`__future__` module.
 
+   The optional argument *flags* also control wether the compiled source is
+   allowed to contain top-level ``await``, ``async for`` and ``async with``.
+   When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the the return code
+   object has ``CO_COROUTINE`` set in ``co_code``, and can be interacitvely
+   executed via ``yield from eval(code_object)``.
+
    The argument *optimize* specifies the optimization level of the compiler; the
    default value of ``-1`` selects the optimization level of the interpreter as
    given by :option:`-O` options.  Explicit levels are ``0`` (no optimization;
@@ -290,6 +296,10 @@ are always available.  They are listed here in alphabetical order.
       Previously, :exc:`TypeError` was raised when null bytes were encountered
       in *source*.
 
+   .. versionadded:: 3.8
+      *flags* can now be used to accept source that would contain a top-level
+      ``await``, ``async for`` or ``async with``.
+
 
 .. class:: complex([real[, imag]])
 
diff --git a/Include/compile.h b/Include/compile.h
index 094c9f51f0685c..a833caa06b9dad 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -23,7 +23,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
 #define PyCF_TYPE_COMMENTS 0x1000
-#define PyCF_AWAIT 0x2000
+#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
 
 #ifndef Py_LIMITED_API
 typedef struct {
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 5674ea89b1c408..7acf781bb8eaba 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -18,6 +18,8 @@
 import unittest
 import warnings
 from contextlib import ExitStack
+from inspect import CO_COROUTINE
+from textwrap import dedent
 from operator import neg
 from test.support import (
     EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink)
@@ -358,6 +360,33 @@ def f(): """doc"""
                 rv = ns['f']()
                 self.assertEqual(rv, tuple(expected))
 
+    def test_compile_top_level_await(self):
+        for mode in ('single', 'exec'):
+            for num,code_sample in enumerate(['''await sleep(0)''',
+                                '''async for i in range(10):
+                                       print(i)''',
+                                '''async with asyncio.Lock() as l:
+                                       pass''']):
+                source = dedent(code_sample)
+                with self.assertRaises(SyntaxError,
+                                  msg='source={!r} mode={!r})'.format(source, mode)):
+                    compile(source, '?' , mode)
+                co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE)
+
+    def test_compile_async_generator(self):
+        co = compile(dedent("""async def ticker():
+                for i in range(10):
+                    yield i
+                    await asyncio.sleep(0)"""), '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+        glob = {}
+        exec(co, glob)
+
+        from types import AsyncGeneratorType
+
+        self.assertEqual(type(glob['ticker']()), AsyncGeneratorType)
+
+
     def test_delattr(self):
         sys.spam = 1
         delattr(sys, 'spam')
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst
new file mode 100644
index 00000000000000..c264d21bd01610
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst	
@@ -0,0 +1 @@
+The ``compile()`` builtin functions now support the ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` flag,  which allow to compile sources that  contains top-level ``await``, ``async with`` or ``async for``. This is useful to evaluate async-code from with an already async functions; for example in a custom REPL.
\ No newline at end of file
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 4091b6db638cd6..cb0e6d7f9df26a 100644
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1000,6 +1000,8 @@ def visitModule(self, mod):
         self.emit("if (!m) return NULL;", 1)
         self.emit("d = PyModule_GetDict(m);", 1)
         self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1)
+        self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)', 1)
+        self.emit("return NULL;", 2)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1)
         self.emit("return NULL;", 2)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index cb53a41cdf35bd..552750584480b7 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -8776,6 +8776,8 @@ PyInit__ast(void)
     if (!m) return NULL;
     d = PyModule_GetDict(m);
     if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;
+    if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)
+        return NULL;
     if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)
         return NULL;
     if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)
diff --git a/Python/compile.c b/Python/compile.c
index ea3ae4ff971ab0..bf78f24e6baf5d 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -2609,10 +2609,13 @@ static int
 compiler_async_for(struct compiler *c, stmt_ty s)
 {
     basicblock *start, *except, *end;
-    if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
+    if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+        c->u->u_ste->ste_coroutine = 1;
+    } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
         return compiler_error(c, "'async for' outside async function");
     }
 
+
     start = compiler_new_block(c);
     except = compiler_new_block(c);
     end = compiler_new_block(c);
@@ -4564,10 +4567,10 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
     withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
 
     assert(s->kind == AsyncWith_kind);
-    if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
-        if (!(c->c_flags->cf_flags & 0x2000)) {
-            return compiler_error(c, "'async with' outside async function");
-        }
+    if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+        c->u->u_ste->ste_coroutine = 1;
+    } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){
+        return compiler_error(c, "'async with' outside async function");
     }
 
     block = compiler_new_block(c);
@@ -4775,18 +4778,17 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
         ADDOP(c, YIELD_FROM);
         break;
     case Await_kind:
-        if (c->u->u_ste->ste_type != FunctionBlock) {
-            if (!(c->c_flags->cf_flags & 0x2000)) {
+        if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+            c->u->u_ste->ste_coroutine = 1;
+        } else {
+            if (c->u->u_ste->ste_type != FunctionBlock){
                 return compiler_error(c, "'await' outside function");
             }
 
-            c->u->u_ste->ste_coroutine = 1;
-        }
-
-        if (!(c->c_flags->cf_flags & 0x2000)) {
-            if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
-                    c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
+            if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && 
+                    c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){
                 return compiler_error(c, "'await' outside async function");
+            }
         }
 
         VISIT(c, expr, e->v.Await.value);
@@ -5721,7 +5723,7 @@ compute_code_flags(struct compiler *c)
     /* (Only) inherit compilerflags in PyCF_MASK */
     flags |= (c->c_flags->cf_flags & PyCF_MASK);
 
-    if ((c->c_flags->cf_flags & 0x2000) && c->u->u_ste->ste_coroutine) {
+    if ((c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) && ste->ste_coroutine && !ste->ste_generator) {
         flags |= CO_COROUTINE;
     }
 

From 1e480cb247f9d0b76006a0f02919440dd7a118da Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 7 May 2019 13:24:50 -0700
Subject: [PATCH 03/19] remove some unnecessary already existing coroutine
 flags

---
 Lib/test/test_builtin.py | 2 +-
 Python/compile.c         | 9 +++------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 7acf781bb8eaba..0f301d251018eb 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -372,7 +372,7 @@ def test_compile_top_level_await(self):
                                   msg='source={!r} mode={!r})'.format(source, mode)):
                     compile(source, '?' , mode)
                 co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
-                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE)
+                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, msg='source={!r} mode={!r})'.format(source, mode))
 
     def test_compile_async_generator(self):
         co = compile(dedent("""async def ticker():
diff --git a/Python/compile.c b/Python/compile.c
index bf78f24e6baf5d..1deccfe02e7e60 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -4567,9 +4567,8 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
     withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
 
     assert(s->kind == AsyncWith_kind);
-    if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
-        c->u->u_ste->ste_coroutine = 1;
-    } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){
+    if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
+        !(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){
         return compiler_error(c, "'async with' outside async function");
     }
 
@@ -4778,9 +4777,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
         ADDOP(c, YIELD_FROM);
         break;
     case Await_kind:
-        if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
-            c->u->u_ste->ste_coroutine = 1;
-        } else {
+        if (! (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){
             if (c->u->u_ste->ste_type != FunctionBlock){
                 return compiler_error(c, "'await' outside function");
             }

From f48d70ca4a16bbf330b7b9ac4839556ef14f4941 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 7 May 2019 13:53:31 -0700
Subject: [PATCH 04/19] add actual necessary ste_couroutines

---
 Python/compile.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Python/compile.c b/Python/compile.c
index 1deccfe02e7e60..156dbe7409eab4 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -4567,8 +4567,9 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
     withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
 
     assert(s->kind == AsyncWith_kind);
-    if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
-        !(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){
+    if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+        c->u->u_ste->ste_coroutine = 1;
+    } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){
         return compiler_error(c, "'async with' outside async function");
     }
 
@@ -4777,7 +4778,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
         ADDOP(c, YIELD_FROM);
         break;
     case Await_kind:
-        if (! (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){
+        if (!(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){
             if (c->u->u_ste->ste_type != FunctionBlock){
                 return compiler_error(c, "'await' outside function");
             }

From ecde54004f1c0d2ec0902b055392cb2b2cb548f0 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 7 May 2019 14:12:07 -0700
Subject: [PATCH 05/19] stry new line

---
 Python/compile.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Python/compile.c b/Python/compile.c
index 156dbe7409eab4..d80254a295c72a 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -2615,7 +2615,6 @@ compiler_async_for(struct compiler *c, stmt_ty s)
         return compiler_error(c, "'async for' outside async function");
     }
 
-
     start = compiler_new_block(c);
     except = compiler_new_block(c);
     end = compiler_new_block(c);

From 25dda358422ae31a4c7ebf962aef7fd3b8186ebf Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 7 May 2019 14:26:30 -0700
Subject: [PATCH 06/19] Reformat some code

---
 Lib/test/test_builtin.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 0f301d251018eb..4f7d527b1c35fe 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -20,6 +20,7 @@
 from contextlib import ExitStack
 from inspect import CO_COROUTINE
 from textwrap import dedent
+from types import AsyncGeneratorType
 from operator import neg
 from test.support import (
     EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink)
@@ -371,8 +372,10 @@ def test_compile_top_level_await(self):
                 with self.assertRaises(SyntaxError,
                                   msg='source={!r} mode={!r})'.format(source, mode)):
                     compile(source, '?' , mode)
+
                 co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
-                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, msg='source={!r} mode={!r})'.format(source, mode))
+                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
+                                 msg='source={!r} mode={!r})'.format(source, mode))
 
     def test_compile_async_generator(self):
         co = compile(dedent("""async def ticker():
@@ -381,9 +384,6 @@ def test_compile_async_generator(self):
                     await asyncio.sleep(0)"""), '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
         glob = {}
         exec(co, glob)
-
-        from types import AsyncGeneratorType
-
         self.assertEqual(type(glob['ticker']()), AsyncGeneratorType)
 
 

From d208fdfd80e7fb8bbdbe99d0f271e48649f2d12d Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Sat, 11 May 2019 13:34:30 -0700
Subject: [PATCH 07/19] Apply grammar corrections from code review

thanks to tirkarthi

Co-Authored-By: Xtreak <tir.karthi@gmail.com>
---
 Doc/library/functions.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index e0b5a47030abd1..1bf5b19ed0e5a6 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -257,10 +257,10 @@ are always available.  They are listed here in alphabetical order.
    can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on
    the :class:`~__future__._Feature` instance in the :mod:`__future__` module.
 
-   The optional argument *flags* also control wether the compiled source is
+   The optional argument *flags* also controls whether the compiled source is
    allowed to contain top-level ``await``, ``async for`` and ``async with``.
    When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the the return code
-   object has ``CO_COROUTINE`` set in ``co_code``, and can be interacitvely
+   object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively
    executed via ``yield from eval(code_object)``.
 
    The argument *optimize* specifies the optimization level of the compiler; the

From 8107bea1609bf0fcefab89c4f7c25bbe8b2ecc89 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Sat, 11 May 2019 13:43:42 -0700
Subject: [PATCH 08/19] Cleanup test suite code formatting.

Use itertools product to remove one level of nesting, and make things a
tiny bit more readable.
---
 Lib/test/test_builtin.py | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 4f7d527b1c35fe..19eeb915ccaed6 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -19,6 +19,7 @@
 import warnings
 from contextlib import ExitStack
 from inspect import CO_COROUTINE
+from itertools import product
 from textwrap import dedent
 from types import AsyncGeneratorType
 from operator import neg
@@ -362,20 +363,27 @@ def f(): """doc"""
                 self.assertEqual(rv, tuple(expected))
 
     def test_compile_top_level_await(self):
-        for mode in ('single', 'exec'):
-            for num,code_sample in enumerate(['''await sleep(0)''',
-                                '''async for i in range(10):
-                                       print(i)''',
-                                '''async with asyncio.Lock() as l:
-                                       pass''']):
-                source = dedent(code_sample)
-                with self.assertRaises(SyntaxError,
-                                  msg='source={!r} mode={!r})'.format(source, mode)):
-                    compile(source, '?' , mode)
-
-                co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
-                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
-                                 msg='source={!r} mode={!r})'.format(source, mode))
+        """Test whether code some top level await can be compiled.
+
+        Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set,
+        and make sure the generated code object has the CO_COROUTINE flag set in
+        order to execute it with `yield from eval(.....)` instead of exec.
+        """
+        modes = ('single', 'exec')
+        code_samples = ['''await sleep(0)''',
+        '''async for i in range(10):
+               print(i)''',
+        '''async with asyncio.Lock() as l:
+               pass''']
+        for mode, code_sample in product(modes,code_samples):
+            source = dedent(code_sample)
+            with self.assertRaises(SyntaxError,
+                                   msg='source={!r} mode={!r})'.format(source, mode)):
+                compile(source, '?' , mode)
+
+            co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+            self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
+                             msg='source={!r} mode={!r})'.format(source, mode))
 
     def test_compile_async_generator(self):
         co = compile(dedent("""async def ticker():

From 371fa8ff9b5f2eee1a6b2dadff4eefd5dc5fb84c Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Sat, 11 May 2019 13:46:45 -0700
Subject: [PATCH 09/19] document an extra test

---
 Lib/test/test_builtin.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 19eeb915ccaed6..3b4ad03f7f2d86 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -386,6 +386,10 @@ def test_compile_top_level_await(self):
                              msg='source={!r} mode={!r})'.format(source, mode))
 
     def test_compile_async_generator(self):
+        """
+        With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to
+        make sure AsyncGenerators are still properly not marked with CO_COROUTINE
+        """
         co = compile(dedent("""async def ticker():
                 for i in range(10):
                     yield i

From 0efd6fb39b153029b3e21a985039f27e90842865 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Mon, 20 May 2019 19:14:50 -0700
Subject: [PATCH 10/19] better formatting

---
 Lib/test/test_builtin.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 3b4ad03f7f2d86..607b5dc52db48d 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -377,13 +377,14 @@ def test_compile_top_level_await(self):
                pass''']
         for mode, code_sample in product(modes,code_samples):
             source = dedent(code_sample)
-            with self.assertRaises(SyntaxError,
-                                   msg='source={!r} mode={!r})'.format(source, mode)):
+            msg = 'source={!r} mode={!r})'
+            with self.assertRaises(SyntaxError, msg=msg.format(source, mode)):
                 compile(source, '?' , mode)
 
             co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+
             self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
-                             msg='source={!r} mode={!r})'.format(source, mode))
+                             msg=msg.format(source, mode))
 
     def test_compile_async_generator(self):
         """

From 7c1b7eeefa3e5966a0bfe7b185b5a0e04737756b Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Mon, 20 May 2019 19:18:31 -0700
Subject: [PATCH 11/19] use debug-f-string

---
 Lib/test/test_builtin.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 607b5dc52db48d..214a50c8b837af 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -377,14 +377,13 @@ def test_compile_top_level_await(self):
                pass''']
         for mode, code_sample in product(modes,code_samples):
             source = dedent(code_sample)
-            msg = 'source={!r} mode={!r})'
-            with self.assertRaises(SyntaxError, msg=msg.format(source, mode)):
+            with self.assertRaises(SyntaxError, msg=f"{source=} {mode=}"):
                 compile(source, '?' , mode)
 
             co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
 
             self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
-                             msg=msg.format(source, mode))
+                             msg=f"{source=} {mode=}")
 
     def test_compile_async_generator(self):
         """

From fd44a147532bf826e4c39d6707b5e74ae60b28ea Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Tue, 21 May 2019 09:56:08 -0700
Subject: [PATCH 12/19] Update Doc/library/functions.rst

Co-Authored-By: Yury Selivanov <yury@magic.io>
---
 Doc/library/functions.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 1bf5b19ed0e5a6..b809d46e6c6cca 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -261,7 +261,7 @@ are always available.  They are listed here in alphabetical order.
    allowed to contain top-level ``await``, ``async for`` and ``async with``.
    When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the the return code
    object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively
-   executed via ``yield from eval(code_object)``.
+   executed via ``await eval(code_object)``.
 
    The argument *optimize* specifies the optimization level of the compiler; the
    default value of ``-1`` selects the optimization level of the interpreter as

From 74bea865bbcbfe07f5414c8ec1beb24cb93728d8 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 21 May 2019 09:58:41 -0700
Subject: [PATCH 13/19] code cleanup

---
 Doc/library/functions.rst | 4 ++--
 Lib/test/test_builtin.py  | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index b809d46e6c6cca..288997964a4bce 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -297,8 +297,8 @@ are always available.  They are listed here in alphabetical order.
       in *source*.
 
    .. versionadded:: 3.8
-      *flags* can now be used to accept source that would contain a top-level
-      ``await``, ``async for`` or ``async with``.
+      ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
+      support for top-level await, async for, and async with.
 
 
 .. class:: complex([real[, imag]])
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 214a50c8b837af..b4c4ab4749b639 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -390,10 +390,12 @@ def test_compile_async_generator(self):
         With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to
         make sure AsyncGenerators are still properly not marked with CO_COROUTINE
         """
-        co = compile(dedent("""async def ticker():
+        code = dedent("""async def ticker():
                 for i in range(10):
                     yield i
-                    await asyncio.sleep(0)"""), '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+                    await asyncio.sleep(0)""")
+
+        co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
         glob = {}
         exec(co, glob)
         self.assertEqual(type(glob['ticker']()), AsyncGeneratorType)

From 48c7a7fa32d4ed5c691459d6e9839104e6e960b3 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 21 May 2019 10:49:18 -0700
Subject: [PATCH 14/19] Redo-some functional testing.

Make the the top=level-await codeobjects can actually be put into t a
FunctionType and run, and/or put inside `await eval()` and do actually
run.
---
 Lib/test/test_builtin.py | 57 +++++++++++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 13 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index b4c4ab4749b639..7bfd7162371274 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -1,6 +1,7 @@
 # Python test set -- built-in functions
 
 import ast
+import asyncio
 import builtins
 import collections
 import decimal
@@ -21,10 +22,11 @@
 from inspect import CO_COROUTINE
 from itertools import product
 from textwrap import dedent
-from types import AsyncGeneratorType
+from types import AsyncGeneratorType, FunctionType
 from operator import neg
 from test.support import (
-    EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink)
+    EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink,
+    maybe_get_event_loop_policy)
 from test.support.script_helper import assert_python_ok
 from unittest.mock import MagicMock, patch
 try:
@@ -369,21 +371,50 @@ def test_compile_top_level_await(self):
         and make sure the generated code object has the CO_COROUTINE flag set in
         order to execute it with `yield from eval(.....)` instead of exec.
         """
+
+        # helper function just to check we can run top=level async-for
+        async def arange(n):
+            for i in range(n):
+                yield i
+
+        async def async_exec(co, *args):
+            await eval(co, *args)
+
         modes = ('single', 'exec')
-        code_samples = ['''await sleep(0)''',
-        '''async for i in range(10):
-               print(i)''',
+        code_samples = ['''a = await asyncio.sleep(0, result=1)''',
+        '''async for i in arange(1):
+               a = 1''',
         '''async with asyncio.Lock() as l:
-               pass''']
-        for mode, code_sample in product(modes,code_samples):
-            source = dedent(code_sample)
-            with self.assertRaises(SyntaxError, msg=f"{source=} {mode=}"):
-                compile(source, '?' , mode)
+               a = 1''']
+        policy = maybe_get_event_loop_policy()
+        try:
+            for mode, code_sample in product(modes,code_samples):
+                source = dedent(code_sample)
+                with self.assertRaises(SyntaxError, msg=f"{source=} {mode=}"):
+                    compile(source, '?' , mode)
+
+                co = compile(source,
+                             '?',
+                             mode,
+                             flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
 
-            co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
+                                 msg=f"{source=} {mode=}")
 
-            self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
-                             msg=f"{source=} {mode=}")
+
+                # test we can create and  advance a fucntion type
+
+                globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
+                async_f = FunctionType(co, globals_)
+                asyncio.run(async_f())
+                self.assertEqual(globals_['a'], 1)
+
+                globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
+                asyncio.run(async_exec(co, globals_))
+                self.assertEqual(globals_['a'], 1)
+        except Exception:
+            asyncio.set_event_loop_policy(policy)
+            raise
 
     def test_compile_async_generator(self):
         """

From 267d0ad4fb9a620493e5a636a775b02005e15655 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 21 May 2019 10:52:16 -0700
Subject: [PATCH 15/19] Fix line lenght

---
 Python/compile.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Python/compile.c b/Python/compile.c
index d80254a295c72a..d276152c42a8c6 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -5720,7 +5720,9 @@ compute_code_flags(struct compiler *c)
     /* (Only) inherit compilerflags in PyCF_MASK */
     flags |= (c->c_flags->cf_flags & PyCF_MASK);
 
-    if ((c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) && ste->ste_coroutine && !ste->ste_generator) {
+    if ((c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) &&
+         ste->ste_coroutine &&
+         !ste->ste_generator) {
         flags |= CO_COROUTINE;
     }
 

From 09c453cceec8d67b2e636fcb2c7cd7cdff27d289 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 21 May 2019 10:55:40 -0700
Subject: [PATCH 16/19] skip await-eval

---
 Lib/test/test_builtin.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 7bfd7162371274..27f54c14f5353a 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -377,9 +377,6 @@ async def arange(n):
             for i in range(n):
                 yield i
 
-        async def async_exec(co, *args):
-            await eval(co, *args)
-
         modes = ('single', 'exec')
         code_samples = ['''a = await asyncio.sleep(0, result=1)''',
         '''async for i in arange(1):
@@ -410,7 +407,7 @@ async def async_exec(co, *args):
                 self.assertEqual(globals_['a'], 1)
 
                 globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
-                asyncio.run(async_exec(co, globals_))
+                asyncio.run(eval(co, globals_))
                 self.assertEqual(globals_['a'], 1)
         except Exception:
             asyncio.set_event_loop_policy(policy)

From d8c3b9654ea209749620b67906a4130062f41a81 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 21 May 2019 10:58:18 -0700
Subject: [PATCH 17/19] fix docs

---
 Lib/test/test_builtin.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 27f54c14f5353a..6cef39a76609af 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -369,7 +369,8 @@ def test_compile_top_level_await(self):
 
         Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set,
         and make sure the generated code object has the CO_COROUTINE flag set in
-        order to execute it with `yield from eval(.....)` instead of exec.
+        order to execute it with  `await eval(.....)` instead of exec, or via a
+        FunctionType.
         """
 
         # helper function just to check we can run top=level async-for
@@ -399,13 +400,13 @@ async def arange(n):
                                  msg=f"{source=} {mode=}")
 
 
-                # test we can create and  advance a fucntion type
-
+                # test we can create and  advance a function type
                 globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
                 async_f = FunctionType(co, globals_)
                 asyncio.run(async_f())
                 self.assertEqual(globals_['a'], 1)
 
+                # test we can await-eval,
                 globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
                 asyncio.run(eval(co, globals_))
                 self.assertEqual(globals_['a'], 1)

From 41345f4a07d4fcd139a8835560e3a58ba6706039 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 21 May 2019 11:06:15 -0700
Subject: [PATCH 18/19] better documentation formatting

---
 Doc/library/functions.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 288997964a4bce..1a9a8b5beeeebe 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -259,7 +259,7 @@ are always available.  They are listed here in alphabetical order.
 
    The optional argument *flags* also controls whether the compiled source is
    allowed to contain top-level ``await``, ``async for`` and ``async with``.
-   When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the the return code
+   When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code
    object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively
    executed via ``await eval(code_object)``.
 
@@ -298,7 +298,7 @@ are always available.  They are listed here in alphabetical order.
 
    .. versionadded:: 3.8
       ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
-      support for top-level await, async for, and async with.
+      support for top-level ``await``, ``async for``, and ``async with``.
 
 
 .. class:: complex([real[, imag]])

From f8c8b4ccb1abd5a4e5d51e2941fa1b6e46478c3c Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <mbussonnier@ucmerced.edu>
Date: Tue, 21 May 2019 11:21:16 -0700
Subject: [PATCH 19/19] restaure ev policy in finally

---
 Lib/test/test_builtin.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 6cef39a76609af..4a358e89d1c2e5 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -410,9 +410,8 @@ async def arange(n):
                 globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
                 asyncio.run(eval(co, globals_))
                 self.assertEqual(globals_['a'], 1)
-        except Exception:
+        finally:
             asyncio.set_event_loop_policy(policy)
-            raise
 
     def test_compile_async_generator(self):
         """