Implement unparsing the AST back to string form

This is required for PEP 563 and as such only implements a part of the unparsing process that covers expressions.
python · ambv · Jan 26, 2018 · Nov 10, 2017 · Nov 10, 2017 · Nov 10, 2017
commit bc3ad37b1ca035752f567f978bf7933cf708fe2e
diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py
@@ -1,7 +1,9 @@
 # Test various flavors of legal and illegal future statements
 
+from functools import partial
 import unittest
 from test import support
+from textwrap import dedent
 import os
 import re
 
@@ -102,6 +104,160 @@ def test_unicode_literals_exec(self):
         exec("from __future__ import unicode_literals; x = ''", {}, scope)
         self.assertIsInstance(scope["x"], str)
 
+class AnnotationsFutureTestCase(unittest.TestCase):
+    template = dedent(
+        """
+        from __future__ import annotations
+        def f() -> {ann}:
+            ...
+        var: {ann}
+        var2: {ann} = None
+        """
+    )
+
+    def getActual(self, annotation):
+        scope = {}
+        exec(self.template.format(ann=annotation), {}, scope)
+        func_ann = scope['f'].__annotations__['return']
+        var_ann1 = scope['__annotations__']['var']
+        var_ann2 = scope['__annotations__']['var2']
+        self.assertEqual(func_ann, var_ann1)
+        self.assertEqual(func_ann, var_ann2)
+        return func_ann
+
+    def assertAnnotationEqual(
+        self, annotation, drop_parens=False, is_tuple=False
+    ):
+        actual = self.getActual(annotation)
+        if annotation.startswith("'"):
+            # strings are passed verbatim
+            expected = eval(annotation)
+        elif is_tuple:
+            expected = annotation[1:-1]
+        else:
+            expected = annotation
+        if drop_parens:
+            self.assertNotEqual(actual, expected)
+            actual = actual.replace("(", "").replace(")", "")
+
+        self.assertEqual(actual, expected)
+
+    def test_annotations(self):
+        eq = self.assertAnnotationEqual
+        eq('...')
+        eq("'some_string'")
+        eq("b'\\xa3'")
+        eq('Name')
+        eq('None')
+        eq('True')
+        eq('False')
+        eq('1')
+        eq('1.0')
+        eq('1j')
+        eq('True or False')
+        eq('True or False or None')
+        eq('True and False')
+        eq('True and False and None')
+        eq('(Name1 and Name2) or Name3')
+        eq('Name1 or (Name2 and Name3)')
+        eq('(Name1 and Name2) or (Name3 and Name4)')
+        eq('Name1 or (Name2 and Name3) or Name4')
+        eq('1 << 2')
+        eq('1 >> 2')
+        eq('((1 + 2) - (3 * 4)) ^ (((5 ** 6) / 7) // 8)')
+        eq('not True')
+        eq('~True')
+        eq('+1')
+        eq('-1')
+        eq('(~int) and (not ((True ^ (123 + 2)) | True))')
+        eq('lambda arg: None')
+        eq('lambda a=True: a')
+        eq('lambda a, b, c=True: a')
+        eq("lambda a, b, c=True, *, d=(1 << 2), e='str': a")
+        eq("lambda a, b, c=True, *vararg, d=(1 << 2), e='str', **kwargs: a + b")
+        eq('1 if True else 2')
+        eq('(str or None) if True else (str or bytes or None)')
+        eq('(str or None) if (1 if True else 2) else (str or bytes or None)')
+        eq("{'2.7': dead, '3.7': (long_live or die_hard)}")
+        eq("{'2.7': dead, '3.7': (long_live or die_hard), **{'3.6': verygood}}")
+        eq("{**a, **b, **c}")
+        eq("{'2.7', '3.6', '3.7', '3.8', '3.9', ('4.0' if gilectomy else '3.10')}")
+        eq("({'a': 'b'}, (True or False), (+1), 'string', b'bytes') or None")
+        eq("()")
+        eq("(1, )")
+        eq("[]")
+        eq("[1, 2, 3, 4, 5, 6, 7, 8, 9, (10 or A), (11 or B), (12 or C)]")
+        eq("{i for i in (1, 2, 3)}")
+        eq("{(i ** 2) for i in (1, 2, 3)}")
+        eq("{(i ** 2) for i, _ in [(1, 'a'), (2, 'b'), (3, 'c')]}")
+        eq("{((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)}")
+        eq("[i for i in (1, 2, 3)]")
+        eq("[(i ** 2) for i in (1, 2, 3)]")
+        eq("[(i ** 2) for i, _ in [(1, 'a'), (2, 'b'), (3, 'c')]]")
+        eq("[((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)]")
+        eq("{i: 0 for i in (1, 2, 3)}")
+        eq("{i: j for i, j in [(1, 'a'), (2, 'b'), (3, 'c')]}")
+        eq("Python3 > Python2 > COBOL")
+        eq("Life is Life")
+        eq("call()")
+        eq("call(arg)")
+        eq("call(kwarg='hey')")
+        eq("call(arg, kwarg='hey')")
+        eq("call(arg, another, kwarg='hey', **kwargs)")
+        eq("lukasz.langa.pl")
+        eq("call.me(maybe)")
+        eq("list[str]")
+        eq("dict[str, int]")
+        eq("tuple[str, ...]")
+        eq("tuple[str, int, float, dict[str, int]]")
+        eq('(str or None) if (sys.version_info[0] > (3, )) else (str or bytes or None)')
+
+    def test_annotations_no_fstring_support_implemented(self):
+        # FIXME: Add f-string support in ast_unparse.c.
+        eq = self.assertAnnotationEqual
+        with self.assertRaises(SystemError) as err:
+            eq("""f'some f-string with {a} {few():.2f} {formatted.values!r}'""")
+        self.assertEqual(
+            str(err.exception),
+            "f-string support in annotations not implemented yet",
+        )
+
+    def test_annotations_inexact(self):
+        """Source formatting is not always preserved
+
+        This is due to reconstruction from AST.  We *need to* put the parens
+        in nested expressions because we don't know if the source code
+        had them in the first place or not.
+        """
+        eq = partial(self.assertAnnotationEqual, drop_parens=True)
+        eq('Name1 and Name2 or Name3')
+        eq('Name1 or Name2 and Name3')
+        eq('Name1 and Name2 or Name3 and Name4')
+        eq('Name1 or Name2 and Name3 or Name4')
+        eq('1 + 2 - 3 * 4 ^ 5 ** 6 / 7 // 8')
+        eq('~int and not True ^ 123 + 2 | True')
+        eq('str or None if True else str or bytes or None')
+        eq("{'2.7': dead, '3.7': long_live or die_hard}")
+        eq("{'2.7', '3.6', '3.7', '3.8', '3.9', '4.0' if gilectomy else '3.10'}")
+        eq("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10 or A, 11 or B, 12 or C]")
+        # Consequently, we always drop unnecessary parens if they were given in
+        # the outer scope:
+        some_name = self.getActual("(SomeName)")
+        self.assertEqual(some_name, 'SomeName')
+        # Interestingly, in the case of tuples (and generator expressions) the
+        # parens are *required* by the Python syntax in the annotation context.
+        # But there's no point storing that detail in __annotations__ so we're
+        # fine with the parens-less form.
+        eq = partial(self.assertAnnotationEqual, is_tuple=True)
+        eq("(Good, Bad, Ugly)")
+        eq("(i for i in (1, 2, 3))")
+        eq("((i ** 2) for i in (1, 2, 3))")
+        eq("((i ** 2) for i, _ in [(1, 'a'), (2, 'b'), (3, 'c')])")
+        eq("(((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3))")
+        eq("(*starred)")
+        eq('(yield from outside_of_generator)')
+        eq('(yield)')
+        eq('(await some.complicated[0].call(with_args=(True or (1 is not 1))))')
 
 
 if __name__ == "__main__":

diff --git a/Makefile.pre.in b/Makefile.pre.in
@@ -320,6 +320,7 @@ PYTHON_OBJS=	\
 		Python/asdl.o \
 		Python/ast.o \
 		Python/ast_opt.o \
+		Python/ast_unparse.o \
 		Python/bltinmodule.o \
 		Python/ceval.o \
 		Python/compile.o \
@@ -833,7 +834,7 @@ regen-opcode:
 		$(srcdir)/Include/opcode.h.new
 	$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
 
-Python/compile.o Python/symtable.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
+Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c

diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
@@ -357,6 +357,7 @@
     <ClCompile Include="..\Python\asdl.c" />
     <ClCompile Include="..\Python\ast.c" />
     <ClCompile Include="..\Python\ast_opt.c" />
+    <ClCompile Include="..\Python\ast_unparse.c" />
     <ClCompile Include="..\Python\bltinmodule.c" />
     <ClCompile Include="..\Python\bootstrap_hash.c" />
     <ClCompile Include="..\Python\ceval.c" />