Skip to content

Commit bd260da

Browse files
loewisloewis
committed
Generate code to recursively copy an AST into
a tree of Python objects. Expose this through compile().
1 parent 23b0dc5 commit bd260da

File tree

8 files changed

+1946
-785
lines changed

8 files changed

+1946
-785
lines changed

‎Include/Python-ast.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,3 +404,4 @@ arguments_ty arguments(asdl_seq * args, identifier vararg, identifier kwarg,
404404
keyword_ty keyword(identifier arg, expr_ty value, PyArena *arena);
405405
alias_ty alias(identifier name, identifier asname, PyArena *arena);
406406

407+
PyObject* PyAST_mod2obj(mod_ty t);

‎Include/pythonrun.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ extern "C" {
1111
#define PyCF_MASK_OBSOLETE (CO_NESTED)
1212
#define PyCF_SOURCE_IS_UTF8 0x0100
1313
#define PyCF_DONT_IMPLY_DEDENT 0x0200
14+
#define PyCF_ONLY_AST 0x0400
1415

1516
typedef struct {
1617
int cf_flags; /* bitmask of CO_xxx flags relevant to future */

‎Lib/test/test_ast.py‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import sys, itertools
2+
3+
def to_tuple(t):
4+
if t is None or isinstance(t, (str, int, long)):
5+
return t
6+
elif isinstance(t, list):
7+
return [to_tuple(e) for e in t]
8+
result = [t.__class__.__name__]
9+
if t._fields is None:
10+
return tuple(result)
11+
for f in t._fields:
12+
result.append(to_tuple(getattr(t, f)))
13+
return tuple(result)
14+
15+
# These tests are compiled through "exec"
16+
# There should be atleast one test per statement
17+
exec_tests = [
18+
# FunctionDef
19+
"def f(): pass",
20+
# ClassDef
21+
"class C:pass",
22+
# Return
23+
"def f():return 1",
24+
# Delete
25+
"del v",
26+
# Assign
27+
"v = 1",
28+
# AugAssign
29+
"v += 1",
30+
# Print
31+
"print >>f, 1, ",
32+
# For
33+
"for v in v:pass",
34+
# While
35+
"while v:pass",
36+
# If
37+
"if v:pass",
38+
# Raise
39+
"raise Exception, 'string'",
40+
# TryExcept
41+
"try:\n pass\nexcept Exception:\n pass",
42+
# TryFinally
43+
"try:\n pass\nfinally:\n pass",
44+
# Assert
45+
"assert v",
46+
# Import
47+
"import sys",
48+
# ImportFrom
49+
"from sys import v",
50+
# Exec
51+
"exec 'v'",
52+
# Global
53+
"global v",
54+
# Expr
55+
"1",
56+
# Pass,
57+
"pass",
58+
# Break
59+
"break",
60+
# Continue
61+
"continue",
62+
]
63+
64+
# These are compiled through "single"
65+
# because of overlap with "eval", it just tests what
66+
# can't be tested with "eval"
67+
single_tests = [
68+
"1+2"
69+
]
70+
71+
# These are compiled through "eval"
72+
# It should test all expressions
73+
eval_tests = [
74+
# BoolOp
75+
"a and b",
76+
# BinOp
77+
"a + b",
78+
# UnaryOp
79+
"not v",
80+
# Lambda
81+
"lambda:None",
82+
# Dict
83+
"{ 1:2 }",
84+
# ListComp
85+
"[a for b in c if d]",
86+
# GeneratorExp
87+
"(a for b in c if d)",
88+
# Yield
89+
#"def f():yield 3",
90+
# Compare
91+
"1 < 2 < 3",
92+
# Call
93+
"f(1,2,c=3,*d,**e)",
94+
# Repr
95+
"`v`",
96+
# Num
97+
"10L",
98+
# Str
99+
"'string'",
100+
# Attribute
101+
"a.b",
102+
# Subscript
103+
"a[b:c]",
104+
# Name
105+
"v",
106+
# List
107+
"[1,2,3]",
108+
# Tuple
109+
"1,2,3"
110+
]
111+
112+
# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
113+
# excepthandler, arguments, keywords, alias
114+
115+
if __name__=='__main__' and sys.argv[1:] == ['-g']:
116+
for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), (eval_tests, "eval")):
117+
print kind+"_results = ["
118+
for s in statements:
119+
print repr(to_tuple(compile(s, "?", kind, 0x400)))+","
120+
print "]"
121+
print "run_tests()"
122+
raise SystemExit
123+
124+
def run_tests():
125+
for input, output, kind in ((exec_tests, exec_results, "exec"),
126+
(single_tests, single_results, "single"),
127+
(eval_tests, eval_results, "eval")):
128+
for i, o in itertools.izip(input, output):
129+
assert to_tuple(compile(i, "?", kind, 0x400)) == o
130+
131+
#### EVERYTHING BELOW IS GENERATED #####
132+
exec_results = [
133+
('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Pass',)], [])]),
134+
('Module', [('ClassDef', 'C', [], [('Pass',)])]),
135+
('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Return', ('Num', 1))], [])]),
136+
('Module', [('Delete', [('Name', 'v', ('Del',))])]),
137+
('Module', [('Assign', [('Name', 'v', ('Store',))], ('Num', 1))]),
138+
('Module', [('AugAssign', ('Name', 'v', ('Load',)), ('Add',), ('Num', 1))]),
139+
('Module', [('Print', ('Name', 'f', ('Load',)), [('Num', 1)], False)]),
140+
('Module', [('For', ('Name', 'v', ('Store',)), ('Name', 'v', ('Load',)), [('Pass',)], [])]),
141+
('Module', [('While', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
142+
('Module', [('If', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
143+
('Module', [('Raise', ('Name', 'Exception', ('Load',)), ('Str', 'string'), None)]),
144+
('Module', [('TryExcept', [('Pass',)], [('excepthandler', ('Name', 'Exception', ('Load',)), None, [('Pass',)])], [])]),
145+
('Module', [('TryFinally', [('Pass',)], [('Pass',)])]),
146+
('Module', [('Assert', ('Name', 'v', ('Load',)), None)]),
147+
('Module', [('Import', [('alias', 'sys', None)])]),
148+
('Module', [('ImportFrom', 'sys', [('alias', 'v', None)])]),
149+
('Module', [('Exec', ('Str', 'v'), None, None)]),
150+
('Module', [('Global', ['v'])]),
151+
('Module', [('Expr', ('Num', 1))]),
152+
('Module', [('Pass',)]),
153+
('Module', [('Break',)]),
154+
('Module', [('Continue',)]),
155+
]
156+
single_results = [
157+
('Interactive', [('Expr', ('BinOp', ('Num', 1), ('Add',), ('Num', 2)))]),
158+
]
159+
eval_results = [
160+
('Expression', ('BoolOp', ('And',), [('Name', 'a', ('Load',)), ('Name', 'b', ('Load',))])),
161+
('Expression', ('BinOp', ('Name', 'a', ('Load',)), ('Add',), ('Name', 'b', ('Load',)))),
162+
('Expression', ('UnaryOp', ('Not',), ('Name', 'v', ('Load',)))),
163+
('Expression', ('Lambda', ('arguments', [], None, None, []), ('Name', 'None', ('Load',)))),
164+
('Expression', ('Dict', [('Num', 1)], [('Num', 2)])),
165+
('Expression', ('ListComp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
166+
('Expression', ('GeneratorExp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
167+
('Expression', ('Compare', ('Num', 1), [('Lt',), ('Lt',)], [('Num', 2), ('Num', 3)])),
168+
('Expression', ('Call', ('Name', 'f', ('Load',)), [('Num', 1), ('Num', 2)], [('keyword', 'c', ('Num', 3))], ('Name', 'd', ('Load',)), ('Name', 'e', ('Load',)))),
169+
('Expression', ('Repr', ('Name', 'v', ('Load',)))),
170+
('Expression', ('Num', 10L)),
171+
('Expression', ('Str', 'string')),
172+
('Expression', ('Attribute', ('Name', 'a', ('Load',)), 'b', ('Load',))),
173+
('Expression', ('Subscript', ('Name', 'a', ('Load',)), ('Slice', ('Name', 'b', ('Load',)), ('Name', 'c', ('Load',)), None), ('Load',))),
174+
('Expression', ('Name', 'v', ('Load',))),
175+
('Expression', ('List', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
176+
('Expression', ('Tuple', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
177+
]
178+
run_tests()
179+

‎Misc/NEWS‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,9 @@ Core and builtins
7474

7575
- Speed up some Unicode operations.
7676

77-
- A new AST parser implementation was completed.
77+
- A new AST parser implementation was completed. The abstract
78+
syntax tree is available for read-only (non-compile) access
79+
to Python code.
7880

7981
- SF bug #1167751: fix incorrect code being for generator expressions.
8082
The following code now raises a SyntaxError: foo(a = i for i in range(10))

0 commit comments

Comments
 (0)