Skip to content

Commit cda75be

Browse files
benjaminpbenjaminp
committed
unify some ast.argument's attrs; change Attribute column offset (closes python#16795)
Patch from Sven Brauch.
1 parent c45e041 commit cda75be

File tree

11 files changed

+234
-219
lines changed

11 files changed

+234
-219
lines changed

‎Include/Python-ast.h‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -364,18 +364,18 @@ struct _excepthandler {
364364

365365
struct _arguments {
366366
asdl_seq *args;
367-
identifier vararg;
368-
expr_ty varargannotation;
367+
arg_ty vararg;
369368
asdl_seq *kwonlyargs;
370-
identifier kwarg;
371-
expr_ty kwargannotation;
372-
asdl_seq *defaults;
373369
asdl_seq *kw_defaults;
370+
arg_ty kwarg;
371+
asdl_seq *defaults;
374372
};
375373

376374
struct _arg {
377375
identifier arg;
378376
expr_ty annotation;
377+
int lineno;
378+
int col_offset;
379379
};
380380

381381
struct _keyword {
@@ -550,11 +550,10 @@ comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq *
550550
excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq *
551551
body, int lineno, int col_offset, PyArena
552552
*arena);
553-
#define arguments(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_arguments(a0, a1, a2, a3, a4, a5, a6, a7, a8)
554-
arguments_ty _Py_arguments(asdl_seq * args, identifier vararg, expr_ty
555-
varargannotation, asdl_seq * kwonlyargs, identifier
556-
kwarg, expr_ty kwargannotation, asdl_seq * defaults,
557-
asdl_seq * kw_defaults, PyArena *arena);
553+
#define arguments(a0, a1, a2, a3, a4, a5, a6) _Py_arguments(a0, a1, a2, a3, a4, a5, a6)
554+
arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq *
555+
kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg,
556+
asdl_seq * defaults, PyArena *arena);
558557
#define arg(a0, a1, a2) _Py_arg(a0, a1, a2)
559558
arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena);
560559
#define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)

‎Lib/test/test_ast.py‎

Lines changed: 45 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -180,20 +180,36 @@ def to_tuple(t):
180180

181181
class AST_Tests(unittest.TestCase):
182182

183-
def _assertTrueorder(self, ast_node, parent_pos):
183+
def _assertTrueorder(self, ast_node, parent_pos, reverse_check = False):
184+
def should_reverse_check(parent, child):
185+
# In some situations, the children of nodes occur before
186+
# their parents, for example in a.b.c, a occurs before b
187+
# but a is a child of b.
188+
if isinstance(parent, ast.Call):
189+
if parent.func == child:
190+
return True
191+
if isinstance(parent, (ast.Attribute, ast.Subscript)):
192+
return True
193+
return False
194+
184195
if not isinstance(ast_node, ast.AST) or ast_node._fields is None:
185196
return
186197
if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)):
187198
node_pos = (ast_node.lineno, ast_node.col_offset)
188-
self.assertTrue(node_pos >= parent_pos)
199+
if reverse_check:
200+
self.assertTrue(node_pos <= parent_pos)
201+
else:
202+
self.assertTrue(node_pos >= parent_pos)
189203
parent_pos = (ast_node.lineno, ast_node.col_offset)
190204
for name in ast_node._fields:
191205
value = getattr(ast_node, name)
192206
if isinstance(value, list):
193207
for child in value:
194-
self._assertTrueorder(child, parent_pos)
208+
self._assertTrueorder(child, parent_pos,
209+
should_reverse_check(ast_node, child))
195210
elif value is not None:
196-
self._assertTrueorder(value, parent_pos)
211+
self._assertTrueorder(value, parent_pos,
212+
should_reverse_check(ast_node, value))
197213

198214
def test_AST_objects(self):
199215
x = ast.AST()
@@ -262,14 +278,14 @@ def test_field_attr_existence(self):
262278

263279
def test_arguments(self):
264280
x = ast.arguments()
265-
self.assertEqual(x._fields, ('args', 'vararg', 'varargannotation',
266-
'kwonlyargs', 'kwarg', 'kwargannotation',
267-
'defaults', 'kw_defaults'))
281+
self.assertEqual(x._fields, ('args', 'vararg',
282+
'kwonlyargs', 'kw_defaults',
283+
'kwarg', 'defaults'))
268284

269285
with self.assertRaises(AttributeError):
270286
x.vararg
271287

272-
x = ast.arguments(*range(1, 9))
288+
x = ast.arguments(*range(1, 7))
273289
self.assertEqual(x.vararg, 2)
274290

275291
def test_field_attr_writable(self):
@@ -439,7 +455,7 @@ def test_dump(self):
439455
"lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
440456
"lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
441457
"col_offset=11)], keywords=[], starargs=None, kwargs=None, "
442-
"lineno=1, col_offset=0), lineno=1, col_offset=0)])"
458+
"lineno=1, col_offset=4), lineno=1, col_offset=0)])"
443459
)
444460

445461
def test_copy_location(self):
@@ -460,7 +476,7 @@ def test_fix_missing_locations(self):
460476
"Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), "
461477
"lineno=1, col_offset=0), args=[Str(s='spam', lineno=1, "
462478
"col_offset=6)], keywords=[], starargs=None, kwargs=None, "
463-
"lineno=1, col_offset=0), lineno=1, col_offset=0), "
479+
"lineno=1, col_offset=5), lineno=1, col_offset=0), "
464480
"Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
465481
"col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], "
466482
"keywords=[], starargs=None, kwargs=None, lineno=1, "
@@ -560,8 +576,8 @@ def test_module(self):
560576
self.mod(m, "must have Load context", "eval")
561577

562578
def _check_arguments(self, fac, check):
563-
def arguments(args=None, vararg=None, varargannotation=None,
564-
kwonlyargs=None, kwarg=None, kwargannotation=None,
579+
def arguments(args=None, vararg=None,
580+
kwonlyargs=None, kwarg=None,
565581
defaults=None, kw_defaults=None):
566582
if args is None:
567583
args = []
@@ -571,20 +587,12 @@ def arguments(args=None, vararg=None, varargannotation=None,
571587
defaults = []
572588
if kw_defaults is None:
573589
kw_defaults = []
574-
args = ast.arguments(args, vararg, varargannotation, kwonlyargs,
575-
kwarg, kwargannotation, defaults, kw_defaults)
590+
args = ast.arguments(args, vararg, kwonlyargs, kw_defaults,
591+
kwarg, defaults)
576592
return fac(args)
577593
args = [ast.arg("x", ast.Name("x", ast.Store()))]
578594
check(arguments(args=args), "must have Load context")
579-
check(arguments(varargannotation=ast.Num(3)),
580-
"varargannotation but no vararg")
581-
check(arguments(varargannotation=ast.Name("x", ast.Store()), vararg="x"),
582-
"must have Load context")
583595
check(arguments(kwonlyargs=args), "must have Load context")
584-
check(arguments(kwargannotation=ast.Num(42)),
585-
"kwargannotation but no kwarg")
586-
check(arguments(kwargannotation=ast.Name("x", ast.Store()),
587-
kwarg="x"), "must have Load context")
588596
check(arguments(defaults=[ast.Num(3)]),
589597
"more positional defaults than args")
590598
check(arguments(kw_defaults=[ast.Num(4)]),
@@ -599,7 +607,7 @@ def arguments(args=None, vararg=None, varargannotation=None,
599607
"must have Load context")
600608

601609
def test_funcdef(self):
602-
a = ast.arguments([], None, None, [], None, None, [], [])
610+
a = ast.arguments([], None, [], [], None, [])
603611
f = ast.FunctionDef("x", a, [], [], None)
604612
self.stmt(f, "empty body on FunctionDef")
605613
f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())],
@@ -770,7 +778,7 @@ def test_unaryop(self):
770778
self.expr(u, "must have Load context")
771779

772780
def test_lambda(self):
773-
a = ast.arguments([], None, None, [], None, None, [], [])
781+
a = ast.arguments([], None, [], [], None, [])
774782
self.expr(ast.Lambda(a, ast.Name("x", ast.Store())),
775783
"must have Load context")
776784
def fac(args):
@@ -963,15 +971,15 @@ def main():
963971
#### EVERYTHING BELOW IS GENERATED #####
964972
exec_results = [
965973
('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]),
966-
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]),
967-
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [], []), [('Pass', (1, 10))], [], None)]),
968-
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [('Num', (1, 8), 0)], []), [('Pass', (1, 12))], [], None)]),
969-
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, [], None, None, [], []), [('Pass', (1, 14))], [], None)]),
970-
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], 'kwargs', None, [], []), [('Pass', (1, 17))], [], None)]),
971-
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]),
974+
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]),
975+
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]),
976+
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]),
977+
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]),
978+
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]),
979+
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [], [], ('arg', (1, 43), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 52))], [], None)]),
972980
('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [])]),
973981
('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], None, None, [('Pass', (1, 17))], [])]),
974-
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
982+
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
975983
('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
976984
('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
977985
('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
@@ -980,7 +988,7 @@ def main():
980988
('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
981989
('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]),
982990
('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]),
983-
('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]),
991+
('Module', [('Raise', (1, 0), ('Call', (1, 15), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]),
984992
('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]),
985993
('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]),
986994
('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
@@ -1009,25 +1017,25 @@ def main():
10091017
('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
10101018
('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
10111019
('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
1012-
('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('NameConstant', (1, 7), None))),
1020+
('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], [], None, []), ('NameConstant', (1, 7), None))),
10131021
('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
10141022
('Expression', ('Dict', (1, 0), [], [])),
10151023
('Expression', ('Set', (1, 0), [('NameConstant', (1, 1), None)])),
10161024
('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
10171025
('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
10181026
('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
10191027
('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
1020-
('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
1028+
('Expression', ('Call', (1, 1), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
10211029
('Expression', ('Num', (1, 0), 10)),
10221030
('Expression', ('Str', (1, 0), 'string')),
1023-
('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
1024-
('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
1031+
('Expression', ('Attribute', (1, 2), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
1032+
('Expression', ('Subscript', (1, 2), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
10251033
('Expression', ('Name', (1, 0), 'v', ('Load',))),
10261034
('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
10271035
('Expression', ('List', (1, 0), [], ('Load',))),
10281036
('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
10291037
('Expression', ('Tuple', (1, 1), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
10301038
('Expression', ('Tuple', (1, 0), [], ('Load',))),
1031-
('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
1039+
('Expression', ('Call', (1, 7), ('Attribute', (1, 6), ('Attribute', (1, 4), ('Attribute', (1, 2), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 12), ('Attribute', (1, 10), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
10321040
]
10331041
main()

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ What's New in Python 3.4.0 Alpha 1?
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #16795: On the ast.arguments object, unify vararg with varargannotation
14+
and kwarg and kwargannotation. Change the column offset of ast.Attribute to be
15+
at the attribute name.
16+
1317
- Issue #17434: Properly raise a SyntaxError when a string occurs between future
1418
imports.
1519

‎Parser/Python.asdl‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,11 +103,11 @@ module Python
103103
excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)
104104
attributes (int lineno, int col_offset)
105105

106-
arguments = (arg* args, identifier? vararg, expr? varargannotation,
107-
arg* kwonlyargs, identifier? kwarg,
108-
expr? kwargannotation, expr* defaults,
109-
expr* kw_defaults)
106+
arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults,
107+
arg? kwarg, expr* defaults)
108+
110109
arg = (identifier arg, expr? annotation)
110+
attributes (int lineno, int col_offset)
111111

112112
-- keyword arguments supplied to call
113113
keyword = (identifier arg, expr value)

‎Parser/asdl.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -158,11 +158,19 @@ def p_type_2(self, info):
158158
msg="expected attributes, found %s" % id)
159159
return Sum(sum, attributes)
160160

161-
def p_product(self, info):
161+
def p_product_0(self, info):
162162
" product ::= ( fields ) "
163163
_0, fields, _1 = info
164164
return Product(fields)
165165

166+
def p_product_1(self, info):
167+
" product ::= ( fields ) Id ( fields ) "
168+
_0, fields, _1, id, _2, attributes, _3 = info
169+
if id.value != "attributes":
170+
raise ASDLSyntaxError(id.lineno,
171+
msg="expected attributes, found %s" % id)
172+
return Product(fields, attributes)
173+
166174
def p_sum_0(self, constructor):
167175
" sum ::= constructor "
168176
return [constructor[0]]
@@ -289,11 +297,15 @@ def __repr__(self):
289297
return "Sum(%s, %s)" % (self.types, self.attributes)
290298

291299
class Product(AST):
292-
def __init__(self, fields):
300+
def __init__(self, fields, attributes=None):
293301
self.fields = fields
302+
self.attributes = attributes or []
294303

295304
def __repr__(self):
296-
return "Product(%s)" % self.fields
305+
if self.attributes is None:
306+
return "Product(%s)" % self.fields
307+
else:
308+
return "Product(%s, %s)" % (self.fields, self.attributes)
297309

298310
class VisitorBase(object):
299311

0 commit comments

Comments
 (0)