Skip to content

Commit 2acc293

Browse files
zsolzsol
authored
Fix whitespace, fstring, walrus related parse errors (#939, #938, #937, #936, #935, #934, #933, #932, #931)
* Allow walrus in slices See python/cpython#23317 Raised in #930. * Fix parsing of nested f-string specifiers For an expression like `f"{one:{two:}{three}}"`, `three` is not in an f-string spec, and should be tokenized accordingly. This PR fixes the `format_spec_count` bookkeeping in the tokenizer, so it properly decrements it when a closing `}` is encountered but only if the `}` closes a format_spec. Reported in #930. * Fix tokenizing `0else` This is an obscure one. `_ if 0else _` failed to parse with some very weird errors. It turns out that the tokenizer tries to parse `0else` as a single number, but when it encounters `l` it realizes it can't be a single number and it backtracks. Unfortunately the backtracking logic was broken, and it failed to correctly backtrack one of the offsets used for whitespace parsing (the byte offset since the start of the line). This caused whitespace nodes to refer to incorrect parts of the input text, eventually resulting in the above behavior. This PR fixes the bookkeeping when the tokenizer backtracks. Reported in #930. * Allow no whitespace between lambda keyword and params in certain cases Python accepts code where `lambda` follows a `*`, so this PR relaxes validation rules for Lambdas. Raised in #930. * Allow any expression in comprehensions' evaluated expression This PR relaxes the accepted types for the `elt` field in `ListComp`, `SetComp`, and `GenExp`, as well as the `key` and `value` fields in `DictComp`. Fixes #500. * Allow no space around an ifexp in certain cases For example in `_ if _ else""if _ else _`. Raised in #930. Also fixes #854. * Allow no spaces after `as` in a contextmanager in certain cases Like in `with foo()as():pass` Raised in #930. * Allow no spaces around walrus in certain cases Like in `[_:=''for _ in _]` Raised in #930. * Allow no whitespace after lambda body in certain cases Like in `[lambda:()for _ in _]` Reported in #930.
1 parent 648e161 commit 2acc293

19 files changed

Lines changed: 23698 additions & 23493 deletions

File tree

‎libcst/_nodes/expression.py‎

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1983,6 +1983,25 @@ def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Parameters":
19831983
star_kwarg=visit_optional(self, "star_kwarg", self.star_kwarg, visitor),
19841984
)
19851985

1986+
def _safe_to_join_with_lambda(self) -> bool:
1987+
"""
1988+
Determine if Parameters need a space after the `lambda` keyword. Returns True
1989+
iff it's safe to omit the space between `lambda` and these Parameters.
1990+
1991+
See also `BaseExpression._safe_to_use_with_word_operator`.
1992+
1993+
For example: `lambda*_: pass`
1994+
"""
1995+
if len(self.posonly_params) != 0:
1996+
return False
1997+
1998+
# posonly_ind can't appear if above condition is false
1999+
2000+
if len(self.params) > 0 and self.params[0].star not in {"*", "**"}:
2001+
return False
2002+
2003+
return True
2004+
19862005
def _codegen_impl(self, state: CodegenState) -> None: # noqa: C901
19872006
# Compute the star existence first so we can ask about whether
19882007
# each element is the last in the list or not.
@@ -2088,6 +2107,13 @@ class Lambda(BaseExpression):
20882107
BaseParenthesizableWhitespace, MaybeSentinel
20892108
] = MaybeSentinel.DEFAULT
20902109

2110+
def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
2111+
if position == ExpressionPosition.LEFT:
2112+
return len(self.rpar) > 0 or self.body._safe_to_use_with_word_operator(
2113+
position
2114+
)
2115+
return super()._safe_to_use_with_word_operator(position)
2116+
20912117
def _validate(self) -> None:
20922118
# Validate parents
20932119
super(Lambda, self)._validate()
@@ -2115,6 +2141,7 @@ def _validate(self) -> None:
21152141
if (
21162142
isinstance(whitespace_after_lambda, BaseParenthesizableWhitespace)
21172143
and whitespace_after_lambda.empty
2144+
and not self.params._safe_to_join_with_lambda()
21182145
):
21192146
raise CSTValidationError(
21202147
"Must have at least one space after lambda when specifying params"
@@ -2492,6 +2519,12 @@ class IfExp(BaseExpression):
24922519
#: Whitespace after the ``else`` keyword, but before the ``orelse`` expression.
24932520
whitespace_after_else: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
24942521

2522+
def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
2523+
if position == ExpressionPosition.RIGHT:
2524+
return self.body._safe_to_use_with_word_operator(position)
2525+
else:
2526+
return self.orelse._safe_to_use_with_word_operator(position)
2527+
24952528
def _validate(self) -> None:
24962529
# Paren validation and such
24972530
super(IfExp, self)._validate()
@@ -3495,7 +3528,7 @@ class BaseSimpleComp(BaseComp, ABC):
34953528
#: The expression evaluated during each iteration of the comprehension. This
34963529
#: lexically comes before the ``for_in`` clause, but it is semantically the
34973530
#: inner-most element, evaluated inside the ``for_in`` clause.
3498-
elt: BaseAssignTargetExpression
3531+
elt: BaseExpression
34993532

35003533
#: The ``for ... in ... if ...`` clause that lexically comes after ``elt``. This may
35013534
#: be a nested structure for nested comprehensions. See :class:`CompFor` for
@@ -3528,7 +3561,7 @@ class GeneratorExp(BaseSimpleComp):
35283561
"""
35293562

35303563
#: The expression evaluated and yielded during each iteration of the generator.
3531-
elt: BaseAssignTargetExpression
3564+
elt: BaseExpression
35323565

35333566
#: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
35343567
#: nested structure for nested comprehensions. See :class:`CompFor` for details.
@@ -3579,7 +3612,7 @@ class ListComp(BaseList, BaseSimpleComp):
35793612
"""
35803613

35813614
#: The expression evaluated and stored during each iteration of the comprehension.
3582-
elt: BaseAssignTargetExpression
3615+
elt: BaseExpression
35833616

35843617
#: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
35853618
#: nested structure for nested comprehensions. See :class:`CompFor` for details.
@@ -3621,7 +3654,7 @@ class SetComp(BaseSet, BaseSimpleComp):
36213654
"""
36223655

36233656
#: The expression evaluated and stored during each iteration of the comprehension.
3624-
elt: BaseAssignTargetExpression
3657+
elt: BaseExpression
36253658

36263659
#: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
36273660
#: nested structure for nested comprehensions. See :class:`CompFor` for details.
@@ -3663,10 +3696,10 @@ class DictComp(BaseDict, BaseComp):
36633696
"""
36643697

36653698
#: The key inserted into the dictionary during each iteration of the comprehension.
3666-
key: BaseAssignTargetExpression
3699+
key: BaseExpression
36673700
#: The value associated with the ``key`` inserted into the dictionary during each
36683701
#: iteration of the comprehension.
3669-
value: BaseAssignTargetExpression
3702+
value: BaseExpression
36703703

36713704
#: The ``for ... in ... if ...`` clause that lexically comes after ``key`` and
36723705
#: ``value``. This may be a nested structure for nested comprehensions. See
@@ -3770,6 +3803,15 @@ def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "NamedExpr":
37703803
rpar=visit_sequence(self, "rpar", self.rpar, visitor),
37713804
)
37723805

3806+
def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
3807+
if position == ExpressionPosition.LEFT:
3808+
return len(self.rpar) > 0 or self.value._safe_to_use_with_word_operator(
3809+
position
3810+
)
3811+
return len(self.lpar) > 0 or self.target._safe_to_use_with_word_operator(
3812+
position
3813+
)
3814+
37733815
def _codegen_impl(self, state: CodegenState) -> None:
37743816
with self._parenthesize(state):
37753817
self.target._codegen(state)

‎libcst/_nodes/statement.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -745,7 +745,10 @@ class AsName(CSTNode):
745745
whitespace_after_as: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
746746

747747
def _validate(self) -> None:
748-
if self.whitespace_after_as.empty:
748+
if (
749+
self.whitespace_after_as.empty
750+
and not self.name._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
751+
):
749752
raise CSTValidationError(
750753
"There must be at least one space between 'as' and name."
751754
)

‎libcst/_nodes/tests/test_dict_comp.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,17 @@ class DictCompTest(CSTNodeTest):
2626
"parser": parse_expression,
2727
"expected_position": CodeRange((1, 0), (1, 17)),
2828
},
29+
# non-trivial keys & values in DictComp
30+
{
31+
"node": cst.DictComp(
32+
cst.BinaryOperation(cst.Name("k1"), cst.Add(), cst.Name("k2")),
33+
cst.BinaryOperation(cst.Name("v1"), cst.Add(), cst.Name("v2")),
34+
cst.CompFor(target=cst.Name("a"), iter=cst.Name("b")),
35+
),
36+
"code": "{k1 + k2: v1 + v2 for a in b}",
37+
"parser": parse_expression,
38+
"expected_position": CodeRange((1, 0), (1, 29)),
39+
},
2940
# custom whitespace around colon
3041
{
3142
"node": cst.DictComp(

‎libcst/_nodes/tests/test_ifexp.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,41 @@ class IfExpTest(CSTNodeTest):
5252
"(foo)if(bar)else(baz)",
5353
CodeRange((1, 0), (1, 21)),
5454
),
55+
(
56+
cst.IfExp(
57+
body=cst.Name("foo"),
58+
whitespace_before_if=cst.SimpleWhitespace(" "),
59+
whitespace_after_if=cst.SimpleWhitespace(" "),
60+
test=cst.Name("bar"),
61+
whitespace_before_else=cst.SimpleWhitespace(" "),
62+
whitespace_after_else=cst.SimpleWhitespace(""),
63+
orelse=cst.IfExp(
64+
body=cst.SimpleString("''"),
65+
whitespace_before_if=cst.SimpleWhitespace(""),
66+
test=cst.Name("bar"),
67+
orelse=cst.Name("baz"),
68+
),
69+
),
70+
"foo if bar else''if bar else baz",
71+
CodeRange((1, 0), (1, 32)),
72+
),
73+
(
74+
cst.GeneratorExp(
75+
elt=cst.IfExp(
76+
body=cst.Name("foo"),
77+
test=cst.Name("bar"),
78+
orelse=cst.SimpleString("''"),
79+
whitespace_after_else=cst.SimpleWhitespace(""),
80+
),
81+
for_in=cst.CompFor(
82+
target=cst.Name("_"),
83+
iter=cst.Name("_"),
84+
whitespace_before=cst.SimpleWhitespace(""),
85+
),
86+
),
87+
"(foo if bar else''for _ in _)",
88+
CodeRange((1, 1), (1, 28)),
89+
),
5590
# Make sure that spacing works
5691
(
5792
cst.IfExp(

‎libcst/_nodes/tests/test_lambda.py‎

Lines changed: 47 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -303,30 +303,6 @@ def test_valid(
303303
),
304304
"at least one space after lambda",
305305
),
306-
(
307-
lambda: cst.Lambda(
308-
cst.Parameters(star_arg=cst.Param(cst.Name("arg"))),
309-
cst.Integer("5"),
310-
whitespace_after_lambda=cst.SimpleWhitespace(""),
311-
),
312-
"at least one space after lambda",
313-
),
314-
(
315-
lambda: cst.Lambda(
316-
cst.Parameters(kwonly_params=(cst.Param(cst.Name("arg")),)),
317-
cst.Integer("5"),
318-
whitespace_after_lambda=cst.SimpleWhitespace(""),
319-
),
320-
"at least one space after lambda",
321-
),
322-
(
323-
lambda: cst.Lambda(
324-
cst.Parameters(star_kwarg=cst.Param(cst.Name("arg"))),
325-
cst.Integer("5"),
326-
whitespace_after_lambda=cst.SimpleWhitespace(""),
327-
),
328-
"at least one space after lambda",
329-
),
330306
(
331307
lambda: cst.Lambda(
332308
cst.Parameters(
@@ -944,6 +920,53 @@ class LambdaParserTest(CSTNodeTest):
944920
),
945921
"( lambda : 5 )",
946922
),
923+
# No space between lambda and params
924+
(
925+
cst.Lambda(
926+
cst.Parameters(star_arg=cst.Param(cst.Name("args"), star="*")),
927+
cst.Integer("5"),
928+
whitespace_after_lambda=cst.SimpleWhitespace(""),
929+
),
930+
"lambda*args: 5",
931+
),
932+
(
933+
cst.Lambda(
934+
cst.Parameters(star_kwarg=cst.Param(cst.Name("kwargs"), star="**")),
935+
cst.Integer("5"),
936+
whitespace_after_lambda=cst.SimpleWhitespace(""),
937+
),
938+
"lambda**kwargs: 5",
939+
),
940+
(
941+
cst.Lambda(
942+
cst.Parameters(
943+
star_arg=cst.ParamStar(
944+
comma=cst.Comma(
945+
cst.SimpleWhitespace(""), cst.SimpleWhitespace("")
946+
)
947+
),
948+
kwonly_params=[cst.Param(cst.Name("args"), star="")],
949+
),
950+
cst.Integer("5"),
951+
whitespace_after_lambda=cst.SimpleWhitespace(""),
952+
),
953+
"lambda*,args: 5",
954+
),
955+
(
956+
cst.ListComp(
957+
elt=cst.Lambda(
958+
params=cst.Parameters(),
959+
body=cst.Tuple(()),
960+
colon=cst.Colon(),
961+
),
962+
for_in=cst.CompFor(
963+
target=cst.Name("_"),
964+
iter=cst.Name("_"),
965+
whitespace_before=cst.SimpleWhitespace(""),
966+
),
967+
),
968+
"[lambda:()for _ in _]",
969+
),
947970
)
948971
)
949972
def test_valid(

‎libcst/_nodes/tests/test_namedexpr.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,22 @@ class NamedExprTest(CSTNodeTest):
166166
"parser": _parse_expression_force_38,
167167
"expected_position": None,
168168
},
169+
{
170+
"node": cst.ListComp(
171+
elt=cst.NamedExpr(
172+
cst.Name("_"),
173+
cst.SimpleString("''"),
174+
whitespace_after_walrus=cst.SimpleWhitespace(""),
175+
whitespace_before_walrus=cst.SimpleWhitespace(""),
176+
),
177+
for_in=cst.CompFor(
178+
target=cst.Name("_"),
179+
iter=cst.Name("_"),
180+
whitespace_before=cst.SimpleWhitespace(""),
181+
),
182+
),
183+
"code": "[_:=''for _ in _]",
184+
},
169185
)
170186
)
171187
def test_valid(self, **kwargs: Any) -> None:

‎libcst/_nodes/tests/test_simple_comp.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,33 @@ class SimpleCompTest(CSTNodeTest):
4141
"code": "{a for b in c}",
4242
"parser": parse_expression,
4343
},
44+
# non-trivial elt in GeneratorExp
45+
{
46+
"node": cst.GeneratorExp(
47+
cst.BinaryOperation(cst.Name("a1"), cst.Add(), cst.Name("a2")),
48+
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
49+
),
50+
"code": "(a1 + a2 for b in c)",
51+
"parser": parse_expression,
52+
},
53+
# non-trivial elt in ListComp
54+
{
55+
"node": cst.ListComp(
56+
cst.BinaryOperation(cst.Name("a1"), cst.Add(), cst.Name("a2")),
57+
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
58+
),
59+
"code": "[a1 + a2 for b in c]",
60+
"parser": parse_expression,
61+
},
62+
# non-trivial elt in SetComp
63+
{
64+
"node": cst.SetComp(
65+
cst.BinaryOperation(cst.Name("a1"), cst.Add(), cst.Name("a2")),
66+
cst.CompFor(target=cst.Name("b"), iter=cst.Name("c")),
67+
),
68+
"code": "{a1 + a2 for b in c}",
69+
"parser": parse_expression,
70+
},
4471
# async GeneratorExp
4572
{
4673
"node": cst.GeneratorExp(

‎libcst/_nodes/tests/test_with.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,23 @@ class WithTest(CSTNodeTest):
102102
"code": "with context_mgr() as ctx: pass\n",
103103
"parser": parse_statement,
104104
},
105+
{
106+
"node": cst.With(
107+
(
108+
cst.WithItem(
109+
cst.Call(cst.Name("context_mgr")),
110+
cst.AsName(
111+
cst.Tuple(()),
112+
whitespace_after_as=cst.SimpleWhitespace(""),
113+
whitespace_before_as=cst.SimpleWhitespace(""),
114+
),
115+
),
116+
),
117+
cst.SimpleStatementSuite((cst.Pass(),)),
118+
),
119+
"code": "with context_mgr()as(): pass\n",
120+
"parser": parse_statement,
121+
},
105122
# indentation
106123
{
107124
"node": DummyIndentedBlock(

0 commit comments

Comments
 (0)