diff --git a/Grammar/python.gram b/Grammar/python.gram
index f2c9993aa4f80a..d223d05caf83b3 100644
--- a/Grammar/python.gram
+++ b/Grammar/python.gram
@@ -97,17 +97,11 @@ func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMA
statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) }
statement[asdl_stmt_seq*]:
- | a=compound_stmt { _PyPegen_register_stmts(p ,
- (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a)
- ) }
+ | a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) }
| a[asdl_stmt_seq*]=simple_stmts { a }
-single_compound_stmt[asdl_stmt_seq*]:
- | a=compound_stmt {
- _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a)) }
-
statement_newline[asdl_stmt_seq*]:
- | a=single_compound_stmt NEWLINE { a }
+ | a=compound_stmt NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) }
| simple_stmts
| NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, CHECK(stmt_ty, _PyAST_Pass(EXTRA))) }
| ENDMARKER { _PyPegen_interactive_exit(p) }
diff --git a/Lib/codeop.py b/Lib/codeop.py
index 140e960b35f4ef..33e4b226f82f07 100644
--- a/Lib/codeop.py
+++ b/Lib/codeop.py
@@ -47,7 +47,7 @@
PyCF_ONLY_AST = 0x400
PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
-def _maybe_compile(compiler, source, filename, symbol, flags):
+def _maybe_compile(compiler, source, filename, symbol):
# Check for source consisting of only blank lines and comments.
for line in source.split("\n"):
line = line.strip()
@@ -61,7 +61,7 @@ def _maybe_compile(compiler, source, filename, symbol, flags):
with warnings.catch_warnings():
warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
try:
- compiler(source, filename, symbol, flags=flags)
+ compiler(source, filename, symbol)
except _IncompleteInputError:
return None
except SyntaxError: # Let other compile() errors propagate.
@@ -69,13 +69,14 @@ def _maybe_compile(compiler, source, filename, symbol, flags):
return compiler(source, filename, symbol, incomplete_input=False)
-def _compile(source, filename, symbol, incomplete_input=True, *, flags=0):
+def _compile(source, filename, symbol, incomplete_input=True):
+ flags = 0
if incomplete_input:
flags |= PyCF_ALLOW_INCOMPLETE_INPUT
flags |= PyCF_DONT_IMPLY_DEDENT
return compile(source, filename, symbol, flags)
-def compile_command(source, filename="", symbol="single", flags=0):
+def compile_command(source, filename="", symbol="single"):
r"""Compile a command and determine whether it is incomplete.
Arguments:
@@ -94,7 +95,7 @@ def compile_command(source, filename="", symbol="single", flags=0):
syntax error (OverflowError and ValueError can be produced by
malformed literals).
"""
- return _maybe_compile(_compile, source, filename, symbol, flags)
+ return _maybe_compile(_compile, source, filename, symbol)
class Compile:
"""Instances of this class behave much like the built-in compile
@@ -146,4 +147,4 @@ def __call__(self, source, filename="", symbol="single"):
syntax error (OverflowError and ValueError can be produced by
malformed literals).
"""
- return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags)
+ return _maybe_compile(self.compiler, source, filename, symbol)
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
index 39ab53bb1f8b5f..4a7726d59177d3 100644
--- a/Lib/test/test_syntax.py
+++ b/Lib/test/test_syntax.py
@@ -287,11 +287,11 @@
>>> [x for x if range(1)]
Traceback (most recent call last):
-SyntaxError: 'in' expected after for-loop variables
+SyntaxError: invalid syntax. Did you mean 'in'?
>>> tuple(x for x if range(1))
Traceback (most recent call last):
-SyntaxError: 'in' expected after for-loop variables
+SyntaxError: invalid syntax. Did you mean 'in'?
>>> [x for x() in a]
Traceback (most recent call last):
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index d5b00a3bb628cb..19b0ec5c24b7e1 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -1815,6 +1815,10 @@ class TestKeywordTypoSuggestions(unittest.TestCase):
("[x for x\nin range(3)\nof x]", "if"),
("[123 fur x\nin range(3)\nif x]", "for"),
("for x im n:\n pass", "in"),
+ ("for x is n:\n pass", "in"),
+ ("is x in y:\n pass", "if"),
+ ("try:\n pass\nfinaly:\n pass", "finally"),
+ ("result = (one\n adn two)", "and"),
("mach x:", "match"),
("math x:", "match"),
("match 1:\n cse 1:", "case"),
@@ -1847,6 +1851,15 @@ def test_keyword_suggestions_from_command_string(self):
stderr_text = stderr.decode('utf-8')
self.assertIn(f"Did you mean '{expected_kw}'", stderr_text)
+ @cpython_only
+ def test_keyword_suggestions_from_single_input(self):
+ for source, keyword in self.TYPO_CASES:
+ for ending in ('', '\n'):
+ with self.subTest(source=source, ending=ending):
+ with self.assertRaises(SyntaxError) as caught:
+ compile(source + ending, "", "single")
+ self.assertEqual(caught.exception._metadata[0], keyword)
+
def test_no_keyword_suggestion_for_comma_errors(self):
# When the parser identifies a missing comma, don't suggest
# bogus keyword replacements like 'print' -> 'not'
@@ -1858,11 +1871,136 @@ def test_no_keyword_suggestion_for_comma_errors(self):
file=sys.stderr
)
'''
- source = textwrap.dedent(code).strip()
- rc, stdout, stderr = assert_python_failure('-c', source)
- stderr_text = stderr.decode('utf-8')
- self.assertIn("Perhaps you forgot a comma", stderr_text)
- self.assertNotIn("Did you mean", stderr_text)
+ for source in (textwrap.dedent(code).strip(), "[foo bar]", "f(foo bar)"):
+ with self.subTest(source=source):
+ rc, stdout, stderr = assert_python_failure('-c', source)
+ stderr_text = stderr.decode('utf-8')
+ self.assertIn("Perhaps you forgot a comma", stderr_text)
+ self.assertNotIn("Did you mean", stderr_text)
+
+ @cpython_only
+ def test_parser_suggestions_need_no_formatting_work(self):
+ cases = [
+ ("def f():\n returm value", "return", 2, 5),
+ ("a=b=c=d=e=f=g=h=i=0\ndef fn():\n retrun True", "return", 3, 5),
+ ("def f(a,b):\n if b:\n yield\n retrun a+b", "return", 4, 5),
+ ("def f():\n for i in range(10):\n yield i\n\ndfe x():", "def", 5, 1),
+ ("é = 1; impor math", "import", 1, 8),
+ ("for café is items:\n pass", "in", 1, 10),
+ ("a = 1\nif True:\n pass\nelso:\n pass", "else", 4, 1),
+ ("[123 fur x\nin range(3)\nif x]", "for", 1, 6),
+ ("mach x:\n case 1: pass", "match", 1, 1),
+ ("match x:\n cse 1: pass", "case", 2, 5),
+ ("typ Alias = int", "type", 1, 1),
+ ("lazi import math", "lazy", 1, 1),
+ ("long_name = 1\n" * 85 + "whille True:\n pass", "while", 86, 1),
+ ]
+ for source, keyword, lineno, offset in cases:
+ with self.subTest(source=source):
+ with self.assertRaises(SyntaxError) as caught:
+ compile(source, "", "exec")
+ error = caught.exception
+ original = error.args
+ with (unittest.mock.patch("builtins.compile") as compile_mock,
+ unittest.mock.patch("builtins.open") as open_mock):
+ exc = traceback.TracebackException.from_exception(error)
+ first = ''.join(exc.format_exception_only())
+ self.assertEqual(first, ''.join(exc.format_exception_only()))
+ compile_mock.assert_not_called()
+ open_mock.assert_not_called()
+ self.assertIn(f"Did you mean '{keyword}'?", first)
+ self.assertEqual((exc.lineno, exc.offset), (str(lineno), offset))
+ self.assertEqual(error.args, original)
+
+ @cpython_only
+ def test_keyword_trials_do_not_repeat_warnings(self):
+ import warnings
+ for prefix in ("value = '\\q'\n", "from . lazy import x\n"):
+ with self.subTest(prefix=prefix), warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ with self.assertRaises(SyntaxError) as error:
+ compile(prefix + "impor math", "", "exec")
+ self.assertEqual(len(caught), 1)
+ self.assertEqual(error.exception._metadata[0], "import")
+
+ @cpython_only
+ def test_keyword_replay_requires_complete_valid_input(self):
+ for source in ("dell 1", "impor math\nother =", "whille True:\npass",
+ "switch +:\n case:", "function f(=):\n pass", "(mat x)",
+ "impor math\n" + "pass\n" * 20000):
+ with self.subTest(source=source):
+ with self.assertRaises(SyntaxError) as caught:
+ compile(source, "", "exec")
+ # Action errors and a cache which stopped before EOF must not
+ # count as successful repairs.
+ self.assertIsNone(caught.exception._metadata)
+
+ @cpython_only
+ def test_keyword_replay_respects_feature_version(self):
+ import ast
+ for source in ("mach x:\n case 1: pass", "mach x:", "switch x:\n case:"):
+ with self.subTest(source=source), self.assertRaises(SyntaxError) as caught:
+ ast.parse(source, feature_version=(3, 9))
+ self.assertIsNone(caught.exception._metadata)
+
+ @cpython_only
+ def test_keyword_trials_do_not_call_normalizer(self):
+ with unittest.mock.patch("unicodedata.normalize") as normalize:
+ with self.assertRaises(SyntaxError) as caught:
+ compile("whille True:\n café = 1", "", "exec")
+ normalize.assert_not_called()
+ self.assertIsNone(caught.exception._metadata)
+
+ @cpython_only
+ @requires_subprocess()
+ def test_keyword_trials_do_not_intern_names(self):
+ assert_python_ok('-c', textwrap.dedent('''
+ import sys
+
+ def check(index, suffix):
+ source = (f"impor math\\nidentifier_after_typo_{index} = 1\\n"
+ f"import module_after_typo_{index}.child" + suffix)
+ try:
+ compile(source, "", "exec")
+ except SyntaxError as error:
+ assert (error._metadata is not None) == (not suffix)
+ else:
+ raise AssertionError("expected SyntaxError")
+
+ for suffix in ("", "\\nother ="):
+ check(0, suffix)
+ before = sys.getunicodeinternedsize(_only_immortal=True)
+ for index in range(1, 20):
+ check(index, suffix)
+ assert sys.getunicodeinternedsize(_only_immortal=True) == before
+ '''))
+
+ @cpython_only
+ def test_keyword_replay_source_limit(self):
+ with tempfile.TemporaryDirectory() as script_dir:
+ for size in (256 * 1024 - 512, 256 * 1024 + 512):
+ literal = "value = " + repr("a" * size) + "\n"
+ for source in (literal + "impor math", "impor math\n" + literal):
+ with self.subTest(size=size, typo_first=source.startswith("impor")):
+ with self.assertRaises(SyntaxError) as caught:
+ compile(source, "", "exec")
+ expected = size < 256 * 1024
+ self.assertEqual(caught.exception._metadata is not None, expected)
+ script = make_script(script_dir, "large_keyword_typo", source)
+ _, _, stderr = assert_python_failure(script)
+ self.assertEqual(b"Did you mean 'import'" in stderr, expected)
+ # The size limit applies only to optional hints.
+ compile("value = " + repr("a" * (1024 * 1024)), "", "exec")
+
+ @cpython_only
+ def test_keyword_replay_work_limit(self):
+ # Several candidates must share the budget, including failed passes.
+ for count, expected in ((100, "lambda"), (2000, None)):
+ source = "x = 1\n" * count + "f(" + "fron, " * 8 + "lamda x: x)"
+ with self.subTest(count=count), self.assertRaises(SyntaxError) as caught:
+ compile(source, "", "exec")
+ metadata = caught.exception._metadata
+ self.assertEqual(metadata[0] if metadata else None, expected)
@requires_debug_ranges()
@force_not_colorized_test_class
diff --git a/Lib/traceback.py b/Lib/traceback.py
index dcdab1f12e9a16..a93bcbe779274d 100644
--- a/Lib/traceback.py
+++ b/Lib/traceback.py
@@ -10,10 +10,6 @@
import textwrap
import types
import warnings
-import codeop
-import keyword
-import tokenize
-import io
import importlib.util
import pathlib
@@ -1182,7 +1178,13 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None,
self.end_offset = exc_value.end_offset
self.msg = exc_value.msg
self._is_syntax_error = True
- self._exc_metadata = getattr(exc_value, "_metadata", None)
+ metadata = getattr(exc_value, "_metadata", None)
+ if isinstance(metadata, tuple) and len(metadata) == 6:
+ (suggestion, self.text, lineno, self.offset,
+ end_lineno, self.end_offset) = metadata
+ self.lineno = str(lineno)
+ self.end_lineno = str(end_lineno)
+ self.msg = f"invalid syntax. Did you mean '{suggestion}'?"
elif exc_type and issubclass(exc_type, ImportError) and \
getattr(exc_value, "name_from", None) is not None:
wrong_name = getattr(exc_value, "name_from", None)
@@ -1413,122 +1415,6 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs):
for ex in self.exceptions:
yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize)
- def _find_keyword_typos(self):
- assert self._is_syntax_error
- try:
- import _suggestions
- except ImportError:
- _suggestions = None
-
- # Only try to find keyword typos if there is no custom message
- if self.msg != "invalid syntax" and "Perhaps you forgot a comma" not in self.msg:
- return
-
- if not self._exc_metadata:
- return
-
- line, offset, source = self._exc_metadata
- end_line = int(self.lineno) if self.lineno is not None else 0
- lines = None
- from_filename = False
-
- if source is None:
- if self.filename:
- try:
- with open(self.filename) as f:
- lines = f.read().splitlines()
- except Exception:
- line, end_line, offset = 0,1,0
- else:
- from_filename = True
- lines = lines if lines is not None else self.text.splitlines()
- else:
- lines = source.splitlines()
-
- error_code = lines[line -1 if line > 0 else 0:end_line]
- error_code = textwrap.dedent('\n'.join(error_code))
-
- # Do not continue if the source is too large
- if len(error_code) > 1024:
- return
-
- # If the original code doesn't raise SyntaxError, we can't validate
- # that a keyword replacement actually fixes anything
- try:
- codeop.compile_command(error_code, symbol="exec", flags=codeop.PyCF_ONLY_AST)
- except SyntaxError:
- pass # Good - the original code has a syntax error we might fix
- else:
- return # Original code compiles or is incomplete - can't validate fixes
-
- error_lines = error_code.splitlines()
- tokens = tokenize.generate_tokens(io.StringIO(error_code).readline)
- tokens_left_to_process = 10
- import difflib
- for token in tokens:
- start, end = token.start, token.end
- if token.type != tokenize.NAME:
- continue
- # Only consider NAME tokens on the same line as the error
- the_end = end_line if line == 0 else end_line + 1
- if from_filename and token.start[0]+line != the_end:
- continue
- wrong_name = token.string
- if wrong_name in keyword.kwlist:
- continue
-
- # Limit the number of valid tokens to consider to not spend
- # to much time in this function
- tokens_left_to_process -= 1
- if tokens_left_to_process < 0:
- break
- # Limit the number of possible matches to try
- max_matches = 3
- matches = []
-
- hint = _get_cross_language_keyword_hint(wrong_name)
- if hint:
- matches.append(hint)
- if _suggestions is not None:
- suggestion = _suggestions._generate_suggestions(keyword.kwlist + keyword.softkwlist, wrong_name)
- if suggestion:
- matches.append(suggestion)
- matches.extend(
- difflib.get_close_matches(
- wrong_name,
- keyword.kwlist + keyword.softkwlist,
- n=max_matches,
- cutoff=0.5
- )
- )
- matches = matches[:max_matches]
- for suggestion in matches:
- if not suggestion or suggestion == wrong_name:
- continue
- # Try to replace the token with the keyword
- the_lines = error_lines.copy()
- the_line = the_lines[start[0] - 1][:]
- chars = list(the_line)
- chars[token.start[1]:token.end[1]] = suggestion
- the_lines[start[0] - 1] = ''.join(chars)
- code = '\n'.join(the_lines)
-
- # Check if it works
- try:
- codeop.compile_command(code, symbol="exec", flags=codeop.PyCF_ONLY_AST)
- except SyntaxError:
- continue
-
- # Keep token.line but handle offsets correctly
- self.text = token.line
- self.offset = token.start[1] + 1
- self.end_offset = token.end[1] + 1
- self.lineno = start[0]
- self.end_lineno = end[0]
- self.msg = f"invalid syntax. Did you mean '{suggestion}'?"
- return
-
-
def _format_syntax_error(self, stype, **kwargs):
"""Format SyntaxError exceptions (internal helper)."""
# Show exactly where the problem was found.
@@ -1555,9 +1441,6 @@ def _format_syntax_error(self, stype, **kwargs):
# text = " foo\n"
# rtext = " foo"
# ltext = "foo"
- with suppress(Exception):
- self._find_keyword_typos()
- text = self.text
rtext = text.rstrip('\n')
ltext = rtext.lstrip(' \n\f')
spaces = len(rtext) - len(ltext)
@@ -1798,17 +1681,6 @@ def print(self, *, file=None, chain=True, **kwargs):
})
-# Cross-language keyword suggestions.
-_CROSS_LANGUAGE_KEYWORD_HINTS = frozendict({
- # C/C++ equivalents
- 'switch': 'match',
- 'delete': 'del',
- # function define equivalents
- 'function': 'def',
- 'func': 'def',
- 'void': 'def',
-})
-
def _substitution_cost(ch_a, ch_b):
if ch_a == ch_b:
return 0
@@ -1888,12 +1760,6 @@ def _get_cross_language_hint(obj, wrong_name):
return None
-def _get_cross_language_keyword_hint(wrong_name):
- """Check if wrong_name is a common keyword from another language
- """
- return _CROSS_LANGUAGE_KEYWORD_HINTS.get(wrong_name)
-
-
def _get_safe___dir__(obj):
# Use obj.__dir__() to avoid a TypeError when calling dir(obj).
# See gh-131001 and gh-139933.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2026-10-18-13-00-00.gh-issue-159672.keyword-typos.rst b/Misc/NEWS.d/next/Core and Builtins/2026-10-18-13-00-00.gh-issue-159672.keyword-typos.rst
new file mode 100644
index 00000000000000..44ff0e377e7a84
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2026-10-18-13-00-00.gh-issue-159672.keyword-typos.rst
@@ -0,0 +1,3 @@
+The parser now generates keyword typo suggestions from cached tokens. The
+:mod:`traceback` formatter displays these suggestions without source
+recompilation.
diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
index 2f8a5d84dd9755..4a35900ecdb6dc 100644
--- a/Parser/action_helpers.c
+++ b/Parser/action_helpers.c
@@ -128,8 +128,10 @@ _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
if (!uni) {
return NULL;
}
- PyInterpreterState *interp = _PyInterpreterState_GET();
- _PyUnicode_InternImmortal(interp, &uni);
+ if (p->keyword_trial == NULL) {
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ _PyUnicode_InternImmortal(interp, &uni);
+ }
if (_PyArena_AddPyObject(p->arena, uni) < 0) {
Py_DECREF(uni);
return NULL;
@@ -986,6 +988,7 @@ _PyPegen_raise_error_for_missing_comma(Parser *p, expr_ty a, expr_ty b)
if (p->tokens[p->mark - 1]->level == 0) {
return NULL;
}
+ p->missing_comma = 1;
// For multi-line expressions (like string concatenations), point to the
// last line instead of the first for a more helpful error message.
// Use a->col_offset as the starting column since all strings in the
@@ -2072,7 +2075,7 @@ _warn_relative_import_of_lazy(Parser *p, asdl_seq *dots, expr_ty module)
// the module name is insignificant, so this is parsed exactly like
// `from .lazy import x` (an import of the relative module "lazy"), but it
// is most likely a transposition of `lazy from . import x` (PEP 810).
- if (p->call_invalid_rules) {
+ if (p->call_invalid_rules || p->keyword_trial != NULL) {
return 0;
}
@@ -2134,7 +2137,9 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name,
alias_ty alias = asdl_seq_GET(names, i);
if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) {
p->flags |= PyPARSE_BARRY_AS_BDFL;
- p->tok->barry_as_bdfl = 1;
+ if (p->keyword_trial == NULL) {
+ p->tok->barry_as_bdfl = 1;
+ }
}
}
}
@@ -2150,23 +2155,3 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name,
return _PyAST_ImportFrom(module, names, level, lazy_token ? 1 : 0, lineno,
col_offset, end_lineno, end_col_offset, arena);
}
-
-asdl_stmt_seq*
-_PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts) {
- if (!p->call_invalid_rules) {
- return stmts;
- }
- Py_ssize_t len = asdl_seq_LEN(stmts);
- if (len == 0) {
- return stmts;
- }
- stmt_ty last_stmt = asdl_seq_GET(stmts, len - 1);
- if (p->last_stmt_location.lineno > last_stmt->lineno) {
- return stmts;
- }
- p->last_stmt_location.lineno = last_stmt->lineno;
- p->last_stmt_location.col_offset = last_stmt->col_offset;
- p->last_stmt_location.end_lineno = last_stmt->end_lineno;
- p->last_stmt_location.end_col_offset = last_stmt->end_col_offset;
- return stmts;
-}
diff --git a/Parser/parser.c b/Parser/parser.c
index 979907f4873b9d..cd000fea2d6f5c 100644
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -92,443 +92,442 @@ static char *soft_keywords[] = {
#define func_type_type 1003
#define statements_type 1004
#define statement_type 1005
-#define single_compound_stmt_type 1006
-#define statement_newline_type 1007
-#define simple_stmts_type 1008
-#define simple_stmt_type 1009
-#define compound_stmt_type 1010
-#define assignment_type 1011
-#define annotated_rhs_type 1012
-#define augassign_type 1013
-#define return_stmt_type 1014
-#define raise_stmt_type 1015
-#define pass_stmt_type 1016
-#define break_stmt_type 1017
-#define continue_stmt_type 1018
-#define global_stmt_type 1019
-#define nonlocal_stmt_type 1020
-#define del_stmt_type 1021
-#define yield_stmt_type 1022
-#define assert_stmt_type 1023
-#define import_stmt_type 1024
-#define import_name_type 1025
-#define import_from_type 1026
-#define import_from_targets_type 1027
-#define import_from_as_names_type 1028
-#define import_from_as_name_type 1029
-#define dotted_as_names_type 1030
-#define dotted_as_name_type 1031
-#define dotted_name_type 1032 // Left-recursive
-#define block_type 1033
-#define decorators_type 1034
-#define class_def_type 1035
-#define class_def_raw_type 1036
-#define function_def_type 1037
-#define function_def_raw_type 1038
-#define params_type 1039
-#define parameters_type 1040
-#define slash_no_default_type 1041
-#define slash_with_default_type 1042
-#define star_etc_type 1043
-#define kwds_type 1044
-#define param_no_default_type 1045
-#define param_no_default_star_annotation_type 1046
-#define param_with_default_type 1047
-#define param_maybe_default_type 1048
-#define param_type 1049
-#define param_star_annotation_type 1050
-#define annotation_type 1051
-#define star_annotation_type 1052
-#define default_type 1053
-#define if_stmt_type 1054
-#define elif_stmt_type 1055
-#define else_block_type 1056
-#define while_stmt_type 1057
-#define for_stmt_type 1058
-#define with_stmt_type 1059
-#define with_item_type 1060
-#define try_stmt_type 1061
-#define except_block_type 1062
-#define except_star_block_type 1063
-#define finally_block_type 1064
-#define match_stmt_type 1065
-#define subject_expr_type 1066
-#define case_block_type 1067
-#define guard_type 1068
-#define patterns_type 1069
-#define pattern_type 1070
-#define as_pattern_type 1071
-#define or_pattern_type 1072
-#define closed_pattern_type 1073
-#define literal_pattern_type 1074
-#define literal_expr_type 1075
-#define complex_number_type 1076
-#define signed_number_type 1077
-#define signed_real_number_type 1078
-#define real_number_type 1079
-#define imaginary_number_type 1080
-#define capture_pattern_type 1081
-#define pattern_capture_target_type 1082
-#define wildcard_pattern_type 1083
-#define value_pattern_type 1084
-#define attr_type 1085 // Left-recursive
-#define name_or_attr_type 1086 // Left-recursive
-#define group_pattern_type 1087
-#define sequence_pattern_type 1088
-#define open_sequence_pattern_type 1089
-#define maybe_sequence_pattern_type 1090
-#define maybe_star_pattern_type 1091
-#define star_pattern_type 1092
-#define mapping_pattern_type 1093
-#define items_pattern_type 1094
-#define key_value_pattern_type 1095
-#define double_star_pattern_type 1096
-#define class_pattern_type 1097
-#define positional_patterns_type 1098
-#define keyword_patterns_type 1099
-#define keyword_pattern_type 1100
-#define type_alias_type 1101
-#define type_params_type 1102
-#define type_param_seq_type 1103
-#define type_param_type 1104
-#define type_param_bound_type 1105
-#define type_param_default_type 1106
-#define type_param_starred_default_type 1107
-#define expressions_type 1108
-#define expression_type 1109
-#define if_expression_type 1110
-#define yield_expr_type 1111
-#define star_expressions_type 1112
-#define star_expression_type 1113
-#define star_named_expressions_type 1114
-#define star_named_expression_type 1115
-#define assignment_expression_type 1116
-#define named_expression_type 1117
-#define disjunction_type 1118
-#define conjunction_type 1119
-#define inversion_type 1120
-#define comparison_type 1121
-#define compare_op_bitwise_or_pair_type 1122
-#define bitwise_or_type 1123 // Left-recursive
-#define bitwise_xor_type 1124 // Left-recursive
-#define bitwise_and_type 1125 // Left-recursive
-#define shift_expr_type 1126 // Left-recursive
-#define sum_type 1127 // Left-recursive
-#define term_type 1128 // Left-recursive
-#define factor_type 1129
-#define power_type 1130
-#define await_primary_type 1131
-#define primary_type 1132 // Left-recursive
-#define slices_type 1133
-#define slice_type 1134
-#define atom_type 1135
-#define group_type 1136
-#define lambdef_type 1137
-#define lambda_params_type 1138
-#define lambda_parameters_type 1139
-#define lambda_slash_no_default_type 1140
-#define lambda_slash_with_default_type 1141
-#define lambda_star_etc_type 1142
-#define lambda_kwds_type 1143
-#define lambda_param_no_default_type 1144
-#define lambda_param_with_default_type 1145
-#define lambda_param_maybe_default_type 1146
-#define lambda_param_type 1147
-#define fstring_middle_type 1148
-#define fstring_replacement_field_type 1149
-#define fstring_conversion_type 1150
-#define fstring_full_format_spec_type 1151
-#define fstring_format_spec_type 1152
-#define fstring_type 1153
-#define tstring_format_spec_replacement_field_type 1154
-#define tstring_format_spec_type 1155
-#define tstring_full_format_spec_type 1156
-#define tstring_replacement_field_type 1157
-#define tstring_middle_type 1158
-#define tstring_type 1159
-#define string_type 1160
-#define strings_type 1161
-#define display_items_type 1162
-#define display_item_type 1163
-#define list_type 1164
-#define tuple_type 1165
-#define set_type 1166
-#define dict_type 1167
-#define double_starred_kvpairs_type 1168
-#define double_starred_kvpair_type 1169
-#define kvpair_type 1170
-#define for_if_clauses_type 1171
-#define for_if_clause_type 1172
-#define listcomp_type 1173
-#define setcomp_type 1174
-#define genexp_type 1175
-#define dictcomp_type 1176
-#define arguments_type 1177
-#define args_type 1178
-#define kwargs_type 1179
-#define starred_expression_type 1180
-#define kwarg_or_starred_type 1181
-#define kwarg_or_double_starred_type 1182
-#define star_targets_type 1183
-#define star_target_type 1184
-#define target_type 1185
-#define single_target_type 1186
-#define attribute_or_subscript_target_type 1187
-#define target_value_type 1188
-#define del_targets_type 1189
-#define del_target_type 1190
-#define type_expressions_type 1191
-#define func_type_comment_type 1192
-#define invalid_arguments_type 1193
-#define invalid_kwarg_type 1194
-#define expression_without_invalid_type 1195
-#define invalid_legacy_expression_type 1196
-#define invalid_type_param_type 1197
-#define invalid_expression_type 1198
-#define invalid_if_expression_type 1199
-#define invalid_named_expression_type 1200
-#define invalid_assignment_type 1201
-#define invalid_ann_assign_target_type 1202
-#define invalid_raise_stmt_type 1203
-#define invalid_del_stmt_type 1204
-#define invalid_assert_stmt_type 1205
-#define invalid_block_type 1206
-#define invalid_comprehension_type 1207
-#define invalid_parameters_type 1208
-#define invalid_default_type 1209
-#define invalid_star_etc_type 1210
-#define invalid_kwds_type 1211
-#define invalid_parameters_helper_type 1212
-#define invalid_lambda_parameters_type 1213
-#define invalid_lambda_parameters_helper_type 1214
-#define invalid_lambda_star_etc_type 1215
-#define invalid_lambda_kwds_type 1216
-#define invalid_double_type_comments_type 1217
-#define invalid_with_item_type 1218
-#define invalid_for_if_clause_type 1219
-#define invalid_for_target_type 1220
-#define invalid_group_type 1221
-#define invalid_import_type 1222
-#define invalid_dotted_as_name_type 1223
-#define invalid_import_from_as_name_type 1224
-#define invalid_import_from_type 1225
-#define invalid_import_from_targets_type 1226
-#define invalid_with_stmt_type 1227
-#define invalid_with_stmt_indent_type 1228
-#define invalid_try_stmt_type 1229
-#define invalid_except_stmt_type 1230
-#define invalid_except_star_stmt_type 1231
-#define invalid_finally_stmt_type 1232
-#define invalid_except_stmt_indent_type 1233
-#define invalid_except_star_stmt_indent_type 1234
-#define invalid_match_stmt_type 1235
-#define invalid_case_block_type 1236
-#define invalid_as_pattern_type 1237
-#define invalid_class_pattern_type 1238
-#define invalid_mapping_pattern_type 1239
-#define invalid_class_argument_pattern_type 1240
-#define invalid_if_stmt_type 1241
-#define invalid_elif_stmt_type 1242
-#define invalid_else_stmt_type 1243
-#define invalid_while_stmt_type 1244
-#define invalid_for_stmt_type 1245
-#define invalid_def_raw_type 1246
-#define invalid_class_def_raw_type 1247
-#define invalid_double_starred_kvpairs_type 1248
-#define invalid_kvpair_unpacking_type 1249
-#define invalid_kvpair_type 1250
-#define invalid_starred_expression_unpacking_type 1251
-#define invalid_starred_expression_unpacking_sequence_type 1252
-#define invalid_starred_expression_type 1253
-#define invalid_fstring_replacement_field_type 1254
-#define invalid_fstring_conversion_character_type 1255
-#define invalid_tstring_replacement_field_type 1256
-#define invalid_tstring_conversion_character_type 1257
-#define invalid_string_tstring_concat_type 1258
-#define invalid_arithmetic_type 1259 // Left-recursive
-#define invalid_factor_type 1260
-#define invalid_type_params_type 1261
-#define invalid_bitwise_and_type 1262 // Left-recursive
-#define invalid_bitwise_or_type 1263 // Left-recursive
-#define invalid_noteq_type 1264
-#define _loop0_1_type 1265
-#define _loop1_2_type 1266
-#define _loop0_3_type 1267
-#define _gather_4_type 1268
-#define _tmp_5_type 1269
-#define _tmp_6_type 1270
-#define _tmp_7_type 1271
-#define _tmp_8_type 1272
-#define _tmp_9_type 1273
-#define _tmp_10_type 1274
-#define _tmp_11_type 1275
-#define _loop1_12_type 1276
-#define _loop0_13_type 1277
-#define _gather_14_type 1278
-#define _tmp_15_type 1279
-#define _tmp_16_type 1280
-#define _loop0_17_type 1281
-#define _loop1_18_type 1282
-#define _loop0_19_type 1283
-#define _gather_20_type 1284
-#define _tmp_21_type 1285
-#define _loop0_22_type 1286
-#define _gather_23_type 1287
-#define _loop1_24_type 1288
-#define _tmp_25_type 1289
-#define _tmp_26_type 1290
-#define _loop0_27_type 1291
-#define _loop0_28_type 1292
-#define _loop1_29_type 1293
-#define _loop1_30_type 1294
-#define _loop0_31_type 1295
-#define _loop1_32_type 1296
-#define _loop0_33_type 1297
-#define _gather_34_type 1298
-#define _tmp_35_type 1299
-#define _loop1_36_type 1300
-#define _loop1_37_type 1301
-#define _loop1_38_type 1302
-#define _loop0_39_type 1303
-#define _gather_40_type 1304
-#define _tmp_41_type 1305
-#define _tmp_42_type 1306
-#define _tmp_43_type 1307
-#define _loop0_44_type 1308
-#define _gather_45_type 1309
-#define _loop0_46_type 1310
-#define _gather_47_type 1311
-#define _tmp_48_type 1312
-#define _loop0_49_type 1313
-#define _gather_50_type 1314
-#define _loop0_51_type 1315
-#define _gather_52_type 1316
-#define _loop0_53_type 1317
-#define _gather_54_type 1318
-#define _loop1_55_type 1319
-#define _loop1_56_type 1320
-#define _loop0_57_type 1321
-#define _gather_58_type 1322
-#define _loop1_59_type 1323
-#define _loop1_60_type 1324
-#define _loop1_61_type 1325
-#define _tmp_62_type 1326
-#define _loop0_63_type 1327
-#define _gather_64_type 1328
-#define _tmp_65_type 1329
-#define _tmp_66_type 1330
-#define _tmp_67_type 1331
-#define _tmp_68_type 1332
-#define _tmp_69_type 1333
-#define _loop0_70_type 1334
-#define _loop0_71_type 1335
-#define _loop1_72_type 1336
-#define _loop1_73_type 1337
-#define _loop0_74_type 1338
-#define _loop1_75_type 1339
-#define _loop0_76_type 1340
-#define _loop0_77_type 1341
-#define _loop0_78_type 1342
-#define _loop0_79_type 1343
-#define _loop1_80_type 1344
-#define _loop1_81_type 1345
-#define _loop0_82_type 1346
-#define _gather_83_type 1347
-#define _tmp_84_type 1348
-#define _loop0_85_type 1349
-#define _gather_86_type 1350
-#define _loop1_87_type 1351
-#define _loop0_88_type 1352
-#define _tmp_89_type 1353
-#define _loop0_90_type 1354
-#define _gather_91_type 1355
-#define _tmp_92_type 1356
-#define _loop0_93_type 1357
-#define _gather_94_type 1358
-#define _tmp_95_type 1359
-#define _loop0_96_type 1360
-#define _gather_97_type 1361
-#define _loop0_98_type 1362
-#define _tmp_99_type 1363
-#define _tmp_100_type 1364
-#define _loop0_101_type 1365
-#define _gather_102_type 1366
-#define _loop0_103_type 1367
-#define _gather_104_type 1368
-#define _tmp_105_type 1369
-#define _tmp_106_type 1370
-#define _loop0_107_type 1371
-#define _gather_108_type 1372
-#define _tmp_109_type 1373
-#define _tmp_110_type 1374
-#define _tmp_111_type 1375
-#define _tmp_112_type 1376
-#define _loop1_113_type 1377
-#define _tmp_114_type 1378
-#define _tmp_115_type 1379
-#define _tmp_116_type 1380
-#define _tmp_117_type 1381
-#define _tmp_118_type 1382
-#define _loop0_119_type 1383
-#define _loop0_120_type 1384
-#define _tmp_121_type 1385
-#define _tmp_122_type 1386
-#define _tmp_123_type 1387
-#define _tmp_124_type 1388
-#define _tmp_125_type 1389
-#define _tmp_126_type 1390
-#define _tmp_127_type 1391
-#define _tmp_128_type 1392
-#define _loop0_129_type 1393
-#define _gather_130_type 1394
-#define _tmp_131_type 1395
-#define _tmp_132_type 1396
-#define _tmp_133_type 1397
-#define _tmp_134_type 1398
-#define _loop0_135_type 1399
-#define _gather_136_type 1400
-#define _tmp_137_type 1401
-#define _loop0_138_type 1402
-#define _gather_139_type 1403
-#define _loop0_140_type 1404
-#define _gather_141_type 1405
-#define _tmp_142_type 1406
-#define _loop0_143_type 1407
-#define _tmp_144_type 1408
-#define _tmp_145_type 1409
-#define _tmp_146_type 1410
-#define _tmp_147_type 1411
-#define _tmp_148_type 1412
-#define _tmp_149_type 1413
-#define _tmp_150_type 1414
-#define _tmp_151_type 1415
-#define _tmp_152_type 1416
-#define _tmp_153_type 1417
-#define _tmp_154_type 1418
-#define _tmp_155_type 1419
-#define _tmp_156_type 1420
-#define _tmp_157_type 1421
-#define _tmp_158_type 1422
-#define _tmp_159_type 1423
-#define _tmp_160_type 1424
-#define _tmp_161_type 1425
-#define _tmp_162_type 1426
-#define _tmp_163_type 1427
-#define _tmp_164_type 1428
-#define _tmp_165_type 1429
-#define _tmp_166_type 1430
-#define _tmp_167_type 1431
-#define _tmp_168_type 1432
-#define _tmp_169_type 1433
-#define _tmp_170_type 1434
-#define _tmp_171_type 1435
-#define _loop0_172_type 1436
-#define _tmp_173_type 1437
-#define _tmp_174_type 1438
-#define _tmp_175_type 1439
-#define _tmp_176_type 1440
-#define _tmp_177_type 1441
-#define _tmp_178_type 1442
+#define statement_newline_type 1006
+#define simple_stmts_type 1007
+#define simple_stmt_type 1008
+#define compound_stmt_type 1009
+#define assignment_type 1010
+#define annotated_rhs_type 1011
+#define augassign_type 1012
+#define return_stmt_type 1013
+#define raise_stmt_type 1014
+#define pass_stmt_type 1015
+#define break_stmt_type 1016
+#define continue_stmt_type 1017
+#define global_stmt_type 1018
+#define nonlocal_stmt_type 1019
+#define del_stmt_type 1020
+#define yield_stmt_type 1021
+#define assert_stmt_type 1022
+#define import_stmt_type 1023
+#define import_name_type 1024
+#define import_from_type 1025
+#define import_from_targets_type 1026
+#define import_from_as_names_type 1027
+#define import_from_as_name_type 1028
+#define dotted_as_names_type 1029
+#define dotted_as_name_type 1030
+#define dotted_name_type 1031 // Left-recursive
+#define block_type 1032
+#define decorators_type 1033
+#define class_def_type 1034
+#define class_def_raw_type 1035
+#define function_def_type 1036
+#define function_def_raw_type 1037
+#define params_type 1038
+#define parameters_type 1039
+#define slash_no_default_type 1040
+#define slash_with_default_type 1041
+#define star_etc_type 1042
+#define kwds_type 1043
+#define param_no_default_type 1044
+#define param_no_default_star_annotation_type 1045
+#define param_with_default_type 1046
+#define param_maybe_default_type 1047
+#define param_type 1048
+#define param_star_annotation_type 1049
+#define annotation_type 1050
+#define star_annotation_type 1051
+#define default_type 1052
+#define if_stmt_type 1053
+#define elif_stmt_type 1054
+#define else_block_type 1055
+#define while_stmt_type 1056
+#define for_stmt_type 1057
+#define with_stmt_type 1058
+#define with_item_type 1059
+#define try_stmt_type 1060
+#define except_block_type 1061
+#define except_star_block_type 1062
+#define finally_block_type 1063
+#define match_stmt_type 1064
+#define subject_expr_type 1065
+#define case_block_type 1066
+#define guard_type 1067
+#define patterns_type 1068
+#define pattern_type 1069
+#define as_pattern_type 1070
+#define or_pattern_type 1071
+#define closed_pattern_type 1072
+#define literal_pattern_type 1073
+#define literal_expr_type 1074
+#define complex_number_type 1075
+#define signed_number_type 1076
+#define signed_real_number_type 1077
+#define real_number_type 1078
+#define imaginary_number_type 1079
+#define capture_pattern_type 1080
+#define pattern_capture_target_type 1081
+#define wildcard_pattern_type 1082
+#define value_pattern_type 1083
+#define attr_type 1084 // Left-recursive
+#define name_or_attr_type 1085 // Left-recursive
+#define group_pattern_type 1086
+#define sequence_pattern_type 1087
+#define open_sequence_pattern_type 1088
+#define maybe_sequence_pattern_type 1089
+#define maybe_star_pattern_type 1090
+#define star_pattern_type 1091
+#define mapping_pattern_type 1092
+#define items_pattern_type 1093
+#define key_value_pattern_type 1094
+#define double_star_pattern_type 1095
+#define class_pattern_type 1096
+#define positional_patterns_type 1097
+#define keyword_patterns_type 1098
+#define keyword_pattern_type 1099
+#define type_alias_type 1100
+#define type_params_type 1101
+#define type_param_seq_type 1102
+#define type_param_type 1103
+#define type_param_bound_type 1104
+#define type_param_default_type 1105
+#define type_param_starred_default_type 1106
+#define expressions_type 1107
+#define expression_type 1108
+#define if_expression_type 1109
+#define yield_expr_type 1110
+#define star_expressions_type 1111
+#define star_expression_type 1112
+#define star_named_expressions_type 1113
+#define star_named_expression_type 1114
+#define assignment_expression_type 1115
+#define named_expression_type 1116
+#define disjunction_type 1117
+#define conjunction_type 1118
+#define inversion_type 1119
+#define comparison_type 1120
+#define compare_op_bitwise_or_pair_type 1121
+#define bitwise_or_type 1122 // Left-recursive
+#define bitwise_xor_type 1123 // Left-recursive
+#define bitwise_and_type 1124 // Left-recursive
+#define shift_expr_type 1125 // Left-recursive
+#define sum_type 1126 // Left-recursive
+#define term_type 1127 // Left-recursive
+#define factor_type 1128
+#define power_type 1129
+#define await_primary_type 1130
+#define primary_type 1131 // Left-recursive
+#define slices_type 1132
+#define slice_type 1133
+#define atom_type 1134
+#define group_type 1135
+#define lambdef_type 1136
+#define lambda_params_type 1137
+#define lambda_parameters_type 1138
+#define lambda_slash_no_default_type 1139
+#define lambda_slash_with_default_type 1140
+#define lambda_star_etc_type 1141
+#define lambda_kwds_type 1142
+#define lambda_param_no_default_type 1143
+#define lambda_param_with_default_type 1144
+#define lambda_param_maybe_default_type 1145
+#define lambda_param_type 1146
+#define fstring_middle_type 1147
+#define fstring_replacement_field_type 1148
+#define fstring_conversion_type 1149
+#define fstring_full_format_spec_type 1150
+#define fstring_format_spec_type 1151
+#define fstring_type 1152
+#define tstring_format_spec_replacement_field_type 1153
+#define tstring_format_spec_type 1154
+#define tstring_full_format_spec_type 1155
+#define tstring_replacement_field_type 1156
+#define tstring_middle_type 1157
+#define tstring_type 1158
+#define string_type 1159
+#define strings_type 1160
+#define display_items_type 1161
+#define display_item_type 1162
+#define list_type 1163
+#define tuple_type 1164
+#define set_type 1165
+#define dict_type 1166
+#define double_starred_kvpairs_type 1167
+#define double_starred_kvpair_type 1168
+#define kvpair_type 1169
+#define for_if_clauses_type 1170
+#define for_if_clause_type 1171
+#define listcomp_type 1172
+#define setcomp_type 1173
+#define genexp_type 1174
+#define dictcomp_type 1175
+#define arguments_type 1176
+#define args_type 1177
+#define kwargs_type 1178
+#define starred_expression_type 1179
+#define kwarg_or_starred_type 1180
+#define kwarg_or_double_starred_type 1181
+#define star_targets_type 1182
+#define star_target_type 1183
+#define target_type 1184
+#define single_target_type 1185
+#define attribute_or_subscript_target_type 1186
+#define target_value_type 1187
+#define del_targets_type 1188
+#define del_target_type 1189
+#define type_expressions_type 1190
+#define func_type_comment_type 1191
+#define invalid_arguments_type 1192
+#define invalid_kwarg_type 1193
+#define expression_without_invalid_type 1194
+#define invalid_legacy_expression_type 1195
+#define invalid_type_param_type 1196
+#define invalid_expression_type 1197
+#define invalid_if_expression_type 1198
+#define invalid_named_expression_type 1199
+#define invalid_assignment_type 1200
+#define invalid_ann_assign_target_type 1201
+#define invalid_raise_stmt_type 1202
+#define invalid_del_stmt_type 1203
+#define invalid_assert_stmt_type 1204
+#define invalid_block_type 1205
+#define invalid_comprehension_type 1206
+#define invalid_parameters_type 1207
+#define invalid_default_type 1208
+#define invalid_star_etc_type 1209
+#define invalid_kwds_type 1210
+#define invalid_parameters_helper_type 1211
+#define invalid_lambda_parameters_type 1212
+#define invalid_lambda_parameters_helper_type 1213
+#define invalid_lambda_star_etc_type 1214
+#define invalid_lambda_kwds_type 1215
+#define invalid_double_type_comments_type 1216
+#define invalid_with_item_type 1217
+#define invalid_for_if_clause_type 1218
+#define invalid_for_target_type 1219
+#define invalid_group_type 1220
+#define invalid_import_type 1221
+#define invalid_dotted_as_name_type 1222
+#define invalid_import_from_as_name_type 1223
+#define invalid_import_from_type 1224
+#define invalid_import_from_targets_type 1225
+#define invalid_with_stmt_type 1226
+#define invalid_with_stmt_indent_type 1227
+#define invalid_try_stmt_type 1228
+#define invalid_except_stmt_type 1229
+#define invalid_except_star_stmt_type 1230
+#define invalid_finally_stmt_type 1231
+#define invalid_except_stmt_indent_type 1232
+#define invalid_except_star_stmt_indent_type 1233
+#define invalid_match_stmt_type 1234
+#define invalid_case_block_type 1235
+#define invalid_as_pattern_type 1236
+#define invalid_class_pattern_type 1237
+#define invalid_mapping_pattern_type 1238
+#define invalid_class_argument_pattern_type 1239
+#define invalid_if_stmt_type 1240
+#define invalid_elif_stmt_type 1241
+#define invalid_else_stmt_type 1242
+#define invalid_while_stmt_type 1243
+#define invalid_for_stmt_type 1244
+#define invalid_def_raw_type 1245
+#define invalid_class_def_raw_type 1246
+#define invalid_double_starred_kvpairs_type 1247
+#define invalid_kvpair_unpacking_type 1248
+#define invalid_kvpair_type 1249
+#define invalid_starred_expression_unpacking_type 1250
+#define invalid_starred_expression_unpacking_sequence_type 1251
+#define invalid_starred_expression_type 1252
+#define invalid_fstring_replacement_field_type 1253
+#define invalid_fstring_conversion_character_type 1254
+#define invalid_tstring_replacement_field_type 1255
+#define invalid_tstring_conversion_character_type 1256
+#define invalid_string_tstring_concat_type 1257
+#define invalid_arithmetic_type 1258 // Left-recursive
+#define invalid_factor_type 1259
+#define invalid_type_params_type 1260
+#define invalid_bitwise_and_type 1261 // Left-recursive
+#define invalid_bitwise_or_type 1262 // Left-recursive
+#define invalid_noteq_type 1263
+#define _loop0_1_type 1264
+#define _loop1_2_type 1265
+#define _loop0_3_type 1266
+#define _gather_4_type 1267
+#define _tmp_5_type 1268
+#define _tmp_6_type 1269
+#define _tmp_7_type 1270
+#define _tmp_8_type 1271
+#define _tmp_9_type 1272
+#define _tmp_10_type 1273
+#define _tmp_11_type 1274
+#define _loop1_12_type 1275
+#define _loop0_13_type 1276
+#define _gather_14_type 1277
+#define _tmp_15_type 1278
+#define _tmp_16_type 1279
+#define _loop0_17_type 1280
+#define _loop1_18_type 1281
+#define _loop0_19_type 1282
+#define _gather_20_type 1283
+#define _tmp_21_type 1284
+#define _loop0_22_type 1285
+#define _gather_23_type 1286
+#define _loop1_24_type 1287
+#define _tmp_25_type 1288
+#define _tmp_26_type 1289
+#define _loop0_27_type 1290
+#define _loop0_28_type 1291
+#define _loop1_29_type 1292
+#define _loop1_30_type 1293
+#define _loop0_31_type 1294
+#define _loop1_32_type 1295
+#define _loop0_33_type 1296
+#define _gather_34_type 1297
+#define _tmp_35_type 1298
+#define _loop1_36_type 1299
+#define _loop1_37_type 1300
+#define _loop1_38_type 1301
+#define _loop0_39_type 1302
+#define _gather_40_type 1303
+#define _tmp_41_type 1304
+#define _tmp_42_type 1305
+#define _tmp_43_type 1306
+#define _loop0_44_type 1307
+#define _gather_45_type 1308
+#define _loop0_46_type 1309
+#define _gather_47_type 1310
+#define _tmp_48_type 1311
+#define _loop0_49_type 1312
+#define _gather_50_type 1313
+#define _loop0_51_type 1314
+#define _gather_52_type 1315
+#define _loop0_53_type 1316
+#define _gather_54_type 1317
+#define _loop1_55_type 1318
+#define _loop1_56_type 1319
+#define _loop0_57_type 1320
+#define _gather_58_type 1321
+#define _loop1_59_type 1322
+#define _loop1_60_type 1323
+#define _loop1_61_type 1324
+#define _tmp_62_type 1325
+#define _loop0_63_type 1326
+#define _gather_64_type 1327
+#define _tmp_65_type 1328
+#define _tmp_66_type 1329
+#define _tmp_67_type 1330
+#define _tmp_68_type 1331
+#define _tmp_69_type 1332
+#define _loop0_70_type 1333
+#define _loop0_71_type 1334
+#define _loop1_72_type 1335
+#define _loop1_73_type 1336
+#define _loop0_74_type 1337
+#define _loop1_75_type 1338
+#define _loop0_76_type 1339
+#define _loop0_77_type 1340
+#define _loop0_78_type 1341
+#define _loop0_79_type 1342
+#define _loop1_80_type 1343
+#define _loop1_81_type 1344
+#define _loop0_82_type 1345
+#define _gather_83_type 1346
+#define _tmp_84_type 1347
+#define _loop0_85_type 1348
+#define _gather_86_type 1349
+#define _loop1_87_type 1350
+#define _loop0_88_type 1351
+#define _tmp_89_type 1352
+#define _loop0_90_type 1353
+#define _gather_91_type 1354
+#define _tmp_92_type 1355
+#define _loop0_93_type 1356
+#define _gather_94_type 1357
+#define _tmp_95_type 1358
+#define _loop0_96_type 1359
+#define _gather_97_type 1360
+#define _loop0_98_type 1361
+#define _tmp_99_type 1362
+#define _tmp_100_type 1363
+#define _loop0_101_type 1364
+#define _gather_102_type 1365
+#define _loop0_103_type 1366
+#define _gather_104_type 1367
+#define _tmp_105_type 1368
+#define _tmp_106_type 1369
+#define _loop0_107_type 1370
+#define _gather_108_type 1371
+#define _tmp_109_type 1372
+#define _tmp_110_type 1373
+#define _tmp_111_type 1374
+#define _tmp_112_type 1375
+#define _loop1_113_type 1376
+#define _tmp_114_type 1377
+#define _tmp_115_type 1378
+#define _tmp_116_type 1379
+#define _tmp_117_type 1380
+#define _tmp_118_type 1381
+#define _loop0_119_type 1382
+#define _loop0_120_type 1383
+#define _tmp_121_type 1384
+#define _tmp_122_type 1385
+#define _tmp_123_type 1386
+#define _tmp_124_type 1387
+#define _tmp_125_type 1388
+#define _tmp_126_type 1389
+#define _tmp_127_type 1390
+#define _tmp_128_type 1391
+#define _loop0_129_type 1392
+#define _gather_130_type 1393
+#define _tmp_131_type 1394
+#define _tmp_132_type 1395
+#define _tmp_133_type 1396
+#define _tmp_134_type 1397
+#define _loop0_135_type 1398
+#define _gather_136_type 1399
+#define _tmp_137_type 1400
+#define _loop0_138_type 1401
+#define _gather_139_type 1402
+#define _loop0_140_type 1403
+#define _gather_141_type 1404
+#define _tmp_142_type 1405
+#define _loop0_143_type 1406
+#define _tmp_144_type 1407
+#define _tmp_145_type 1408
+#define _tmp_146_type 1409
+#define _tmp_147_type 1410
+#define _tmp_148_type 1411
+#define _tmp_149_type 1412
+#define _tmp_150_type 1413
+#define _tmp_151_type 1414
+#define _tmp_152_type 1415
+#define _tmp_153_type 1416
+#define _tmp_154_type 1417
+#define _tmp_155_type 1418
+#define _tmp_156_type 1419
+#define _tmp_157_type 1420
+#define _tmp_158_type 1421
+#define _tmp_159_type 1422
+#define _tmp_160_type 1423
+#define _tmp_161_type 1424
+#define _tmp_162_type 1425
+#define _tmp_163_type 1426
+#define _tmp_164_type 1427
+#define _tmp_165_type 1428
+#define _tmp_166_type 1429
+#define _tmp_167_type 1430
+#define _tmp_168_type 1431
+#define _tmp_169_type 1432
+#define _tmp_170_type 1433
+#define _tmp_171_type 1434
+#define _loop0_172_type 1435
+#define _tmp_173_type 1436
+#define _tmp_174_type 1437
+#define _tmp_175_type 1438
+#define _tmp_176_type 1439
+#define _tmp_177_type 1440
+#define _tmp_178_type 1441
static mod_ty file_rule(Parser *p);
static mod_ty interactive_rule(Parser *p);
@@ -536,7 +535,6 @@ static mod_ty eval_rule(Parser *p);
static mod_ty func_type_rule(Parser *p);
static asdl_stmt_seq* statements_rule(Parser *p);
static asdl_stmt_seq* statement_rule(Parser *p);
-static asdl_stmt_seq* single_compound_stmt_rule(Parser *p);
static asdl_stmt_seq* statement_newline_rule(Parser *p);
static asdl_stmt_seq* simple_stmts_rule(Parser *p);
static stmt_ty simple_stmt_rule(Parser *p);
@@ -1242,7 +1240,7 @@ statement_rule(Parser *p)
)
{
D(fprintf(stderr, "%*c+ statement[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt"));
- _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ) );
+ _res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a );
if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {
p->error_indicator = 1;
p->level--;
@@ -1284,50 +1282,7 @@ statement_rule(Parser *p)
return _res;
}
-// single_compound_stmt: compound_stmt
-static asdl_stmt_seq*
-single_compound_stmt_rule(Parser *p)
-{
- if (p->level++ == MAXSTACK || _PyPegen_stack_exhausted(p)) {
- _Pypegen_stack_overflow(p);
- }
- if (p->error_indicator) {
- p->level--;
- return NULL;
- }
- asdl_stmt_seq* _res = NULL;
- int _mark = p->mark;
- { // compound_stmt
- if (p->error_indicator) {
- p->level--;
- return NULL;
- }
- D(fprintf(stderr, "%*c> single_compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compound_stmt"));
- stmt_ty a;
- if (
- (a = compound_stmt_rule(p)) // compound_stmt
- )
- {
- D(fprintf(stderr, "%*c+ single_compound_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt"));
- _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ) );
- if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {
- p->error_indicator = 1;
- p->level--;
- return NULL;
- }
- goto done;
- }
- p->mark = _mark;
- D(fprintf(stderr, "%*c%s single_compound_stmt[%d-%d]: %s failed!\n", p->level, ' ',
- p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compound_stmt"));
- }
- _res = NULL;
- done:
- p->level--;
- return _res;
-}
-
-// statement_newline: single_compound_stmt NEWLINE | simple_stmts | NEWLINE | $
+// statement_newline: compound_stmt NEWLINE | simple_stmts | NEWLINE | $
static asdl_stmt_seq*
statement_newline_rule(Parser *p)
{
@@ -1349,22 +1304,22 @@ statement_newline_rule(Parser *p)
UNUSED(_start_lineno); // Only used by EXTRA macro
int _start_col_offset = p->tokens[_mark]->col_offset;
UNUSED(_start_col_offset); // Only used by EXTRA macro
- { // single_compound_stmt NEWLINE
+ { // compound_stmt NEWLINE
if (p->error_indicator) {
p->level--;
return NULL;
}
- D(fprintf(stderr, "%*c> statement_newline[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "single_compound_stmt NEWLINE"));
- asdl_stmt_seq* a;
+ D(fprintf(stderr, "%*c> statement_newline[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compound_stmt NEWLINE"));
+ stmt_ty a;
Token * newline_var;
if (
- (a = single_compound_stmt_rule(p)) // single_compound_stmt
+ (a = compound_stmt_rule(p)) // compound_stmt
&&
(newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE'
)
{
- D(fprintf(stderr, "%*c+ statement_newline[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "single_compound_stmt NEWLINE"));
- _res = a;
+ D(fprintf(stderr, "%*c+ statement_newline[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt NEWLINE"));
+ _res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a );
if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {
p->error_indicator = 1;
p->level--;
@@ -1374,7 +1329,7 @@ statement_newline_rule(Parser *p)
}
p->mark = _mark;
D(fprintf(stderr, "%*c%s statement_newline[%d-%d]: %s failed!\n", p->level, ' ',
- p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "single_compound_stmt NEWLINE"));
+ p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compound_stmt NEWLINE"));
}
{ // simple_stmts
if (p->error_indicator) {
diff --git a/Parser/pegen.c b/Parser/pegen.c
index bb14a1e78c5ad3..898cefd70120c1 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -327,9 +327,35 @@ _resize_tokens_array(Parser *p) {
return 0;
}
+int
+_PyPegen_append_token(Parser *p, struct token *token)
+{
+ int type = token->type;
+ // Single input inserts a final newline and may still need dedents. Apply
+ // the same normalization when the error pass fills the replay cache.
+ if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
+ type = NEWLINE;
+ p->parsing_started = 0;
+ if (!(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
+ _PyTokenizer_ImplyDedents(p->tok);
+ }
+ }
+ else {
+ p->parsing_started = 1;
+ }
+ if (p->fill == p->size && _resize_tokens_array(p) < 0) {
+ return -1;
+ }
+ return initialize_token(p, p->tokens[p->fill], token, type);
+}
+
int
_PyPegen_fill_token(Parser *p)
{
+ if (p->keyword_trial != NULL) {
+ // A trial may only consume the cached input, including its ENDMARKER.
+ return -1;
+ }
struct token new_token;
_PyToken_Init(&new_token);
_PyTokenizer_Get(p->tok, &new_token);
@@ -363,26 +389,7 @@ _PyPegen_fill_token(Parser *p)
p->input_exhausted = 1;
}
- // If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
- if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
- type = NEWLINE; /* Add an extra newline */
- p->parsing_started = 0;
-
- if (!(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
- _PyTokenizer_ImplyDedents(p->tok);
- }
- }
- else {
- p->parsing_started = 1;
- }
-
- // Check if we are at the limit of the token array capacity and resize if needed
- if ((p->fill == p->size) && (_resize_tokens_array(p) != 0)) {
- goto error;
- }
-
- Token *t = p->tokens[p->fill];
- int result = initialize_token(p, t, &new_token, type);
+ int result = _PyPegen_append_token(p, &new_token);
_PyToken_Free(&new_token);
return result;
error:
@@ -604,6 +611,12 @@ _PyPegen_new_identifier(Parser *p, const char *n)
identifier; if so, normalize to NFKC. */
if (!PyUnicode_IS_ASCII(id))
{
+ // Trials reuse identifiers already normalized by the original parse.
+ // A new one would import or call Python code during speculation.
+ if (p->keyword_trial != NULL) {
+ Py_DECREF(id);
+ goto error;
+ }
if (!init_normalization(p))
{
Py_DECREF(id);
@@ -649,8 +662,11 @@ _PyPegen_new_identifier(Parser *p, const char *n)
goto error;
}
}
- PyInterpreterState *interp = _PyInterpreterState_GET();
- _PyUnicode_InternImmortal(interp, &id);
+ // Trial-only identifiers must die with the trial arena.
+ if (p->keyword_trial == NULL) {
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ _PyUnicode_InternImmortal(interp, &id);
+ }
if (_PyArena_AddPyObject(p->arena, id) < 0)
{
Py_DECREF(id);
@@ -674,7 +690,7 @@ get_cached_identifier(Parser *p, PyObject *bytes)
}
// Identifiers repeat constantly; a small span-keyed cache skips the
// UTF-8 decode + intern for repeated occurrences. Keys point into
- // arena-owned token bytes and values are arena-owned interned strings,
+ // arena-owned token bytes and values are arena-owned strings,
// so borrowed references are valid for the lifetime of the parse
// (including the second error pass, which reuses parser and arena).
Py_ssize_t len = PyBytes_GET_SIZE(bytes);
@@ -955,7 +971,7 @@ compute_parser_flags(PyCompilerFlags *flags)
Parser *
_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
- int feature_version, int *errcode, const char* source, PyArena *arena)
+ int feature_version, int *errcode, PyArena *arena)
{
Parser *p = PyMem_Malloc(sizeof(Parser));
if (p == NULL) {
@@ -1009,11 +1025,9 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
#endif
p->level = 0;
p->call_invalid_rules = 0;
+ p->missing_comma = 0;
p->input_exhausted = 0;
- p->last_stmt_location.lineno = 0;
- p->last_stmt_location.col_offset = 0;
- p->last_stmt_location.end_lineno = 0;
- p->last_stmt_location.end_col_offset = 0;
+ p->keyword_trial = NULL;
#ifdef Py_DEBUG
p->debug = _Py_GetConfig()->parser_debug;
#endif
@@ -1035,60 +1049,413 @@ _PyPegen_Parser_Free(Parser *p)
PyMem_Free(p);
}
+int
+_PyPegen_stack_exhausted_slow(Parser *p)
+{
+ if (p->keyword_trial != NULL) {
+ if (p->keyword_trial->budget == 0) {
+ return 1;
+ }
+ p->keyword_trial->budget--;
+ p->keyword_trial->furthest = Py_MAX(p->keyword_trial->furthest, p->mark);
+ }
+ return _Py_ReachedRecursionLimitWithMargin(p->tstate, 1);
+}
+
static void
reset_parser_state_for_error_pass(Parser *p)
{
- p->last_stmt_location.lineno = 0;
- p->last_stmt_location.col_offset = 0;
- p->last_stmt_location.end_lineno = 0;
- p->last_stmt_location.end_col_offset = 0;
for (int i = 0; i < p->fill; i++) {
p->tokens[i]->memo = NULL;
p->tokens[i]->memo_mask = 0;
}
p->mark = 0;
p->flags &= ~PyPARSE_ALLOW_INCOMPLETE_INPUT;
- _PyTokenizer_AllowIncompleteInput(p->tok, 0);
p->call_invalid_rules = 1;
- _PyTokenizer_StopInteractive(p->tok);
+ if (p->keyword_trial == NULL) {
+ _PyTokenizer_AllowIncompleteInput(p->tok, 0);
+ _PyTokenizer_StopInteractive(p->tok);
+ }
+}
+
+typedef struct {
+ int mark;
+ const char *keyword;
+ Py_ssize_t cost;
+ int is_alias;
+} KeywordTypo;
+
+#define MAX_KEYWORD_TYPO_CANDIDATES 32
+#define MAX_KEYWORD_TYPO_STEPS 5000000
+
+static int
+compare_keyword_typos(const void *a, const void *b)
+{
+ const KeywordTypo *left = a, *right = b;
+ if (left->cost != right->cost) {
+ return left->cost < right->cost ? -1 : 1;
+ }
+ int order = strcmp(left->keyword, right->keyword);
+ return order ? order : left->mark - right->mark;
+}
+
+static int
+add_keyword_typo(KeywordTypo *candidates, int *count, int mark,
+ PyObject *name, const char *keyword, Py_ssize_t max_cost)
+{
+ PyObject *word = PyUnicode_FromString(keyword);
+ if (word == NULL) {
+ return -1;
+ }
+ if (PyUnicode_GET_LENGTH(name) <= 2) {
+ max_cost = Py_MIN(max_cost, 2);
+ }
+ Py_ssize_t cost = _Py_UTF8_Edit_Cost(name, word, max_cost);
+ // Adjacent transpositions are one spelling error. The shared Levenshtein
+ // distance counts them as two substitutions.
+ if (cost > 2 && PyUnicode_IS_ASCII(name) &&
+ PyUnicode_GET_LENGTH(name) == PyUnicode_GET_LENGTH(word))
+ {
+ const char *text = (const char *)PyUnicode_1BYTE_DATA(name);
+ Py_ssize_t i = 0;
+ while (text[i] && text[i] == keyword[i]) {
+ i++;
+ }
+ if (text[i] && text[i + 1] && text[i] == keyword[i + 1] &&
+ text[i + 1] == keyword[i] && strcmp(text + i + 2, keyword + i + 2) == 0)
+ {
+ cost = 2;
+ }
+ }
+ Py_DECREF(word);
+ if (cost < 0) {
+ return -1;
+ }
+ if (cost == 0 || cost > max_cost) {
+ return 0;
+ }
+ if (*count == MAX_KEYWORD_TYPO_CANDIDATES) {
+ return -1;
+ }
+ candidates[(*count)++] = (KeywordTypo){mark, keyword, cost, 0};
+ return 0;
+}
+
+static int
+keyword_trial_accepts(Parser *p, const KeywordTypo *candidate)
+{
+ int accepted = _PyPegen_parse(p) != NULL;
+ if (p->error_indicator || PyErr_Occurred()) {
+ return 0;
+ }
+ for (int i = p->mark; accepted && i < p->fill; i++) {
+ int type = p->tokens[i]->type;
+ accepted = type == NEWLINE || type == ENDMARKER;
+ }
+ if (accepted) {
+ return 1;
+ }
+ // Version checks on an enclosing rule can run after its suite. For older
+ // grammars, require a complete repair so those checks execute.
+ if (p->feature_version != PY_MINOR_VERSION) {
+ return 0;
+ }
+ // Known foreign keywords can identify a valid header even when its body
+ // contains another error (for example, "switch x:\n case:"). Require the
+ // ordinary grammar to advance into a later line before accepting that hint.
+ if (candidate->is_alias) {
+ for (int i = candidate->mark; i < p->keyword_trial->furthest; i++) {
+ if (p->tokens[i]->type == NEWLINE) {
+ return 1;
+ }
+ }
+ }
+ reset_parser_state_for_error_pass(p);
+ _PyPegen_parse(p);
+ return p->keyword_trial->incomplete_suite;
+}
+
+// Return 1 for an accepted repair, 0 for a rejected repair, or -1 to stop the
+// search. Raw allocation failures need not set an exception.
+static int
+try_keyword_typo(Parser *p, const KeywordTypo *candidate, KeywordTrial *state)
+{
+ // Token text belongs to the original arena. Token headers and memo entries
+ // must be private: even a failed trial can memoize nodes containing its edit.
+ Token *tokens = PyMem_Malloc(p->fill * sizeof(*tokens));
+ Token **index = PyMem_Malloc(p->fill * sizeof(*index));
+ Parser trial = *p;
+ trial.identifier_cache = p->identifier_cache == NULL ? NULL :
+ PyMem_Malloc(IDENTIFIER_CACHE_SIZE * sizeof(*trial.identifier_cache));
+ PyArena *arena = _PyArena_New();
+ PyObject *replacement = PyBytes_FromString(candidate->keyword);
+ int accepted = -1;
+ if (tokens == NULL || index == NULL || arena == NULL || replacement == NULL ||
+ (p->identifier_cache != NULL && trial.identifier_cache == NULL))
+ {
+ goto done;
+ }
+ for (int i = 0; i < p->fill; i++) {
+ tokens[i] = *p->tokens[i];
+ tokens[i].memo = NULL;
+ tokens[i].memo_mask = 0;
+ index[i] = &tokens[i];
+ }
+ if (trial.identifier_cache != NULL) {
+ memcpy(trial.identifier_cache, p->identifier_cache,
+ IDENTIFIER_CACHE_SIZE * sizeof(*trial.identifier_cache));
+ }
+ Token *edited = &tokens[candidate->mark];
+ edited->type = _get_keyword_or_name_type(
+ p, candidate->keyword, strlen(candidate->keyword));
+ edited->bytes = replacement;
+
+ state->furthest = 0;
+ state->incomplete_suite = 0;
+ trial.tokens = index;
+ trial.arena = arena;
+ trial.size = trial.fill;
+ trial.mark = 0;
+ trial.level = 0;
+ trial.error_indicator = 0;
+ trial.call_invalid_rules = 0;
+ trial.keyword_trial = state;
+ // Force trials through the existing stack-check slow path to count rule
+ // entries without adding a branch to normal parsing. The slow path still
+ // checks the thread's real stack limit.
+#if _Py_STACK_GROWS_DOWN
+ trial.stack_soft_limit = UINTPTR_MAX;
+#else
+ trial.stack_soft_limit = 0;
+#endif
+ trial.known_err_token = NULL;
+ trial.errcode = NULL;
+ trial.normalize = NULL;
+ accepted = keyword_trial_accepts(&trial, candidate);
+done:
+ // No Parser_Free: the trial borrows the tokenizer and type-ignore comments,
+ // and its token headers are a single allocation.
+ PyMem_Free(trial.identifier_cache);
+ if (arena != NULL) {
+ _PyArena_Free(arena);
+ }
+ Py_XDECREF(replacement);
+ PyMem_Free(index);
+ PyMem_Free(tokens);
+ return PyErr_Occurred() ? -1 : accepted;
+}
+
+static const char *
+keyword_typo_text(Parser *p, Token *token, int at_error, int *is_keyword)
+{
+ // A keyword can itself be a typo, as in "for x is y". Consider an existing
+ // keyword only at the reported error, not throughout the statement.
+ *is_keyword = token->type != NAME;
+ if (*is_keyword) {
+ if (!at_error) {
+ return NULL;
+ }
+ // Hard keywords have no token text. Use the generated keyword table.
+ for (int length = 0; length < p->n_keyword_lists; length++) {
+ for (KeywordToken *k = p->keywords[length];
+ k != NULL && k->type != -1; k++)
+ {
+ if (k->type == token->type) {
+ return k->str;
+ }
+ }
+ }
+ return NULL;
+ }
+ const char *text = PyBytes_AS_STRING(token->bytes);
+ for (char **k = p->soft_keywords; k != NULL && *k != NULL; k++) {
+ if (strcmp(text, *k) == 0) {
+ *is_keyword = 1;
+ return at_error ? text : NULL;
+ }
+ }
+ return text;
+}
+
+// Return the candidate count, or -1 if allocation or capacity prevents a
+// complete search. Capacity exhaustion does not set an exception.
+static int
+collect_keyword_typos(Parser *p, PySyntaxErrorObject *error, KeywordTypo *candidates)
+{
+ long error_line = PyLong_AsLong(error->lineno);
+ long error_offset = PyLong_AsLong(error->offset);
+ if (error_line < 0 || error_offset < 0) {
+ return -1;
+ }
+ Py_ssize_t error_col = -1;
+ if (error_offset > 0 && error->text != NULL && PyUnicode_Check(error->text)) {
+ PyObject *prefix = PyUnicode_Substring(error->text, 0, error_offset - 1);
+ if (prefix == NULL) {
+ return -1;
+ }
+ int failed = PyUnicode_AsUTF8AndSize(prefix, &error_col) == NULL;
+ Py_DECREF(prefix);
+ if (failed) {
+ return -1;
+ }
+ }
+ int count = 0;
+ int start = 0;
+ for (int i = 0; i < p->fill && p->tokens[i]->lineno < error_line; i++) {
+ if (p->tokens[i]->type == NEWLINE) {
+ start = i + 1;
+ }
+ }
+ static const struct { const char *name, *keyword; } aliases[] = {
+ {"switch", "match"},
+ {"delete", "del"},
+ {"function", "def"},
+ {"func", "def"},
+ {"void", "def"},
+ };
+ for (int i = start; i < p->fill; i++) {
+ Token *token = p->tokens[i];
+ // Error spans can start before the typo in a multiline expression.
+ // Keep the complete logical statement as the candidate range.
+ if (token->type == NEWLINE) {
+ break;
+ }
+ int at_error = token->lineno == error_line && token->col_offset == error_col;
+ int is_keyword;
+ const char *text = keyword_typo_text(p, token, at_error, &is_keyword);
+ if (text == NULL) {
+ continue;
+ }
+ Py_ssize_t text_length = token->type == NAME ?
+ PyBytes_GET_SIZE(token->bytes) : (Py_ssize_t)strlen(text);
+ if (text_length < 2 || text_length > 10) {
+ continue;
+ }
+ const char *alias = NULL;
+ for (size_t j = 0; j < Py_ARRAY_LENGTH(aliases); j++) {
+ if (strcmp(text, aliases[j].name) == 0) {
+ alias = aliases[j].keyword;
+ break;
+ }
+ }
+ if (alias != NULL) {
+ if (count == MAX_KEYWORD_TYPO_CANDIDATES) {
+ return -1;
+ }
+ candidates[count++] = (KeywordTypo){i, alias, 0, 1};
+ continue;
+ }
+ PyObject *name = PyUnicode_DecodeUTF8(text, text_length, NULL);
+ if (name == NULL) {
+ return -1;
+ }
+ // Preserve missing-comma diagnostics and correctly spelled keywords
+ // unless one edit repairs the input. For keywords, only substitute a
+ // character: deleting one from "for" must not suggest "or".
+ Py_ssize_t max_cost = p->missing_comma || is_keyword ? 2 : 4;
+ int result = 0;
+ for (int length = 0; length < p->n_keyword_lists && result == 0; length++) {
+ if (is_keyword && length != text_length) {
+ continue;
+ }
+ for (KeywordToken *k = p->keywords[length];
+ k != NULL && k->type != -1 && result == 0; k++)
+ {
+ result = add_keyword_typo(candidates, &count, i, name, k->str, max_cost);
+ }
+ }
+ for (char **k = p->soft_keywords; k != NULL && *k != NULL && result == 0; k++) {
+ if (is_keyword && (Py_ssize_t)strlen(*k) != text_length) {
+ continue;
+ }
+ result = add_keyword_typo(candidates, &count, i, name, *k, max_cost);
+ }
+ Py_DECREF(name);
+ if (result < 0) {
+ return -1;
+ }
+ }
+ return count;
+}
+
+static PyObject *
+build_keyword_typo_metadata(Parser *p, const KeywordTypo *candidate)
+{
+ Token *token = p->tokens[candidate->mark];
+ PyObject *line;
+ if (_PyTokenizer_RetainedSource(p->tok) != NULL) {
+ Py_ssize_t size;
+ const char *text = _PyTokenizer_LineView(p->tok, token->lineno, &size);
+ line = PyUnicode_DecodeUTF8(text, size, "replace");
+ }
+ else {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ line = _PyErr_ProgramDecodedTextObject(
+ info.filename, token->lineno, info.encoding);
+ }
+ if (line == NULL) {
+ return NULL;
+ }
+ Py_ssize_t start = _PyPegen_byte_offset_to_character_offset(line, token->col_offset);
+ Py_ssize_t end = _PyPegen_byte_offset_to_character_offset(line, token->end_col_offset);
+ PyObject *metadata = NULL;
+ if (start >= 0 && end >= 0) {
+ metadata = Py_BuildValue("(sOinin)", candidate->keyword, line,
+ token->lineno, start + 1,
+ token->end_lineno, end + 1);
+ }
+ Py_DECREF(line);
+ return metadata;
+}
+
+static PyObject *
+keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error)
+{
+ // A real ENDMARKER is required even for an unfinished suite. A truncated
+ // token cache is never evidence that a replacement is valid.
+ if ((p->start_rule != Py_file_input && p->start_rule != Py_eval_input &&
+ p->start_rule != Py_single_input) ||
+ p->fill == 0 || p->fill > MAX_KEYWORD_TYPO_TOKENS ||
+ _PyTokenizer_GetInfo(p->tok).input_span.end > MAX_KEYWORD_TYPO_BYTES ||
+ p->tokens[p->fill - 1]->type != ENDMARKER ||
+ _PyTokenizer_IsInteractive(p->tok))
+ {
+ return NULL;
+ }
+ KeywordTypo candidates[MAX_KEYWORD_TYPO_CANDIDATES];
+ int count = collect_keyword_typos(p, error, candidates);
+ if (count <= 0) {
+ return NULL;
+ }
+ qsort(candidates, count, sizeof(*candidates), compare_keyword_typos);
+ // All candidates and both parser passes share this work budget.
+ KeywordTrial state = {.budget = MAX_KEYWORD_TYPO_STEPS};
+ for (int i = 0; i < count && state.budget > 0; i++) {
+ int accepted = try_keyword_typo(p, &candidates[i], &state);
+ if (accepted < 0) {
+ return NULL;
+ }
+ if (accepted) {
+ return build_keyword_typo_metadata(p, &candidates[i]);
+ }
+ }
+ return NULL;
}
static void
-_PyPegen_set_syntax_error_metadata(Parser *p) {
+_PyPegen_set_syntax_error_metadata(Parser *p)
+{
PyObject *exc = PyErr_GetRaisedException();
- if (!exc || !PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_SyntaxError)) {
- PyErr_SetRaisedException(exc);
- return;
- }
- const char *source = _PyTokenizer_RetainedSource(p->tok);
- const char *encoding = _PyTokenizer_GetInfo(p->tok).encoding;
- PyObject* the_source = NULL;
- if (source) {
- if (encoding == NULL) {
- the_source = PyUnicode_FromString(source);
- } else {
- the_source = PyUnicode_Decode(source, strlen(source), encoding, NULL);
+ if (exc != NULL && Py_IS_TYPE(exc, (PyTypeObject *)PyExc_SyntaxError)) {
+ PySyntaxErrorObject *syntax_error = (PySyntaxErrorObject *)exc;
+ PyObject *hint = keyword_typo_metadata(p, syntax_error);
+ if (hint != NULL) {
+ Py_XSETREF(syntax_error->metadata, hint);
}
- }
- if (!the_source) {
+ // Suggestions must never replace the original exception, including
+ // when a trial runs out of memory or exhausts its rule budget.
PyErr_Clear();
- the_source = Py_None;
- Py_INCREF(the_source);
- }
- PyObject* metadata = Py_BuildValue(
- "(iiN)",
- p->last_stmt_location.lineno,
- p->last_stmt_location.col_offset,
- the_source // N gives ownership to metadata
- );
- if (!metadata) {
- PyErr_Clear();
- return;
}
- PySyntaxErrorObject *syntax_error = (PySyntaxErrorObject *)exc;
-
- Py_XDECREF(syntax_error->metadata);
- syntax_error->metadata = metadata;
PyErr_SetRaisedException(exc);
}
@@ -1123,7 +1490,7 @@ _PyPegen_run_parser(Parser *p)
// point.
_Pypegen_set_syntax_error(p, last_token);
- // Set the metadata in the exception from p->last_stmt_location
+ // Attach an optional, already-validated keyword hint.
if (PyErr_ExceptionMatches(PyExc_SyntaxError)) {
_PyPegen_set_syntax_error_metadata(p);
}
@@ -1179,7 +1546,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
Py_DECREF(module);
Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
- errcode, NULL, arena);
+ errcode, arena);
if (p == NULL) {
goto error;
}
@@ -1240,7 +1607,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
flags->cf_feature_version : PY_MINOR_VERSION;
Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
- NULL, str, arena);
+ NULL, arena);
if (p == NULL) {
goto error;
}
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 44f99310132839..aa3f7058013908 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -29,6 +29,10 @@
#define CURRENT_POS (-5)
+// Error-only token replay is bounded independently of the source length.
+#define MAX_KEYWORD_TYPO_TOKENS 32768
+#define MAX_KEYWORD_TYPO_BYTES (256 * 1024)
+
typedef struct _memo {
int type;
void *node;
@@ -69,11 +73,10 @@ typedef struct {
} growable_comment_array;
typedef struct {
- int lineno;
- int col_offset;
- int end_lineno;
- int end_col_offset;
-} location;
+ int budget;
+ int furthest;
+ int incomplete_suite;
+} KeywordTrial;
typedef struct _identifier_cache_entry IdentifierCacheEntry;
@@ -100,8 +103,9 @@ typedef struct {
Token *known_err_token;
int level;
int call_invalid_rules;
+ int missing_comma;
+ KeywordTrial *keyword_trial;
int debug;
- location last_stmt_location;
IdentifierCacheEntry *identifier_cache;
PyThreadState *tstate;
// Cached copy of tstate->c_stack_soft_limit offset by one margin toward
@@ -109,9 +113,11 @@ typedef struct {
uintptr_t stack_soft_limit;
} Parser;
+int _PyPegen_stack_exhausted_slow(Parser *p);
+
// C-stack exhaustion check run on every rule entry. The fast path compares
// the machine stack pointer against the soft limit cached in the Parser;
-// only when the stack is nearly exhausted is the full check called.
+// trials force the slow path to account for their rule budget.
static inline int
_PyPegen_stack_exhausted(Parser *p)
{
@@ -123,7 +129,7 @@ _PyPegen_stack_exhausted(Parser *p)
assert(!_Py_ReachedRecursionLimitWithMargin(p->tstate, 1));
return 0;
}
- return _Py_ReachedRecursionLimitWithMargin(p->tstate, 1);
+ return _PyPegen_stack_exhausted_slow(p);
}
typedef struct {
@@ -193,6 +199,7 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p);
expr_ty _PyPegen_fstring_middle_token(Parser* p);
Token *_PyPegen_get_last_nonnwhitespace_token(Parser *);
int _PyPegen_fill_token(Parser *p);
+int _PyPegen_append_token(Parser *p, struct token *token);
// Return -1 on failure, leaving the buffer and capacity unchanged.
int _PyPegen_grow_loop_buffer(void ***buffer, Py_ssize_t *capacity);
expr_ty _PyPegen_name_token(Parser *p);
@@ -415,12 +422,10 @@ stmt_ty _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_n
asdl_alias_seq *names, expr_ty lazy_token, int lineno,
int col_offset, int end_lineno, int end_col_offset,
PyArena *arena);
-asdl_stmt_seq* _PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts);
-stmt_ty _PyPegen_register_stmt(Parser *p, stmt_ty s);
// Parser API
-Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, const char*, PyArena *);
+Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
void _PyPegen_Parser_Free(Parser *);
mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
const char *, const char *, PyCompilerFlags *, int *, PyObject **,
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index 007397fa035822..9db7bf510c7c9e 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -154,11 +154,31 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
Py_ssize_t current_err_line = current_token->lineno;
int ret = 0;
+ int cache_tokens = 1;
struct token new_token;
_PyToken_Init(&new_token);
for (;;) {
_PyTokenizer_Get(p->tok, &new_token);
+ if (cache_tokens &&
+ _PyTokenizer_GetInfo(p->tok).input_span.end > MAX_KEYWORD_TYPO_BYTES)
+ {
+ // Logical offsets include discarded input in file mode. Stop
+ // before copying a large token into the parser's arena.
+ cache_tokens = 0;
+ }
+ // Retain the tokens this error check already reads for bounded typo
+ // trials. Never treat a truncated cache as complete input.
+ if (cache_tokens && p->fill < MAX_KEYWORD_TYPO_TOKENS &&
+ (p->fill == 0 || p->tokens[p->fill - 1]->type != ENDMARKER) &&
+ new_token.type != ERRORTOKEN && new_token.type != TYPE_IGNORE &&
+ _PyPegen_append_token(p, &new_token) < 0)
+ {
+ // Caching is optional; its allocation failures must not replace
+ // the syntax error we are already reporting.
+ PyErr_Clear();
+ cache_tokens = 0;
+ }
switch (new_token.type) {
case ERRORTOKEN: {
if (PyErr_Occurred()) {
@@ -177,6 +197,14 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
break;
}
case ENDMARKER:
+ if (cache_tokens && p->fill < MAX_KEYWORD_TYPO_TOKENS &&
+ p->start_rule == Py_single_input && p->fill > 0 &&
+ p->tokens[p->fill - 1]->type == NEWLINE)
+ {
+ // append_token inserted single input's implicit newline;
+ // retain any pending dedents and the actual end marker.
+ continue;
+ }
break;
default:
continue;
@@ -204,6 +232,19 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
void *
_PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...)
{
+ if (p->keyword_trial != NULL) {
+ if (errtype == PyExc_IndentationError) {
+ int mark = p->mark;
+ while (mark < p->fill && (p->tokens[mark]->type == DEDENT ||
+ p->tokens[mark]->type == NEWLINE)) {
+ mark++;
+ }
+ p->keyword_trial->incomplete_suite =
+ mark < p->fill && p->tokens[mark]->type == ENDMARKER;
+ }
+ p->error_indicator = 1;
+ return NULL;
+ }
// Bail out if we already have an error set.
if (p->error_indicator && PyErr_Occurred()) {
return NULL;
@@ -268,6 +309,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
const char *errmsg, va_list va)
{
+ if (p->keyword_trial != NULL) {
+ p->error_indicator = 1;
+ return NULL;
+ }
// Bail out if we already have an error set.
if (p->error_indicator && PyErr_Occurred()) {
return NULL;
diff --git a/Parser/string_parser.c b/Parser/string_parser.c
index 5df7ae8bec04b2..66bee7da7e42ad 100644
--- a/Parser/string_parser.c
+++ b/Parser/string_parser.c
@@ -12,9 +12,8 @@
static int
warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_invalid_escape, Token *t)
{
- if (p->call_invalid_rules) {
- // Do not report warnings if we are in the second pass of the parser
- // to avoid showing the warning twice.
+ if (p->call_invalid_rules || p->keyword_trial != NULL) {
+ // Error passes and keyword trials must not repeat parser warnings.
return 0;
}
unsigned char c = (unsigned char)*first_invalid_escape;