From 07539c858fd21cf2ade7d632957e84209fa2a6a4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 13:05:42 +0100 Subject: [PATCH 01/10] gh-159672: Retain tokens for syntax error diagnostics --- Parser/pegen.c | 43 +++++++++++++++++++++++-------------------- Parser/pegen.h | 4 ++++ Parser/pegen_errors.c | 21 +++++++++++++++++++++ 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index bb14a1e78c5ad35..8b5e4eaa9b77e39 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -327,6 +327,28 @@ _resize_tokens_array(Parser *p) { return 0; } +int +_PyPegen_append_token(Parser *p, struct token *token) +{ + int type = token->type; + // Single input inserts a final newline and may still need dedents. Apply + // the same normalization when the error pass fills the replay cache. + if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) { + type = NEWLINE; + p->parsing_started = 0; + if (!(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) { + _PyTokenizer_ImplyDedents(p->tok); + } + } + else { + p->parsing_started = 1; + } + if (p->fill == p->size && _resize_tokens_array(p) < 0) { + return -1; + } + return initialize_token(p, p->tokens[p->fill], token, type); +} + int _PyPegen_fill_token(Parser *p) { @@ -363,26 +385,7 @@ _PyPegen_fill_token(Parser *p) p->input_exhausted = 1; } - // If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing - if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) { - type = NEWLINE; /* Add an extra newline */ - p->parsing_started = 0; - - if (!(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) { - _PyTokenizer_ImplyDedents(p->tok); - } - } - else { - p->parsing_started = 1; - } - - // Check if we are at the limit of the token array capacity and resize if needed - if ((p->fill == p->size) && (_resize_tokens_array(p) != 0)) { - goto error; - } - - Token *t = p->tokens[p->fill]; - int result = initialize_token(p, t, &new_token, type); + int result = _PyPegen_append_token(p, &new_token); _PyToken_Free(&new_token); return result; error: diff --git a/Parser/pegen.h b/Parser/pegen.h index 44f993101328397..4dabd4b8c9b3fd5 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -29,6 +29,9 @@ #define CURRENT_POS (-5) +// Error-only token replay is bounded independently of the source length. +#define MAX_KEYWORD_TYPO_TOKENS 4096 + typedef struct _memo { int type; void *node; @@ -193,6 +196,7 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p); expr_ty _PyPegen_fstring_middle_token(Parser* p); Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); int _PyPegen_fill_token(Parser *p); +int _PyPegen_append_token(Parser *p, struct token *token); // Return -1 on failure, leaving the buffer and capacity unchanged. int _PyPegen_grow_loop_buffer(void ***buffer, Py_ssize_t *capacity); expr_ty _PyPegen_name_token(Parser *p); diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 007397fa0358229..613e5540a72b05a 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -154,11 +154,24 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { Py_ssize_t current_err_line = current_token->lineno; int ret = 0; + int cache_tokens = 1; struct token new_token; _PyToken_Init(&new_token); for (;;) { _PyTokenizer_Get(p->tok, &new_token); + // Retain the tokens this error check already reads for bounded typo + // trials. Never treat a truncated cache as complete input. + if (cache_tokens && p->fill < MAX_KEYWORD_TYPO_TOKENS && + (p->fill == 0 || p->tokens[p->fill - 1]->type != ENDMARKER) && + new_token.type != ERRORTOKEN && new_token.type != TYPE_IGNORE && + _PyPegen_append_token(p, &new_token) < 0) + { + // Caching is optional; its allocation failures must not replace + // the syntax error we are already reporting. + PyErr_Clear(); + cache_tokens = 0; + } switch (new_token.type) { case ERRORTOKEN: { if (PyErr_Occurred()) { @@ -177,6 +190,14 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { break; } case ENDMARKER: + if (cache_tokens && p->fill < MAX_KEYWORD_TYPO_TOKENS && + p->start_rule == Py_single_input && p->fill > 0 && + p->tokens[p->fill - 1]->type == NEWLINE) + { + // append_token inserted single input's implicit newline; + // retain any pending dedents and the actual end marker. + continue; + } break; default: continue; From 30d819981df47ea9dc2f3603319abe347facbbcd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 13:08:05 +0100 Subject: [PATCH 02/10] gh-159672: Generate keyword typo hints in the parser --- Lib/test/test_traceback.py | 84 +++++ Lib/traceback.py | 148 +-------- ...13-00-00.gh-issue-159672.keyword-typos.rst | 3 + Parser/action_helpers.c | 6 +- Parser/pegen.c | 310 ++++++++++++++++-- Parser/pegen.h | 14 + Parser/pegen_errors.c | 17 + Parser/string_parser.c | 2 +- 8 files changed, 410 insertions(+), 174 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2026-10-18-13-00-00.gh-issue-159672.keyword-typos.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index d5b00a3bb628cb1..854bcf2f264c91a 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1847,6 +1847,21 @@ def test_keyword_suggestions_from_command_string(self): stderr_text = stderr.decode('utf-8') self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + @cpython_only + def test_keyword_suggestions_from_single_input(self): + for source, keyword in self.TYPO_CASES: + for ending in ('', '\n'): + with self.subTest(source=source, ending=ending): + with self.assertRaises(SyntaxError) as caught: + compile(source + ending, "", "single") + self.assertEqual(caught.exception._metadata[0], keyword) + with (unittest.mock.patch("builtins.compile") as compile_mock, + unittest.mock.patch("builtins.open") as open_mock): + text = ''.join(traceback.format_exception_only(caught.exception)) + self.assertIn(f"Did you mean '{keyword}'", text) + compile_mock.assert_not_called() + open_mock.assert_not_called() + def test_no_keyword_suggestion_for_comma_errors(self): # When the parser identifies a missing comma, don't suggest # bogus keyword replacements like 'print' -> 'not' @@ -1864,6 +1879,75 @@ def test_no_keyword_suggestion_for_comma_errors(self): self.assertIn("Perhaps you forgot a comma", stderr_text) self.assertNotIn("Did you mean", stderr_text) + @cpython_only + def test_parser_suggestions_need_no_formatting_work(self): + cases = [ + ("def f():\n returm value", "return", 2, 5), + ("é = 1; impor math", "import", 1, 8), + ("a = 1\nif True:\n pass\nelso:\n pass", "else", 4, 1), + ("[123 fur x\nin range(3)\nif x]", "for", 1, 6), + ("mach x:\n case 1: pass", "match", 1, 1), + ("match x:\n cse 1: pass", "case", 2, 5), + ("typ Alias = int", "type", 1, 1), + ("lazi import math", "lazy", 1, 1), + ("long_name = 1\n" * 85 + "whille True:\n pass", "while", 86, 1), + ] + for source, keyword, lineno, offset in cases: + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as caught: + compile(source, "", "exec") + error = caught.exception + original = error.args + with (unittest.mock.patch("builtins.compile") as compile_mock, + unittest.mock.patch("builtins.open") as open_mock): + exc = traceback.TracebackException.from_exception(error) + first = ''.join(exc.format_exception_only()) + self.assertEqual(first, ''.join(exc.format_exception_only())) + compile_mock.assert_not_called() + open_mock.assert_not_called() + self.assertIn(f"Did you mean '{keyword}'?", first) + self.assertEqual((exc.lineno, exc.offset), (str(lineno), offset)) + self.assertEqual(error.args, original) + + @cpython_only + def test_keyword_trials_do_not_repeat_warnings(self): + import warnings + for prefix in ("value = '\\q'\n", "from . lazy import x\n"): + with self.subTest(prefix=prefix), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with self.assertRaises(SyntaxError) as error: + compile(prefix + "impor math", "", "exec") + self.assertEqual(len(caught), 1) + self.assertEqual(error.exception._metadata[0], "import") + + @cpython_only + def test_keyword_replay_requires_complete_valid_input(self): + for source in ("dell 1", "impor math\nother =", "whille True:\npass", + "switch +:\n case:", "function f(=):\n pass", "(mat x)", + "impor math\n" + "pass\n" * 3000): + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as caught: + compile(source, "", "exec") + # Action errors and a cache which stopped before EOF must not + # count as successful repairs. + self.assertIsNone(caught.exception._metadata) + + @cpython_only + def test_keyword_replay_respects_feature_version(self): + import ast + for source in ("mach x:\n case 1: pass", "mach x:", "switch x:\n case:"): + with self.subTest(source=source), self.assertRaises(SyntaxError) as caught: + ast.parse(source, feature_version=(3, 9)) + self.assertIsNone(caught.exception._metadata) + + @cpython_only + def test_keyword_trials_do_not_call_normalizer(self): + with unittest.mock.patch("unicodedata.normalize") as normalize: + with self.assertRaises(SyntaxError) as caught: + compile("whille True:\n café = 1", "", "exec") + normalize.assert_not_called() + self.assertIsNone(caught.exception._metadata) + @requires_debug_ranges() @force_not_colorized_test_class class PurePythonTracebackErrorCaretTests( diff --git a/Lib/traceback.py b/Lib/traceback.py index dcdab1f12e9a168..a93bcbe779274d7 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -10,10 +10,6 @@ import textwrap import types import warnings -import codeop -import keyword -import tokenize -import io import importlib.util import pathlib @@ -1182,7 +1178,13 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.end_offset = exc_value.end_offset self.msg = exc_value.msg self._is_syntax_error = True - self._exc_metadata = getattr(exc_value, "_metadata", None) + metadata = getattr(exc_value, "_metadata", None) + if isinstance(metadata, tuple) and len(metadata) == 6: + (suggestion, self.text, lineno, self.offset, + end_lineno, self.end_offset) = metadata + self.lineno = str(lineno) + self.end_lineno = str(end_lineno) + self.msg = f"invalid syntax. Did you mean '{suggestion}'?" elif exc_type and issubclass(exc_type, ImportError) and \ getattr(exc_value, "name_from", None) is not None: wrong_name = getattr(exc_value, "name_from", None) @@ -1413,122 +1415,6 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): for ex in self.exceptions: yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize) - def _find_keyword_typos(self): - assert self._is_syntax_error - try: - import _suggestions - except ImportError: - _suggestions = None - - # Only try to find keyword typos if there is no custom message - if self.msg != "invalid syntax" and "Perhaps you forgot a comma" not in self.msg: - return - - if not self._exc_metadata: - return - - line, offset, source = self._exc_metadata - end_line = int(self.lineno) if self.lineno is not None else 0 - lines = None - from_filename = False - - if source is None: - if self.filename: - try: - with open(self.filename) as f: - lines = f.read().splitlines() - except Exception: - line, end_line, offset = 0,1,0 - else: - from_filename = True - lines = lines if lines is not None else self.text.splitlines() - else: - lines = source.splitlines() - - error_code = lines[line -1 if line > 0 else 0:end_line] - error_code = textwrap.dedent('\n'.join(error_code)) - - # Do not continue if the source is too large - if len(error_code) > 1024: - return - - # If the original code doesn't raise SyntaxError, we can't validate - # that a keyword replacement actually fixes anything - try: - codeop.compile_command(error_code, symbol="exec", flags=codeop.PyCF_ONLY_AST) - except SyntaxError: - pass # Good - the original code has a syntax error we might fix - else: - return # Original code compiles or is incomplete - can't validate fixes - - error_lines = error_code.splitlines() - tokens = tokenize.generate_tokens(io.StringIO(error_code).readline) - tokens_left_to_process = 10 - import difflib - for token in tokens: - start, end = token.start, token.end - if token.type != tokenize.NAME: - continue - # Only consider NAME tokens on the same line as the error - the_end = end_line if line == 0 else end_line + 1 - if from_filename and token.start[0]+line != the_end: - continue - wrong_name = token.string - if wrong_name in keyword.kwlist: - continue - - # Limit the number of valid tokens to consider to not spend - # to much time in this function - tokens_left_to_process -= 1 - if tokens_left_to_process < 0: - break - # Limit the number of possible matches to try - max_matches = 3 - matches = [] - - hint = _get_cross_language_keyword_hint(wrong_name) - if hint: - matches.append(hint) - if _suggestions is not None: - suggestion = _suggestions._generate_suggestions(keyword.kwlist + keyword.softkwlist, wrong_name) - if suggestion: - matches.append(suggestion) - matches.extend( - difflib.get_close_matches( - wrong_name, - keyword.kwlist + keyword.softkwlist, - n=max_matches, - cutoff=0.5 - ) - ) - matches = matches[:max_matches] - for suggestion in matches: - if not suggestion or suggestion == wrong_name: - continue - # Try to replace the token with the keyword - the_lines = error_lines.copy() - the_line = the_lines[start[0] - 1][:] - chars = list(the_line) - chars[token.start[1]:token.end[1]] = suggestion - the_lines[start[0] - 1] = ''.join(chars) - code = '\n'.join(the_lines) - - # Check if it works - try: - codeop.compile_command(code, symbol="exec", flags=codeop.PyCF_ONLY_AST) - except SyntaxError: - continue - - # Keep token.line but handle offsets correctly - self.text = token.line - self.offset = token.start[1] + 1 - self.end_offset = token.end[1] + 1 - self.lineno = start[0] - self.end_lineno = end[0] - self.msg = f"invalid syntax. Did you mean '{suggestion}'?" - return - - def _format_syntax_error(self, stype, **kwargs): """Format SyntaxError exceptions (internal helper).""" # Show exactly where the problem was found. @@ -1555,9 +1441,6 @@ def _format_syntax_error(self, stype, **kwargs): # text = " foo\n" # rtext = " foo" # ltext = "foo" - with suppress(Exception): - self._find_keyword_typos() - text = self.text rtext = text.rstrip('\n') ltext = rtext.lstrip(' \n\f') spaces = len(rtext) - len(ltext) @@ -1798,17 +1681,6 @@ def print(self, *, file=None, chain=True, **kwargs): }) -# Cross-language keyword suggestions. -_CROSS_LANGUAGE_KEYWORD_HINTS = frozendict({ - # C/C++ equivalents - 'switch': 'match', - 'delete': 'del', - # function define equivalents - 'function': 'def', - 'func': 'def', - 'void': 'def', -}) - def _substitution_cost(ch_a, ch_b): if ch_a == ch_b: return 0 @@ -1888,12 +1760,6 @@ def _get_cross_language_hint(obj, wrong_name): return None -def _get_cross_language_keyword_hint(wrong_name): - """Check if wrong_name is a common keyword from another language - """ - return _CROSS_LANGUAGE_KEYWORD_HINTS.get(wrong_name) - - def _get_safe___dir__(obj): # Use obj.__dir__() to avoid a TypeError when calling dir(obj). # See gh-131001 and gh-139933. diff --git a/Misc/NEWS.d/next/Core and Builtins/2026-10-18-13-00-00.gh-issue-159672.keyword-typos.rst b/Misc/NEWS.d/next/Core and Builtins/2026-10-18-13-00-00.gh-issue-159672.keyword-typos.rst new file mode 100644 index 000000000000000..44ff0e377e7a84a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2026-10-18-13-00-00.gh-issue-159672.keyword-typos.rst @@ -0,0 +1,3 @@ +The parser now generates keyword typo suggestions from cached tokens. The +:mod:`traceback` formatter displays these suggestions without source +recompilation. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 2f8a5d84dd9755b..3081533c59f6869 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -2072,7 +2072,7 @@ _warn_relative_import_of_lazy(Parser *p, asdl_seq *dots, expr_ty module) // the module name is insignificant, so this is parsed exactly like // `from .lazy import x` (an import of the relative module "lazy"), but it // is most likely a transposition of `lazy from . import x` (PEP 810). - if (p->call_invalid_rules) { + if (p->call_invalid_rules || p->keyword_trial != NULL) { return 0; } @@ -2134,7 +2134,9 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name, alias_ty alias = asdl_seq_GET(names, i); if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) { p->flags |= PyPARSE_BARRY_AS_BDFL; - p->tok->barry_as_bdfl = 1; + if (p->keyword_trial == NULL) { + p->tok->barry_as_bdfl = 1; + } } } } diff --git a/Parser/pegen.c b/Parser/pegen.c index 8b5e4eaa9b77e39..9b4e4e2033e01d3 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -352,6 +352,10 @@ _PyPegen_append_token(Parser *p, struct token *token) int _PyPegen_fill_token(Parser *p) { + if (p->keyword_trial != NULL) { + // A trial may only consume the cached input, including its ENDMARKER. + return -1; + } struct token new_token; _PyToken_Init(&new_token); _PyTokenizer_Get(p->tok, &new_token); @@ -607,6 +611,12 @@ _PyPegen_new_identifier(Parser *p, const char *n) identifier; if so, normalize to NFKC. */ if (!PyUnicode_IS_ASCII(id)) { + // Trials reuse identifiers already normalized by the original parse. + // A new one would import or call Python code during speculation. + if (p->keyword_trial != NULL) { + Py_DECREF(id); + goto error; + } if (!init_normalization(p)) { Py_DECREF(id); @@ -1017,6 +1027,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, p->last_stmt_location.col_offset = 0; p->last_stmt_location.end_lineno = 0; p->last_stmt_location.end_col_offset = 0; + p->keyword_trial = NULL; #ifdef Py_DEBUG p->debug = _Py_GetConfig()->parser_debug; #endif @@ -1051,9 +1062,267 @@ reset_parser_state_for_error_pass(Parser *p) } p->mark = 0; p->flags &= ~PyPARSE_ALLOW_INCOMPLETE_INPUT; - _PyTokenizer_AllowIncompleteInput(p->tok, 0); p->call_invalid_rules = 1; - _PyTokenizer_StopInteractive(p->tok); + if (p->keyword_trial == NULL) { + _PyTokenizer_AllowIncompleteInput(p->tok, 0); + _PyTokenizer_StopInteractive(p->tok); + } +} + +typedef struct { + int mark; + const char *keyword; + Py_ssize_t cost; + int is_alias; +} KeywordTypo; + +#define MAX_KEYWORD_TYPO_CANDIDATES 32 +#define MAX_KEYWORD_TYPO_STEPS 100000 + +static int +compare_keyword_typos(const void *a, const void *b) +{ + const KeywordTypo *left = a, *right = b; + if (left->cost != right->cost) { + return left->cost < right->cost ? -1 : 1; + } + int order = strcmp(left->keyword, right->keyword); + return order ? order : left->mark - right->mark; +} + +static int +add_keyword_typo(KeywordTypo *candidates, int *count, int mark, + PyObject *name, const char *keyword, Py_ssize_t max_cost) +{ + PyObject *word = PyUnicode_FromString(keyword); + if (word == NULL) { + return -1; + } + if (PyUnicode_GET_LENGTH(name) <= 2) { + max_cost = Py_MIN(max_cost, 2); + } + Py_ssize_t cost = _Py_UTF8_Edit_Cost(name, word, max_cost); + Py_DECREF(word); + if (cost < 0) { + return -1; + } + if (cost == 0 || cost > max_cost) { + return 0; + } + if (*count == MAX_KEYWORD_TYPO_CANDIDATES) { + return -1; + } + candidates[(*count)++] = (KeywordTypo){mark, keyword, cost, 0}; + return 0; +} + +static int +try_keyword_typo(Parser *p, KeywordTypo *candidate) +{ + // Token text belongs to the original arena. Token headers and memo entries + // must be private: even a failed trial can memoize nodes containing its edit. + Token *tokens = PyMem_Malloc(p->fill * sizeof(*tokens)); + Token **index = PyMem_Malloc(p->fill * sizeof(*index)); + IdentifierCacheEntry *identifiers = p->identifier_cache == NULL ? NULL : + PyMem_Malloc(IDENTIFIER_CACHE_SIZE * sizeof(*identifiers)); + PyArena *arena = _PyArena_New(); + PyObject *replacement = PyBytes_FromString(candidate->keyword); + if (tokens == NULL || index == NULL || arena == NULL || replacement == NULL || + (p->identifier_cache != NULL && identifiers == NULL)) + { + PyMem_Free(tokens); + PyMem_Free(index); + PyMem_Free(identifiers); + if (arena != NULL) { + _PyArena_Free(arena); + } + Py_XDECREF(replacement); + return -1; + } + for (int i = 0; i < p->fill; i++) { + tokens[i] = *p->tokens[i]; + tokens[i].memo = NULL; + tokens[i].memo_mask = 0; + index[i] = &tokens[i]; + } + if (identifiers != NULL) { + memcpy(identifiers, p->identifier_cache, + IDENTIFIER_CACHE_SIZE * sizeof(*identifiers)); + } + Token *edited = &tokens[candidate->mark]; + edited->type = _get_keyword_or_name_type( + p, candidate->keyword, strlen(candidate->keyword)); + edited->bytes = replacement; + + KeywordTrial state = {.budget = MAX_KEYWORD_TYPO_STEPS}; + Parser trial = *p; + trial.tokens = index; + trial.arena = arena; + trial.size = trial.fill; + trial.mark = 0; + trial.level = 0; + trial.error_indicator = 0; + trial.call_invalid_rules = 0; + trial.keyword_trial = &state; + trial.known_err_token = NULL; + trial.errcode = NULL; + trial.identifier_cache = identifiers; + trial.normalize = NULL; + int accepted = _PyPegen_parse(&trial) != NULL && + !trial.error_indicator && !PyErr_Occurred(); + for (int i = trial.mark; accepted && i < trial.fill; i++) { + accepted = tokens[i].type == NEWLINE || tokens[i].type == ENDMARKER; + } + // Known foreign keywords may identify a valid header even when its body + // contains another error (for example, "switch x:\n case:"). Require the + // ordinary grammar to advance into a later line before accepting that hint. + if (!accepted && !trial.error_indicator && candidate->is_alias && + p->feature_version == PY_MINOR_VERSION) + { + for (int i = candidate->mark; i < state.furthest; i++) { + if (tokens[i].type == NEWLINE) { + accepted = 1; + break; + } + } + } + if (!accepted && !trial.error_indicator && !PyErr_Occurred()) { + reset_parser_state_for_error_pass(&trial); + _PyPegen_parse(&trial); + // Version checks on an enclosing rule can run after its suite. For + // older grammars, require a complete repair so those checks execute. + accepted = state.incomplete_suite && p->feature_version == PY_MINOR_VERSION; + } + // No Parser_Free: the trial borrows the tokenizer and type-ignore comments, + // and its token headers are a single allocation. + PyMem_Free(trial.identifier_cache); + _PyArena_Free(arena); + Py_DECREF(replacement); + PyMem_Free(index); + PyMem_Free(tokens); + return PyErr_Occurred() ? -1 : accepted; +} + +static PyObject * +keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) +{ + // A real ENDMARKER is required even for an unfinished suite. A truncated + // token cache is never evidence that a replacement is valid. + if ((p->start_rule != Py_file_input && p->start_rule != Py_eval_input && + p->start_rule != Py_single_input) || + p->fill == 0 || p->fill > MAX_KEYWORD_TYPO_TOKENS || + p->tokens[p->fill - 1]->type != ENDMARKER || + _PyTokenizer_IsInteractive(p->tok) || !PyUnicode_Check(error->msg)) + { + return NULL; + } + const char *message = PyUnicode_AsUTF8(error->msg); + if (message == NULL || (strcmp(message, "invalid syntax") != 0 && + strstr(message, "Perhaps you forgot a comma") == NULL)) + { + return NULL; + } + // Prefer an existing missing-comma diagnostic unless the spelling differs + // by only one edit (the shared edit-cost function charges two per edit). + Py_ssize_t max_cost = strcmp(message, "invalid syntax") == 0 ? 4 : 2; + long error_line = PyLong_AsLong(error->lineno); + if (error_line < 0) { + return NULL; + } + KeywordTypo candidates[MAX_KEYWORD_TYPO_CANDIDATES]; + int count = 0; + int start = 0; + for (int i = 0; i < p->fill && p->tokens[i]->lineno < error_line; i++) { + if (p->tokens[i]->type == NEWLINE) { + start = i + 1; + } + } + static const struct { const char *name, *keyword; } aliases[] = { + {"switch", "match"}, + {"delete", "del"}, + {"function", "def"}, + {"func", "def"}, + {"void", "def"}, + }; + for (int i = start; i < p->fill; i++) { + Token *token = p->tokens[i]; + if (token->type != NAME || token->lineno > error_line || + PyBytes_GET_SIZE(token->bytes) < 2 || + PyBytes_GET_SIZE(token->bytes) > 10) + { + continue; + } + const char *alias = NULL; + for (size_t j = 0; j < Py_ARRAY_LENGTH(aliases); j++) { + if (strcmp(PyBytes_AS_STRING(token->bytes), aliases[j].name) == 0) { + alias = aliases[j].keyword; + break; + } + } + if (alias != NULL) { + if (count == MAX_KEYWORD_TYPO_CANDIDATES) { + return NULL; + } + candidates[count++] = (KeywordTypo){i, alias, 0, 1}; + continue; + } + PyObject *name = PyUnicode_DecodeUTF8( + PyBytes_AS_STRING(token->bytes), PyBytes_GET_SIZE(token->bytes), NULL); + if (name == NULL) { + return NULL; + } + int result = 0; + for (int length = 0; length < p->n_keyword_lists && result == 0; length++) { + for (KeywordToken *k = p->keywords[length]; + k != NULL && k->type != -1 && result == 0; k++) + { + result = add_keyword_typo(candidates, &count, i, name, k->str, max_cost); + } + } + for (char **k = p->soft_keywords; k != NULL && *k != NULL && result == 0; k++) { + result = add_keyword_typo(candidates, &count, i, name, *k, max_cost); + } + Py_DECREF(name); + if (result < 0) { + return NULL; + } + } + qsort(candidates, count, sizeof(*candidates), compare_keyword_typos); + for (int i = 0; i < count; i++) { + int accepted = try_keyword_typo(p, &candidates[i]); + if (accepted < 0) { + return NULL; + } + if (!accepted) { + continue; + } + Token *token = p->tokens[candidates[i].mark]; + PyObject *line; + if (_PyTokenizer_RetainedSource(p->tok) != NULL) { + Py_ssize_t size; + const char *text = _PyTokenizer_LineView(p->tok, token->lineno, &size); + line = PyUnicode_DecodeUTF8(text, size, "replace"); + } + else { + _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); + line = _PyErr_ProgramDecodedTextObject( + info.filename, token->lineno, info.encoding); + } + if (line == NULL) { + return NULL; + } + Py_ssize_t start = _PyPegen_byte_offset_to_character_offset(line, token->col_offset); + Py_ssize_t end = _PyPegen_byte_offset_to_character_offset(line, token->end_col_offset); + PyObject *metadata = NULL; + if (start >= 0 && end >= 0) { + metadata = Py_BuildValue("(sOinin)", candidates[i].keyword, line, + token->lineno, start + 1, + token->end_lineno, end + 1); + } + Py_DECREF(line); + return metadata; + } + return NULL; } static void @@ -1063,35 +1332,16 @@ _PyPegen_set_syntax_error_metadata(Parser *p) { PyErr_SetRaisedException(exc); return; } - const char *source = _PyTokenizer_RetainedSource(p->tok); - const char *encoding = _PyTokenizer_GetInfo(p->tok).encoding; - PyObject* the_source = NULL; - if (source) { - if (encoding == NULL) { - the_source = PyUnicode_FromString(source); - } else { - the_source = PyUnicode_Decode(source, strlen(source), encoding, NULL); - } - } - if (!the_source) { - PyErr_Clear(); - the_source = Py_None; - Py_INCREF(the_source); - } - PyObject* metadata = Py_BuildValue( - "(iiN)", - p->last_stmt_location.lineno, - p->last_stmt_location.col_offset, - the_source // N gives ownership to metadata - ); - if (!metadata) { - PyErr_Clear(); + PySyntaxErrorObject *syntax_error = (PySyntaxErrorObject *)exc; + PyObject *hint = keyword_typo_metadata(p, syntax_error); + if (hint != NULL) { + Py_XSETREF(syntax_error->metadata, hint); + PyErr_SetRaisedException(exc); return; } - PySyntaxErrorObject *syntax_error = (PySyntaxErrorObject *)exc; - - Py_XDECREF(syntax_error->metadata); - syntax_error->metadata = metadata; + // Suggestions must never replace the original exception, including when + // a trial runs out of memory or exhausts its rule budget. + PyErr_Clear(); PyErr_SetRaisedException(exc); } @@ -1126,7 +1376,7 @@ _PyPegen_run_parser(Parser *p) // point. _Pypegen_set_syntax_error(p, last_token); - // Set the metadata in the exception from p->last_stmt_location + // Attach an optional, already-validated keyword hint. if (PyErr_ExceptionMatches(PyExc_SyntaxError)) { _PyPegen_set_syntax_error_metadata(p); } diff --git a/Parser/pegen.h b/Parser/pegen.h index 4dabd4b8c9b3fd5..deb3550f3e36e0c 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -78,6 +78,12 @@ typedef struct { int end_col_offset; } location; +typedef struct { + int budget; + int furthest; + int incomplete_suite; +} KeywordTrial; + typedef struct _identifier_cache_entry IdentifierCacheEntry; typedef struct { @@ -103,6 +109,7 @@ typedef struct { Token *known_err_token; int level; int call_invalid_rules; + KeywordTrial *keyword_trial; int debug; location last_stmt_location; IdentifierCacheEntry *identifier_cache; @@ -118,6 +125,13 @@ typedef struct { static inline int _PyPegen_stack_exhausted(Parser *p) { + if (p->keyword_trial != NULL) { + if (p->keyword_trial->budget == 0) { + return 1; + } + p->keyword_trial->budget--; + p->keyword_trial->furthest = Py_MAX(p->keyword_trial->furthest, p->mark); + } #if _Py_STACK_GROWS_DOWN if (_Py_get_machine_stack_pointer() > p->stack_soft_limit) { #else diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 613e5540a72b05a..7266cfad2b245d1 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -225,6 +225,19 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { void * _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...) { + if (p->keyword_trial != NULL) { + if (errtype == PyExc_IndentationError) { + int mark = p->mark; + while (mark < p->fill && (p->tokens[mark]->type == DEDENT || + p->tokens[mark]->type == NEWLINE)) { + mark++; + } + p->keyword_trial->incomplete_suite = + mark < p->fill && p->tokens[mark]->type == ENDMARKER; + } + p->error_indicator = 1; + return NULL; + } // Bail out if we already have an error set. if (p->error_indicator && PyErr_Occurred()) { return NULL; @@ -289,6 +302,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t end_lineno, Py_ssize_t end_col_offset, const char *errmsg, va_list va) { + if (p->keyword_trial != NULL) { + p->error_indicator = 1; + return NULL; + } // Bail out if we already have an error set. if (p->error_indicator && PyErr_Occurred()) { return NULL; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 5df7ae8bec04b25..3a2017ce4af51d5 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -12,7 +12,7 @@ static int warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_invalid_escape, Token *t) { - if (p->call_invalid_rules) { + if (p->call_invalid_rules || p->keyword_trial != NULL) { // Do not report warnings if we are in the second pass of the parser // to avoid showing the warning twice. return 0; From 24cd768bba1c803ef30db4d14aa3b8621791a05a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 13:09:12 +0100 Subject: [PATCH 03/10] gh-159672: Remove obsolete keyword typo support --- Grammar/python.gram | 10 +- Lib/codeop.py | 13 +- Parser/action_helpers.c | 20 - Parser/parser.c | 935 +++++++++++++++++++--------------------- Parser/pegen.c | 14 +- Parser/pegen.h | 12 +- 6 files changed, 458 insertions(+), 546 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index f2c9993aa4f80a7..d223d05caf83b3a 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -97,17 +97,11 @@ func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMA statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } statement[asdl_stmt_seq*]: - | a=compound_stmt { _PyPegen_register_stmts(p , - (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) - ) } + | a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | a[asdl_stmt_seq*]=simple_stmts { a } -single_compound_stmt[asdl_stmt_seq*]: - | a=compound_stmt { - _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a)) } - statement_newline[asdl_stmt_seq*]: - | a=single_compound_stmt NEWLINE { a } + | a=compound_stmt NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | simple_stmts | NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, CHECK(stmt_ty, _PyAST_Pass(EXTRA))) } | ENDMARKER { _PyPegen_interactive_exit(p) } diff --git a/Lib/codeop.py b/Lib/codeop.py index 140e960b35f4ef7..33e4b226f82f07f 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -47,7 +47,7 @@ PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 -def _maybe_compile(compiler, source, filename, symbol, flags): +def _maybe_compile(compiler, source, filename, symbol): # Check for source consisting of only blank lines and comments. for line in source.split("\n"): line = line.strip() @@ -61,7 +61,7 @@ def _maybe_compile(compiler, source, filename, symbol, flags): with warnings.catch_warnings(): warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning)) try: - compiler(source, filename, symbol, flags=flags) + compiler(source, filename, symbol) except _IncompleteInputError: return None except SyntaxError: # Let other compile() errors propagate. @@ -69,13 +69,14 @@ def _maybe_compile(compiler, source, filename, symbol, flags): return compiler(source, filename, symbol, incomplete_input=False) -def _compile(source, filename, symbol, incomplete_input=True, *, flags=0): +def _compile(source, filename, symbol, incomplete_input=True): + flags = 0 if incomplete_input: flags |= PyCF_ALLOW_INCOMPLETE_INPUT flags |= PyCF_DONT_IMPLY_DEDENT return compile(source, filename, symbol, flags) -def compile_command(source, filename="", symbol="single", flags=0): +def compile_command(source, filename="", symbol="single"): r"""Compile a command and determine whether it is incomplete. Arguments: @@ -94,7 +95,7 @@ def compile_command(source, filename="", symbol="single", flags=0): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(_compile, source, filename, symbol, flags) + return _maybe_compile(_compile, source, filename, symbol) class Compile: """Instances of this class behave much like the built-in compile @@ -146,4 +147,4 @@ def __call__(self, source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags) + return _maybe_compile(self.compiler, source, filename, symbol) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 3081533c59f6869..f7c556976dbe313 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -2152,23 +2152,3 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name, return _PyAST_ImportFrom(module, names, level, lazy_token ? 1 : 0, lineno, col_offset, end_lineno, end_col_offset, arena); } - -asdl_stmt_seq* -_PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts) { - if (!p->call_invalid_rules) { - return stmts; - } - Py_ssize_t len = asdl_seq_LEN(stmts); - if (len == 0) { - return stmts; - } - stmt_ty last_stmt = asdl_seq_GET(stmts, len - 1); - if (p->last_stmt_location.lineno > last_stmt->lineno) { - return stmts; - } - p->last_stmt_location.lineno = last_stmt->lineno; - p->last_stmt_location.col_offset = last_stmt->col_offset; - p->last_stmt_location.end_lineno = last_stmt->end_lineno; - p->last_stmt_location.end_col_offset = last_stmt->end_col_offset; - return stmts; -} diff --git a/Parser/parser.c b/Parser/parser.c index 979907f4873b9dd..cd000fea2d6f5c4 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -92,443 +92,442 @@ static char *soft_keywords[] = { #define func_type_type 1003 #define statements_type 1004 #define statement_type 1005 -#define single_compound_stmt_type 1006 -#define statement_newline_type 1007 -#define simple_stmts_type 1008 -#define simple_stmt_type 1009 -#define compound_stmt_type 1010 -#define assignment_type 1011 -#define annotated_rhs_type 1012 -#define augassign_type 1013 -#define return_stmt_type 1014 -#define raise_stmt_type 1015 -#define pass_stmt_type 1016 -#define break_stmt_type 1017 -#define continue_stmt_type 1018 -#define global_stmt_type 1019 -#define nonlocal_stmt_type 1020 -#define del_stmt_type 1021 -#define yield_stmt_type 1022 -#define assert_stmt_type 1023 -#define import_stmt_type 1024 -#define import_name_type 1025 -#define import_from_type 1026 -#define import_from_targets_type 1027 -#define import_from_as_names_type 1028 -#define import_from_as_name_type 1029 -#define dotted_as_names_type 1030 -#define dotted_as_name_type 1031 -#define dotted_name_type 1032 // Left-recursive -#define block_type 1033 -#define decorators_type 1034 -#define class_def_type 1035 -#define class_def_raw_type 1036 -#define function_def_type 1037 -#define function_def_raw_type 1038 -#define params_type 1039 -#define parameters_type 1040 -#define slash_no_default_type 1041 -#define slash_with_default_type 1042 -#define star_etc_type 1043 -#define kwds_type 1044 -#define param_no_default_type 1045 -#define param_no_default_star_annotation_type 1046 -#define param_with_default_type 1047 -#define param_maybe_default_type 1048 -#define param_type 1049 -#define param_star_annotation_type 1050 -#define annotation_type 1051 -#define star_annotation_type 1052 -#define default_type 1053 -#define if_stmt_type 1054 -#define elif_stmt_type 1055 -#define else_block_type 1056 -#define while_stmt_type 1057 -#define for_stmt_type 1058 -#define with_stmt_type 1059 -#define with_item_type 1060 -#define try_stmt_type 1061 -#define except_block_type 1062 -#define except_star_block_type 1063 -#define finally_block_type 1064 -#define match_stmt_type 1065 -#define subject_expr_type 1066 -#define case_block_type 1067 -#define guard_type 1068 -#define patterns_type 1069 -#define pattern_type 1070 -#define as_pattern_type 1071 -#define or_pattern_type 1072 -#define closed_pattern_type 1073 -#define literal_pattern_type 1074 -#define literal_expr_type 1075 -#define complex_number_type 1076 -#define signed_number_type 1077 -#define signed_real_number_type 1078 -#define real_number_type 1079 -#define imaginary_number_type 1080 -#define capture_pattern_type 1081 -#define pattern_capture_target_type 1082 -#define wildcard_pattern_type 1083 -#define value_pattern_type 1084 -#define attr_type 1085 // Left-recursive -#define name_or_attr_type 1086 // Left-recursive -#define group_pattern_type 1087 -#define sequence_pattern_type 1088 -#define open_sequence_pattern_type 1089 -#define maybe_sequence_pattern_type 1090 -#define maybe_star_pattern_type 1091 -#define star_pattern_type 1092 -#define mapping_pattern_type 1093 -#define items_pattern_type 1094 -#define key_value_pattern_type 1095 -#define double_star_pattern_type 1096 -#define class_pattern_type 1097 -#define positional_patterns_type 1098 -#define keyword_patterns_type 1099 -#define keyword_pattern_type 1100 -#define type_alias_type 1101 -#define type_params_type 1102 -#define type_param_seq_type 1103 -#define type_param_type 1104 -#define type_param_bound_type 1105 -#define type_param_default_type 1106 -#define type_param_starred_default_type 1107 -#define expressions_type 1108 -#define expression_type 1109 -#define if_expression_type 1110 -#define yield_expr_type 1111 -#define star_expressions_type 1112 -#define star_expression_type 1113 -#define star_named_expressions_type 1114 -#define star_named_expression_type 1115 -#define assignment_expression_type 1116 -#define named_expression_type 1117 -#define disjunction_type 1118 -#define conjunction_type 1119 -#define inversion_type 1120 -#define comparison_type 1121 -#define compare_op_bitwise_or_pair_type 1122 -#define bitwise_or_type 1123 // Left-recursive -#define bitwise_xor_type 1124 // Left-recursive -#define bitwise_and_type 1125 // Left-recursive -#define shift_expr_type 1126 // Left-recursive -#define sum_type 1127 // Left-recursive -#define term_type 1128 // Left-recursive -#define factor_type 1129 -#define power_type 1130 -#define await_primary_type 1131 -#define primary_type 1132 // Left-recursive -#define slices_type 1133 -#define slice_type 1134 -#define atom_type 1135 -#define group_type 1136 -#define lambdef_type 1137 -#define lambda_params_type 1138 -#define lambda_parameters_type 1139 -#define lambda_slash_no_default_type 1140 -#define lambda_slash_with_default_type 1141 -#define lambda_star_etc_type 1142 -#define lambda_kwds_type 1143 -#define lambda_param_no_default_type 1144 -#define lambda_param_with_default_type 1145 -#define lambda_param_maybe_default_type 1146 -#define lambda_param_type 1147 -#define fstring_middle_type 1148 -#define fstring_replacement_field_type 1149 -#define fstring_conversion_type 1150 -#define fstring_full_format_spec_type 1151 -#define fstring_format_spec_type 1152 -#define fstring_type 1153 -#define tstring_format_spec_replacement_field_type 1154 -#define tstring_format_spec_type 1155 -#define tstring_full_format_spec_type 1156 -#define tstring_replacement_field_type 1157 -#define tstring_middle_type 1158 -#define tstring_type 1159 -#define string_type 1160 -#define strings_type 1161 -#define display_items_type 1162 -#define display_item_type 1163 -#define list_type 1164 -#define tuple_type 1165 -#define set_type 1166 -#define dict_type 1167 -#define double_starred_kvpairs_type 1168 -#define double_starred_kvpair_type 1169 -#define kvpair_type 1170 -#define for_if_clauses_type 1171 -#define for_if_clause_type 1172 -#define listcomp_type 1173 -#define setcomp_type 1174 -#define genexp_type 1175 -#define dictcomp_type 1176 -#define arguments_type 1177 -#define args_type 1178 -#define kwargs_type 1179 -#define starred_expression_type 1180 -#define kwarg_or_starred_type 1181 -#define kwarg_or_double_starred_type 1182 -#define star_targets_type 1183 -#define star_target_type 1184 -#define target_type 1185 -#define single_target_type 1186 -#define attribute_or_subscript_target_type 1187 -#define target_value_type 1188 -#define del_targets_type 1189 -#define del_target_type 1190 -#define type_expressions_type 1191 -#define func_type_comment_type 1192 -#define invalid_arguments_type 1193 -#define invalid_kwarg_type 1194 -#define expression_without_invalid_type 1195 -#define invalid_legacy_expression_type 1196 -#define invalid_type_param_type 1197 -#define invalid_expression_type 1198 -#define invalid_if_expression_type 1199 -#define invalid_named_expression_type 1200 -#define invalid_assignment_type 1201 -#define invalid_ann_assign_target_type 1202 -#define invalid_raise_stmt_type 1203 -#define invalid_del_stmt_type 1204 -#define invalid_assert_stmt_type 1205 -#define invalid_block_type 1206 -#define invalid_comprehension_type 1207 -#define invalid_parameters_type 1208 -#define invalid_default_type 1209 -#define invalid_star_etc_type 1210 -#define invalid_kwds_type 1211 -#define invalid_parameters_helper_type 1212 -#define invalid_lambda_parameters_type 1213 -#define invalid_lambda_parameters_helper_type 1214 -#define invalid_lambda_star_etc_type 1215 -#define invalid_lambda_kwds_type 1216 -#define invalid_double_type_comments_type 1217 -#define invalid_with_item_type 1218 -#define invalid_for_if_clause_type 1219 -#define invalid_for_target_type 1220 -#define invalid_group_type 1221 -#define invalid_import_type 1222 -#define invalid_dotted_as_name_type 1223 -#define invalid_import_from_as_name_type 1224 -#define invalid_import_from_type 1225 -#define invalid_import_from_targets_type 1226 -#define invalid_with_stmt_type 1227 -#define invalid_with_stmt_indent_type 1228 -#define invalid_try_stmt_type 1229 -#define invalid_except_stmt_type 1230 -#define invalid_except_star_stmt_type 1231 -#define invalid_finally_stmt_type 1232 -#define invalid_except_stmt_indent_type 1233 -#define invalid_except_star_stmt_indent_type 1234 -#define invalid_match_stmt_type 1235 -#define invalid_case_block_type 1236 -#define invalid_as_pattern_type 1237 -#define invalid_class_pattern_type 1238 -#define invalid_mapping_pattern_type 1239 -#define invalid_class_argument_pattern_type 1240 -#define invalid_if_stmt_type 1241 -#define invalid_elif_stmt_type 1242 -#define invalid_else_stmt_type 1243 -#define invalid_while_stmt_type 1244 -#define invalid_for_stmt_type 1245 -#define invalid_def_raw_type 1246 -#define invalid_class_def_raw_type 1247 -#define invalid_double_starred_kvpairs_type 1248 -#define invalid_kvpair_unpacking_type 1249 -#define invalid_kvpair_type 1250 -#define invalid_starred_expression_unpacking_type 1251 -#define invalid_starred_expression_unpacking_sequence_type 1252 -#define invalid_starred_expression_type 1253 -#define invalid_fstring_replacement_field_type 1254 -#define invalid_fstring_conversion_character_type 1255 -#define invalid_tstring_replacement_field_type 1256 -#define invalid_tstring_conversion_character_type 1257 -#define invalid_string_tstring_concat_type 1258 -#define invalid_arithmetic_type 1259 // Left-recursive -#define invalid_factor_type 1260 -#define invalid_type_params_type 1261 -#define invalid_bitwise_and_type 1262 // Left-recursive -#define invalid_bitwise_or_type 1263 // Left-recursive -#define invalid_noteq_type 1264 -#define _loop0_1_type 1265 -#define _loop1_2_type 1266 -#define _loop0_3_type 1267 -#define _gather_4_type 1268 -#define _tmp_5_type 1269 -#define _tmp_6_type 1270 -#define _tmp_7_type 1271 -#define _tmp_8_type 1272 -#define _tmp_9_type 1273 -#define _tmp_10_type 1274 -#define _tmp_11_type 1275 -#define _loop1_12_type 1276 -#define _loop0_13_type 1277 -#define _gather_14_type 1278 -#define _tmp_15_type 1279 -#define _tmp_16_type 1280 -#define _loop0_17_type 1281 -#define _loop1_18_type 1282 -#define _loop0_19_type 1283 -#define _gather_20_type 1284 -#define _tmp_21_type 1285 -#define _loop0_22_type 1286 -#define _gather_23_type 1287 -#define _loop1_24_type 1288 -#define _tmp_25_type 1289 -#define _tmp_26_type 1290 -#define _loop0_27_type 1291 -#define _loop0_28_type 1292 -#define _loop1_29_type 1293 -#define _loop1_30_type 1294 -#define _loop0_31_type 1295 -#define _loop1_32_type 1296 -#define _loop0_33_type 1297 -#define _gather_34_type 1298 -#define _tmp_35_type 1299 -#define _loop1_36_type 1300 -#define _loop1_37_type 1301 -#define _loop1_38_type 1302 -#define _loop0_39_type 1303 -#define _gather_40_type 1304 -#define _tmp_41_type 1305 -#define _tmp_42_type 1306 -#define _tmp_43_type 1307 -#define _loop0_44_type 1308 -#define _gather_45_type 1309 -#define _loop0_46_type 1310 -#define _gather_47_type 1311 -#define _tmp_48_type 1312 -#define _loop0_49_type 1313 -#define _gather_50_type 1314 -#define _loop0_51_type 1315 -#define _gather_52_type 1316 -#define _loop0_53_type 1317 -#define _gather_54_type 1318 -#define _loop1_55_type 1319 -#define _loop1_56_type 1320 -#define _loop0_57_type 1321 -#define _gather_58_type 1322 -#define _loop1_59_type 1323 -#define _loop1_60_type 1324 -#define _loop1_61_type 1325 -#define _tmp_62_type 1326 -#define _loop0_63_type 1327 -#define _gather_64_type 1328 -#define _tmp_65_type 1329 -#define _tmp_66_type 1330 -#define _tmp_67_type 1331 -#define _tmp_68_type 1332 -#define _tmp_69_type 1333 -#define _loop0_70_type 1334 -#define _loop0_71_type 1335 -#define _loop1_72_type 1336 -#define _loop1_73_type 1337 -#define _loop0_74_type 1338 -#define _loop1_75_type 1339 -#define _loop0_76_type 1340 -#define _loop0_77_type 1341 -#define _loop0_78_type 1342 -#define _loop0_79_type 1343 -#define _loop1_80_type 1344 -#define _loop1_81_type 1345 -#define _loop0_82_type 1346 -#define _gather_83_type 1347 -#define _tmp_84_type 1348 -#define _loop0_85_type 1349 -#define _gather_86_type 1350 -#define _loop1_87_type 1351 -#define _loop0_88_type 1352 -#define _tmp_89_type 1353 -#define _loop0_90_type 1354 -#define _gather_91_type 1355 -#define _tmp_92_type 1356 -#define _loop0_93_type 1357 -#define _gather_94_type 1358 -#define _tmp_95_type 1359 -#define _loop0_96_type 1360 -#define _gather_97_type 1361 -#define _loop0_98_type 1362 -#define _tmp_99_type 1363 -#define _tmp_100_type 1364 -#define _loop0_101_type 1365 -#define _gather_102_type 1366 -#define _loop0_103_type 1367 -#define _gather_104_type 1368 -#define _tmp_105_type 1369 -#define _tmp_106_type 1370 -#define _loop0_107_type 1371 -#define _gather_108_type 1372 -#define _tmp_109_type 1373 -#define _tmp_110_type 1374 -#define _tmp_111_type 1375 -#define _tmp_112_type 1376 -#define _loop1_113_type 1377 -#define _tmp_114_type 1378 -#define _tmp_115_type 1379 -#define _tmp_116_type 1380 -#define _tmp_117_type 1381 -#define _tmp_118_type 1382 -#define _loop0_119_type 1383 -#define _loop0_120_type 1384 -#define _tmp_121_type 1385 -#define _tmp_122_type 1386 -#define _tmp_123_type 1387 -#define _tmp_124_type 1388 -#define _tmp_125_type 1389 -#define _tmp_126_type 1390 -#define _tmp_127_type 1391 -#define _tmp_128_type 1392 -#define _loop0_129_type 1393 -#define _gather_130_type 1394 -#define _tmp_131_type 1395 -#define _tmp_132_type 1396 -#define _tmp_133_type 1397 -#define _tmp_134_type 1398 -#define _loop0_135_type 1399 -#define _gather_136_type 1400 -#define _tmp_137_type 1401 -#define _loop0_138_type 1402 -#define _gather_139_type 1403 -#define _loop0_140_type 1404 -#define _gather_141_type 1405 -#define _tmp_142_type 1406 -#define _loop0_143_type 1407 -#define _tmp_144_type 1408 -#define _tmp_145_type 1409 -#define _tmp_146_type 1410 -#define _tmp_147_type 1411 -#define _tmp_148_type 1412 -#define _tmp_149_type 1413 -#define _tmp_150_type 1414 -#define _tmp_151_type 1415 -#define _tmp_152_type 1416 -#define _tmp_153_type 1417 -#define _tmp_154_type 1418 -#define _tmp_155_type 1419 -#define _tmp_156_type 1420 -#define _tmp_157_type 1421 -#define _tmp_158_type 1422 -#define _tmp_159_type 1423 -#define _tmp_160_type 1424 -#define _tmp_161_type 1425 -#define _tmp_162_type 1426 -#define _tmp_163_type 1427 -#define _tmp_164_type 1428 -#define _tmp_165_type 1429 -#define _tmp_166_type 1430 -#define _tmp_167_type 1431 -#define _tmp_168_type 1432 -#define _tmp_169_type 1433 -#define _tmp_170_type 1434 -#define _tmp_171_type 1435 -#define _loop0_172_type 1436 -#define _tmp_173_type 1437 -#define _tmp_174_type 1438 -#define _tmp_175_type 1439 -#define _tmp_176_type 1440 -#define _tmp_177_type 1441 -#define _tmp_178_type 1442 +#define statement_newline_type 1006 +#define simple_stmts_type 1007 +#define simple_stmt_type 1008 +#define compound_stmt_type 1009 +#define assignment_type 1010 +#define annotated_rhs_type 1011 +#define augassign_type 1012 +#define return_stmt_type 1013 +#define raise_stmt_type 1014 +#define pass_stmt_type 1015 +#define break_stmt_type 1016 +#define continue_stmt_type 1017 +#define global_stmt_type 1018 +#define nonlocal_stmt_type 1019 +#define del_stmt_type 1020 +#define yield_stmt_type 1021 +#define assert_stmt_type 1022 +#define import_stmt_type 1023 +#define import_name_type 1024 +#define import_from_type 1025 +#define import_from_targets_type 1026 +#define import_from_as_names_type 1027 +#define import_from_as_name_type 1028 +#define dotted_as_names_type 1029 +#define dotted_as_name_type 1030 +#define dotted_name_type 1031 // Left-recursive +#define block_type 1032 +#define decorators_type 1033 +#define class_def_type 1034 +#define class_def_raw_type 1035 +#define function_def_type 1036 +#define function_def_raw_type 1037 +#define params_type 1038 +#define parameters_type 1039 +#define slash_no_default_type 1040 +#define slash_with_default_type 1041 +#define star_etc_type 1042 +#define kwds_type 1043 +#define param_no_default_type 1044 +#define param_no_default_star_annotation_type 1045 +#define param_with_default_type 1046 +#define param_maybe_default_type 1047 +#define param_type 1048 +#define param_star_annotation_type 1049 +#define annotation_type 1050 +#define star_annotation_type 1051 +#define default_type 1052 +#define if_stmt_type 1053 +#define elif_stmt_type 1054 +#define else_block_type 1055 +#define while_stmt_type 1056 +#define for_stmt_type 1057 +#define with_stmt_type 1058 +#define with_item_type 1059 +#define try_stmt_type 1060 +#define except_block_type 1061 +#define except_star_block_type 1062 +#define finally_block_type 1063 +#define match_stmt_type 1064 +#define subject_expr_type 1065 +#define case_block_type 1066 +#define guard_type 1067 +#define patterns_type 1068 +#define pattern_type 1069 +#define as_pattern_type 1070 +#define or_pattern_type 1071 +#define closed_pattern_type 1072 +#define literal_pattern_type 1073 +#define literal_expr_type 1074 +#define complex_number_type 1075 +#define signed_number_type 1076 +#define signed_real_number_type 1077 +#define real_number_type 1078 +#define imaginary_number_type 1079 +#define capture_pattern_type 1080 +#define pattern_capture_target_type 1081 +#define wildcard_pattern_type 1082 +#define value_pattern_type 1083 +#define attr_type 1084 // Left-recursive +#define name_or_attr_type 1085 // Left-recursive +#define group_pattern_type 1086 +#define sequence_pattern_type 1087 +#define open_sequence_pattern_type 1088 +#define maybe_sequence_pattern_type 1089 +#define maybe_star_pattern_type 1090 +#define star_pattern_type 1091 +#define mapping_pattern_type 1092 +#define items_pattern_type 1093 +#define key_value_pattern_type 1094 +#define double_star_pattern_type 1095 +#define class_pattern_type 1096 +#define positional_patterns_type 1097 +#define keyword_patterns_type 1098 +#define keyword_pattern_type 1099 +#define type_alias_type 1100 +#define type_params_type 1101 +#define type_param_seq_type 1102 +#define type_param_type 1103 +#define type_param_bound_type 1104 +#define type_param_default_type 1105 +#define type_param_starred_default_type 1106 +#define expressions_type 1107 +#define expression_type 1108 +#define if_expression_type 1109 +#define yield_expr_type 1110 +#define star_expressions_type 1111 +#define star_expression_type 1112 +#define star_named_expressions_type 1113 +#define star_named_expression_type 1114 +#define assignment_expression_type 1115 +#define named_expression_type 1116 +#define disjunction_type 1117 +#define conjunction_type 1118 +#define inversion_type 1119 +#define comparison_type 1120 +#define compare_op_bitwise_or_pair_type 1121 +#define bitwise_or_type 1122 // Left-recursive +#define bitwise_xor_type 1123 // Left-recursive +#define bitwise_and_type 1124 // Left-recursive +#define shift_expr_type 1125 // Left-recursive +#define sum_type 1126 // Left-recursive +#define term_type 1127 // Left-recursive +#define factor_type 1128 +#define power_type 1129 +#define await_primary_type 1130 +#define primary_type 1131 // Left-recursive +#define slices_type 1132 +#define slice_type 1133 +#define atom_type 1134 +#define group_type 1135 +#define lambdef_type 1136 +#define lambda_params_type 1137 +#define lambda_parameters_type 1138 +#define lambda_slash_no_default_type 1139 +#define lambda_slash_with_default_type 1140 +#define lambda_star_etc_type 1141 +#define lambda_kwds_type 1142 +#define lambda_param_no_default_type 1143 +#define lambda_param_with_default_type 1144 +#define lambda_param_maybe_default_type 1145 +#define lambda_param_type 1146 +#define fstring_middle_type 1147 +#define fstring_replacement_field_type 1148 +#define fstring_conversion_type 1149 +#define fstring_full_format_spec_type 1150 +#define fstring_format_spec_type 1151 +#define fstring_type 1152 +#define tstring_format_spec_replacement_field_type 1153 +#define tstring_format_spec_type 1154 +#define tstring_full_format_spec_type 1155 +#define tstring_replacement_field_type 1156 +#define tstring_middle_type 1157 +#define tstring_type 1158 +#define string_type 1159 +#define strings_type 1160 +#define display_items_type 1161 +#define display_item_type 1162 +#define list_type 1163 +#define tuple_type 1164 +#define set_type 1165 +#define dict_type 1166 +#define double_starred_kvpairs_type 1167 +#define double_starred_kvpair_type 1168 +#define kvpair_type 1169 +#define for_if_clauses_type 1170 +#define for_if_clause_type 1171 +#define listcomp_type 1172 +#define setcomp_type 1173 +#define genexp_type 1174 +#define dictcomp_type 1175 +#define arguments_type 1176 +#define args_type 1177 +#define kwargs_type 1178 +#define starred_expression_type 1179 +#define kwarg_or_starred_type 1180 +#define kwarg_or_double_starred_type 1181 +#define star_targets_type 1182 +#define star_target_type 1183 +#define target_type 1184 +#define single_target_type 1185 +#define attribute_or_subscript_target_type 1186 +#define target_value_type 1187 +#define del_targets_type 1188 +#define del_target_type 1189 +#define type_expressions_type 1190 +#define func_type_comment_type 1191 +#define invalid_arguments_type 1192 +#define invalid_kwarg_type 1193 +#define expression_without_invalid_type 1194 +#define invalid_legacy_expression_type 1195 +#define invalid_type_param_type 1196 +#define invalid_expression_type 1197 +#define invalid_if_expression_type 1198 +#define invalid_named_expression_type 1199 +#define invalid_assignment_type 1200 +#define invalid_ann_assign_target_type 1201 +#define invalid_raise_stmt_type 1202 +#define invalid_del_stmt_type 1203 +#define invalid_assert_stmt_type 1204 +#define invalid_block_type 1205 +#define invalid_comprehension_type 1206 +#define invalid_parameters_type 1207 +#define invalid_default_type 1208 +#define invalid_star_etc_type 1209 +#define invalid_kwds_type 1210 +#define invalid_parameters_helper_type 1211 +#define invalid_lambda_parameters_type 1212 +#define invalid_lambda_parameters_helper_type 1213 +#define invalid_lambda_star_etc_type 1214 +#define invalid_lambda_kwds_type 1215 +#define invalid_double_type_comments_type 1216 +#define invalid_with_item_type 1217 +#define invalid_for_if_clause_type 1218 +#define invalid_for_target_type 1219 +#define invalid_group_type 1220 +#define invalid_import_type 1221 +#define invalid_dotted_as_name_type 1222 +#define invalid_import_from_as_name_type 1223 +#define invalid_import_from_type 1224 +#define invalid_import_from_targets_type 1225 +#define invalid_with_stmt_type 1226 +#define invalid_with_stmt_indent_type 1227 +#define invalid_try_stmt_type 1228 +#define invalid_except_stmt_type 1229 +#define invalid_except_star_stmt_type 1230 +#define invalid_finally_stmt_type 1231 +#define invalid_except_stmt_indent_type 1232 +#define invalid_except_star_stmt_indent_type 1233 +#define invalid_match_stmt_type 1234 +#define invalid_case_block_type 1235 +#define invalid_as_pattern_type 1236 +#define invalid_class_pattern_type 1237 +#define invalid_mapping_pattern_type 1238 +#define invalid_class_argument_pattern_type 1239 +#define invalid_if_stmt_type 1240 +#define invalid_elif_stmt_type 1241 +#define invalid_else_stmt_type 1242 +#define invalid_while_stmt_type 1243 +#define invalid_for_stmt_type 1244 +#define invalid_def_raw_type 1245 +#define invalid_class_def_raw_type 1246 +#define invalid_double_starred_kvpairs_type 1247 +#define invalid_kvpair_unpacking_type 1248 +#define invalid_kvpair_type 1249 +#define invalid_starred_expression_unpacking_type 1250 +#define invalid_starred_expression_unpacking_sequence_type 1251 +#define invalid_starred_expression_type 1252 +#define invalid_fstring_replacement_field_type 1253 +#define invalid_fstring_conversion_character_type 1254 +#define invalid_tstring_replacement_field_type 1255 +#define invalid_tstring_conversion_character_type 1256 +#define invalid_string_tstring_concat_type 1257 +#define invalid_arithmetic_type 1258 // Left-recursive +#define invalid_factor_type 1259 +#define invalid_type_params_type 1260 +#define invalid_bitwise_and_type 1261 // Left-recursive +#define invalid_bitwise_or_type 1262 // Left-recursive +#define invalid_noteq_type 1263 +#define _loop0_1_type 1264 +#define _loop1_2_type 1265 +#define _loop0_3_type 1266 +#define _gather_4_type 1267 +#define _tmp_5_type 1268 +#define _tmp_6_type 1269 +#define _tmp_7_type 1270 +#define _tmp_8_type 1271 +#define _tmp_9_type 1272 +#define _tmp_10_type 1273 +#define _tmp_11_type 1274 +#define _loop1_12_type 1275 +#define _loop0_13_type 1276 +#define _gather_14_type 1277 +#define _tmp_15_type 1278 +#define _tmp_16_type 1279 +#define _loop0_17_type 1280 +#define _loop1_18_type 1281 +#define _loop0_19_type 1282 +#define _gather_20_type 1283 +#define _tmp_21_type 1284 +#define _loop0_22_type 1285 +#define _gather_23_type 1286 +#define _loop1_24_type 1287 +#define _tmp_25_type 1288 +#define _tmp_26_type 1289 +#define _loop0_27_type 1290 +#define _loop0_28_type 1291 +#define _loop1_29_type 1292 +#define _loop1_30_type 1293 +#define _loop0_31_type 1294 +#define _loop1_32_type 1295 +#define _loop0_33_type 1296 +#define _gather_34_type 1297 +#define _tmp_35_type 1298 +#define _loop1_36_type 1299 +#define _loop1_37_type 1300 +#define _loop1_38_type 1301 +#define _loop0_39_type 1302 +#define _gather_40_type 1303 +#define _tmp_41_type 1304 +#define _tmp_42_type 1305 +#define _tmp_43_type 1306 +#define _loop0_44_type 1307 +#define _gather_45_type 1308 +#define _loop0_46_type 1309 +#define _gather_47_type 1310 +#define _tmp_48_type 1311 +#define _loop0_49_type 1312 +#define _gather_50_type 1313 +#define _loop0_51_type 1314 +#define _gather_52_type 1315 +#define _loop0_53_type 1316 +#define _gather_54_type 1317 +#define _loop1_55_type 1318 +#define _loop1_56_type 1319 +#define _loop0_57_type 1320 +#define _gather_58_type 1321 +#define _loop1_59_type 1322 +#define _loop1_60_type 1323 +#define _loop1_61_type 1324 +#define _tmp_62_type 1325 +#define _loop0_63_type 1326 +#define _gather_64_type 1327 +#define _tmp_65_type 1328 +#define _tmp_66_type 1329 +#define _tmp_67_type 1330 +#define _tmp_68_type 1331 +#define _tmp_69_type 1332 +#define _loop0_70_type 1333 +#define _loop0_71_type 1334 +#define _loop1_72_type 1335 +#define _loop1_73_type 1336 +#define _loop0_74_type 1337 +#define _loop1_75_type 1338 +#define _loop0_76_type 1339 +#define _loop0_77_type 1340 +#define _loop0_78_type 1341 +#define _loop0_79_type 1342 +#define _loop1_80_type 1343 +#define _loop1_81_type 1344 +#define _loop0_82_type 1345 +#define _gather_83_type 1346 +#define _tmp_84_type 1347 +#define _loop0_85_type 1348 +#define _gather_86_type 1349 +#define _loop1_87_type 1350 +#define _loop0_88_type 1351 +#define _tmp_89_type 1352 +#define _loop0_90_type 1353 +#define _gather_91_type 1354 +#define _tmp_92_type 1355 +#define _loop0_93_type 1356 +#define _gather_94_type 1357 +#define _tmp_95_type 1358 +#define _loop0_96_type 1359 +#define _gather_97_type 1360 +#define _loop0_98_type 1361 +#define _tmp_99_type 1362 +#define _tmp_100_type 1363 +#define _loop0_101_type 1364 +#define _gather_102_type 1365 +#define _loop0_103_type 1366 +#define _gather_104_type 1367 +#define _tmp_105_type 1368 +#define _tmp_106_type 1369 +#define _loop0_107_type 1370 +#define _gather_108_type 1371 +#define _tmp_109_type 1372 +#define _tmp_110_type 1373 +#define _tmp_111_type 1374 +#define _tmp_112_type 1375 +#define _loop1_113_type 1376 +#define _tmp_114_type 1377 +#define _tmp_115_type 1378 +#define _tmp_116_type 1379 +#define _tmp_117_type 1380 +#define _tmp_118_type 1381 +#define _loop0_119_type 1382 +#define _loop0_120_type 1383 +#define _tmp_121_type 1384 +#define _tmp_122_type 1385 +#define _tmp_123_type 1386 +#define _tmp_124_type 1387 +#define _tmp_125_type 1388 +#define _tmp_126_type 1389 +#define _tmp_127_type 1390 +#define _tmp_128_type 1391 +#define _loop0_129_type 1392 +#define _gather_130_type 1393 +#define _tmp_131_type 1394 +#define _tmp_132_type 1395 +#define _tmp_133_type 1396 +#define _tmp_134_type 1397 +#define _loop0_135_type 1398 +#define _gather_136_type 1399 +#define _tmp_137_type 1400 +#define _loop0_138_type 1401 +#define _gather_139_type 1402 +#define _loop0_140_type 1403 +#define _gather_141_type 1404 +#define _tmp_142_type 1405 +#define _loop0_143_type 1406 +#define _tmp_144_type 1407 +#define _tmp_145_type 1408 +#define _tmp_146_type 1409 +#define _tmp_147_type 1410 +#define _tmp_148_type 1411 +#define _tmp_149_type 1412 +#define _tmp_150_type 1413 +#define _tmp_151_type 1414 +#define _tmp_152_type 1415 +#define _tmp_153_type 1416 +#define _tmp_154_type 1417 +#define _tmp_155_type 1418 +#define _tmp_156_type 1419 +#define _tmp_157_type 1420 +#define _tmp_158_type 1421 +#define _tmp_159_type 1422 +#define _tmp_160_type 1423 +#define _tmp_161_type 1424 +#define _tmp_162_type 1425 +#define _tmp_163_type 1426 +#define _tmp_164_type 1427 +#define _tmp_165_type 1428 +#define _tmp_166_type 1429 +#define _tmp_167_type 1430 +#define _tmp_168_type 1431 +#define _tmp_169_type 1432 +#define _tmp_170_type 1433 +#define _tmp_171_type 1434 +#define _loop0_172_type 1435 +#define _tmp_173_type 1436 +#define _tmp_174_type 1437 +#define _tmp_175_type 1438 +#define _tmp_176_type 1439 +#define _tmp_177_type 1440 +#define _tmp_178_type 1441 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -536,7 +535,6 @@ static mod_ty eval_rule(Parser *p); static mod_ty func_type_rule(Parser *p); static asdl_stmt_seq* statements_rule(Parser *p); static asdl_stmt_seq* statement_rule(Parser *p); -static asdl_stmt_seq* single_compound_stmt_rule(Parser *p); static asdl_stmt_seq* statement_newline_rule(Parser *p); static asdl_stmt_seq* simple_stmts_rule(Parser *p); static stmt_ty simple_stmt_rule(Parser *p); @@ -1242,7 +1240,7 @@ statement_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ statement[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt")); - _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ) ); + _res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ); if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -1284,50 +1282,7 @@ statement_rule(Parser *p) return _res; } -// single_compound_stmt: compound_stmt -static asdl_stmt_seq* -single_compound_stmt_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _PyPegen_stack_exhausted(p)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - asdl_stmt_seq* _res = NULL; - int _mark = p->mark; - { // compound_stmt - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> single_compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compound_stmt")); - stmt_ty a; - if ( - (a = compound_stmt_rule(p)) // compound_stmt - ) - { - D(fprintf(stderr, "%*c+ single_compound_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt")); - _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ) ); - if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s single_compound_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compound_stmt")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// statement_newline: single_compound_stmt NEWLINE | simple_stmts | NEWLINE | $ +// statement_newline: compound_stmt NEWLINE | simple_stmts | NEWLINE | $ static asdl_stmt_seq* statement_newline_rule(Parser *p) { @@ -1349,22 +1304,22 @@ statement_newline_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // single_compound_stmt NEWLINE + { // compound_stmt NEWLINE if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> statement_newline[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "single_compound_stmt NEWLINE")); - asdl_stmt_seq* a; + D(fprintf(stderr, "%*c> statement_newline[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compound_stmt NEWLINE")); + stmt_ty a; Token * newline_var; if ( - (a = single_compound_stmt_rule(p)) // single_compound_stmt + (a = compound_stmt_rule(p)) // compound_stmt && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ statement_newline[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "single_compound_stmt NEWLINE")); - _res = a; + D(fprintf(stderr, "%*c+ statement_newline[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt NEWLINE")); + _res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ); if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -1374,7 +1329,7 @@ statement_newline_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s statement_newline[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "single_compound_stmt NEWLINE")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compound_stmt NEWLINE")); } { // simple_stmts if (p->error_indicator) { diff --git a/Parser/pegen.c b/Parser/pegen.c index 9b4e4e2033e01d3..895b7576e86c5bc 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -968,7 +968,7 @@ compute_parser_flags(PyCompilerFlags *flags) Parser * _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, - int feature_version, int *errcode, const char* source, PyArena *arena) + int feature_version, int *errcode, PyArena *arena) { Parser *p = PyMem_Malloc(sizeof(Parser)); if (p == NULL) { @@ -1023,10 +1023,6 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, p->level = 0; p->call_invalid_rules = 0; p->input_exhausted = 0; - p->last_stmt_location.lineno = 0; - p->last_stmt_location.col_offset = 0; - p->last_stmt_location.end_lineno = 0; - p->last_stmt_location.end_col_offset = 0; p->keyword_trial = NULL; #ifdef Py_DEBUG p->debug = _Py_GetConfig()->parser_debug; @@ -1052,10 +1048,6 @@ _PyPegen_Parser_Free(Parser *p) static void reset_parser_state_for_error_pass(Parser *p) { - p->last_stmt_location.lineno = 0; - p->last_stmt_location.col_offset = 0; - p->last_stmt_location.end_lineno = 0; - p->last_stmt_location.end_col_offset = 0; for (int i = 0; i < p->fill; i++) { p->tokens[i]->memo = NULL; p->tokens[i]->memo_mask = 0; @@ -1432,7 +1424,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena Py_DECREF(module); Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION, - errcode, NULL, arena); + errcode, arena); if (p == NULL) { goto error; } @@ -1493,7 +1485,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ? flags->cf_feature_version : PY_MINOR_VERSION; Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version, - NULL, str, arena); + NULL, arena); if (p == NULL) { goto error; } diff --git a/Parser/pegen.h b/Parser/pegen.h index deb3550f3e36e0c..d9ef39dcd7b9ff8 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -71,13 +71,6 @@ typedef struct { size_t num_items; } growable_comment_array; -typedef struct { - int lineno; - int col_offset; - int end_lineno; - int end_col_offset; -} location; - typedef struct { int budget; int furthest; @@ -111,7 +104,6 @@ typedef struct { int call_invalid_rules; KeywordTrial *keyword_trial; int debug; - location last_stmt_location; IdentifierCacheEntry *identifier_cache; PyThreadState *tstate; // Cached copy of tstate->c_stack_soft_limit offset by one margin toward @@ -433,12 +425,10 @@ stmt_ty _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_n asdl_alias_seq *names, expr_ty lazy_token, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -asdl_stmt_seq* _PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts); -stmt_ty _PyPegen_register_stmt(Parser *p, stmt_ty s); // Parser API -Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, const char*, PyArena *); +Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *); void _PyPegen_Parser_Free(Parser *); mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *, const char *, const char *, PyCompilerFlags *, int *, PyObject **, From c80fa5cd0aaa758baa81054e1d49ba72b6f48645 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 13:36:53 +0100 Subject: [PATCH 04/10] gh-159672: Keep typo budgets off the normal parser path --- Parser/pegen.c | 21 +++++++++++++++++++++ Parser/pegen.h | 13 ++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 895b7576e86c5bc..8e8320865e42a06 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1045,6 +1045,19 @@ _PyPegen_Parser_Free(Parser *p) PyMem_Free(p); } +int +_PyPegen_stack_exhausted_slow(Parser *p) +{ + if (p->keyword_trial != NULL) { + if (p->keyword_trial->budget == 0) { + return 1; + } + p->keyword_trial->budget--; + p->keyword_trial->furthest = Py_MAX(p->keyword_trial->furthest, p->mark); + } + return _Py_ReachedRecursionLimitWithMargin(p->tstate, 1); +} + static void reset_parser_state_for_error_pass(Parser *p) { @@ -1156,6 +1169,14 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate) trial.error_indicator = 0; trial.call_invalid_rules = 0; trial.keyword_trial = &state; + // Force trials through the existing stack-check slow path to count rule + // entries without adding a branch to normal parsing. The slow path still + // checks the thread's real stack limit. +#if _Py_STACK_GROWS_DOWN + trial.stack_soft_limit = UINTPTR_MAX; +#else + trial.stack_soft_limit = 0; +#endif trial.known_err_token = NULL; trial.errcode = NULL; trial.identifier_cache = identifiers; diff --git a/Parser/pegen.h b/Parser/pegen.h index d9ef39dcd7b9ff8..7516a140e7a59af 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -111,19 +111,14 @@ typedef struct { uintptr_t stack_soft_limit; } Parser; +int _PyPegen_stack_exhausted_slow(Parser *p); + // C-stack exhaustion check run on every rule entry. The fast path compares // the machine stack pointer against the soft limit cached in the Parser; -// only when the stack is nearly exhausted is the full check called. +// trials force the slow path to account for their rule budget. static inline int _PyPegen_stack_exhausted(Parser *p) { - if (p->keyword_trial != NULL) { - if (p->keyword_trial->budget == 0) { - return 1; - } - p->keyword_trial->budget--; - p->keyword_trial->furthest = Py_MAX(p->keyword_trial->furthest, p->mark); - } #if _Py_STACK_GROWS_DOWN if (_Py_get_machine_stack_pointer() > p->stack_soft_limit) { #else @@ -132,7 +127,7 @@ _PyPegen_stack_exhausted(Parser *p) assert(!_Py_ReachedRecursionLimitWithMargin(p->tstate, 1)); return 0; } - return _Py_ReachedRecursionLimitWithMargin(p->tstate, 1); + return _PyPegen_stack_exhausted_slow(p); } typedef struct { From 0fcbf7069ee8229f8940b194f56532b11efe7180 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 13:37:02 +0100 Subject: [PATCH 05/10] gh-159672: Bound keyword replay source and total work --- Lib/test/test_traceback.py | 27 +++++++++++++++++++++++++++ Parser/pegen.c | 16 ++++++++++------ Parser/pegen.h | 1 + Parser/pegen_errors.c | 7 +++++++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 854bcf2f264c91a..5f8654cd1ef1251 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1948,6 +1948,33 @@ def test_keyword_trials_do_not_call_normalizer(self): normalize.assert_not_called() self.assertIsNone(caught.exception._metadata) + @cpython_only + def test_keyword_replay_source_limit(self): + with tempfile.TemporaryDirectory() as script_dir: + for size in (64 * 1024 - 512, 64 * 1024 + 512): + literal = "value = " + repr("a" * size) + "\n" + for source in (literal + "impor math", "impor math\n" + literal): + with self.subTest(size=size, typo_first=source.startswith("impor")): + with self.assertRaises(SyntaxError) as caught: + compile(source, "", "exec") + expected = size < 64 * 1024 + self.assertEqual(caught.exception._metadata is not None, expected) + script = make_script(script_dir, "large_keyword_typo", source) + _, _, stderr = assert_python_failure(script) + self.assertEqual(b"Did you mean 'import'" in stderr, expected) + # The size limit applies only to optional hints. + compile("value = " + repr("a" * (1024 * 1024)), "", "exec") + + @cpython_only + def test_keyword_replay_work_limit(self): + # Several candidates must share the budget, including failed passes. + for count, expected in ((100, "in"), (300, None)): + source = "x = 1\n" * count + "for x im seq:\n pass" + with self.subTest(count=count), self.assertRaises(SyntaxError) as caught: + compile(source, "", "exec") + metadata = caught.exception._metadata + self.assertEqual(metadata[0] if metadata else None, expected) + @requires_debug_ranges() @force_not_colorized_test_class class PurePythonTracebackErrorCaretTests( diff --git a/Parser/pegen.c b/Parser/pegen.c index 8e8320865e42a06..b32487ac9dec89c 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1122,7 +1122,7 @@ add_keyword_typo(KeywordTypo *candidates, int *count, int mark, } static int -try_keyword_typo(Parser *p, KeywordTypo *candidate) +try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) { // Token text belongs to the original arena. Token headers and memo entries // must be private: even a failed trial can memoize nodes containing its edit. @@ -1159,7 +1159,8 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate) p, candidate->keyword, strlen(candidate->keyword)); edited->bytes = replacement; - KeywordTrial state = {.budget = MAX_KEYWORD_TYPO_STEPS}; + state->furthest = 0; + state->incomplete_suite = 0; Parser trial = *p; trial.tokens = index; trial.arena = arena; @@ -1168,7 +1169,7 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate) trial.level = 0; trial.error_indicator = 0; trial.call_invalid_rules = 0; - trial.keyword_trial = &state; + trial.keyword_trial = state; // Force trials through the existing stack-check slow path to count rule // entries without adding a branch to normal parsing. The slow path still // checks the thread's real stack limit. @@ -1192,7 +1193,7 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate) if (!accepted && !trial.error_indicator && candidate->is_alias && p->feature_version == PY_MINOR_VERSION) { - for (int i = candidate->mark; i < state.furthest; i++) { + for (int i = candidate->mark; i < state->furthest; i++) { if (tokens[i].type == NEWLINE) { accepted = 1; break; @@ -1204,7 +1205,7 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate) _PyPegen_parse(&trial); // Version checks on an enclosing rule can run after its suite. For // older grammars, require a complete repair so those checks execute. - accepted = state.incomplete_suite && p->feature_version == PY_MINOR_VERSION; + accepted = state->incomplete_suite && p->feature_version == PY_MINOR_VERSION; } // No Parser_Free: the trial borrows the tokenizer and type-ignore comments, // and its token headers are a single allocation. @@ -1224,6 +1225,7 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) if ((p->start_rule != Py_file_input && p->start_rule != Py_eval_input && p->start_rule != Py_single_input) || p->fill == 0 || p->fill > MAX_KEYWORD_TYPO_TOKENS || + _PyTokenizer_GetInfo(p->tok).input_span.end > MAX_KEYWORD_TYPO_BYTES || p->tokens[p->fill - 1]->type != ENDMARKER || _PyTokenizer_IsInteractive(p->tok) || !PyUnicode_Check(error->msg)) { @@ -1301,8 +1303,10 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) } } qsort(candidates, count, sizeof(*candidates), compare_keyword_typos); + // All candidates and both parser passes share this work budget. + KeywordTrial state = {.budget = MAX_KEYWORD_TYPO_STEPS}; for (int i = 0; i < count; i++) { - int accepted = try_keyword_typo(p, &candidates[i]); + int accepted = try_keyword_typo(p, &candidates[i], &state); if (accepted < 0) { return NULL; } diff --git a/Parser/pegen.h b/Parser/pegen.h index 7516a140e7a59af..f9fc68ae34ad187 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -31,6 +31,7 @@ // Error-only token replay is bounded independently of the source length. #define MAX_KEYWORD_TYPO_TOKENS 4096 +#define MAX_KEYWORD_TYPO_BYTES (64 * 1024) typedef struct _memo { int type; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 7266cfad2b245d1..9db7bf510c7c9eb 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -160,6 +160,13 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { for (;;) { _PyTokenizer_Get(p->tok, &new_token); + if (cache_tokens && + _PyTokenizer_GetInfo(p->tok).input_span.end > MAX_KEYWORD_TYPO_BYTES) + { + // Logical offsets include discarded input in file mode. Stop + // before copying a large token into the parser's arena. + cache_tokens = 0; + } // Retain the tokens this error check already reads for bounded typo // trials. Never treat a truncated cache as complete input. if (cache_tokens && p->fill < MAX_KEYWORD_TYPO_TOKENS && From a3d51fa55bc32d5c8877ff3ed3859a67dfc62e04 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 14:12:06 +0100 Subject: [PATCH 06/10] gh-159672: Size keyword replay limits for stdlib modules --- Lib/test/test_traceback.py | 10 +++++----- Parser/pegen.c | 2 +- Parser/pegen.h | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5f8654cd1ef1251..e10ce8cb2c761bb 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1924,7 +1924,7 @@ def test_keyword_trials_do_not_repeat_warnings(self): def test_keyword_replay_requires_complete_valid_input(self): for source in ("dell 1", "impor math\nother =", "whille True:\npass", "switch +:\n case:", "function f(=):\n pass", "(mat x)", - "impor math\n" + "pass\n" * 3000): + "impor math\n" + "pass\n" * 20000): with self.subTest(source=source): with self.assertRaises(SyntaxError) as caught: compile(source, "", "exec") @@ -1951,13 +1951,13 @@ def test_keyword_trials_do_not_call_normalizer(self): @cpython_only def test_keyword_replay_source_limit(self): with tempfile.TemporaryDirectory() as script_dir: - for size in (64 * 1024 - 512, 64 * 1024 + 512): + for size in (256 * 1024 - 512, 256 * 1024 + 512): literal = "value = " + repr("a" * size) + "\n" for source in (literal + "impor math", "impor math\n" + literal): with self.subTest(size=size, typo_first=source.startswith("impor")): with self.assertRaises(SyntaxError) as caught: compile(source, "", "exec") - expected = size < 64 * 1024 + expected = size < 256 * 1024 self.assertEqual(caught.exception._metadata is not None, expected) script = make_script(script_dir, "large_keyword_typo", source) _, _, stderr = assert_python_failure(script) @@ -1968,8 +1968,8 @@ def test_keyword_replay_source_limit(self): @cpython_only def test_keyword_replay_work_limit(self): # Several candidates must share the budget, including failed passes. - for count, expected in ((100, "in"), (300, None)): - source = "x = 1\n" * count + "for x im seq:\n pass" + for count, expected in ((100, "lambda"), (2000, None)): + source = "x = 1\n" * count + "f(" + "fron, " * 8 + "lamda x: x)" with self.subTest(count=count), self.assertRaises(SyntaxError) as caught: compile(source, "", "exec") metadata = caught.exception._metadata diff --git a/Parser/pegen.c b/Parser/pegen.c index b32487ac9dec89c..ae0518ed607a0f7 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1082,7 +1082,7 @@ typedef struct { } KeywordTypo; #define MAX_KEYWORD_TYPO_CANDIDATES 32 -#define MAX_KEYWORD_TYPO_STEPS 100000 +#define MAX_KEYWORD_TYPO_STEPS 5000000 static int compare_keyword_typos(const void *a, const void *b) diff --git a/Parser/pegen.h b/Parser/pegen.h index f9fc68ae34ad187..682ea2c838c7259 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -30,8 +30,8 @@ #define CURRENT_POS (-5) // Error-only token replay is bounded independently of the source length. -#define MAX_KEYWORD_TYPO_TOKENS 4096 -#define MAX_KEYWORD_TYPO_BYTES (64 * 1024) +#define MAX_KEYWORD_TYPO_TOKENS 32768 +#define MAX_KEYWORD_TYPO_BYTES (256 * 1024) typedef struct _memo { int type; From 889a092668ccb70a9345e2dfe44dbb74e31e941d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 14:12:33 +0100 Subject: [PATCH 07/10] gh-159672: Cover keyword repairs across logical statements --- Lib/test/test_syntax.py | 4 +- Lib/test/test_traceback.py | 19 ++++++-- Parser/action_helpers.c | 1 + Parser/pegen.c | 96 +++++++++++++++++++++++++++++++------- Parser/pegen.h | 1 + 5 files changed, 96 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 39ab53bb1f8b5fe..4a7726d59177d3a 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -287,11 +287,11 @@ >>> [x for x if range(1)] Traceback (most recent call last): -SyntaxError: 'in' expected after for-loop variables +SyntaxError: invalid syntax. Did you mean 'in'? >>> tuple(x for x if range(1)) Traceback (most recent call last): -SyntaxError: 'in' expected after for-loop variables +SyntaxError: invalid syntax. Did you mean 'in'? >>> [x for x() in a] Traceback (most recent call last): diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index e10ce8cb2c761bb..b889090a4f2628f 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1815,6 +1815,10 @@ class TestKeywordTypoSuggestions(unittest.TestCase): ("[x for x\nin range(3)\nof x]", "if"), ("[123 fur x\nin range(3)\nif x]", "for"), ("for x im n:\n pass", "in"), + ("for x is n:\n pass", "in"), + ("is x in y:\n pass", "if"), + ("try:\n pass\nfinaly:\n pass", "finally"), + ("result = (one\n adn two)", "and"), ("mach x:", "match"), ("math x:", "match"), ("match 1:\n cse 1:", "case"), @@ -1873,17 +1877,22 @@ def test_no_keyword_suggestion_for_comma_errors(self): file=sys.stderr ) ''' - source = textwrap.dedent(code).strip() - rc, stdout, stderr = assert_python_failure('-c', source) - stderr_text = stderr.decode('utf-8') - self.assertIn("Perhaps you forgot a comma", stderr_text) - self.assertNotIn("Did you mean", stderr_text) + for source in (textwrap.dedent(code).strip(), "[foo bar]", "f(foo bar)"): + with self.subTest(source=source): + rc, stdout, stderr = assert_python_failure('-c', source) + stderr_text = stderr.decode('utf-8') + self.assertIn("Perhaps you forgot a comma", stderr_text) + self.assertNotIn("Did you mean", stderr_text) @cpython_only def test_parser_suggestions_need_no_formatting_work(self): cases = [ ("def f():\n returm value", "return", 2, 5), + ("a=b=c=d=e=f=g=h=i=0\ndef fn():\n retrun True", "return", 3, 5), + ("def f(a,b):\n if b:\n yield\n retrun a+b", "return", 4, 5), + ("def f():\n for i in range(10):\n yield i\n\ndfe x():", "def", 5, 1), ("é = 1; impor math", "import", 1, 8), + ("for café is items:\n pass", "in", 1, 10), ("a = 1\nif True:\n pass\nelso:\n pass", "else", 4, 1), ("[123 fur x\nin range(3)\nif x]", "for", 1, 6), ("mach x:\n case 1: pass", "match", 1, 1), diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index f7c556976dbe313..cbf1ae41882d45b 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -986,6 +986,7 @@ _PyPegen_raise_error_for_missing_comma(Parser *p, expr_ty a, expr_ty b) if (p->tokens[p->mark - 1]->level == 0) { return NULL; } + p->missing_comma = 1; // For multi-line expressions (like string concatenations), point to the // last line instead of the first for a more helpful error message. // Use a->col_offset as the starting column since all strings in the diff --git a/Parser/pegen.c b/Parser/pegen.c index ae0518ed607a0f7..2a10d681ac63d80 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1022,6 +1022,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, #endif p->level = 0; p->call_invalid_rules = 0; + p->missing_comma = 0; p->input_exhausted = 0; p->keyword_trial = NULL; #ifdef Py_DEBUG @@ -1107,6 +1108,22 @@ add_keyword_typo(KeywordTypo *candidates, int *count, int mark, max_cost = Py_MIN(max_cost, 2); } Py_ssize_t cost = _Py_UTF8_Edit_Cost(name, word, max_cost); + // Adjacent transpositions are one spelling error. The shared Levenshtein + // distance counts them as two substitutions. + if (cost > 2 && PyUnicode_IS_ASCII(name) && + PyUnicode_GET_LENGTH(name) == PyUnicode_GET_LENGTH(word)) + { + const char *text = (const char *)PyUnicode_1BYTE_DATA(name); + Py_ssize_t i = 0; + while (text[i] && text[i] == keyword[i]) { + i++; + } + if (text[i] && text[i + 1] && text[i] == keyword[i + 1] && + text[i + 1] == keyword[i] && strcmp(text + i + 2, keyword + i + 2) == 0) + { + cost = 2; + } + } Py_DECREF(word); if (cost < 0) { return -1; @@ -1227,23 +1244,28 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) p->fill == 0 || p->fill > MAX_KEYWORD_TYPO_TOKENS || _PyTokenizer_GetInfo(p->tok).input_span.end > MAX_KEYWORD_TYPO_BYTES || p->tokens[p->fill - 1]->type != ENDMARKER || - _PyTokenizer_IsInteractive(p->tok) || !PyUnicode_Check(error->msg)) + _PyTokenizer_IsInteractive(p->tok) || + !Py_IS_TYPE(error, (PyTypeObject *)PyExc_SyntaxError)) { return NULL; } - const char *message = PyUnicode_AsUTF8(error->msg); - if (message == NULL || (strcmp(message, "invalid syntax") != 0 && - strstr(message, "Perhaps you forgot a comma") == NULL)) - { - return NULL; - } - // Prefer an existing missing-comma diagnostic unless the spelling differs - // by only one edit (the shared edit-cost function charges two per edit). - Py_ssize_t max_cost = strcmp(message, "invalid syntax") == 0 ? 4 : 2; long error_line = PyLong_AsLong(error->lineno); - if (error_line < 0) { + long error_offset = PyLong_AsLong(error->offset); + if (error_line < 0 || error_offset < 0) { return NULL; } + Py_ssize_t error_col = -1; + if (error_offset > 0 && error->text != NULL && PyUnicode_Check(error->text)) { + PyObject *prefix = PyUnicode_Substring(error->text, 0, error_offset - 1); + if (prefix == NULL) { + return NULL; + } + const char *text = PyUnicode_AsUTF8AndSize(prefix, &error_col); + Py_DECREF(prefix); + if (text == NULL) { + return NULL; + } + } KeywordTypo candidates[MAX_KEYWORD_TYPO_CANDIDATES]; int count = 0; int start = 0; @@ -1261,15 +1283,44 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) }; for (int i = start; i < p->fill; i++) { Token *token = p->tokens[i]; - if (token->type != NAME || token->lineno > error_line || - PyBytes_GET_SIZE(token->bytes) < 2 || - PyBytes_GET_SIZE(token->bytes) > 10) - { + // Error spans can start before the typo in a multiline expression. + // Keep the complete logical statement as the candidate range. + if (token->type == NEWLINE) { + break; + } + const char *text = token->type == NAME ? PyBytes_AS_STRING(token->bytes) : NULL; + Py_ssize_t text_length = text == NULL ? 0 : PyBytes_GET_SIZE(token->bytes); + int is_keyword = token->type != NAME; + // A keyword can itself be a typo, as in "for x is y". Consider an + // existing keyword only at the reported error, not throughout the line. + if (text == NULL && token->lineno == error_line && token->col_offset == error_col) { + for (int length = 0; length < p->n_keyword_lists && text == NULL; length++) { + for (KeywordToken *k = p->keywords[length]; + k != NULL && k->type != -1; k++) + { + if (k->type == token->type) { + text = k->str; + text_length = length; + break; + } + } + } + } + if (text == NULL || text_length < 2 || text_length > 10) { + continue; + } + for (char **k = p->soft_keywords; k != NULL && *k != NULL; k++) { + if (strcmp(text, *k) == 0) { + is_keyword = 1; + break; + } + } + if (is_keyword && (token->lineno != error_line || token->col_offset != error_col)) { continue; } const char *alias = NULL; for (size_t j = 0; j < Py_ARRAY_LENGTH(aliases); j++) { - if (strcmp(PyBytes_AS_STRING(token->bytes), aliases[j].name) == 0) { + if (strcmp(text, aliases[j].name) == 0) { alias = aliases[j].keyword; break; } @@ -1281,13 +1332,19 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) candidates[count++] = (KeywordTypo){i, alias, 0, 1}; continue; } - PyObject *name = PyUnicode_DecodeUTF8( - PyBytes_AS_STRING(token->bytes), PyBytes_GET_SIZE(token->bytes), NULL); + PyObject *name = PyUnicode_DecodeUTF8(text, text_length, NULL); if (name == NULL) { return NULL; } + // Preserve missing-comma diagnostics and correctly spelled keywords + // unless one edit repairs the input. For keywords, only substitute a + // character: deleting one from "for" must not suggest "or". + Py_ssize_t max_cost = p->missing_comma || is_keyword ? 2 : 4; int result = 0; for (int length = 0; length < p->n_keyword_lists && result == 0; length++) { + if (is_keyword && length != text_length) { + continue; + } for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1 && result == 0; k++) { @@ -1295,6 +1352,9 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) } } for (char **k = p->soft_keywords; k != NULL && *k != NULL && result == 0; k++) { + if (is_keyword && (Py_ssize_t)strlen(*k) != text_length) { + continue; + } result = add_keyword_typo(candidates, &count, i, name, *k, max_cost); } Py_DECREF(name); diff --git a/Parser/pegen.h b/Parser/pegen.h index 682ea2c838c7259..aa3f7058013908c 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -103,6 +103,7 @@ typedef struct { Token *known_err_token; int level; int call_invalid_rules; + int missing_comma; KeywordTrial *keyword_trial; int debug; IdentifierCacheEntry *identifier_cache; From 05fb3a1c720cf28829b18e6f2a2cee95a95ac776 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 15:21:27 +0100 Subject: [PATCH 08/10] gh-159672: Simplify keyword trial cleanup --- Lib/test/test_traceback.py | 6 ------ Parser/pegen.c | 38 +++++++++++++++++--------------------- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b889090a4f2628f..e37504178258600 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1859,12 +1859,6 @@ def test_keyword_suggestions_from_single_input(self): with self.assertRaises(SyntaxError) as caught: compile(source + ending, "", "single") self.assertEqual(caught.exception._metadata[0], keyword) - with (unittest.mock.patch("builtins.compile") as compile_mock, - unittest.mock.patch("builtins.open") as open_mock): - text = ''.join(traceback.format_exception_only(caught.exception)) - self.assertIn(f"Did you mean '{keyword}'", text) - compile_mock.assert_not_called() - open_mock.assert_not_called() def test_no_keyword_suggestion_for_comma_errors(self): # When the parser identifies a missing comma, don't suggest diff --git a/Parser/pegen.c b/Parser/pegen.c index 2a10d681ac63d80..32c14367c39de63 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1145,21 +1145,16 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) // must be private: even a failed trial can memoize nodes containing its edit. Token *tokens = PyMem_Malloc(p->fill * sizeof(*tokens)); Token **index = PyMem_Malloc(p->fill * sizeof(*index)); - IdentifierCacheEntry *identifiers = p->identifier_cache == NULL ? NULL : - PyMem_Malloc(IDENTIFIER_CACHE_SIZE * sizeof(*identifiers)); + Parser trial = *p; + trial.identifier_cache = p->identifier_cache == NULL ? NULL : + PyMem_Malloc(IDENTIFIER_CACHE_SIZE * sizeof(*trial.identifier_cache)); PyArena *arena = _PyArena_New(); PyObject *replacement = PyBytes_FromString(candidate->keyword); + int accepted = -1; if (tokens == NULL || index == NULL || arena == NULL || replacement == NULL || - (p->identifier_cache != NULL && identifiers == NULL)) + (p->identifier_cache != NULL && trial.identifier_cache == NULL)) { - PyMem_Free(tokens); - PyMem_Free(index); - PyMem_Free(identifiers); - if (arena != NULL) { - _PyArena_Free(arena); - } - Py_XDECREF(replacement); - return -1; + goto done; } for (int i = 0; i < p->fill; i++) { tokens[i] = *p->tokens[i]; @@ -1167,9 +1162,9 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) tokens[i].memo_mask = 0; index[i] = &tokens[i]; } - if (identifiers != NULL) { - memcpy(identifiers, p->identifier_cache, - IDENTIFIER_CACHE_SIZE * sizeof(*identifiers)); + if (trial.identifier_cache != NULL) { + memcpy(trial.identifier_cache, p->identifier_cache, + IDENTIFIER_CACHE_SIZE * sizeof(*trial.identifier_cache)); } Token *edited = &tokens[candidate->mark]; edited->type = _get_keyword_or_name_type( @@ -1178,7 +1173,6 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) state->furthest = 0; state->incomplete_suite = 0; - Parser trial = *p; trial.tokens = index; trial.arena = arena; trial.size = trial.fill; @@ -1197,10 +1191,9 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) #endif trial.known_err_token = NULL; trial.errcode = NULL; - trial.identifier_cache = identifiers; trial.normalize = NULL; - int accepted = _PyPegen_parse(&trial) != NULL && - !trial.error_indicator && !PyErr_Occurred(); + accepted = _PyPegen_parse(&trial) != NULL && + !trial.error_indicator && !PyErr_Occurred(); for (int i = trial.mark; accepted && i < trial.fill; i++) { accepted = tokens[i].type == NEWLINE || tokens[i].type == ENDMARKER; } @@ -1224,11 +1217,14 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) // older grammars, require a complete repair so those checks execute. accepted = state->incomplete_suite && p->feature_version == PY_MINOR_VERSION; } +done: // No Parser_Free: the trial borrows the tokenizer and type-ignore comments, // and its token headers are a single allocation. PyMem_Free(trial.identifier_cache); - _PyArena_Free(arena); - Py_DECREF(replacement); + if (arena != NULL) { + _PyArena_Free(arena); + } + Py_XDECREF(replacement); PyMem_Free(index); PyMem_Free(tokens); return PyErr_Occurred() ? -1 : accepted; @@ -1365,7 +1361,7 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) qsort(candidates, count, sizeof(*candidates), compare_keyword_typos); // All candidates and both parser passes share this work budget. KeywordTrial state = {.budget = MAX_KEYWORD_TYPO_STEPS}; - for (int i = 0; i < count; i++) { + for (int i = 0; i < count && state.budget > 0; i++) { int accepted = try_keyword_typo(p, &candidates[i], &state); if (accepted < 0) { return NULL; From daa773fd39dc936495f89a6e328e109e6c56e1e6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 16:13:03 +0100 Subject: [PATCH 09/10] gh-159672: Release trial identifiers with their arena Skip immortal interning for replay-only identifiers and dotted names. The parser can then release those strings after each trial. --- Lib/test/test_traceback.py | 24 ++++++++++++++++++++++++ Parser/action_helpers.c | 6 ++++-- Parser/pegen.c | 9 ++++++--- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index e37504178258600..19b0ec5c24b7e1c 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1951,6 +1951,30 @@ def test_keyword_trials_do_not_call_normalizer(self): normalize.assert_not_called() self.assertIsNone(caught.exception._metadata) + @cpython_only + @requires_subprocess() + def test_keyword_trials_do_not_intern_names(self): + assert_python_ok('-c', textwrap.dedent(''' + import sys + + def check(index, suffix): + source = (f"impor math\\nidentifier_after_typo_{index} = 1\\n" + f"import module_after_typo_{index}.child" + suffix) + try: + compile(source, "", "exec") + except SyntaxError as error: + assert (error._metadata is not None) == (not suffix) + else: + raise AssertionError("expected SyntaxError") + + for suffix in ("", "\\nother ="): + check(0, suffix) + before = sys.getunicodeinternedsize(_only_immortal=True) + for index in range(1, 20): + check(index, suffix) + assert sys.getunicodeinternedsize(_only_immortal=True) == before + ''')) + @cpython_only def test_keyword_replay_source_limit(self): with tempfile.TemporaryDirectory() as script_dir: diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index cbf1ae41882d45b..4a35900ecdb6dcf 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -128,8 +128,10 @@ _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name) if (!uni) { return NULL; } - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUnicode_InternImmortal(interp, &uni); + if (p->keyword_trial == NULL) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &uni); + } if (_PyArena_AddPyObject(p->arena, uni) < 0) { Py_DECREF(uni); return NULL; diff --git a/Parser/pegen.c b/Parser/pegen.c index 32c14367c39de63..e230dd46adf53a5 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -662,8 +662,11 @@ _PyPegen_new_identifier(Parser *p, const char *n) goto error; } } - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUnicode_InternImmortal(interp, &id); + // Trial-only identifiers must die with the trial arena. + if (p->keyword_trial == NULL) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &id); + } if (_PyArena_AddPyObject(p->arena, id) < 0) { Py_DECREF(id); @@ -687,7 +690,7 @@ get_cached_identifier(Parser *p, PyObject *bytes) } // Identifiers repeat constantly; a small span-keyed cache skips the // UTF-8 decode + intern for repeated occurrences. Keys point into - // arena-owned token bytes and values are arena-owned interned strings, + // arena-owned token bytes and values are arena-owned strings, // so borrowed references are valid for the lifetime of the parse // (including the second error pass, which reuses parser and arena). Py_ssize_t len = PyBytes_GET_SIZE(bytes); From cadefcb88b5056eb3a701c146c956d1c77a6e007 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 16:13:03 +0100 Subject: [PATCH 10/10] gh-159672: Separate keyword replay stages Separate candidate collection, trial acceptance, and hint construction. Keep trial allocation and cleanup together, and restore the original syntax error through one exit path. --- Parser/pegen.c | 268 +++++++++++++++++++++++------------------ Parser/string_parser.c | 3 +- 2 files changed, 154 insertions(+), 117 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index e230dd46adf53a5..898cefd70120c1d 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1142,7 +1142,43 @@ add_keyword_typo(KeywordTypo *candidates, int *count, int mark, } static int -try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) +keyword_trial_accepts(Parser *p, const KeywordTypo *candidate) +{ + int accepted = _PyPegen_parse(p) != NULL; + if (p->error_indicator || PyErr_Occurred()) { + return 0; + } + for (int i = p->mark; accepted && i < p->fill; i++) { + int type = p->tokens[i]->type; + accepted = type == NEWLINE || type == ENDMARKER; + } + if (accepted) { + return 1; + } + // Version checks on an enclosing rule can run after its suite. For older + // grammars, require a complete repair so those checks execute. + if (p->feature_version != PY_MINOR_VERSION) { + return 0; + } + // Known foreign keywords can identify a valid header even when its body + // contains another error (for example, "switch x:\n case:"). Require the + // ordinary grammar to advance into a later line before accepting that hint. + if (candidate->is_alias) { + for (int i = candidate->mark; i < p->keyword_trial->furthest; i++) { + if (p->tokens[i]->type == NEWLINE) { + return 1; + } + } + } + reset_parser_state_for_error_pass(p); + _PyPegen_parse(p); + return p->keyword_trial->incomplete_suite; +} + +// Return 1 for an accepted repair, 0 for a rejected repair, or -1 to stop the +// search. Raw allocation failures need not set an exception. +static int +try_keyword_typo(Parser *p, const KeywordTypo *candidate, KeywordTrial *state) { // Token text belongs to the original arena. Token headers and memo entries // must be private: even a failed trial can memoize nodes containing its edit. @@ -1195,31 +1231,7 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) trial.known_err_token = NULL; trial.errcode = NULL; trial.normalize = NULL; - accepted = _PyPegen_parse(&trial) != NULL && - !trial.error_indicator && !PyErr_Occurred(); - for (int i = trial.mark; accepted && i < trial.fill; i++) { - accepted = tokens[i].type == NEWLINE || tokens[i].type == ENDMARKER; - } - // Known foreign keywords may identify a valid header even when its body - // contains another error (for example, "switch x:\n case:"). Require the - // ordinary grammar to advance into a later line before accepting that hint. - if (!accepted && !trial.error_indicator && candidate->is_alias && - p->feature_version == PY_MINOR_VERSION) - { - for (int i = candidate->mark; i < state->furthest; i++) { - if (tokens[i].type == NEWLINE) { - accepted = 1; - break; - } - } - } - if (!accepted && !trial.error_indicator && !PyErr_Occurred()) { - reset_parser_state_for_error_pass(&trial); - _PyPegen_parse(&trial); - // Version checks on an enclosing rule can run after its suite. For - // older grammars, require a complete repair so those checks execute. - accepted = state->incomplete_suite && p->feature_version == PY_MINOR_VERSION; - } + accepted = keyword_trial_accepts(&trial, candidate); done: // No Parser_Free: the trial borrows the tokenizer and type-ignore comments, // and its token headers are a single allocation. @@ -1233,39 +1245,60 @@ try_keyword_typo(Parser *p, KeywordTypo *candidate, KeywordTrial *state) return PyErr_Occurred() ? -1 : accepted; } -static PyObject * -keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) +static const char * +keyword_typo_text(Parser *p, Token *token, int at_error, int *is_keyword) { - // A real ENDMARKER is required even for an unfinished suite. A truncated - // token cache is never evidence that a replacement is valid. - if ((p->start_rule != Py_file_input && p->start_rule != Py_eval_input && - p->start_rule != Py_single_input) || - p->fill == 0 || p->fill > MAX_KEYWORD_TYPO_TOKENS || - _PyTokenizer_GetInfo(p->tok).input_span.end > MAX_KEYWORD_TYPO_BYTES || - p->tokens[p->fill - 1]->type != ENDMARKER || - _PyTokenizer_IsInteractive(p->tok) || - !Py_IS_TYPE(error, (PyTypeObject *)PyExc_SyntaxError)) - { + // A keyword can itself be a typo, as in "for x is y". Consider an existing + // keyword only at the reported error, not throughout the statement. + *is_keyword = token->type != NAME; + if (*is_keyword) { + if (!at_error) { + return NULL; + } + // Hard keywords have no token text. Use the generated keyword table. + for (int length = 0; length < p->n_keyword_lists; length++) { + for (KeywordToken *k = p->keywords[length]; + k != NULL && k->type != -1; k++) + { + if (k->type == token->type) { + return k->str; + } + } + } return NULL; } + const char *text = PyBytes_AS_STRING(token->bytes); + for (char **k = p->soft_keywords; k != NULL && *k != NULL; k++) { + if (strcmp(text, *k) == 0) { + *is_keyword = 1; + return at_error ? text : NULL; + } + } + return text; +} + +// Return the candidate count, or -1 if allocation or capacity prevents a +// complete search. Capacity exhaustion does not set an exception. +static int +collect_keyword_typos(Parser *p, PySyntaxErrorObject *error, KeywordTypo *candidates) +{ long error_line = PyLong_AsLong(error->lineno); long error_offset = PyLong_AsLong(error->offset); if (error_line < 0 || error_offset < 0) { - return NULL; + return -1; } Py_ssize_t error_col = -1; if (error_offset > 0 && error->text != NULL && PyUnicode_Check(error->text)) { PyObject *prefix = PyUnicode_Substring(error->text, 0, error_offset - 1); if (prefix == NULL) { - return NULL; + return -1; } - const char *text = PyUnicode_AsUTF8AndSize(prefix, &error_col); + int failed = PyUnicode_AsUTF8AndSize(prefix, &error_col) == NULL; Py_DECREF(prefix); - if (text == NULL) { - return NULL; + if (failed) { + return -1; } } - KeywordTypo candidates[MAX_KEYWORD_TYPO_CANDIDATES]; int count = 0; int start = 0; for (int i = 0; i < p->fill && p->tokens[i]->lineno < error_line; i++) { @@ -1287,34 +1320,15 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) if (token->type == NEWLINE) { break; } - const char *text = token->type == NAME ? PyBytes_AS_STRING(token->bytes) : NULL; - Py_ssize_t text_length = text == NULL ? 0 : PyBytes_GET_SIZE(token->bytes); - int is_keyword = token->type != NAME; - // A keyword can itself be a typo, as in "for x is y". Consider an - // existing keyword only at the reported error, not throughout the line. - if (text == NULL && token->lineno == error_line && token->col_offset == error_col) { - for (int length = 0; length < p->n_keyword_lists && text == NULL; length++) { - for (KeywordToken *k = p->keywords[length]; - k != NULL && k->type != -1; k++) - { - if (k->type == token->type) { - text = k->str; - text_length = length; - break; - } - } - } - } - if (text == NULL || text_length < 2 || text_length > 10) { + int at_error = token->lineno == error_line && token->col_offset == error_col; + int is_keyword; + const char *text = keyword_typo_text(p, token, at_error, &is_keyword); + if (text == NULL) { continue; } - for (char **k = p->soft_keywords; k != NULL && *k != NULL; k++) { - if (strcmp(text, *k) == 0) { - is_keyword = 1; - break; - } - } - if (is_keyword && (token->lineno != error_line || token->col_offset != error_col)) { + Py_ssize_t text_length = token->type == NAME ? + PyBytes_GET_SIZE(token->bytes) : (Py_ssize_t)strlen(text); + if (text_length < 2 || text_length > 10) { continue; } const char *alias = NULL; @@ -1326,14 +1340,14 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) } if (alias != NULL) { if (count == MAX_KEYWORD_TYPO_CANDIDATES) { - return NULL; + return -1; } candidates[count++] = (KeywordTypo){i, alias, 0, 1}; continue; } PyObject *name = PyUnicode_DecodeUTF8(text, text_length, NULL); if (name == NULL) { - return NULL; + return -1; } // Preserve missing-comma diagnostics and correctly spelled keywords // unless one edit repairs the input. For keywords, only substitute a @@ -1358,9 +1372,61 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) } Py_DECREF(name); if (result < 0) { - return NULL; + return -1; } } + return count; +} + +static PyObject * +build_keyword_typo_metadata(Parser *p, const KeywordTypo *candidate) +{ + Token *token = p->tokens[candidate->mark]; + PyObject *line; + if (_PyTokenizer_RetainedSource(p->tok) != NULL) { + Py_ssize_t size; + const char *text = _PyTokenizer_LineView(p->tok, token->lineno, &size); + line = PyUnicode_DecodeUTF8(text, size, "replace"); + } + else { + _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); + line = _PyErr_ProgramDecodedTextObject( + info.filename, token->lineno, info.encoding); + } + if (line == NULL) { + return NULL; + } + Py_ssize_t start = _PyPegen_byte_offset_to_character_offset(line, token->col_offset); + Py_ssize_t end = _PyPegen_byte_offset_to_character_offset(line, token->end_col_offset); + PyObject *metadata = NULL; + if (start >= 0 && end >= 0) { + metadata = Py_BuildValue("(sOinin)", candidate->keyword, line, + token->lineno, start + 1, + token->end_lineno, end + 1); + } + Py_DECREF(line); + return metadata; +} + +static PyObject * +keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) +{ + // A real ENDMARKER is required even for an unfinished suite. A truncated + // token cache is never evidence that a replacement is valid. + if ((p->start_rule != Py_file_input && p->start_rule != Py_eval_input && + p->start_rule != Py_single_input) || + p->fill == 0 || p->fill > MAX_KEYWORD_TYPO_TOKENS || + _PyTokenizer_GetInfo(p->tok).input_span.end > MAX_KEYWORD_TYPO_BYTES || + p->tokens[p->fill - 1]->type != ENDMARKER || + _PyTokenizer_IsInteractive(p->tok)) + { + return NULL; + } + KeywordTypo candidates[MAX_KEYWORD_TYPO_CANDIDATES]; + int count = collect_keyword_typos(p, error, candidates); + if (count <= 0) { + return NULL; + } qsort(candidates, count, sizeof(*candidates), compare_keyword_typos); // All candidates and both parser passes share this work budget. KeywordTrial state = {.budget = MAX_KEYWORD_TYPO_STEPS}; @@ -1369,55 +1435,27 @@ keyword_typo_metadata(Parser *p, PySyntaxErrorObject *error) if (accepted < 0) { return NULL; } - if (!accepted) { - continue; - } - Token *token = p->tokens[candidates[i].mark]; - PyObject *line; - if (_PyTokenizer_RetainedSource(p->tok) != NULL) { - Py_ssize_t size; - const char *text = _PyTokenizer_LineView(p->tok, token->lineno, &size); - line = PyUnicode_DecodeUTF8(text, size, "replace"); - } - else { - _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); - line = _PyErr_ProgramDecodedTextObject( - info.filename, token->lineno, info.encoding); + if (accepted) { + return build_keyword_typo_metadata(p, &candidates[i]); } - if (line == NULL) { - return NULL; - } - Py_ssize_t start = _PyPegen_byte_offset_to_character_offset(line, token->col_offset); - Py_ssize_t end = _PyPegen_byte_offset_to_character_offset(line, token->end_col_offset); - PyObject *metadata = NULL; - if (start >= 0 && end >= 0) { - metadata = Py_BuildValue("(sOinin)", candidates[i].keyword, line, - token->lineno, start + 1, - token->end_lineno, end + 1); - } - Py_DECREF(line); - return metadata; } return NULL; } static void -_PyPegen_set_syntax_error_metadata(Parser *p) { +_PyPegen_set_syntax_error_metadata(Parser *p) +{ PyObject *exc = PyErr_GetRaisedException(); - if (!exc || !PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_SyntaxError)) { - PyErr_SetRaisedException(exc); - return; - } - PySyntaxErrorObject *syntax_error = (PySyntaxErrorObject *)exc; - PyObject *hint = keyword_typo_metadata(p, syntax_error); - if (hint != NULL) { - Py_XSETREF(syntax_error->metadata, hint); - PyErr_SetRaisedException(exc); - return; - } - // Suggestions must never replace the original exception, including when - // a trial runs out of memory or exhausts its rule budget. - PyErr_Clear(); + if (exc != NULL && Py_IS_TYPE(exc, (PyTypeObject *)PyExc_SyntaxError)) { + PySyntaxErrorObject *syntax_error = (PySyntaxErrorObject *)exc; + PyObject *hint = keyword_typo_metadata(p, syntax_error); + if (hint != NULL) { + Py_XSETREF(syntax_error->metadata, hint); + } + // Suggestions must never replace the original exception, including + // when a trial runs out of memory or exhausts its rule budget. + PyErr_Clear(); + } PyErr_SetRaisedException(exc); } diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 3a2017ce4af51d5..66bee7da7e42ad0 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -13,8 +13,7 @@ static int warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_invalid_escape, Token *t) { if (p->call_invalid_rules || p->keyword_trial != NULL) { - // Do not report warnings if we are in the second pass of the parser - // to avoid showing the warning twice. + // Error passes and keyword trials must not repeat parser warnings. return 0; } unsigned char c = (unsigned char)*first_invalid_escape;