From 8dec429e49c5e22f6b40796400da0480641b9489 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 18 Oct 2026 12:39:05 +0100 Subject: [PATCH] gh-159465: Preserve escape errors in unfinished f-strings and t-strings --- Lib/test/test_codeop.py | 50 +++++++++++++++++++ ...-20-00.gh-issue-159465.partial-escapes.rst | 2 + Parser/action_helpers.c | 22 ++++++++ Parser/lexer/state.c | 1 + Parser/lexer/string.c | 11 +++- Parser/pegen.c | 1 + Parser/pegen.h | 7 +-- Parser/string_parser.c | 8 ++- Parser/tokenizer/tokenizer.h | 8 +-- 9 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 69dbc2ddfb6bbe4..c54cb0d72fcd7b5 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -176,6 +176,56 @@ def test_string_continuation(self, symbol, prefix): with self.assertRaises(_IncompleteInputError): Compile()(source, '', symbol) + @subTests('symbol', ('single', 'exec', 'eval')) + @subTests('prefix', ('f', 't')) + @subTests('quote', ('"', '"""')) + def test_invalid_escape_in_incomplete_string(self, symbol, prefix, quote): + for escape in (r'\xZZ', r'\uZZZZ', r'\U00110000', r'\x', r'\u12'): + for body in (escape, escape + '{0}', '{0}' + escape, + '{0:' + escape): + source = prefix + quote + body + '\\' + for compiler in COMPILERS: + with self.subTest(source=source, compiler=compiler): + with self.assertRaises(SyntaxError) as cm: + compiler(source, '', symbol) + self.assertNotIsInstance(cm.exception, _IncompleteInputError) + + @subTests('symbol', ('single', 'exec', 'eval')) + @subTests('prefix', ('f', 't')) + @subTests('quote', ('"', '"""')) + def test_valid_escape_in_incomplete_string(self, symbol, prefix, quote): + for escape in (r'\x41', r'\u00e9', r'\U0001f600', r'\\xZZ'): + source = prefix + quote + escape + '\\' + with self.subTest(escape=escape): + self.assertIsNone(compile_command(source, symbol=symbol)) + with self.assertRaises(_IncompleteInputError): + Compile()(source, '', symbol) + for compiler in COMPILERS: + self.assertValid(source + '\n' + quote, symbol, + compiler=compiler) + + source = 'r' + prefix + quote + r'\xZZ' + '\\' + self.assertIsNone(compile_command(source, symbol=symbol)) + with self.assertRaises(_IncompleteInputError): + Compile()(source, '', symbol) + for compiler in COMPILERS: + self.assertValid(source + '\n' + quote, symbol, compiler=compiler) + + @subTests('symbol', ('single', 'exec', 'eval')) + @subTests('prefix', ('f', 't')) + def test_unfinished_escape_in_incomplete_string(self, symbol, prefix): + for escape, suffix in ((r'\x', '41'), (r'\u12', '34'), + (r'\U0001', 'f600'), + (r'\N{LATIN', ' SMALL LETTER A}')): + source = prefix + '"""' + escape + with self.subTest(escape=escape): + self.assertIsNone(compile_command(source, symbol=symbol)) + with self.assertRaises(_IncompleteInputError): + Compile()(source, '', symbol) + for compiler in COMPILERS: + self.assertValid(source + suffix + '"""', symbol, + compiler=compiler) + @subTests('symbol', ('single', 'exec', 'eval')) @subTests('newline', ('\n', '\r', '\r\n')) def test_continued_final_line(self, symbol, newline): diff --git a/Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst b/Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst new file mode 100644 index 000000000000000..ecddc0f540761d1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst @@ -0,0 +1,2 @@ +Preserve :exc:`SyntaxError` for malformed escapes in unfinished f-strings and +t-strings passed to :func:`codeop.compile_command`. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 2f8a5d84dd9755b..0047de66b3a2341 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1507,8 +1507,15 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok); if (str == NULL) { + if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) || tok->is_incomplete) { + _Pypegen_raise_decode_error(p); + } return NULL; } + if (tok->is_incomplete) { + Py_DECREF(str); + return _PyPegen_raise_incomplete_input(p); + } if (_PyArena_AddPyObject(p->arena, str) < 0) { Py_DECREF(str); return NULL; @@ -1523,6 +1530,21 @@ expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) { return _PyPegen_raise_incomplete_input(p); } const char *bstr = PyBytes_AS_STRING(tok->bytes); + Py_ssize_t bsize = PyBytes_GET_SIZE(tok->bytes); + if (((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) || tok->is_incomplete) && + !tok->is_raw && memchr(bstr, '\\', bsize) != NULL) { + // Check escapes before the string is complete, without emitting warnings. + PyObject *decoded = _PyPegen_decode_string( + p, 0, bstr, bsize, tok->is_incomplete ? tok : NULL); + if (decoded == NULL) { + _Pypegen_raise_decode_error(p); + return NULL; + } + Py_DECREF(decoded); + } + if (tok->is_incomplete) { + return _PyPegen_raise_incomplete_input(p); + } PyObject* str = PyUnicode_FromString(bstr); if (str == NULL) { return NULL; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 71f1c86461c4bfb..33d334f98fe3739 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -86,6 +86,7 @@ _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTo { token->level = tok->level; token->is_raw = 0; + token->is_incomplete = 0; assert((start == -1 && end == -1) || (start >= 0 && end >= start)); token->span = (_PyTok_Span){start, end}; if (start >= 0) { diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index c2bef9773eab937..cd840195ee41340 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -27,6 +27,7 @@ incomplete_string_token(struct tok_state *tok, struct token *token, { string_error_token(tok, token, start, location); tok->done = E_INCOMPLETE; + token->is_incomplete = 1; return type; } @@ -464,8 +465,16 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok return MAKE_TOKEN(ERRORTOKEN); } if (c == EOF && tok->allow_incomplete_input) { - return incomplete_string_token(tok, token, + int type = incomplete_string_token(tok, token, FTSTRING_MIDDLE(current), current->start, current->start_loc); + // Preserve the actual prefix for incremental escape decoding. + _PyTok_Off end = tok->cur; + if (end > tok->start && tok->implicit_newline) { + end--; + } + token->span = (_PyTok_Span){tok->start, end}; + token->is_raw = raw; + return type; } if (in_format_spec && c == '\n') { diff --git a/Parser/pegen.c b/Parser/pegen.c index bb14a1e78c5ad35..5f638327a7ac932 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -279,6 +279,7 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to parser_token->level = new_token->level; parser_token->is_raw = new_token->is_raw; + parser_token->is_incomplete = new_token->is_incomplete; parser_token->lineno = new_token->start_loc.lineno; parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno ? p->starting_col_offset + new_token->start_loc.byte_col diff --git a/Parser/pegen.h b/Parser/pegen.h index 44f993101328397..20a561f9b86e39e 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -39,11 +39,12 @@ typedef struct _memo { typedef struct { int type; // Text required by grammar actions or helpers; otherwise NULL. - // An unfinished string token also has NULL text. Its grammar action - // requests more input only if the token is valid at that position. + // Unfinished f-string and t-string middle tokens retain their text so + // their grammar action can check escapes before requesting more input. PyObject *bytes; int level; - int is_raw; + unsigned char is_raw; + unsigned char is_incomplete; int lineno, col_offset, end_lineno, end_col_offset; Memo *memo; // Filter over the rule types present in `memo` (bit `type & 63` is set diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 5df7ae8bec04b25..360d4a161db9b62 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -194,13 +194,17 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) int first_invalid_escape_char; const char *first_invalid_escape_ptr; - v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL, NULL, + // Allow a trailing unfinished escape in a partial token. + int incomplete = t != NULL && t->is_incomplete; + Py_ssize_t consumed; + v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL, + incomplete ? &consumed : NULL, &first_invalid_escape_char, &first_invalid_escape_ptr); // HACK: later we can simply pass the line no, since we don't preserve the tokens // when we are decoding the string but we preserve the line numbers. - if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL) { + if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL && !incomplete) { if (warn_invalid_escape_sequence(parser, buf, first_invalid_escape_ptr, t) < 0) { /* We have not deallocated the buffer before because * first_invalid_escape_ptr points inside buf. */ diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index 149bca06b8b7099..1dba5c6eeb2c22b 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -24,7 +24,8 @@ enum { struct token { int type; int level; - int is_raw; + unsigned char is_raw; + unsigned char is_incomplete; _PyTok_Span span; _PyTok_Loc start_loc; _PyTok_Loc end_loc; @@ -69,8 +70,9 @@ typedef struct { /* Get replaces the initialized token, releasing its previous metadata. Errors are returned as ERRORTOKEN, with or without a Python exception. - In incomplete-input mode, an unfinished string retains its token type - with an absent span and E_INCOMPLETE status. No Python exception is set. */ + In incomplete-input mode, an unfinished string retains its token type, + sets is_incomplete and has E_INCOMPLETE status. Only f-string and t-string + middle tokens retain their text span. No Python exception is set. */ void _PyTokenizer_Get(struct tok_state *, struct token *); void _PyTokenizer_Free(struct tok_state *); int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *);