Skip to content

Commit 6ec4360

Browse files
chestnut1717chestnut1717
authored
Fix EOF SyntaxError diagnostics (#8429)
* Fix EOF SyntaxError diagnostics * Fix Scope EOF SyntaxError location conversion * Fix Additional Unexpected Test * Docs Add Annotation source_location_in_code_points()
1 parent 12a0a1e commit 6ec4360

8 files changed

Lines changed: 100 additions & 21 deletions

File tree

‎Lib/test/test_eof.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ def test_EOF_single_quote(self):
1818
self.assertEqual(str(cm.exception), expect)
1919
self.assertEqual(cm.exception.offset, 1)
2020

21-
@unittest.expectedFailure # TODO: RUSTPYTHON
2221
def test_EOFS(self):
2322
expect = ("unterminated triple-quoted string literal (detected at line 3) (<string>, line 1)")
2423
with self.assertRaises(SyntaxError) as cm:
@@ -45,7 +44,6 @@ def test_EOFS(self):
4544
self.assertEqual(cm.exception.text, "ä = '''thîs is ")
4645
self.assertEqual(cm.exception.offset, 5)
4746

48-
@unittest.expectedFailure # TODO: RUSTPYTHON
4947
@force_not_colorized
5048
def test_EOFS_with_file(self):
5149
expect = ("(<string>, line 1)")
@@ -86,15 +84,13 @@ def test_EOFS_with_file(self):
8684
' ^',
8785
'SyntaxError: unterminated triple-quoted string literal (detected at line 4)'])
8886

89-
@unittest.expectedFailure # TODO: RUSTPYTHON
9087
@warnings_helper.ignore_warnings(category=SyntaxWarning)
9188
def test_eof_with_line_continuation(self):
9289
expect = "unexpected EOF while parsing (<string>, line 1)"
9390
with self.assertRaises(SyntaxError) as cm:
9491
compile('"\\Xhh" \\', '<string>', 'exec')
9592
self.assertEqual(str(cm.exception), expect)
9693

97-
@unittest.expectedFailure # TODO: RUSTPYTHON
9894
def test_line_continuation_EOF(self):
9995
"""A continuation at the end of input must be an error; bpo2180."""
10096
expect = 'unexpected EOF while parsing (<string>, line 1)'
@@ -127,7 +123,6 @@ def test_line_continuation_EOF(self):
127123
exec('\\')
128124
self.assertEqual(str(cm.exception), expect)
129125

130-
@unittest.expectedFailure # TODO: RUSTPYTHON
131126
@unittest.skipIf(not sys.executable, "sys.executable required")
132127
@force_not_colorized
133128
def test_line_continuation_EOF_from_file_bpo2180(self):

‎Lib/test/test_exceptions.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2145,7 +2145,6 @@ class AssertionErrorTests(unittest.TestCase):
21452145
def tearDown(self):
21462146
unlink(TESTFN)
21472147

2148-
@unittest.expectedFailure # TODO: RUSTPYTHON
21492148
@force_not_colorized
21502149
def test_assertion_error_location(self):
21512150
cases = [

‎Lib/test/test_tokenize.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1922,7 +1922,6 @@ def test_newline_and_space_at_the_end_of_the_source_without_newline(self):
19221922
tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline))
19231923
self.assertEqual(tokens, expected_tokens)
19241924

1925-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'SyntaxError' not found in b'OSError: stream did not contain valid UTF-8\n'
19261925
def test_invalid_character_in_fstring_middle(self):
19271926
# See gh-103824
19281927
script = b'''F"""

‎crates/compiler/src/lib.rs‎

Lines changed: 53 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,13 @@ pub enum CompileError {
5050

5151
impl CompileError {
5252
#[must_use]
53-
pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self {
53+
pub fn from_ruff_parse_error(
54+
error: parser::ParseError,
55+
source_file: &SourceFile,
56+
mode: Mode,
57+
) -> Self {
5458
let raw_location = error.location;
55-
let diagnostic = match cpython_parse_diagnostic_override(&error, source_file) {
59+
let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) {
5660
Some(diagnostic) => diagnostic,
5761
None => default_parse_diagnostic(error, source_file),
5862
};
@@ -129,6 +133,13 @@ fn source_location(source_file: &SourceFile, offset: TextSize) -> SourceLocation
129133
.source_location(offset, PositionEncoding::Utf8)
130134
}
131135

136+
// Call only with UTF-8 character boundaries for Python-facing offsets.
137+
fn source_location_in_code_points(source_file: &SourceFile, offset: TextSize) -> SourceLocation {
138+
source_file
139+
.to_source_code()
140+
.source_location(offset, PositionEncoding::Utf32)
141+
}
142+
132143
fn source_locations(
133144
source_file: &SourceFile,
134145
start: TextSize,
@@ -175,6 +186,21 @@ impl NormalizedParseDiagnostic {
175186
)
176187
}
177188

189+
fn other_in_code_points(
190+
source_file: &SourceFile,
191+
message: String,
192+
start: usize,
193+
end: usize,
194+
) -> Self {
195+
let start = TextSize::new(start as u32);
196+
let end = TextSize::new(end as u32);
197+
Self::new(
198+
parser::ParseErrorType::OtherError(message),
199+
source_location_in_code_points(source_file, start),
200+
source_location_in_code_points(source_file, end),
201+
)
202+
}
203+
178204
const fn with_unclosed_bracket(mut self, is_unclosed_bracket: bool) -> Self {
179205
self.is_unclosed_bracket = is_unclosed_bracket;
180206
self
@@ -184,6 +210,7 @@ impl NormalizedParseDiagnostic {
184210
fn cpython_parse_diagnostic_override(
185211
error: &parser::ParseError,
186212
source_file: &SourceFile,
213+
mode: Mode,
187214
) -> Option<NormalizedParseDiagnostic> {
188215
let source_text = source_file.source_text();
189216

@@ -223,6 +250,18 @@ fn cpython_parse_diagnostic_override(
223250
&error.error,
224251
parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError)
225252
) {
253+
// Only a backslash at the end of the source is an EOF error.
254+
let terminal_backslash = source_text.len().checked_sub(1);
255+
if !matches!(mode, Mode::Eval)
256+
&& terminal_backslash == Some(error.location.start().to_usize())
257+
{
258+
let loc = source_line_end_location(source_file, error.location.start());
259+
return Some(NormalizedParseDiagnostic::new(
260+
parser::ParseErrorType::OtherError("unexpected EOF while parsing".to_owned()),
261+
loc,
262+
loc,
263+
));
264+
}
226265
let loc = source_location(source_file, error.location.start() + TextSize::from(1));
227266
return Some(NormalizedParseDiagnostic::new(
228267
error.error.clone(),
@@ -231,7 +270,15 @@ fn cpython_parse_diagnostic_override(
231270
));
232271
}
233272

234-
source_error!(unterminated_string_error(source_text));
273+
if let Some((message, start, end)) = unterminated_string_error(source_text) {
274+
// The scanner reports quote positions, which are UTF-8 character boundaries.
275+
return Some(NormalizedParseDiagnostic::other_in_code_points(
276+
source_file,
277+
message,
278+
start,
279+
end,
280+
));
281+
}
235282
source_error!(expected_indented_block_error(error, source_text));
236283

237284
if matches!(
@@ -5176,7 +5223,7 @@ fn _compile_with_syntax_warning_handler<'a>(
51765223
};
51775224
let parser_options = parser::ParseOptions::from(parser_mode);
51785225
let parsed = parser::parse(source_file.source_text(), parser_options)
5179-
.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file))?;
5226+
.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?;
51805227
if opts.dont_imply_dedent
51815228
&& matches!(mode, Mode::Single)
51825229
&& let Some(error) = dont_imply_dedent_source_error(&source_file)
@@ -5235,7 +5282,7 @@ pub fn _compile_symtable(
52355282
let res = match mode {
52365283
Mode::Exec | Mode::Single | Mode::BlockExpr => {
52375284
let ast = ruff_python_parser::parse_module(source_file.source_text())
5238-
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?;
5285+
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?;
52395286
if let Some(error) =
52405287
post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default())
52415288
{
@@ -5254,7 +5301,7 @@ pub fn _compile_symtable(
52545301
source_file.source_text(),
52555302
parser::Mode::Expression.into(),
52565303
)
5257-
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?;
5304+
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?;
52585305
if let Some(error) =
52595306
post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default())
52605307
{

‎crates/vm/src/exceptions.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,11 @@ impl VirtualMachine {
245245
_ => true,
246246
};
247247

248-
if same_line {
248+
// A lone continuation at EOF has no highlighted source span.
249+
let lone_line_continuation =
250+
maybe_end_offset == Some(-1) && l_text.to_string_lossy() == "\\";
251+
252+
if same_line && !lone_line_continuation {
249253
let mut end_offset = match maybe_end_offset {
250254
Some(0) | None => offset,
251255
Some(end_offset) => end_offset,

‎crates/vm/src/stdlib/sys.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -818,8 +818,27 @@ pub mod sys {
818818
vm: &VirtualMachine,
819819
) -> PyResult<()> {
820820
let stderr = super::get_stderr(vm)?;
821+
// Keep runtime SyntaxErrors on the normal traceback path.
822+
let has_traceback = !vm.is_none(&exc_tb);
821823
match vm.normalize_exception(exc_type, exc_val.clone(), exc_tb) {
822824
Ok(exc) => {
825+
let native_syntax_error_display = !has_traceback
826+
&& exc.fast_isinstance(vm.ctx.exceptions.syntax_error)
827+
&& exc
828+
.as_object()
829+
.get_attr("msg", vm)
830+
.ok()
831+
.and_then(|msg| msg.downcast::<PyStr>().ok())
832+
.is_some_and(|msg| msg.to_string_lossy() == "unexpected EOF while parsing")
833+
&& exc
834+
.as_object()
835+
.get_attr("text", vm)
836+
.ok()
837+
.and_then(|text| text.downcast::<PyStr>().ok())
838+
.is_some_and(|text| text.to_string_lossy().trim_end() == "\\");
839+
if native_syntax_error_display {
840+
return vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc);
841+
}
823842
// PyErr_Display: try traceback._print_exception_bltin first
824843
if let Ok(tb_mod) = vm.import("traceback", 0)
825844
&& let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm)

‎crates/vm/src/vm/python_run.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,10 @@ mod file_run {
113113
"source code cannot contain null bytes".into(),
114114
));
115115
}
116+
#[cfg(feature = "parser")]
117+
// Match compile() by honoring BOMs and encoding cookies in files.
118+
let source = self.decode_source_bytes(&source_bytes, path, false)?;
119+
#[cfg(not(feature = "parser"))]
116120
let source = String::from_utf8(source_bytes)
117121
.map_err(|err| self.new_os_error(err.to_string()))?;
118122
let code_obj = self

‎crates/vm/src/vm/vm_new.rs‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ impl VirtualMachine {
752752
Some(line + "\n")
753753
}
754754

755-
let statement = source.and_then(|src| get_statement(src, error.location()));
755+
let mut statement = source.and_then(|src| get_statement(src, error.location()));
756756

757757
let mut msg = error.to_string();
758758
if !msg.starts_with("Exceeds the limit ")
@@ -799,6 +799,16 @@ impl VirtualMachine {
799799
}
800800

801801
let SyntaxErrorInfo { msg, narrow_caret } = syntax_error_info;
802+
let unterminated_triple_quoted_string =
803+
msg.starts_with("unterminated triple-quoted string literal");
804+
let unexpected_eof_error = msg == "unexpected EOF while parsing";
805+
if unterminated_triple_quoted_string
806+
&& let Some(statement) = statement.as_mut()
807+
&& statement.ends_with('\n')
808+
{
809+
// CPython omits the parser-added final newline from SyntaxError.text.
810+
statement.pop();
811+
}
802812
let check_version_suite_error = msg.starts_with("Async functions are")
803813
|| msg.starts_with("Async for loops are")
804814
|| msg.starts_with("Async with statements are")
@@ -820,12 +830,14 @@ impl VirtualMachine {
820830

821831
// Set end_lineno and end_offset if available
822832
if let Some((end_lineno, end_offset)) = error.python_end_location() {
823-
let (end_lineno, end_offset) = if check_version_suite_error
824-
&& statement
825-
.as_deref()
826-
.and_then(|line| line.chars().next())
827-
.is_some_and(|ch| ch.is_ascii_whitespace())
828-
{
833+
// EOF errors have no source span in CPython.
834+
let no_end_offset = unexpected_eof_error
835+
|| (check_version_suite_error
836+
&& statement
837+
.as_deref()
838+
.and_then(|line| line.chars().next())
839+
.is_some_and(|ch| ch.is_ascii_whitespace()));
840+
let (end_lineno, end_offset) = if no_end_offset {
829841
(end_lineno, -1)
830842
} else if line_end_binary_operator_error && end_offset == offset_raw {
831843
(end_lineno, (end_offset + 1) as isize)

0 commit comments

Comments
 (0)