diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index f5a0bc56958..582e5b6de6e 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -18,7 +18,6 @@ def test_EOF_single_quote(self): self.assertEqual(str(cm.exception), expect) self.assertEqual(cm.exception.offset, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_EOFS(self): expect = ("unterminated triple-quoted string literal (detected at line 3) (, line 1)") with self.assertRaises(SyntaxError) as cm: @@ -45,7 +44,6 @@ def test_EOFS(self): self.assertEqual(cm.exception.text, "ä = '''thîs is ") self.assertEqual(cm.exception.offset, 5) - @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_EOFS_with_file(self): expect = ("(, line 1)") @@ -86,7 +84,6 @@ def test_EOFS_with_file(self): ' ^', 'SyntaxError: unterminated triple-quoted string literal (detected at line 4)']) - @unittest.expectedFailure # TODO: RUSTPYTHON @warnings_helper.ignore_warnings(category=SyntaxWarning) def test_eof_with_line_continuation(self): expect = "unexpected EOF while parsing (, line 1)" @@ -94,7 +91,6 @@ def test_eof_with_line_continuation(self): compile('"\\Xhh" \\', '', 'exec') self.assertEqual(str(cm.exception), expect) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_line_continuation_EOF(self): """A continuation at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (, line 1)' @@ -127,7 +123,6 @@ def test_line_continuation_EOF(self): exec('\\') self.assertEqual(str(cm.exception), expect) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(not sys.executable, "sys.executable required") @force_not_colorized def test_line_continuation_EOF_from_file_bpo2180(self): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 7ab4c810a08..7c81c4b3905 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2145,7 +2145,6 @@ class AssertionErrorTests(unittest.TestCase): def tearDown(self): unlink(TESTFN) - @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_assertion_error_location(self): cases = [ diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 5ed844c34f0..0e81c6f6db2 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1922,7 +1922,6 @@ def test_newline_and_space_at_the_end_of_the_source_without_newline(self): tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline)) self.assertEqual(tokens, expected_tokens) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'SyntaxError' not found in b'OSError: stream did not contain valid UTF-8\n' def test_invalid_character_in_fstring_middle(self): # See gh-103824 script = b'''F""" diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 43f7174d804..ab8c4dd96f2 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -50,9 +50,13 @@ pub enum CompileError { impl CompileError { #[must_use] - pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self { + pub fn from_ruff_parse_error( + error: parser::ParseError, + source_file: &SourceFile, + mode: Mode, + ) -> Self { let raw_location = error.location; - let diagnostic = match cpython_parse_diagnostic_override(&error, source_file) { + let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) { Some(diagnostic) => diagnostic, None => default_parse_diagnostic(error, source_file), }; @@ -129,6 +133,13 @@ fn source_location(source_file: &SourceFile, offset: TextSize) -> SourceLocation .source_location(offset, PositionEncoding::Utf8) } +// Call only with UTF-8 character boundaries for Python-facing offsets. +fn source_location_in_code_points(source_file: &SourceFile, offset: TextSize) -> SourceLocation { + source_file + .to_source_code() + .source_location(offset, PositionEncoding::Utf32) +} + fn source_locations( source_file: &SourceFile, start: TextSize, @@ -175,6 +186,21 @@ impl NormalizedParseDiagnostic { ) } + fn other_in_code_points( + source_file: &SourceFile, + message: String, + start: usize, + end: usize, + ) -> Self { + let start = TextSize::new(start as u32); + let end = TextSize::new(end as u32); + Self::new( + parser::ParseErrorType::OtherError(message), + source_location_in_code_points(source_file, start), + source_location_in_code_points(source_file, end), + ) + } + const fn with_unclosed_bracket(mut self, is_unclosed_bracket: bool) -> Self { self.is_unclosed_bracket = is_unclosed_bracket; self @@ -184,6 +210,7 @@ impl NormalizedParseDiagnostic { fn cpython_parse_diagnostic_override( error: &parser::ParseError, source_file: &SourceFile, + mode: Mode, ) -> Option { let source_text = source_file.source_text(); @@ -223,6 +250,18 @@ fn cpython_parse_diagnostic_override( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) ) { + // Only a backslash at the end of the source is an EOF error. + let terminal_backslash = source_text.len().checked_sub(1); + if !matches!(mode, Mode::Eval) + && terminal_backslash == Some(error.location.start().to_usize()) + { + let loc = source_line_end_location(source_file, error.location.start()); + return Some(NormalizedParseDiagnostic::new( + parser::ParseErrorType::OtherError("unexpected EOF while parsing".to_owned()), + loc, + loc, + )); + } let loc = source_location(source_file, error.location.start() + TextSize::from(1)); return Some(NormalizedParseDiagnostic::new( error.error.clone(), @@ -231,7 +270,15 @@ fn cpython_parse_diagnostic_override( )); } - source_error!(unterminated_string_error(source_text)); + if let Some((message, start, end)) = unterminated_string_error(source_text) { + // The scanner reports quote positions, which are UTF-8 character boundaries. + return Some(NormalizedParseDiagnostic::other_in_code_points( + source_file, + message, + start, + end, + )); + } source_error!(expected_indented_block_error(error, source_text)); if matches!( @@ -5176,7 +5223,7 @@ fn _compile_with_syntax_warning_handler<'a>( }; let parser_options = parser::ParseOptions::from(parser_mode); let parsed = parser::parse(source_file.source_text(), parser_options) - .map_err(|err| CompileError::from_ruff_parse_error(err, &source_file))?; + .map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?; if opts.dont_imply_dedent && matches!(mode, Mode::Single) && let Some(error) = dont_imply_dedent_source_error(&source_file) @@ -5235,10 +5282,12 @@ pub fn _compile_symtable( let res = match mode { Mode::Exec | Mode::Single | Mode::BlockExpr => { let ast = ruff_python_parser::parse_module(source_file.source_text()) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?; + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; + if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { + return Err(error); } let ast = ast.into_syntax(); @@ -5254,7 +5303,7 @@ pub fn _compile_symtable( source_file.source_text(), parser::Mode::Expression.into(), ) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?; + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 4facffee15d..ffe5a6f0a41 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -245,7 +245,11 @@ impl VirtualMachine { _ => true, }; - if same_line { + // A lone continuation at EOF has no highlighted source span. + let lone_line_continuation = + maybe_end_offset == Some(-1) && l_text.to_string_lossy() == "\\"; + + if same_line && !lone_line_continuation { let mut end_offset = match maybe_end_offset { Some(0) | None => offset, Some(end_offset) => end_offset, diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 8f264d7a739..71aeccf34da 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -818,8 +818,27 @@ pub mod sys { vm: &VirtualMachine, ) -> PyResult<()> { let stderr = super::get_stderr(vm)?; + // Keep runtime SyntaxErrors on the normal traceback path. + let has_traceback = !vm.is_none(&exc_tb); match vm.normalize_exception(exc_type, exc_val.clone(), exc_tb) { Ok(exc) => { + let native_syntax_error_display = !has_traceback + && exc.fast_isinstance(vm.ctx.exceptions.syntax_error) + && exc + .as_object() + .get_attr("msg", vm) + .ok() + .and_then(|msg| msg.downcast::().ok()) + .is_some_and(|msg| msg.to_string_lossy() == "unexpected EOF while parsing") + && exc + .as_object() + .get_attr("text", vm) + .ok() + .and_then(|text| text.downcast::().ok()) + .is_some_and(|text| text.to_string_lossy().trim_end() == "\\"); + if native_syntax_error_display { + return vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc); + } // PyErr_Display: try traceback._print_exception_bltin first if let Ok(tb_mod) = vm.import("traceback", 0) && let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm) diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index 91d5885e740..a1c2552cef4 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -113,6 +113,10 @@ mod file_run { "source code cannot contain null bytes".into(), )); } + #[cfg(feature = "parser")] + // Match compile() by honoring BOMs and encoding cookies in files. + let source = self.decode_source_bytes(&source_bytes, path, false)?; + #[cfg(not(feature = "parser"))] let source = String::from_utf8(source_bytes) .map_err(|err| self.new_os_error(err.to_string()))?; let code_obj = self diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 48909c1a41e..4df3c639182 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -752,7 +752,7 @@ impl VirtualMachine { Some(line + "\n") } - let statement = source.and_then(|src| get_statement(src, error.location())); + let mut statement = source.and_then(|src| get_statement(src, error.location())); let mut msg = error.to_string(); if !msg.starts_with("Exceeds the limit ") @@ -799,6 +799,16 @@ impl VirtualMachine { } let SyntaxErrorInfo { msg, narrow_caret } = syntax_error_info; + let unterminated_triple_quoted_string = + msg.starts_with("unterminated triple-quoted string literal"); + let unexpected_eof_error = msg == "unexpected EOF while parsing"; + if unterminated_triple_quoted_string + && let Some(statement) = statement.as_mut() + && statement.ends_with('\n') + { + // CPython omits the parser-added final newline from SyntaxError.text. + statement.pop(); + } let check_version_suite_error = msg.starts_with("Async functions are") || msg.starts_with("Async for loops are") || msg.starts_with("Async with statements are") @@ -820,12 +830,14 @@ impl VirtualMachine { // Set end_lineno and end_offset if available if let Some((end_lineno, end_offset)) = error.python_end_location() { - let (end_lineno, end_offset) = if check_version_suite_error - && statement - .as_deref() - .and_then(|line| line.chars().next()) - .is_some_and(|ch| ch.is_ascii_whitespace()) - { + // EOF errors have no source span in CPython. + let no_end_offset = unexpected_eof_error + || (check_version_suite_error + && statement + .as_deref() + .and_then(|line| line.chars().next()) + .is_some_and(|ch| ch.is_ascii_whitespace())); + let (end_lineno, end_offset) = if no_end_offset { (end_lineno, -1) } else if line_end_binary_operator_error && end_offset == offset_raw { (end_lineno, (end_offset + 1) as isize)