From 557bec1adf1c9c64e17c7fea42cbc36d8c09f8fb Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:48:48 +0900 Subject: [PATCH 01/28] sre_engine: resume the tail match at prefix_skip, not past the whole prefix (#8473) `search_info_literal`'s len>1 arm reset the tail-match cursor by advancing one character past the matched prefix. That position equals the INFO block's prefix_skip boundary only when the prefix ends where the skip does; for a pattern like `ab(cd)`, `_get_literal_prefix` reports prefix_len=4 with prefix_skip=2, so the tail resumed two characters too far. `state.cursor` already holds `req.start + skip`, which is what `sre_lib.h` SRE(search) computes as `ptr - (prefix_len - prefix_skip - 1)`, and which the len==1 arm above already uses. Take it unconditionally. Searching "xabcdcd" for `ab(cd)` returned span (1, 7) instead of (1, 5); over a 600-case corpus stamped with CPython's own answers, 94 cases disagreed before this change and none after. The wrong resume also dropped matches outright, not only widened them. Assisted-by: Claude --- crates/sre_engine/src/engine.rs | 10 +++++----- crates/sre_engine/tests/tests.rs | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/sre_engine/src/engine.rs b/crates/sre_engine/src/engine.rs index a110a75d65f..690801e0d9d 100644 --- a/crates/sre_engine/src/engine.rs +++ b/crates/sre_engine/src/engine.rs @@ -998,12 +998,12 @@ fn search_info_literal( return true; } + // `state.cursor` is `req.start + skip`, the position the + // tail match resumes from; advancing past the prefix + // instead would resume at `req.start + len` and only + // agree when the prefix ends at the skip boundary. let mut next_ctx = ctx; - if skip != 0 { - next_ctx.advance_char::(); - } else { - next_ctx.cursor = state.cursor; - } + next_ctx.cursor = state.cursor; if _match(req, state, next_ctx) { return true; diff --git a/crates/sre_engine/tests/tests.rs b/crates/sre_engine/tests/tests.rs index 53f5225d4ad..0a9f45c374e 100644 --- a/crates/sre_engine/tests/tests.rs +++ b/crates/sre_engine/tests/tests.rs @@ -252,4 +252,19 @@ mod tests { #[rustfmt::skip] let p = Pattern { pattern: "\u{e0}+", code: &[14, 4, 0, 1, 4294967295, 24, 6, 1, 4294967295, 16, 224, 1, 1] }; // END GENERATED } + + #[test] + fn search_literal_prefix_longer_than_skip() { + // The INFO block carries prefix_len=4 and prefix_skip=2, so the tail + // match has to resume at the skip boundary rather than past the whole + // prefix. + // pattern p = re.compile('ab(cd)') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "ab(cd)", code: &[14, 14, 1, 4, 4, 4, 2, 97, 98, 99, 100, 0, 0, 0, 0, 16, 97, 16, 98, 17, 0, 16, 99, 16, 100, 17, 1, 1] }; + // END GENERATED + let (req, mut state) = p.state("xabcdcd"); + assert!(state.search(req)); + assert_eq!(state.start, 1); + assert_eq!(state.cursor.position, 5); + } } From 12a0a1ed8b9b4cd3292ca936d01cb18e3dd9779a Mon Sep 17 00:00:00 2001 From: Seonghun An <53287605+shAn-kor@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:58:07 +0900 Subject: [PATCH 02/28] Implement negative zero coercion format option (#8475) Assisted-by: Codex:gpt-5 --- Lib/test/test_format.py | 1 - crates/common/src/format.rs | 186 ++++++++++++++++++++++++++++++++---- crates/vm/src/format.rs | 6 ++ 3 files changed, 173 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index f6452341e1e..aa28108312e 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -558,7 +558,6 @@ def test_unicode_in_error_message(self): with self.assertRaisesRegex(ValueError, str_err): "{a:%ЫйЯЧ}".format(a='a') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_negative_zero(self): ## default behavior self.assertEqual(f"{-0.:.1f}", "-0.0") diff --git a/crates/common/src/format.rs b/crates/common/src/format.rs index b5f062f97a6..1c5c0a9c9de 100644 --- a/crates/common/src/format.rs +++ b/crates/common/src/format.rs @@ -229,6 +229,7 @@ pub struct FormatSpec { align: Option, align_specified: bool, sign: Option, + no_neg_0: bool, alternate_form: bool, width: Option, grouping_option: Option, @@ -285,6 +286,14 @@ fn parse_alternate_form(text: &Wtf8) -> (bool, &Wtf8) { } } +fn parse_no_negative_zero(text: &Wtf8) -> (bool, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('z') => (true, chars.as_wtf8()), + _ => (false, text), + } +} + fn parse_zero(text: &Wtf8) -> (bool, &Wtf8) { let mut chars = text.code_points(); match chars.next().and_then(CodePoint::to_char) { @@ -349,6 +358,7 @@ impl FormatSpec { let (mut fill, mut align, text) = parse_fill_and_align(text); let align_specified = align.is_some(); let (sign, text) = FormatSign::parse(text); + let (no_neg_0, text) = parse_no_negative_zero(text); let (alternate_form, text) = parse_alternate_form(text); let (zero, text) = parse_zero(text); let (width, text) = parse_number(text)?; @@ -378,6 +388,7 @@ impl FormatSpec { align, align_specified, sign, + no_neg_0, alternate_form, width, grouping_option, @@ -505,6 +516,25 @@ impl FormatSpec { Ok(()) } + fn formatted_magnitude_is_zero(magnitude: &str) -> bool { + let mut saw_digit = false; + for byte in magnitude.bytes() { + if byte.is_ascii_digit() { + saw_digit = true; + if byte != b'0' { + return false; + } + } + } + saw_digit + } + + fn is_negative_after_zero_coercion(&self, num: f64, magnitude: &str) -> bool { + num.is_sign_negative() + && !num.is_nan() + && !(self.no_neg_0 && Self::formatted_magnitude_is_zero(magnitude)) + } + fn validate_complex_padding_and_alignment(&self) -> Result<(), FormatSpecError> { match &self.fill.unwrap_or_else(|| ' '.into()).to_char() { Some('0') => Err(FormatSpecError::ZeroPadding), @@ -674,6 +704,9 @@ impl FormatSpec { Some(FormatType::Number(Case::Lower)) => self.format_int_radix(magnitude, 10), _ => return self.format_int(num), }?; + if self.no_neg_0 { + return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")); + } let magnitude_str = Self::apply_locale_formatting(raw_magnitude_str, locale); @@ -723,7 +756,7 @@ impl FormatSpec { let magnitude_str = Self::apply_locale_formatting(raw_magnitude_str, locale); let format_sign = self.sign.unwrap_or(FormatSign::Minus); - let sign_str = if num.is_sign_negative() && !num.is_nan() { + let sign_str = if self.is_negative_after_zero_coercion(num, &magnitude_str) { "-" } else { match format_sign { @@ -812,11 +845,16 @@ impl FormatSpec { self.format_float(x as f64) } None => { + if self.no_neg_0 { + return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")); + } let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase(); Ok(first_letter.collect::() + &input.to_string()[1..]) } - Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(*c, "int")), - _ => Err(FormatSpecError::InvalidFormatSpecifier), + Some(format_type) => { + let ch = char::from(format_type); + Err(FormatSpecError::UnknownFormatCode(ch, "bool")) + } } } @@ -925,8 +963,9 @@ impl FormatSpec { }, }, }; + let raw_magnitude_str = raw_magnitude_str?; let format_sign = self.sign.unwrap_or(FormatSign::Minus); - let sign_str = if num.is_sign_negative() && !num.is_nan() { + let sign_str = if self.is_negative_after_zero_coercion(num, &raw_magnitude_str) { "-" } else { match format_sign { @@ -935,7 +974,7 @@ impl FormatSpec { FormatSign::MinusOrSpace => " ", } }; - let magnitude_str = self.add_magnitude_separators(raw_magnitude_str?, sign_str); + let magnitude_str = self.add_magnitude_separators(raw_magnitude_str, sign_str); let magnitude_str = self.add_frac_separators(magnitude_str); Ok( self.format_sign_and_align( @@ -986,15 +1025,24 @@ impl FormatSpec { Err(FormatSpecError::UnknownFormatCode('N', "int")) } Some(FormatType::String) => Err(FormatSpecError::UnknownFormatCode('s', "int")), - Some(FormatType::Character) => match (self.precision, self.sign, self.alternate_form) { - (Some(_), _, _) => Err(FormatSpecError::PrecisionNotAllowed), - (_, Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), - (_, _, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), - (_, _, _) => match num.to_u32() { - Some(n) if n <= 0x10ffff => Ok(core::char::from_u32(n).unwrap().to_string()), - Some(_) | None => Err(FormatSpecError::CodeNotInRange), - }, - }, + Some(FormatType::Character) => { + if self.precision.is_some() { + Err(FormatSpecError::PrecisionNotAllowed) + } else if self.no_neg_0 { + Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")) + } else { + match (self.sign, self.alternate_form) { + (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), + (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), + _ => match num.to_u32() { + Some(n) if n <= 0x10ffff => { + Ok(core::char::from_u32(n).unwrap().to_string()) + } + Some(_) | None => Err(FormatSpecError::CodeNotInRange), + }, + } + } + } Some( FormatType::GeneralFormat(_) | FormatType::FixedPoint(_) @@ -1007,6 +1055,9 @@ impl FormatSpec { Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(c, "int")), None => self.format_int_radix(magnitude, 10), }?; + if self.no_neg_0 { + return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")); + } let format_sign = self.sign.unwrap_or(FormatSign::Minus); let sign_str = match num.sign() { Sign::Minus => "-", @@ -1032,6 +1083,9 @@ impl FormatSpec { self.validate_format(FormatType::String)?; match self.format_type { Some(FormatType::String) | None => { + if self.no_neg_0 { + return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("string")); + } if self.align == Some(FormatAlign::AfterSign) && self.align_specified { return Err(FormatSpecError::StringAlignmentFlag); } @@ -1074,7 +1128,8 @@ impl FormatSpec { // Format real part let formatted_re = if num.re != 0.0 || num.re.is_negative_zero() || self.format_type.is_some() { - let sign_re = if num.re.is_sign_negative() && !num.is_nan() { + let re = self.format_complex_float(num.re)?; + let sign_re = if self.is_negative_after_zero_coercion(num.re, &re) { "-" } else { match self.sign.unwrap_or(FormatSign::Minus) { @@ -1083,21 +1138,24 @@ impl FormatSpec { FormatSign::MinusOrSpace => " ", } }; - let re = self.format_complex_float(num.re)?; format!("{sign_re}{re}") } else { String::new() }; // Format imaginary part - let sign_im = if num.im.is_sign_negative() && !num.im.is_nan() { + let im = self.format_complex_float(num.im)?; + let sign_im = if self.is_negative_after_zero_coercion(num.im, &im) { "-" } else if formatted_re.is_empty() { - "" + match self.sign.unwrap_or(FormatSign::Minus) { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + } } else { "+" }; - let im = self.format_complex_float(num.im)?; Ok((formatted_re, format!("{sign_im}{im}j"))) } @@ -1274,6 +1332,7 @@ pub enum FormatSpecError { CodeNotInRange, ZeroPadding, AlignmentFlag, + NegativeZeroCoercionNotAllowed(&'static str), StringAlignmentFlag, NotImplemented(char, &'static str), } @@ -1625,6 +1684,7 @@ mod tests { align: None, align_specified: false, sign: None, + no_neg_0: false, alternate_form: false, width: Some(33), grouping_option: None, @@ -1643,6 +1703,7 @@ mod tests { align: Some(FormatAlign::Right), align_specified: true, sign: None, + no_neg_0: false, alternate_form: false, width: Some(33), grouping_option: None, @@ -1661,6 +1722,7 @@ mod tests { align: Some(FormatAlign::Right), align_specified: true, sign: Some(FormatSign::Minus), + no_neg_0: false, alternate_form: true, width: Some(23), grouping_option: Some(FormatGrouping::Comma), @@ -1779,6 +1841,92 @@ mod tests { ); } + #[test] + fn format_negative_zero_coercion() { + let int_spec = FormatSpec::parse("z8").unwrap(); + assert_eq!( + int_spec.format_int(&BigInt::from(-42)), + Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")) + ); + assert_eq!( + FormatSpec::parse("zs") + .unwrap() + .format_int(&BigInt::from(0)), + Err(FormatSpecError::UnknownFormatCode('s', "int")) + ); + assert_eq!( + FormatSpec::parse("z.1d") + .unwrap() + .format_int(&BigInt::from(0)), + Err(FormatSpecError::PrecisionNotAllowed) + ); + assert_eq!( + FormatSpec::parse("+zc") + .unwrap() + .format_int(&BigInt::from(0)), + Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")) + ); + + let float_spec = FormatSpec::parse("z.2f").unwrap(); + assert_eq!(float_spec.format_float(-0.0001), Ok("0.00".to_owned())); + + let complex_spec = FormatSpec::parse("z").unwrap(); + assert_eq!( + complex_spec.format_complex(&Complex64::new(-0.0, -0.0)), + Ok("(0+0j)".to_owned()) + ); + let pure_imaginary = Complex64::new(0.0, -0.0); + assert_eq!( + FormatSpec::parse("+z") + .unwrap() + .format_complex(&pure_imaginary), + Ok("+0j".to_owned()) + ); + assert_eq!( + FormatSpec::parse(" z") + .unwrap() + .format_complex(&pure_imaginary), + Ok(" 0j".to_owned()) + ); + + let string_value = "value".to_owned(); + assert_eq!( + FormatSpec::parse("z").unwrap().format_string(&string_value), + Err(FormatSpecError::NegativeZeroCoercionNotAllowed("string")) + ); + assert_eq!( + FormatSpec::parse("zd") + .unwrap() + .format_string(&string_value), + Err(FormatSpecError::UnknownFormatCode('d', "str")) + ); + assert_eq!( + FormatSpec::parse("zs").unwrap().format_bool(false), + Err(FormatSpecError::UnknownFormatCode('s', "bool")) + ); + + let locale = LocaleInfo { + thousands_sep: ",".to_owned(), + decimal_point: ".".to_owned(), + grouping: vec![3, 0], + }; + let locale_spec = FormatSpec::parse("zn").unwrap(); + assert_eq!( + locale_spec.format_float_locale(-0.0, &locale), + Ok("0".to_owned()) + ); + assert_eq!( + locale_spec.format_complex_locale(&Complex64::new(-0.0, -0.0), &locale), + Ok("0+0j".to_owned()) + ); + assert_eq!( + FormatSpec::parse("z.1n") + .unwrap() + .format_int_locale(&BigInt::from(0), &locale), + Err(FormatSpecError::PrecisionNotAllowed) + ); + } + #[test] fn format_int() { assert_eq!( diff --git a/crates/vm/src/format.rs b/crates/vm/src/format.rs index 80b906505bf..657601e1470 100644 --- a/crates/vm/src/format.rs +++ b/crates/vm/src/format.rs @@ -77,6 +77,12 @@ impl IntoPyException for FormatSpecError { Self::AlignmentFlag => { vm.new_value_error("'=' alignment flag is not allowed in complex format specifier") } + Self::NegativeZeroCoercionNotAllowed(type_name) => { + let msg = format!( + "Negative zero coercion (z) not allowed in {type_name} format specifier" + ); + vm.new_value_error(msg) + } Self::StringAlignmentFlag => { vm.new_value_error("'=' alignment not allowed in string format specifier") } From 6ec43604a293dd207730bf05317dec58edaff9a8 Mon Sep 17 00:00:00 2001 From: chestnut1717 <62554639+chestnut1717@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:58:53 +0900 Subject: [PATCH 03/28] Fix EOF SyntaxError diagnostics (#8429) * Fix EOF SyntaxError diagnostics * Fix Scope EOF SyntaxError location conversion * Fix Additional Unexpected Test * Docs Add Annotation source_location_in_code_points() --- Lib/test/test_eof.py | 5 --- Lib/test/test_exceptions.py | 1 - Lib/test/test_tokenize.py | 1 - crates/compiler/src/lib.rs | 59 ++++++++++++++++++++++++++++++---- crates/vm/src/exceptions.rs | 6 +++- crates/vm/src/stdlib/sys.rs | 19 +++++++++++ crates/vm/src/vm/python_run.rs | 4 +++ crates/vm/src/vm/vm_new.rs | 26 +++++++++++---- 8 files changed, 100 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index f5a0bc56958..582e5b6de6e 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -18,7 +18,6 @@ def test_EOF_single_quote(self): self.assertEqual(str(cm.exception), expect) self.assertEqual(cm.exception.offset, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_EOFS(self): expect = ("unterminated triple-quoted string literal (detected at line 3) (, line 1)") with self.assertRaises(SyntaxError) as cm: @@ -45,7 +44,6 @@ def test_EOFS(self): self.assertEqual(cm.exception.text, "ä = '''thîs is ") self.assertEqual(cm.exception.offset, 5) - @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_EOFS_with_file(self): expect = ("(, line 1)") @@ -86,7 +84,6 @@ def test_EOFS_with_file(self): ' ^', 'SyntaxError: unterminated triple-quoted string literal (detected at line 4)']) - @unittest.expectedFailure # TODO: RUSTPYTHON @warnings_helper.ignore_warnings(category=SyntaxWarning) def test_eof_with_line_continuation(self): expect = "unexpected EOF while parsing (, line 1)" @@ -94,7 +91,6 @@ def test_eof_with_line_continuation(self): compile('"\\Xhh" \\', '', 'exec') self.assertEqual(str(cm.exception), expect) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_line_continuation_EOF(self): """A continuation at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (, line 1)' @@ -127,7 +123,6 @@ def test_line_continuation_EOF(self): exec('\\') self.assertEqual(str(cm.exception), expect) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(not sys.executable, "sys.executable required") @force_not_colorized def test_line_continuation_EOF_from_file_bpo2180(self): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 7ab4c810a08..7c81c4b3905 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2145,7 +2145,6 @@ class AssertionErrorTests(unittest.TestCase): def tearDown(self): unlink(TESTFN) - @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_assertion_error_location(self): cases = [ diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 5ed844c34f0..0e81c6f6db2 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1922,7 +1922,6 @@ def test_newline_and_space_at_the_end_of_the_source_without_newline(self): tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline)) self.assertEqual(tokens, expected_tokens) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'SyntaxError' not found in b'OSError: stream did not contain valid UTF-8\n' def test_invalid_character_in_fstring_middle(self): # See gh-103824 script = b'''F""" diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 43f7174d804..7562e8939b9 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -50,9 +50,13 @@ pub enum CompileError { impl CompileError { #[must_use] - pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self { + pub fn from_ruff_parse_error( + error: parser::ParseError, + source_file: &SourceFile, + mode: Mode, + ) -> Self { let raw_location = error.location; - let diagnostic = match cpython_parse_diagnostic_override(&error, source_file) { + let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) { Some(diagnostic) => diagnostic, None => default_parse_diagnostic(error, source_file), }; @@ -129,6 +133,13 @@ fn source_location(source_file: &SourceFile, offset: TextSize) -> SourceLocation .source_location(offset, PositionEncoding::Utf8) } +// Call only with UTF-8 character boundaries for Python-facing offsets. +fn source_location_in_code_points(source_file: &SourceFile, offset: TextSize) -> SourceLocation { + source_file + .to_source_code() + .source_location(offset, PositionEncoding::Utf32) +} + fn source_locations( source_file: &SourceFile, start: TextSize, @@ -175,6 +186,21 @@ impl NormalizedParseDiagnostic { ) } + fn other_in_code_points( + source_file: &SourceFile, + message: String, + start: usize, + end: usize, + ) -> Self { + let start = TextSize::new(start as u32); + let end = TextSize::new(end as u32); + Self::new( + parser::ParseErrorType::OtherError(message), + source_location_in_code_points(source_file, start), + source_location_in_code_points(source_file, end), + ) + } + const fn with_unclosed_bracket(mut self, is_unclosed_bracket: bool) -> Self { self.is_unclosed_bracket = is_unclosed_bracket; self @@ -184,6 +210,7 @@ impl NormalizedParseDiagnostic { fn cpython_parse_diagnostic_override( error: &parser::ParseError, source_file: &SourceFile, + mode: Mode, ) -> Option { let source_text = source_file.source_text(); @@ -223,6 +250,18 @@ fn cpython_parse_diagnostic_override( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) ) { + // Only a backslash at the end of the source is an EOF error. + let terminal_backslash = source_text.len().checked_sub(1); + if !matches!(mode, Mode::Eval) + && terminal_backslash == Some(error.location.start().to_usize()) + { + let loc = source_line_end_location(source_file, error.location.start()); + return Some(NormalizedParseDiagnostic::new( + parser::ParseErrorType::OtherError("unexpected EOF while parsing".to_owned()), + loc, + loc, + )); + } let loc = source_location(source_file, error.location.start() + TextSize::from(1)); return Some(NormalizedParseDiagnostic::new( error.error.clone(), @@ -231,7 +270,15 @@ fn cpython_parse_diagnostic_override( )); } - source_error!(unterminated_string_error(source_text)); + if let Some((message, start, end)) = unterminated_string_error(source_text) { + // The scanner reports quote positions, which are UTF-8 character boundaries. + return Some(NormalizedParseDiagnostic::other_in_code_points( + source_file, + message, + start, + end, + )); + } source_error!(expected_indented_block_error(error, source_text)); if matches!( @@ -5176,7 +5223,7 @@ fn _compile_with_syntax_warning_handler<'a>( }; let parser_options = parser::ParseOptions::from(parser_mode); let parsed = parser::parse(source_file.source_text(), parser_options) - .map_err(|err| CompileError::from_ruff_parse_error(err, &source_file))?; + .map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?; if opts.dont_imply_dedent && matches!(mode, Mode::Single) && let Some(error) = dont_imply_dedent_source_error(&source_file) @@ -5235,7 +5282,7 @@ pub fn _compile_symtable( let res = match mode { Mode::Exec | Mode::Single | Mode::BlockExpr => { let ast = ruff_python_parser::parse_module(source_file.source_text()) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?; + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { @@ -5254,7 +5301,7 @@ pub fn _compile_symtable( source_file.source_text(), parser::Mode::Expression.into(), ) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?; + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 4facffee15d..ffe5a6f0a41 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -245,7 +245,11 @@ impl VirtualMachine { _ => true, }; - if same_line { + // A lone continuation at EOF has no highlighted source span. + let lone_line_continuation = + maybe_end_offset == Some(-1) && l_text.to_string_lossy() == "\\"; + + if same_line && !lone_line_continuation { let mut end_offset = match maybe_end_offset { Some(0) | None => offset, Some(end_offset) => end_offset, diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 8f264d7a739..71aeccf34da 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -818,8 +818,27 @@ pub mod sys { vm: &VirtualMachine, ) -> PyResult<()> { let stderr = super::get_stderr(vm)?; + // Keep runtime SyntaxErrors on the normal traceback path. + let has_traceback = !vm.is_none(&exc_tb); match vm.normalize_exception(exc_type, exc_val.clone(), exc_tb) { Ok(exc) => { + let native_syntax_error_display = !has_traceback + && exc.fast_isinstance(vm.ctx.exceptions.syntax_error) + && exc + .as_object() + .get_attr("msg", vm) + .ok() + .and_then(|msg| msg.downcast::().ok()) + .is_some_and(|msg| msg.to_string_lossy() == "unexpected EOF while parsing") + && exc + .as_object() + .get_attr("text", vm) + .ok() + .and_then(|text| text.downcast::().ok()) + .is_some_and(|text| text.to_string_lossy().trim_end() == "\\"); + if native_syntax_error_display { + return vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc); + } // PyErr_Display: try traceback._print_exception_bltin first if let Ok(tb_mod) = vm.import("traceback", 0) && let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm) diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index 91d5885e740..a1c2552cef4 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -113,6 +113,10 @@ mod file_run { "source code cannot contain null bytes".into(), )); } + #[cfg(feature = "parser")] + // Match compile() by honoring BOMs and encoding cookies in files. + let source = self.decode_source_bytes(&source_bytes, path, false)?; + #[cfg(not(feature = "parser"))] let source = String::from_utf8(source_bytes) .map_err(|err| self.new_os_error(err.to_string()))?; let code_obj = self diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 48909c1a41e..4df3c639182 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -752,7 +752,7 @@ impl VirtualMachine { Some(line + "\n") } - let statement = source.and_then(|src| get_statement(src, error.location())); + let mut statement = source.and_then(|src| get_statement(src, error.location())); let mut msg = error.to_string(); if !msg.starts_with("Exceeds the limit ") @@ -799,6 +799,16 @@ impl VirtualMachine { } let SyntaxErrorInfo { msg, narrow_caret } = syntax_error_info; + let unterminated_triple_quoted_string = + msg.starts_with("unterminated triple-quoted string literal"); + let unexpected_eof_error = msg == "unexpected EOF while parsing"; + if unterminated_triple_quoted_string + && let Some(statement) = statement.as_mut() + && statement.ends_with('\n') + { + // CPython omits the parser-added final newline from SyntaxError.text. + statement.pop(); + } let check_version_suite_error = msg.starts_with("Async functions are") || msg.starts_with("Async for loops are") || msg.starts_with("Async with statements are") @@ -820,12 +830,14 @@ impl VirtualMachine { // Set end_lineno and end_offset if available if let Some((end_lineno, end_offset)) = error.python_end_location() { - let (end_lineno, end_offset) = if check_version_suite_error - && statement - .as_deref() - .and_then(|line| line.chars().next()) - .is_some_and(|ch| ch.is_ascii_whitespace()) - { + // EOF errors have no source span in CPython. + let no_end_offset = unexpected_eof_error + || (check_version_suite_error + && statement + .as_deref() + .and_then(|line| line.chars().next()) + .is_some_and(|ch| ch.is_ascii_whitespace())); + let (end_lineno, end_offset) = if no_end_offset { (end_lineno, -1) } else if line_end_binary_operator_error && end_offset == offset_raw { (end_lineno, (end_offset + 1) as isize) From 5b3eb41b49daf29a64947ee40aa9332f32ea7001 Mon Sep 17 00:00:00 2001 From: MsfPablo <129399053+MsfPablo@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:06:48 +0200 Subject: [PATCH 04/28] Fix panic in code.replace() with non-interned strings (#8471) code.replace() called as_interned_str().unwrap() on its string arguments, which panics when the caller passes a string that has not been interned. modulefinder's replace_paths_in_code() builds a fresh co_filename, so any use of ModuleFinder(replace_paths=...) aborted the interpreter. Intern the incoming strings instead, and raise TypeError rather than panicking when a non-string appears in co_names/co_varnames/ co_cellvars/co_freevars. Unskips test_modulefinder.test_replace_paths. Co-authored-by: Pablo Garcia --- Lib/test/test_modulefinder.py | 2 -- crates/vm/src/builtins/code.rs | 38 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py index 51f7fd257e0..b64e684f805 100644 --- a/Lib/test/test_modulefinder.py +++ b/Lib/test/test_modulefinder.py @@ -390,8 +390,6 @@ def test_bytecode(self): os.remove(source_path) self._do_test(bytecode_test) - # TODO: RUSTPYTHON; panics at code.rs with 'called Option::unwrap() on a None value' - @unittest.skip("TODO: RUSTPYTHON; panics in co_filename replacement") def test_replace_paths(self): old_path = os.path.join(self.test_dir, 'a', 'module.py') new_path = os.path.join(self.test_dir, 'a', 'spam.py') diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index b1132a55a20..b25c9d5c499 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -1366,19 +1366,25 @@ impl PyCode { OptionalArg::Missing => self.code.instructions.clone(), }; + let intern_all = |objs: Vec, field: &str| -> PyResult> { + objs.into_iter() + .map(|o| { + let s = o.downcast_ref::().ok_or_else(|| { + vm.new_type_error(format!("{field} must be a tuple of strings")) + })?; + Ok(vm.ctx.intern_str(s.as_wtf8())) + }) + .collect::>>() + .map(Vec::into_boxed_slice) + }; + let cellvars = match co_cellvars { - OptionalArg::Present(cellvars) => cellvars - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), + OptionalArg::Present(cellvars) => intern_all(cellvars, "co_cellvars")?, OptionalArg::Missing => self.code.cellvars.clone(), }; let freevars = match co_freevars { - OptionalArg::Present(freevars) => freevars - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), + OptionalArg::Present(freevars) => intern_all(freevars, "co_freevars")?, OptionalArg::Missing => self.code.freevars.clone(), }; @@ -1411,10 +1417,10 @@ impl PyCode { posonlyarg_count, arg_count, kwonlyarg_count, - source_path: source_path.as_object().as_interned_str(vm).unwrap(), + source_path: vm.ctx.intern_str(source_path.as_wtf8()), first_line_number, - obj_name: obj_name.as_object().as_interned_str(vm).unwrap(), - qualname: qualname.as_object().as_interned_str(vm).unwrap(), + obj_name: vm.ctx.intern_str(obj_name.as_wtf8()), + qualname: vm.ctx.intern_str(qualname.as_wtf8()), max_stackdepth, instructions, @@ -1422,14 +1428,8 @@ impl PyCode { // It can be removed once we move every other code to use linetable only. locations: self.code.locations.clone(), constants: constants.into_iter().map(Literal).collect(), - names: names - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), - varnames: varnames - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), + names: intern_all(names, "co_names")?, + varnames: intern_all(varnames, "co_varnames")?, cellvars, freevars, localspluskinds: self.code.localspluskinds.clone(), From 3a1caa1eb5c553dc321be18b0b051ff11116f72b Mon Sep 17 00:00:00 2001 From: Joshua Megnauth <48846352+joshuamegnauth54@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:08:01 -0400 Subject: [PATCH 05/28] Use C string literals and w! instead of allocs (#8474) Rust supports C string literals which automatically create a CStr with a trailing NUL. `windows-sys` provides an analogous macro for wide strings. Both of these avoid allocations which is nice for constants. --- crates/host_env/src/fileutils.rs | 18 ++++++------- crates/host_env/src/nt.rs | 37 ++++++++++++++------------- crates/host_env/src/winapi.rs | 5 ++-- crates/host_env/src/windows.rs | 43 +++++++++++++++++--------------- crates/host_env/src/wmi.rs | 24 +++++++++--------- 5 files changed, 65 insertions(+), 62 deletions(-) diff --git a/crates/host_env/src/fileutils.rs b/crates/host_env/src/fileutils.rs index a4922e7a2fe..a8e56bb1c0b 100644 --- a/crates/host_env/src/fileutils.rs +++ b/crates/host_env/src/fileutils.rs @@ -1,6 +1,8 @@ // Python/fileutils.c in CPython #![allow(non_snake_case)] +use alloc::ffi::CString; + #[cfg(not(windows))] pub use rustix::fs::Stat as StatStruct; @@ -16,9 +18,8 @@ pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result { pub mod windows { use crate::crt_fd; use crate::windows::ToWideString; - use alloc::ffi::CString; use libc::{S_IFCHR, S_IFDIR, S_IFMT}; - use std::ffi::{OsStr, OsString}; + use std::ffi::OsStr; use std::os::windows::io::AsRawHandle; use std::sync::OnceLock; use windows_sys::Win32::Foundation::{ @@ -33,6 +34,7 @@ pub mod windows { use windows_sys::Win32::System::LibraryLoader::{GetProcAddress, LoadLibraryW}; use windows_sys::Win32::System::SystemServices::IO_REPARSE_TAG_SYMLINK; use windows_sys::core::PCWSTR; + use windows_sys::w; pub const S_IFIFO: libc::c_int = 0o010000; pub const S_IFLNK: libc::c_int = 0o120000; @@ -302,16 +304,13 @@ pub mod windows { let GetFileInformationByName = GET_FILE_INFORMATION_BY_NAME .get_or_init(|| { - let library_name = - OsString::from("api-ms-win-core-file-l2-1-4.dll").to_wide_with_nul(); - let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; + let library_name = w!("api-ms-win-core-file-l2-1-4.dll"); + let module = unsafe { LoadLibraryW(library_name) }; if module.is_null() { return None; } - let name = CString::new("GetFileInformationByName").unwrap(); - if let Some(proc) = - unsafe { GetProcAddress(module, name.as_bytes_with_nul().as_ptr()) } - { + let name = c"GetFileInformationByName"; + if let Some(proc) = unsafe { GetProcAddress(module, name.as_ptr().cast()) } { Some(unsafe { core::mem::transmute::< unsafe extern "system" fn() -> isize, @@ -458,7 +457,6 @@ pub unsafe fn fclose(fp: *mut CFile) -> core::ffi::c_int { reason = "false positive: core::io::ErrorKind is unstable (core_io)" )] pub fn fopen(path: &std::path::Path, mode: &str) -> std::io::Result<*mut CFile> { - use alloc::ffi::CString; use std::fs::File; // Currently only supports read mode diff --git a/crates/host_env/src/nt.rs b/crates/host_env/src/nt.rs index 780a75910ea..7e0591600b1 100644 --- a/crates/host_env/src/nt.rs +++ b/crates/host_env/src/nt.rs @@ -22,19 +22,22 @@ use crate::{ windows::{CheckWin32Bool, CheckWin32Handle, CheckWin32Sentinel, HandleToOwned, ToWideString}, }; use libc::intptr_t; -use windows_sys::Win32::{ - Foundation::{ - CloseHandle, ERROR_INVALID_HANDLE, GetLastError, HANDLE, INVALID_HANDLE_VALUE, MAX_PATH, - }, - Globalization::{CP_UTF8, MultiByteToWideChar, WideCharToMultiByte}, - Storage::FileSystem::{ - CreateFileW, FILE_BASIC_INFO, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, - FILE_READ_ATTRIBUTES, FILE_TYPE_UNKNOWN, FileBasicInfo, FindClose, FindFirstFileW, - GetFileAttributesW, GetFileInformationByHandleEx, GetFileType, GetFullPathNameW, - INVALID_FILE_ATTRIBUTES, OPEN_EXISTING, SetFileAttributesW, SetFileInformationByHandle, - WIN32_FIND_DATAW, +use windows_sys::{ + Win32::{ + Foundation::{ + CloseHandle, ERROR_INVALID_HANDLE, GetLastError, HANDLE, INVALID_HANDLE_VALUE, MAX_PATH, + }, + Globalization::{CP_UTF8, MultiByteToWideChar, WideCharToMultiByte}, + Storage::FileSystem::{ + CreateFileW, FILE_BASIC_INFO, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, + FILE_READ_ATTRIBUTES, FILE_TYPE_UNKNOWN, FileBasicInfo, FindClose, FindFirstFileW, + GetFileAttributesW, GetFileInformationByHandleEx, GetFileType, GetFullPathNameW, + INVALID_FILE_ATTRIBUTES, OPEN_EXISTING, SetFileAttributesW, SetFileInformationByHandle, + WIN32_FIND_DATAW, + }, + System::{Console, Threading}, }, - System::{Console, Threading}, + w, }; pub type Handle = HANDLE; @@ -1172,12 +1175,10 @@ pub fn mkdir(path: &widestring::WideCStr, mode: i32) -> io::Result<()> { lpSecurityDescriptor: core::ptr::null_mut(), bInheritHandle: 0, }; - let sddl: Vec = "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)\0" - .encode_utf16() - .collect(); + let sddl = w!("D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"); unsafe { ConvertStringSecurityDescriptorToSecurityDescriptorW( - sddl.as_ptr(), + sddl, SDDL_REVISION_1, &mut sec_attr.lpSecurityDescriptor, core::ptr::null_mut(), @@ -1699,10 +1700,10 @@ pub fn get_terminal_size_handle(h: HANDLE) -> io::Result<(usize, usize)> { if err != windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED { return Err(io::Error::last_os_error()); } - let conout: Vec = "CONOUT$\0".encode_utf16().collect(); + let conout = w!("CONOUT$"); let console_handle = unsafe { CreateFileW( - conout.as_ptr(), + conout, windows_sys::Win32::Foundation::GENERIC_READ | windows_sys::Win32::Foundation::GENERIC_WRITE, windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ diff --git a/crates/host_env/src/winapi.rs b/crates/host_env/src/winapi.rs index af53910089e..19e18f32d3f 100644 --- a/crates/host_env/src/winapi.rs +++ b/crates/host_env/src/winapi.rs @@ -65,6 +65,7 @@ pub use windows_sys::Win32::{ }, UI::WindowsAndMessaging::SW_HIDE, }; +use windows_sys::w; pub type Handle = HANDLE; pub type StdHandle = windows_sys::Win32::System::Console::STD_HANDLE; @@ -1093,14 +1094,14 @@ where return Err(MimeRegistryReadError::Os(err)); } - let content_type_key: Vec = "Content Type\0".encode_utf16().collect(); + let content_type_key = w!("Content Type"); let mut type_buf = [0u16; 256]; let mut cb_type = (type_buf.len() * 2) as u32; let mut reg_type = 0; let err = unsafe { RegQueryValueExW( subkey, - content_type_key.as_ptr(), + content_type_key, core::ptr::null_mut(), &mut reg_type, type_buf.as_mut_ptr().cast(), diff --git a/crates/host_env/src/windows.rs b/crates/host_env/src/windows.rs index bde8d679737..635f12f3f38 100644 --- a/crates/host_env/src/windows.rs +++ b/crates/host_env/src/windows.rs @@ -4,24 +4,27 @@ use std::{ io, os::windows::ffi::{OsStrExt, OsStringExt}, }; -use windows_sys::Win32::{ - Foundation::{ - E_POINTER, ERROR_INSUFFICIENT_BUFFER, ERROR_INVALID_FLAGS, ERROR_NO_UNICODE_TRANSLATION, - MAX_PATH, S_OK, - }, - Networking::WinSock::WSAStartup, - Storage::FileSystem::{ - GetFileVersionInfoSizeW, GetFileVersionInfoW, VS_FIXEDFILEINFO, VerQueryValueW, - }, - System::{ - Diagnostics::Debug::{ - FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM, - FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, +use windows_sys::{ + Win32::{ + Foundation::{ + E_POINTER, ERROR_INSUFFICIENT_BUFFER, ERROR_INVALID_FLAGS, + ERROR_NO_UNICODE_TRANSLATION, MAX_PATH, S_OK, + }, + Networking::WinSock::WSAStartup, + Storage::FileSystem::{ + GetFileVersionInfoSizeW, GetFileVersionInfoW, VS_FIXEDFILEINFO, VerQueryValueW, + }, + System::{ + Diagnostics::Debug::{ + FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM, + FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, + }, + LibraryLoader::{GetModuleFileNameW, GetModuleHandleW}, + SystemInformation::{GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW}, + Threading::{GetCurrentThreadStackLimits, SetThreadStackGuarantee}, }, - LibraryLoader::{GetModuleFileNameW, GetModuleHandleW}, - SystemInformation::{GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW}, - Threading::{GetCurrentThreadStackLimits, SetThreadStackGuarantee}, }, + w, }; /// _MAX_ENV from Windows CRT stdlib.h - maximum environment variable size @@ -154,8 +157,8 @@ pub struct WindowsVersionInfo { fn get_kernel32_version() -> io::Result<(u32, u32, u32)> { unsafe { - let module_name: Vec = OsStr::new("kernel32.dll").to_wide_with_nul(); - let h_kernel32 = GetModuleHandleW(module_name.as_ptr()).check_nonnull()?; + let module_name = w!("kernel32.dll"); + let h_kernel32 = GetModuleHandleW(module_name).check_nonnull()?; let mut kernel32_path = [0u16; MAX_PATH as usize]; let len = GetModuleFileNameW( @@ -181,13 +184,13 @@ fn get_kernel32_version() -> io::Result<(u32, u32, u32)> { ) .check_win32_bool()?; - let sub_block: Vec = OsStr::new("").to_wide_with_nul(); + let sub_block = w!(""); let mut ffi_ptr: *mut VS_FIXEDFILEINFO = core::ptr::null_mut(); let mut ffi_len: u32 = 0; VerQueryValueW( ver_block.as_ptr() as *const _, - sub_block.as_ptr(), + sub_block, &mut ffi_ptr as *mut *mut VS_FIXEDFILEINFO as *mut *mut _, &mut ffi_len as *mut u32, ) diff --git a/crates/host_env/src/wmi.rs b/crates/host_env/src/wmi.rs index 592ffd1dc23..2b46eebcbe5 100644 --- a/crates/host_env/src/wmi.rs +++ b/crates/host_env/src/wmi.rs @@ -7,15 +7,17 @@ use core::ffi::c_void; use core::ptr::{NonNull, null, null_mut}; +use widestring::WideCString; use windows_sys::Win32::Foundation::{ - CloseHandle, ERROR_BROKEN_PIPE, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY, GetLastError, HANDLE, - WAIT_OBJECT_0, WAIT_TIMEOUT, + CloseHandle, ERROR_BROKEN_PIPE, ERROR_INVALID_NAME, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY, + GetLastError, HANDLE, WAIT_OBJECT_0, WAIT_TIMEOUT, }; use windows_sys::Win32::Storage::FileSystem::{ReadFile, WriteFile}; use windows_sys::Win32::System::Pipes::CreatePipe; use windows_sys::Win32::System::Threading::{ CreateEventW, CreateThread, GetExitCodeThread, SetEvent, WaitForSingleObject, }; +use windows_sys::w; use crate::ctypes::wcslen; @@ -256,10 +258,6 @@ const fn failed(hr: HRESULT) -> bool { hr < 0 } -fn wide_str(s: &str) -> Vec { - s.encode_utf16().chain(core::iter::once(0)).collect() -} - unsafe fn wait_event(event: HANDLE, timeout: u32) -> u32 { match unsafe { WaitForSingleObject(event, timeout) } { WAIT_OBJECT_0 => 0, @@ -346,8 +344,8 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { } if succeeded(hr) { - let root_cimv2 = wide_str("ROOT\\CIMV2"); - let bstr_root = unsafe { SysAllocString(root_cimv2.as_ptr()) }; + let root_cimv2 = w!("ROOT\\CIMV2"); + let bstr_root = unsafe { SysAllocString(root_cimv2) }; hr = unsafe { locator_connect_server( locator, @@ -384,8 +382,8 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { }; } if succeeded(hr) { - let wql = wide_str("WQL"); - let bstr_wql = unsafe { SysAllocString(wql.as_ptr()) }; + let wql = w!("WQL"); + let bstr_wql = unsafe { SysAllocString(wql) }; hr = unsafe { services_exec_query( services, @@ -557,7 +555,9 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { } pub fn exec_query(query_str: &str) -> Result { - let query_wide = wide_str(query_str); + let query = WideCString::from_str(query_str) + .map_err(|_| ExecQueryError::Code(ERROR_INVALID_NAME))? + .into(); let mut h_thread: HANDLE = null_mut(); let mut err: u32 = 0; @@ -579,7 +579,7 @@ pub fn exec_query(query_str: &str) -> Result { err = GetLastError(); } else { let thread_data = Box::new(QueryThreadData { - query: query_wide, + query, write_pipe, init_event, connect_event, From 689c8b57e7cc8f9b834dfd041cd5fa493c8018d2 Mon Sep 17 00:00:00 2001 From: Jiwoo Ahn Date: Sun, 9 Aug 2026 15:09:18 +0900 Subject: [PATCH 06/28] wasi: fix os.environb byte handling (#8476) Part of: #4583 This will enable -m test for wasi preview 1 Signed-off-by: Jiwoo Ahn --- crates/host_env/src/os.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/host_env/src/os.rs b/crates/host_env/src/os.rs index 01711687c6c..7af8f586110 100644 --- a/crates/host_env/src/os.rs +++ b/crates/host_env/src/os.rs @@ -519,13 +519,14 @@ pub fn set_errno(value: i32) { #[cfg(not(any(unix, windows, target_os = "wasi")))] pub fn set_errno(_value: i32) {} -#[cfg(unix)] +// WASIp1, like Unix, provides byte-preserving OsStr conversions. +#[cfg(any(unix, all(target_os = "wasi", not(target_env = "p2"))))] pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { - use std::os::unix::ffi::OsStrExt; + use self::ffi::OsStrExt; Ok(std::ffi::OsStr::from_bytes(b)) } -#[cfg(not(unix))] +#[cfg(not(any(unix, all(target_os = "wasi", not(target_env = "p2")))))] pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { Ok(core::str::from_utf8(b)?.as_ref()) } From bcf5c4b193d92244b9229842c50b2d76b5f66804 Mon Sep 17 00:00:00 2001 From: Jeaeun Kim <109906379+kyokuping@users.noreply.github.com> Date: Mon, 10 Aug 2026 00:43:44 +0900 Subject: [PATCH 07/28] ssl: support explicit session reuse (#8466) * ssl: support explicit session reuse Assisted-by: Codex-5.6-sol * ssl: support explicit TLS 1.3 session reuse Assisted-by: Codex-5.6-sol --- Lib/test/test_ssl.py | 2 - crates/stdlib/src/ssl.rs | 388 +++++++++++++++++++++++++-------------- 2 files changed, 247 insertions(+), 143 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 8dad1ba7382..e1759f1aa6b 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4556,7 +4556,6 @@ def test_sendfile(self): s.sendfile(file) self.assertEqual(s.recv(1024), TEST_DATA) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_session(self): client_context, server_context, hostname = testing_context() # TODO: sessions aren't compatible with TLSv1.3 yet @@ -4614,7 +4613,6 @@ def test_session(self): self.assertEqual(sess_stat['accept'], 4) self.assertEqual(sess_stat['hits'], 2) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False != True def test_session_handling(self): client_context, server_context, hostname = testing_context() client_context2, _, _ = testing_context() diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 454fdcba899..04b905d544e 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -137,6 +137,8 @@ mod _ssl { #[pyattr] const PROTOCOL_TLSv1_3: i32 = 6; + static NEXT_SSL_SESSION_NONCE: AtomicUsize = AtomicUsize::new(1); + // Protocol version constants for TLSVersion enum #[pyattr] const PROTO_SSLv3: i32 = 0x0300; @@ -439,6 +441,31 @@ mod _ssl { lifetime: u64, } + impl SessionData { + // NOTE: This is NOT the actual TLS session ID, just a unique identifier. + fn new(server_name: &str, lifetime: u64) -> Self { + let creation_time = SystemTime::now(); + let nonce = NEXT_SSL_SESSION_NONCE.fetch_add(1, Ordering::Relaxed); + let mut hasher = Sha256::new(); + hasher.update(server_name.as_bytes()); + hasher.update( + creation_time + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .to_le_bytes(), + ); + hasher.update(nonce.to_le_bytes()); + + Self { + _server_name: server_name.to_owned(), + session_id: hasher.finalize()[..16].to_vec(), + creation_time, + lifetime, + } + } + } + // Type alias to simplify complex session cache type type SessionCache = Arc, Arc>>>>; @@ -466,20 +493,6 @@ mod _ssl { // ✓ session_reused - tracked via handshake_kind() // ✗ Actual TLS session ID/ticket data - NOT ACCESSIBLE - // Generate a synthetic session ID from server name and timestamp - // NOTE: This is NOT the actual TLS session ID, just a unique identifier - fn generate_session_id_from_metadata(server_name: &str, time: SystemTime) -> Vec { - let mut hasher = Sha256::new(); - hasher.update(server_name.as_bytes()); - hasher.update( - time.duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() - .to_le_bytes(), - ); - hasher.finalize()[..16].to_vec() - } - // Custom ClientSessionStore that tracks session metadata for Python access // NOTE: This wraps ClientSessionMemoryCache and records metadata when sessions are stored #[derive(Debug)] @@ -488,6 +501,39 @@ mod _ssl { session_cache: SessionCache, } + impl PythonClientSessionStore { + fn new(session_cache: SessionCache) -> Self { + Self { + inner: Arc::new(ClientSessionMemoryCache::new(SSL_SESSION_CACHE_SIZE)), + session_cache, + } + } + + fn transfer_session( + &self, + target: &Self, + server_name: &ServerName<'static>, + kind: ClientSessionKind, + ) { + if let Some(group) = self.kx_hint(server_name) { + target.set_kx_hint(server_name.clone(), group); + } + + match kind { + ClientSessionKind::Tls12 => { + if let Some(session) = self.tls12_session(server_name) { + target.set_tls12_session(server_name.clone(), session); + } + } + ClientSessionKind::Tls13 => { + if let Some(ticket) = self.take_tls13_ticket(server_name) { + target.insert_tls13_ticket(server_name.clone(), ticket); + } + } + } + } + } + impl ClientSessionStore for PythonClientSessionStore { fn set_kx_hint(&self, server_name: ServerName<'static>, group: rustls::NamedGroup) { self.inner.set_kx_hint(server_name, group); @@ -508,17 +554,8 @@ mod _ssl { // Record metadata in Python-accessible cache // NOTE: We can't access value.session_id or value.ticket (private fields) // So we generate a synthetic ID from metadata - let creation_time = SystemTime::now(); let server_name_str = server_name.to_str(); - let session_data = SessionData { - _server_name: server_name_str.as_ref().to_string(), - session_id: generate_session_id_from_metadata( - server_name_str.as_ref(), - creation_time, - ), - creation_time, - lifetime: 7200, // TLS 1.2 default session lifetime - }; + let session_data = SessionData::new(server_name_str.as_ref(), 7200); let key = server_name_str.as_bytes().to_vec(); self.session_cache @@ -552,17 +589,8 @@ mod _ssl { // Record metadata in Python-accessible cache // NOTE: We can't access value.ticket or value.lifetime_secs (private fields) // So we use default values - let creation_time = SystemTime::now(); let server_name_str = server_name.to_str(); - let session_data = SessionData { - _server_name: server_name_str.to_string(), - session_id: generate_session_id_from_metadata( - server_name_str.as_ref(), - creation_time, - ), - creation_time, - lifetime: 7200, // Default TLS 1.3 ticket lifetime (Rustls uses this) - }; + let session_data = SessionData::new(server_name_str.as_ref(), 7200); let key = server_name_str.as_bytes().to_vec(); self.session_cache @@ -745,6 +773,8 @@ mod _ssl { #[pyclass(name = "_SSLContext", module = "ssl", traverse)] #[derive(Debug, PyPayload)] struct PySSLContext { + #[pytraverse(skip)] + context_identity: Arc<()>, #[pytraverse(skip)] protocol: i32, #[pytraverse(skip)] @@ -807,9 +837,6 @@ mod _ssl { // Session management #[pytraverse(skip)] client_session_cache: SessionCache, - // Rustls session store for actual TLS session resumption - #[pytraverse(skip)] - rustls_session_store: Arc, // Rustls server session store for server-side session resumption #[pytraverse(skip)] rustls_server_session_store: Arc, @@ -1926,8 +1953,9 @@ mod _ssl { .map(|o| o.downgrade(None, vm)) .transpose()?, ), - // Filter out Python None objects - only store actual SSLSession objects - session: PyRwLock::new(args.session.into_option().filter(|s| !vm.is_none(s))), + session: PyRwLock::new(None), + client_config: PyRwLock::new(None), + client_session_store: PyRwLock::new(None), incoming_bio: None, outgoing_bio: None, sni_state: PyRwLock::new(None), @@ -1945,6 +1973,12 @@ mod _ssl { .into_ref_with_type(vm, vm.class("_ssl", "_SSLSocket")) .map_err(|_| vm.new_type_error("Failed to create SSLSocket"))?; + if let Some(session) = args.session.into_option() + && !vm.is_none(&session) + { + ssl_socket_ref.set_session(session, vm)?; + } + Ok(ssl_socket_ref) } @@ -2008,8 +2042,9 @@ mod _ssl { .map(|o| o.downgrade(None, vm)) .transpose()?, ), - // Filter out Python None objects - only store actual SSLSession objects - session: PyRwLock::new(args.session.into_option().filter(|s| !vm.is_none(s))), + session: PyRwLock::new(None), + client_config: PyRwLock::new(None), + client_session_store: PyRwLock::new(None), incoming_bio: Some(args.incoming), outgoing_bio: Some(args.outgoing), sni_state: PyRwLock::new(None), @@ -2026,6 +2061,12 @@ mod _ssl { .into_ref_with_type(vm, vm.class("_ssl", "_SSLSocket")) .map_err(|_| vm.new_type_error("Failed to create SSLSocket"))?; + if let Some(session) = args.session.into_option() + && !vm.is_none(&session) + { + ssl_socket_ref.set_session(session, vm)?; + } + Ok(ssl_socket_ref) } @@ -2304,19 +2345,11 @@ mod _ssl { _ => (PROTO_MINIMUM_SUPPORTED, PROTO_MAXIMUM_SUPPORTED), // Auto-negotiate }; - // IMPORTANT: Create shared session cache BEFORE PySSLContext - // Both client_session_cache and PythonClientSessionStore.session_cache - // MUST point to the same HashMap to ensure Python-level and Rustls-level - // sessions are synchronized + // Session metadata is scoped to the SSLContext. Each per-connection + // rustls session store records into this shared cache. let shared_session_cache = Arc::new(ParkingRwLock::new(HashMap::new())); - let rustls_client_store = Arc::new(PythonClientSessionStore { - inner: Arc::new(rustls::client::ClientSessionMemoryCache::new( - SSL_SESSION_CACHE_SIZE, - )), - session_cache: shared_session_cache.clone(), - }); - Ok(Self { + context_identity: Arc::new(()), protocol, check_hostname: PyRwLock::new(protocol == PROTOCOL_TLS_CLIENT), verify_mode: PyRwLock::new(default_verify_mode), @@ -2340,7 +2373,6 @@ mod _ssl { x509_cert_count: PyRwLock::new(0), // Use the shared cache created above client_session_cache: shared_session_cache, - rustls_session_store: rustls_client_store, rustls_server_session_store: rustls::server::ServerSessionMemoryCache::new( SSL_SESSION_CACHE_SIZE, ), @@ -2390,6 +2422,13 @@ mod _ssl { owner: PyRwLock>>, // Session for resumption session: PyRwLock>, + // Client configuration used by this connection. Retained so the resulting + // SSLSession can reuse the same verifier and client credentials. + #[pytraverse(skip)] + client_config: PyRwLock>>, + // Per-connection store containing the session selected by this connection. + #[pytraverse(skip)] + client_session_store: PyRwLock>>, // MemoryBIO mode (optional) incoming_bio: Option>, outgoing_bio: Option>, @@ -2487,31 +2526,27 @@ mod _ssl { } // Create and store a session object after successful handshake - fn create_session_after_handshake(&self, vm: &VirtualMachine) { + fn create_session_after_handshake(&self, was_resumed: bool, vm: &VirtualMachine) { // Only create session for client-side connections if self.server_side { return; } - // Check if session already exists - let session_opt = self.session.read().clone(); - if let Some(ref s) = session_opt { - if vm.is_none(s) { - } else { - return; - } - } - // Get server hostname let server_name = self.server_hostname.read().clone(); + let previous_session = self.session.read().clone(); // Try to get session data from context's session cache // IMPORTANT: Acquire and release locks quickly to avoid deadlock - let context = self.context.read(); - let session_cache_arc = context.client_session_cache.clone(); - drop(context); // Release context lock ASAP + let (context_identity, session_cache_arc) = { + let context = self.context.read(); + ( + context.context_identity.clone(), + context.client_session_cache.clone(), + ) + }; - let (session_id, creation_time, lifetime) = if let Some(ref name) = server_name { + let cached_session_data = if let Some(ref name) = server_name { let key = name.as_bytes().to_vec(); // Clone the data we need while holding the lock, then immediately release @@ -2521,29 +2556,57 @@ mod _ssl { }; // Lock released here if let Some(session_data_arc) = session_data_opt { - let data = session_data_arc.lock(); - let result = (data.session_id.clone(), data.creation_time, data.lifetime); - drop(data); // Explicit unlock - result + session_data_arc.lock().clone() } else { - // Create new session ID if not in cache - let time = std::time::SystemTime::now(); - (generate_session_id_from_metadata(name, time), time, 7200) + SessionData::new(name, 7200) } } else { - // No server name, use defaults - let time = std::time::SystemTime::now(); - (vec![0; 16], time, 7200) + SessionData::new("", 7200) + }; + + let session_data = if was_resumed { + previous_session + .as_ref() + .and_then(|session| session.downcast_ref::()) + .map_or(cached_session_data, |session| SessionData { + _server_name: server_name.clone().unwrap_or_default(), + session_id: session.session_id.clone(), + creation_time: session.creation_time, + lifetime: session.lifetime, + }) + } else { + cached_session_data + }; + + let rustls_server_name = server_name.and_then(|name| ServerName::try_from(name).ok()); + let protocol_version = self + .connection + .lock() + .as_ref() + .and_then(|connection| connection.protocol_version()); + let session_kind = match protocol_version { + Some(rustls::ProtocolVersion::TLSv1_2) => ClientSessionKind::Tls12, + Some(rustls::ProtocolVersion::TLSv1_3) => ClientSessionKind::Tls13, + _ => return, + }; + + let Some(client_config) = self.client_config.write().take() else { + return; + }; + let Some(session_store) = self.client_session_store.write().take() else { + return; }; - // Create a new SSLSession object with real metadata let session = PySSLSession { - // Use dummy session data to indicate we have a ticket - // TLS 1.2+ always uses session tickets/resumption - session_data: vec![1], // Non-empty to indicate has_ticket=True - session_id, - creation_time, - lifetime, + context_identity, + client_config, + session_store, + server_name: rustls_server_name, + kind: session_kind, + session_id: session_data.session_id, + creation_time: session_data.creation_time, + lifetime: session_data.lifetime, + has_ticket: true, }; let py_session = session.into_pyobject(vm); @@ -2638,7 +2701,7 @@ mod _ssl { let _ = self.track_used_ca_from_capath(); } - self.create_session_after_handshake(vm); + self.create_session_after_handshake(was_resumed, vm); } // Internal implementation with timeout control @@ -3486,9 +3549,9 @@ mod _ssl { let check_hostname = *ctx.check_hostname.read(); let verify_flags = *ctx.verify_flags.read(); + let context_identity = ctx.context_identity.clone(); - // Get session store before dropping ctx - let session_store = ctx.rustls_session_store.clone(); + let session_cache = ctx.client_session_cache.clone(); // Get CRLs for revocation checking let crls_clone = ctx.crls.read().clone(); @@ -3496,31 +3559,6 @@ mod _ssl { // Drop ctx early to avoid borrow conflicts drop(ctx); - // Build client config using compat helper - let config_options = ClientConfigOptions { - protocol_settings, - root_store: if verify_mode != CERT_NONE { - Some(root_store_clone) - } else { - None - }, - ca_certs_der: ca_certs_der_clone, - cert_chain: if !cert_chain_clone.is_empty() { - Some(cert_chain_clone) - } else { - None - }, - private_key: private_key_opt, - verify_server_cert: verify_mode != CERT_NONE, - check_hostname, - verify_flags, - session_store: Some(session_store), - crls: crls_clone, - }; - - let config = - create_client_config(config_options).map_err(|e| vm.new_value_error(e))?; - // Parse server name for SNI // Convert to ServerName use rustls::pki_types::ServerName; @@ -3539,10 +3577,61 @@ mod _ssl { ) }; - let conn = ClientConnection::new(Arc::new(config), server_name.clone()) - .map_err(|e| { - vm.new_value_error(format!("Failed to create client connection: {e}")) - })?; + let explicit_session = self.session.read().clone(); + let session_store = Arc::new(PythonClientSessionStore::new(session_cache)); + let config = if let Some(session) = explicit_session { + let session = session + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?; + if !Arc::ptr_eq(&session.context_identity, &context_identity) { + return Err( + vm.new_value_error("Session refers to a different SSLContext.") + ); + } + if session.server_name.as_ref() == Some(&server_name) { + session.session_store.transfer_session( + &session_store, + &server_name, + session.kind, + ); + } + let mut config = (*session.client_config).clone(); + config.resumption = + rustls::client::Resumption::store(session_store.clone()); + Arc::new(config) + } else { + let config_options = ClientConfigOptions { + protocol_settings, + root_store: if verify_mode != CERT_NONE { + Some(root_store_clone) + } else { + None + }, + ca_certs_der: ca_certs_der_clone, + cert_chain: if !cert_chain_clone.is_empty() { + Some(cert_chain_clone) + } else { + None + }, + private_key: private_key_opt, + verify_server_cert: verify_mode != CERT_NONE, + check_hostname, + verify_flags, + session_store: Some(session_store.clone()), + crls: crls_clone, + }; + Arc::new( + create_client_config(config_options) + .map_err(|e| vm.new_value_error(e))?, + ) + }; + + *self.client_config.write() = Some(config.clone()); + *self.client_session_store.write() = Some(session_store); + + let conn = ClientConnection::new(config, server_name).map_err(|e| { + vm.new_value_error(format!("Failed to create client connection: {e}")) + })?; *conn_guard = Some(Connection::Client(conn)); } @@ -4046,12 +4135,15 @@ mod _ssl { #[pygetset(setter)] fn set_session(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // Validate that value is an SSLSession - if !value.is(vm.ctx.types.none_type) { - // Try to downcast to SSLSession to validate - let _ = value - .downcast_ref::() - .ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?; + let session = value + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?; + + if !Arc::ptr_eq( + &session.context_identity, + &self.context.read().context_identity, + ) { + return Err(vm.new_value_error("Session refers to a different SSLContext.")); } // Check if this is a client socket @@ -4065,11 +4157,7 @@ mod _ssl { } // Store the session for potential use during handshake - *self.session.write() = if value.is(vm.ctx.types.none_type) { - None - } else { - Some(value) - }; + *self.session.write() = Some(value); Ok(()) } @@ -4751,22 +4839,31 @@ mod _ssl { // SSLSession - represents a cached SSL session // NOTE: This is an EMULATION - actual session data is managed by Rustls internally + #[derive(Debug, Clone, Copy)] + enum ClientSessionKind { + Tls12, + Tls13, + } + #[pyattr] #[pyclass(name = "SSLSession", module = "ssl")] #[derive(Debug, PyPayload)] struct PySSLSession { - // Session data - serialized rustls session (EMULATED - kept empty) - session_data: Vec, + context_identity: Arc<()>, + client_config: Arc, + session_store: Arc, + server_name: Option>, + kind: ClientSessionKind, // Session ID - synthetic ID generated from metadata (NOT actual TLS session ID) - #[allow(dead_code)] session_id: Vec, // Session metadata creation_time: std::time::SystemTime, // Lifetime in seconds (default 7200 = 2 hours) lifetime: u64, + has_ticket: bool, } - #[pyclass(flags(BASETYPE))] + #[pyclass(flags(BASETYPE), with(Comparable))] impl PySSLSession { #[pygetset] fn time(&self) -> i64 { @@ -4791,20 +4888,29 @@ mod _ssl { #[pygetset] fn id(&self, vm: &VirtualMachine) -> PyBytesRef { - // Return session ID (hash of session data for uniqueness) - - let mut hasher = DefaultHasher::new(); - self.session_data.hash(&mut hasher); - let hash = hasher.finish(); - - // Convert hash to bytes - vm.ctx.new_bytes(hash.to_be_bytes().to_vec()) + vm.ctx.new_bytes(self.session_id.clone()) } #[pygetset] fn has_ticket(&self) -> bool { - // For rustls, if we have session data, we have a ticket - !self.session_data.is_empty() + self.has_ticket + } + } + + impl Comparable for PySSLSession { + fn cmp( + zelf: &Py, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult { + op.eq_only(|| { + if let Some(other_session) = other.downcast_ref::() { + Ok((zelf.session_id == other_session.session_id).into()) + } else { + Ok(PyComparisonValue::NotImplemented) + } + }) } } From dca9b09cb3053795864075104989cf49920d8146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=95=88?= Date: Mon, 10 Aug 2026 00:50:46 +0900 Subject: [PATCH 08/28] Add new_payload_exception helper for constructing built-in payload exceptions (#8403) * Add new_payload_exception helper for built-in payload exceptions * Use new_payload_exception for SystemExit raise sites * Use new_payload_exception for BlockingIoError raise sites * Use new_payload_exception for OSError errno dispatch in slot_new * Run rustfmt * Run clippy * Add type guard to new_payload_exception and extract new_system_exit * Extract new_system_exit and fix stale expect messages The four SystemExit raise sites repeated the same construction, so route them through a new_system_exit helper next to new_stop_iteration. The expect messages still described invoke_exception's downcast failure, which no longer applies after moving to new_payload_exception. * Update crates/vm/src/vm/vm_new.rs --------- Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- crates/vm/src/exceptions.rs | 15 ++++---- crates/vm/src/stdlib/_io.rs | 60 +++++++++++++++++++------------- crates/vm/src/stdlib/_thread.rs | 2 +- crates/vm/src/stdlib/builtins.rs | 2 +- crates/vm/src/stdlib/sys.rs | 3 +- crates/vm/src/vm/mod.rs | 2 +- crates/vm/src/vm/vm_new.rs | 49 +++++++++++++++++++------- 7 files changed, 82 insertions(+), 51 deletions(-) diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index ffe5a6f0a41..9c42df966ea 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -1389,14 +1389,8 @@ impl OSErrorBuilder { vec![strerror.to_pyobject(vm)] }; - let payload = PyOSError::py_new(&exc_type, args.clone().into(), vm) - .expect("new_os_error usage error"); - let os_error = payload - .into_ref_with_type_lazy_dict(vm, exc_type) - .expect("new_os_error usage error"); - PyOSError::slot_init(os_error.as_object().to_owned(), args.into(), vm) - .expect("new_os_error usage error"); - os_error + vm.new_payload_exception::(exc_type, args.into()) + .expect("new_os_error usage error") } } @@ -2148,7 +2142,10 @@ pub(super) mod types { .downcast_ref::() .and_then(|errno| errno.try_to_primitive::(vm).ok()) .and_then(|errno| super::errno_to_exc_type(errno, vm)) - .and_then(|typ| vm.invoke_exception(typ, args_vec).ok()) + .and_then(|typ| { + vm.new_payload_exception::(typ.to_owned(), args_vec.into()) + .ok() + }) { return error.to_pyresult(vm); } diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 43a869fbecf..ab1be4297ec 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -20,7 +20,8 @@ cfg_select! { } use crate::{ - AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyModule, + AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyModule, PyOSError}, }; pub use _io::{OpenArgs, io_open as open}; use rustpython_host_env::io as host_io; @@ -943,14 +944,17 @@ mod _io { Some(n) => n, None => { // BlockingIOError(errno, msg, characters_written=0) - return Err(vm.invoke_exception( - vm.ctx.exceptions.blocking_io_error, - vec![ - vm.new_pyobj(EAGAIN), - vm.new_pyobj("write could not complete without blocking"), - vm.new_pyobj(0), - ], - )?); + return Err(vm + .new_payload_exception::( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(0), + ] + .into(), + )? + .upcast()); } }; self.write_pos += n as Offset; @@ -1154,14 +1158,17 @@ mod _io { self.buffer[self.write_end as usize..][..avail].copy_from_slice(&buf[..avail]); self.write_end += avail as Offset; self.pos += avail as Offset; - return Err(vm.invoke_exception( - vm.ctx.exceptions.blocking_io_error, - vec![ - vm.new_pyobj(EAGAIN), - vm.new_pyobj("write could not complete without blocking"), - vm.new_pyobj(avail), - ], - )?); + return Err(vm + .new_payload_exception::( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(avail), + ] + .into(), + )? + .upcast()); } Err(e) => return Err(e), } @@ -1200,14 +1207,17 @@ mod _io { self.write_end = buffer_size; // BlockingIOError(errno, msg, characters_written) let chars_written = written + buffer_len; - return Err(vm.invoke_exception( - vm.ctx.exceptions.blocking_io_error, - vec![ - vm.new_pyobj(EAGAIN), - vm.new_pyobj("write could not complete without blocking"), - vm.new_pyobj(chars_written), - ], - )?); + return Err(vm + .new_payload_exception::( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(chars_written), + ] + .into(), + )? + .upcast()); } None => break, } diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 0a80285dbe3..70304e63980 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -635,7 +635,7 @@ pub(crate) mod _thread { #[pyfunction] fn exit(vm: &VirtualMachine) -> PyResult { - Err(vm.invoke_exception(vm.ctx.exceptions.system_exit, vec![])?) + Err(vm.new_system_exit(vec![].into())) } thread_local!(static SENTINELS: RefCell>> = const { RefCell::new(Vec::new()) }); diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 7817d6ecb97..35f404f0f3b 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -1041,7 +1041,7 @@ mod builtins { #[pyfunction] pub(super) fn exit(exit_code_arg: OptionalArg, vm: &VirtualMachine) -> PyResult { let code = exit_code_arg.unwrap_or_else(|| vm.ctx.new_int(0).into()); - Err(vm.invoke_exception(vm.ctx.exceptions.system_exit, vec![code])?) + Err(vm.new_system_exit(vec![code].into())) } #[derive(Debug, Default, FromArgs)] diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 71aeccf34da..c7cc2fd298a 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -776,8 +776,7 @@ pub mod sys { } else { vec![status] }; - let exc = vm.invoke_exception(vm.ctx.exceptions.system_exit, args)?; - Err(exc) + Err(vm.new_system_exit(args.into())) } #[pyfunction] diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index e0a086c10db..6009f421e12 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -2737,7 +2737,7 @@ impl VirtualMachine { if self.state.finalizing.load(Ordering::Acquire) && !self.is_main_thread() { // once finalization starts, // non-main Python threads should stop running bytecode. - return Err(self.invoke_exception(self.ctx.exceptions.system_exit, vec![])?); + return Err(self.new_system_exit(vec![].into())); } // Suspend this thread if stop-the-world is in progress diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 4df3c639182..6110a3d5b1a 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -12,17 +12,18 @@ use rustpython_compiler::{CompileError, ParseError}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ - PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyOSError, PyStrRef, - PyType, PyTypeRef, + PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyOSError, + PyStopIteration, PyStrRef, PySystemExit, PyType, PyTypeRef, builtin_func::PyNativeFunction, descriptor::PyMethodDescriptor, tuple::{IntoPyTuple, PyTupleRef}, }, convert::{ToPyException, ToPyObject}, exceptions::OSErrorBuilder, - function::{IntoPyNativeFn, PyMethodFlags}, + function::{FuncArgs, IntoPyNativeFn, PyMethodFlags}, scope::Scope, set_attrs, + types::{Constructor, Initializer}, vm::VirtualMachine, }; @@ -353,6 +354,26 @@ impl VirtualMachine { .expect("vm.new_exception() called with an invalid exception type") } + /// Construct a built-in exception type that carries a payload, directly + /// (`py_new` + `slot_init`), without routing through `PyType::call`. + /// Only valid for a built-in `T` whose exact type is known at compile time. + pub fn new_payload_exception(&self, cls: PyTypeRef, args: FuncArgs) -> PyResult> + where + T: Constructor + Initializer, + { + debug_assert_eq!( + cls.slots.basicsize, + size_of::(), + "vm.new_payload_exception::<{}>() called with mismatched type '{}'", + core::any::type_name::(), + cls.name() + ); + let payload = T::py_new(&cls, args.clone(), self)?; + let exc = payload.into_ref_with_type_lazy_dict(self, cls)?; + T::slot_init(exc.as_object().to_owned(), args, self)?; + Ok(exc) + } + pub fn new_os_error(&self, msg: impl ToPyObject) -> PyRef { self.new_os_subtype_error(self.ctx.exceptions.os_error.to_owned(), None, msg) .upcast() @@ -905,16 +926,20 @@ impl VirtualMachine { exc } - pub fn new_stop_iteration(&self, value: Option) -> PyBaseExceptionRef { - let stop_iteration_error = self.ctx.exceptions.stop_iteration; - let args = if let Some(value) = value { - vec![value] - } else { - Vec::new() - }; - let exc = self.invoke_exception(stop_iteration_error, args); + pub fn new_system_exit(&self, args: FuncArgs) -> PyBaseExceptionRef { + self.new_payload_exception::(self.ctx.exceptions.system_exit.to_owned(), args) + .expect("SystemExit construction from internal args is infallible") + .upcast() + } - exc.expect("StopIteration is a BaseException Subclass.") + pub fn new_stop_iteration(&self, value: Option) -> PyBaseExceptionRef { + let args: FuncArgs = value.map(|v| vec![v]).unwrap_or_default().into(); + self.new_payload_exception::( + self.ctx.exceptions.stop_iteration.to_owned(), + args, + ) + .expect("StopIteration construction from internal args is infallible") + .upcast() } fn new_downcast_error( From 365434b5f120440d32d7b218cbc38fa4eb95cea5 Mon Sep 17 00:00:00 2001 From: Jeaeun Kim <109906379+kyokuping@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:45:16 +0900 Subject: [PATCH 09/28] Add JIT support for returning None (#8479) --- crates/jit/src/instructions.rs | 39 ++++++++++++++++++---------------- crates/jit/src/lib.rs | 32 ++++++++++++++-------------- crates/jit/tests/misc_tests.rs | 19 ++++++++--------- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/crates/jit/src/instructions.rs b/crates/jit/src/instructions.rs index 9b4656da5ad..67cf07f6e7f 100644 --- a/crates/jit/src/instructions.rs +++ b/crates/jit/src/instructions.rs @@ -39,6 +39,7 @@ impl JitValue { JitType::Int => Self::Int(val), JitType::Float => Self::Float(val), JitType::Bool => Self::Bool(val), + JitType::None => unreachable!("None cannot be used as an argument type"), } } @@ -47,7 +48,8 @@ impl JitValue { Self::Int(_) => Some(JitType::Int), Self::Float(_) => Some(JitType::Float), Self::Bool(_) => Some(JitType::Bool), - Self::None | Self::Null | Self::Tuple(_) | Self::FuncRef(_) => None, + Self::None => Some(JitType::None), + Self::Null | Self::Tuple(_) | Self::FuncRef(_) => None, } } @@ -112,8 +114,9 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { #[expect(clippy::mut_mut, reason = "This seems like a false positive")] let builder = &mut self.builder; let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; + let cranelift_ty = ty.to_cranelift().ok_or(JitCompileError::NotSupported)?; let local = self.variables[idx].get_or_insert_with(|| { - let var = builder.declare_var(ty.to_cranelift()); + let var = builder.declare_var(cranelift_ty); Local { var, ty: ty.clone(), @@ -328,27 +331,27 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { } fn return_value(&mut self, val: JitValue) -> Result<(), JitCompileError> { - if let Some(ref ty) = self.sig.ret { - // If the signature has a return type, enforce it - if val.to_jit_type().as_ref() != Some(ty) { + let val_type = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; + if let Some(ref ret_type) = self.sig.ret { + if ret_type != &val_type { return Err(JitCompileError::NotSupported); } } else { - // First time we see a return, define it in the signature - let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; - self.sig.ret = Some(ty.clone()); - self.builder - .func - .signature - .returns - .push(AbiParam::new(ty.to_cranelift())); + self.sig.ret = Some(val_type.clone()); + if let Some(val_type) = val_type.to_cranelift() { + self.builder + .func + .signature + .returns + .push(AbiParam::new(val_type)); + } } - // If this is e.g. an Int, Float, or Bool we have a Cranelift `Value`. - // If we have JitValue::None or .Tuple(...) but can't handle that, error out (or handle differently). - let cr_val = val.into_value().ok_or(JitCompileError::NotSupported)?; - - self.builder.ins().return_(&[cr_val]); + if let Some(cr_val) = val.into_value() { + self.builder.ins().return_(&[cr_val]); + } else { + self.builder.ins().return_(&[]); + } Ok(()) } diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index dbaa4a3eb26..0c700e93cf8 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -61,19 +61,12 @@ impl Jit { ret: Option, ) -> Result<(FuncId, JitSig), JitCompileError> { for arg in args { - self.ctx - .func - .signature - .params - .push(AbiParam::new(arg.to_cranelift())); + let arg = arg.to_cranelift().ok_or(JitCompileError::NotSupported)?; + self.ctx.func.signature.params.push(AbiParam::new(arg)); } - if ret.is_some() { - self.ctx - .func - .signature - .returns - .push(AbiParam::new(ret.clone().unwrap().to_cranelift())); + if let Some(ret) = ret.as_ref().and_then(JitType::to_cranelift) { + self.ctx.func.signature.returns.push(AbiParam::new(ret)); } let id = self.module.declare_function( @@ -167,7 +160,10 @@ impl CompiledCode { libffi::middle::CodePtr::from_ptr(self.code as *const _), cif_args, ); - self.sig.ret.as_ref().map(|ty| value.to_typed(ty)) + match self.sig.ret.as_ref() { + Some(JitType::None) | None => None, + Some(ty) => Some(value.to_typed(ty)), + } } } } @@ -193,14 +189,16 @@ pub enum JitType { Int, Float, Bool, + None, } impl JitType { - fn to_cranelift(&self) -> types::Type { + fn to_cranelift(&self) -> Option { match self { - Self::Int => types::I64, - Self::Float => types::F64, - Self::Bool => types::I8, + Self::Int => Some(types::I64), + Self::Float => Some(types::F64), + Self::Bool => Some(types::I8), + Self::None => None, } } @@ -209,6 +207,7 @@ impl JitType { Self::Int => libffi::middle::Type::i64(), Self::Float => libffi::middle::Type::f64(), Self::Bool => libffi::middle::Type::u8(), + Self::None => libffi::middle::Type::void(), } } } @@ -306,6 +305,7 @@ impl UnTypedAbiValue { JitType::Int => AbiValue::Int(self.int), JitType::Float => AbiValue::Float(self.float), JitType::Bool => AbiValue::Bool(self.boolean != 0), + JitType::None => unreachable!("None has no ABI value"), } } } diff --git a/crates/jit/tests/misc_tests.rs b/crates/jit/tests/misc_tests.rs index b73100ad6ec..5404df0a769 100644 --- a/crates/jit/tests/misc_tests.rs +++ b/crates/jit/tests/misc_tests.rs @@ -2,16 +2,15 @@ mod tests { use rustpython_jit::{AbiValue, JitArgumentError}; - // TODO currently broken - // #[test] - // fn test_no_return_value() { - // let func = jit_function! { func() => r##" - // def func(): - // pass - // "## }; - // - // assert_eq!(func(), Ok(())); - // } + #[test] + fn no_return_value() { + let func = jit_function! { func() => r##" + def func(): + pass + "## }; + + assert_eq!(func(), Ok(())); + } #[test] fn invoke() { From 98d060d69d2a8b3b4745cc51cec72c820e055eff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:25:19 +0900 Subject: [PATCH 10/28] build(deps): bump cargo-bins/cargo-binstall from 1.21.0 to 1.21.1 (#8482) Bumps [cargo-bins/cargo-binstall](https://github.com/cargo-bins/cargo-binstall) from 1.21.0 to 1.21.1. - [Release notes](https://github.com/cargo-bins/cargo-binstall/releases) - [Changelog](https://github.com/cargo-bins/cargo-binstall/blob/main/release-plz.toml) - [Commits](https://github.com/cargo-bins/cargo-binstall/compare/ead08b90bd7b2e6d81963fb9cf0b7239f66d5db4...e00d2c94cc0067b77737821097a62d91c0301baa) --- updated-dependencies: - dependency-name: cargo-bins/cargo-binstall dependency-version: 1.21.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae82353c284..3a295f7427d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -526,7 +526,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: cargo-bins/cargo-binstall@ead08b90bd7b2e6d81963fb9cf0b7239f66d5db4 # v1.21.0 + - uses: cargo-bins/cargo-binstall@e00d2c94cc0067b77737821097a62d91c0301baa # v1.21.1 - name: cargo shear run: | From 81187db09bcf5e85c25f5b712f239bb82f5a93ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:25:28 +0900 Subject: [PATCH 11/28] build(deps): bump j178/prek-action from 2.0.6 to 3.0.0 (#8483) Bumps [j178/prek-action](https://github.com/j178/prek-action) from 2.0.6 to 3.0.0. - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/5337cb91e0fa35a7ff31b9ca345126d8bbbcdf16...4e14d07f9231acabce116ccfca13b13dd9755ece) --- updated-dependencies: - dependency-name: j178/prek-action dependency-version: 3.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3a295f7427d..4222579f621 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -567,7 +567,7 @@ jobs: - name: install prek id: prek - uses: j178/prek-action@5337cb91e0fa35a7ff31b9ca345126d8bbbcdf16 # v2.0.6 + uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 with: cache: false show-verbose-logs: false From 0d8260b88528cf43d86b8d97f112e47fd3f4e574 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:25:40 +0900 Subject: [PATCH 12/28] build(deps): bump jiff in the jiff group across 1 directory (#8484) Bumps the jiff group with 1 update in the / directory: [jiff](https://github.com/BurntSushi/jiff). Updates `jiff` from 0.2.31 to 0.2.35 - [Release notes](https://github.com/BurntSushi/jiff/releases) - [Changelog](https://github.com/BurntSushi/jiff/blob/master/CHANGELOG.md) - [Commits](https://github.com/BurntSushi/jiff/compare/jiff-static-0.2.31...jiff-static-0.2.35) --- updated-dependencies: - dependency-name: jiff dependency-version: 0.2.35 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: jiff ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3acba335146..b8762619a97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1768,11 +1768,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.31" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", @@ -1784,12 +1785,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + [[package]] name = "jiff-static" -version = "0.2.31" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.119", From c30748c961e369fa6a18907cc046b1aee328bca8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:25:50 +0900 Subject: [PATCH 13/28] build(deps): bump pyo3 in the pyo3 group across 1 directory (#8485) Bumps the pyo3 group with 1 update in the / directory: [pyo3](https://github.com/pyo3/pyo3). Updates `pyo3` from 0.29.0 to 0.29.2 - [Release notes](https://github.com/pyo3/pyo3/releases) - [Changelog](https://github.com/PyO3/pyo3/blob/main/CHANGELOG.md) - [Commits](https://github.com/pyo3/pyo3/compare/v0.29.0...v0.29.2) --- updated-dependencies: - dependency-name: pyo3 dependency-version: 0.29.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: pyo3 ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8762619a97..4af185c048b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2790,9 +2790,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "libc", "once_cell", @@ -2804,18 +2804,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -2823,9 +2823,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2835,9 +2835,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", From 6ff1c26e6382f17b78d3818275f776c21c4d10b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:25:59 +0900 Subject: [PATCH 14/28] build(deps): bump libz-rs-sys from 0.6.6 to 0.6.7 (#8486) Bumps [libz-rs-sys](https://github.com/trifectatechfoundation/zlib-rs) from 0.6.6 to 0.6.7. - [Release notes](https://github.com/trifectatechfoundation/zlib-rs/releases) - [Changelog](https://github.com/trifectatechfoundation/zlib-rs/blob/main/docs/release.md) - [Commits](https://github.com/trifectatechfoundation/zlib-rs/compare/v0.6.6...v0.6.7) --- updated-dependencies: - dependency-name: libz-rs-sys dependency-version: 0.6.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4af185c048b..26015d39933 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2033,9 +2033,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50474818739ccab820cd57bca432d6b02d090b47f9e85501d963cd05851f82eb" +checksum = "03dcace986b149f29509af6ca70e6182bccce916b644424ecf484faa8ddc899a" dependencies = [ "zlib-rs", ] @@ -5081,9 +5081,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" [[package]] name = "zmij" From 68fa89ac9e52be7b71425075f0c5ec23595502d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:26:10 +0900 Subject: [PATCH 15/28] build(deps): bump github/gh-aw/actions/setup from 0.83.4 to 0.84.3 (#8487) Bumps [github/gh-aw/actions/setup](https://github.com/github/gh-aw) from 0.83.4 to 0.84.3. - [Release notes](https://github.com/github/gh-aw/releases) - [Changelog](https://github.com/github/gh-aw/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/gh-aw/compare/bbb8042878459948333b15b66f27113f4b5c1b9a...53258938b59e0797fefeed05ec0c681514b2a827) --- updated-dependencies: - dependency-name: github/gh-aw/actions/setup dependency-version: 0.84.3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/upgrade-pylib.lock.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml index 4f2944408f2..3ac44585943 100644 --- a/.github/workflows/upgrade-pylib.lock.yml +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -58,7 +58,7 @@ jobs: comment_repo: "" steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Check workflow file timestamps @@ -99,7 +99,7 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Checkout repository @@ -806,7 +806,7 @@ jobs: total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Download agent output artifact @@ -927,7 +927,7 @@ jobs: success: ${{ steps.parse_results.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Download agent artifacts @@ -1039,7 +1039,7 @@ jobs: process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Download agent output artifact From b25b14fffb4c7f96e8c3261bfa684ad0a31a7e42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:26:22 +0900 Subject: [PATCH 16/28] build(deps): bump https://github.com/astral-sh/ruff-pre-commit (#8488) Bumps [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit) from v0.16.0 to 0.16.1. - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.16.0...v0.16.1) --- updated-dependencies: - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.16.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db9869bb3a5..9ffc4b8d4fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: priority: 0 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.0 + rev: v0.16.1 hooks: - id: ruff-format priority: 0 From 05a9873a99d152069bb0418fb7f85a5140d7e8fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:37:49 +0900 Subject: [PATCH 17/28] build(deps): bump zizmorcore/zizmor-action from 0.6.1 to 0.6.2 (#8481) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.1 to 0.6.2. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/6fc4b006235f201fdab3722e17240ab420d580e5...3dc1ecc9bcb9e94e9b2c709687979e1298497054) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4222579f621..d832398ebf6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -557,7 +557,7 @@ jobs: uses: reviewdog/action-actionlint@50842263c20a7c46bd0065b9e624d3c569db061e # v1.73.0 - name: zizmor - uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1 + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 - name: restore prek cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 From e8431490e72bbb864f85c9f806035b4e165b5885 Mon Sep 17 00:00:00 2001 From: William Goode <95141298+william-goode@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:16:10 -0400 Subject: [PATCH 18/28] Fix map no iterables (#8478) * map: raise TypeError when called with no iterables Assisted-by: Claude Code:claude-fable-5 * add test for map with no iterables Assisted-by: Claude Code:claude-fable-5 * removed expectedFailures * removed test for map constructed with no iterables - covered by existing in test_itertools --- Lib/test/test_itertools.py | 1 - crates/vm/src/builtins/map.rs | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index e865c9bf059..585f6611ade 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1136,7 +1136,6 @@ def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) diff --git a/crates/vm/src/builtins/map.rs b/crates/vm/src/builtins/map.rs index 4dda9caf211..cb8db23e640 100644 --- a/crates/vm/src/builtins/map.rs +++ b/crates/vm/src/builtins/map.rs @@ -37,9 +37,12 @@ impl Constructor for PyMap { fn py_new( _cls: &Py, (mapper, iterators, args): Self::Args, - _vm: &VirtualMachine, + vm: &VirtualMachine, ) -> PyResult { let iterators = iterators.into_vec(); + if iterators.is_empty() { + return Err(vm.new_type_error("map() must have at least two arguments.")); + } let strict = Radium::new(args.strict.unwrap_or(false)); Ok(Self { mapper, From 81df1ff12f5cf660d6189b3c0dcf375beb0ab1bd Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:07:02 +0900 Subject: [PATCH 19/28] Replace deleted README demo script reference (#8492) * Initial plan * Fix stale README example Assisted-by: Copilot: GPT-5.6 Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 37b577d0f0e..cb086687d8f 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ needed to prevent stack overflow on Windows): ```bash $ cd RustPython -$ cargo run --release demo_closures.py +$ cargo run --release -- -c 'print("Hello, RustPython!")' Hello, RustPython! ``` From 7612e68357e4d7b8c386eb7785d000f5fcfe6f8d Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Wed, 12 Aug 2026 16:20:37 +0900 Subject: [PATCH 20/28] Reuse stored hashes when building a set from a set/frozenset/dict (#8491) * Reuse stored hashes when building a set from a set/frozenset/dict set and frozenset recomputed __hash__ for every element even when the source object already stored a hash per entry. CPython's set_update_internal branches on PyAnySet_Check / PyDict_CheckExact and feeds set_add_entry the hash read from the source table; RustPython always iterated generically. Split the hash computation out of the Dict entry points so callers can supply a hash they already hold, add keys_with_hashes() to hand out (key, hash) pairs, and take the fast path in the set constructors and in the set operations whose argument is a set/frozenset/exact dict. ArgIterable::as_object() exposes the pre-__iter__ object so the set operations can dispatch on the source type without changing any of their signatures. Closes #8489. dict.fromkeys() is the dict-target counterpart and is tracked in #8490, so test_do_not_rehash_dict_keys keeps its expectedFailure marker until that lands too. Co-Authored-By: Claude Opus 5 (1M context) * Rename the hash-carrying variants to *_known_hash `_with_hash` said nothing about which direction the hash travels, and the same suffix was already used both ways in this file: keys_with_hashes() hands hashes out, while insert_with_hash() takes one in. The pre-existing insert_with_hint()/get_with_hint() pair has the same problem. Follow CPython's "KnownHash" variants (_PyDict_SetItem_KnownHash, _PyDict_Contains_KnownHash, _PyDict_DelItem_KnownHash) instead, so the suffix marks the argument direction and matches the name a reader familiar with CPython already expects. keys_with_hashes() keeps `with` because it really does return the hashes. Co-Authored-By: Claude Opus 5 (1M context) * Narrow visibility of the new helpers and trim their docs from_object() and ArgIterable::as_object() had no caller outside their own file and crate respectively, so drop them to private and pub(crate). The dict_inner helpers stay pub(crate) because builtins::set calls them. Also shorten the doc comments to match the density of the surrounding code; only insert_known_hash keeps a real note, since a wrong hash there corrupts the table silently. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- crates/vm/src/builtins/set.rs | 121 ++++++++++++++++++++++++----- crates/vm/src/dict_inner.rs | 84 +++++++++++++++++++- crates/vm/src/function/protocol.rs | 5 ++ 3 files changed, 190 insertions(+), 20 deletions(-) diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index cd724abc5c1..62bdb0f0da5 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -195,6 +195,26 @@ impl PySetInner { Ok(set) } + /// Build a set from an arbitrary object, reusing stored hashes when the + /// source is a set/frozenset/dict. + fn from_object(iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let set = Self::default(); + set.update_internal(iterable, vm)?; + Ok(set) + } + + /// Elements of `obj` with their stored hashes, or `None` if `obj` keeps + /// none and must be iterated generically. Mirrors the `PyAnySet_Check` / + /// `PyDict_CheckExact` fast paths in CPython's `set_update_internal`. + fn cached_hashes(obj: &PyObject, vm: &VirtualMachine) -> Option> { + if let Some(set) = extract_set(obj) { + Some(set.content.keys_with_hashes()) + } else { + obj.downcast_ref_if_exact::(vm) + .map(|dict| dict._as_dict_inner().keys_with_hashes()) + } + } + fn fold_op( &self, others: impl core::iter::Iterator, @@ -228,6 +248,17 @@ impl PySetInner { Self::wrap_unhashable_error(result, needle, vm) } + /// [`Self::contains`] with a known hash. Such a needle came out of a + /// set/dict, so it is hashable and needs no frozenset retry. + fn contains_known_hash( + &self, + needle: &PyObject, + hash: PyHash, + vm: &VirtualMachine, + ) -> PyResult { + self.content.contains_known_hash(vm, needle, hash) + } + fn compare(&self, other: &Self, op: PyComparisonOp, vm: &VirtualMachine) -> PyResult { if op == PyComparisonOp::Ne { return self.compare(other, PyComparisonOp::Eq, vm).map(|eq| !eq); @@ -251,6 +282,12 @@ impl PySetInner { pub(super) fn union(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = self.clone(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (item, hash) in elements { + set.add_known_hash(item, hash, vm)?; + } + return Ok(set); + } for item in other.iter(vm)? { set.add(item?, vm)?; } @@ -260,6 +297,14 @@ impl PySetInner { pub(super) fn intersection(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = Self::default(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (obj, hash) in elements { + if self.contains_known_hash(&obj, hash, vm)? { + set.add_known_hash(obj, hash, vm)?; + } + } + return Ok(set); + } for item in other.iter(vm)? { let obj = item?; if self.contains(&obj, vm)? { @@ -271,6 +316,12 @@ impl PySetInner { pub(super) fn difference(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = self.copy(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (item, hash) in elements { + set.content.delete_if_exists_known_hash(vm, &*item, hash)?; + } + return Ok(set); + } for item in other.iter(vm)? { set.content.delete_if_exists(vm, &*item?)?; } @@ -284,6 +335,16 @@ impl PySetInner { ) -> PyResult { let new_inner = self.clone(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + // the source is already duplicate-free + for (item, hash) in elements { + new_inner + .content + .delete_or_insert_known_hash(vm, &item, hash, ())?; + } + return Ok(new_inner); + } + // We want to remove duplicates in other let other_set = Self::from_iter(other.iter(vm)?, vm)?; @@ -333,6 +394,12 @@ impl PySetInner { Self::wrap_unhashable_error(result, &item, vm) } + /// [`Self::add`] with a known hash. + fn add_known_hash(&self, item: PyObjectRef, hash: PyHash, vm: &VirtualMachine) -> PyResult<()> { + let result = self.content.insert_known_hash(vm, &*item, hash, ()); + Self::wrap_unhashable_error(result, &item, vm) + } + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let result = self.retry_op_with_frozenset(&item, vm, |item, vm| self.content.delete(vm, item)); @@ -393,15 +460,15 @@ impl PySetInner { } fn merge_set(&self, any_set: AnySet, vm: &VirtualMachine) -> PyResult<()> { - for item in any_set.as_inner().elements() { - self.add(item, vm)?; + for (item, hash) in any_set.as_inner().content.keys_with_hashes() { + self.add_known_hash(item, hash, vm)?; } Ok(()) } fn merge_dict(&self, dict: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { - for (key, _value) in dict { - self.add(key, vm)?; + for (key, hash) in dict._as_dict_inner().keys_with_hashes() { + self.add_known_hash(key, hash, vm)?; } Ok(()) } @@ -413,8 +480,8 @@ impl PySetInner { ) -> PyResult<()> { let temp_inner = self.fold_op(others, Self::intersection, vm)?; self.clear(); - for obj in temp_inner.elements() { - self.add(obj, vm)?; + for (obj, hash) in temp_inner.content.keys_with_hashes() { + self.add_known_hash(obj, hash, vm)?; } Ok(()) } @@ -425,6 +492,12 @@ impl PySetInner { vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { + if let Some(elements) = Self::cached_hashes(iterable.as_object(), vm) { + for (item, hash) in elements { + self.content.delete_if_exists_known_hash(vm, &*item, hash)?; + } + continue; + } let items = iterable.iter(vm)?.collect::, _>>()?; for item in items { self.content.delete_if_exists(vm, &*item)?; @@ -439,6 +512,14 @@ impl PySetInner { vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { + if let Some(elements) = Self::cached_hashes(iterable.as_object(), vm) { + // the source is already duplicate-free + for (item, hash) in elements { + self.content + .delete_or_insert_known_hash(vm, &item, hash, ())?; + } + continue; + } // We want to remove duplicates in iterable let iterable_set = Self::from_iter(iterable.iter(vm)?, vm)?; for item in iterable_set.elements() { @@ -955,7 +1036,7 @@ impl Representable for PySet { } impl Constructor for PyFrozenSet { - type Args = Vec; + type Args = OptionalArg; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { let is_exact_frozenset = cls.is(vm.ctx.types.frozenset_type); @@ -988,11 +1069,11 @@ impl Constructor for PyFrozenSet { return Ok(input.clone()); } - iterable.into_option() + iterable } else { match &args.args[..] { - [] => None, - [iterable] => Some(iterable.clone()), + [] => OptionalArg::Missing, + [iterable] => OptionalArg::Present(iterable.clone()), slice => { return Err(vm.new_type_error(format!( "frozenset expected at most 1 argument, got {}", @@ -1002,23 +1083,25 @@ impl Constructor for PyFrozenSet { } }; - let elements = if let Some(iterable) = iterable_opt { - iterable.try_to_value(vm)? - } else { - vec![] - }; + let payload = Self::py_new(&cls, iterable_opt, vm)?; // Return empty frozenset singleton - if is_exact_frozenset && elements.is_empty() { + if is_exact_frozenset && payload.inner.len() == 0 { return Ok(vm.ctx.empty_frozenset.clone().into()); } - let payload = Self::py_new(&cls, elements, vm)?; payload.into_ref_with_type(vm, cls).map(Into::into) } - fn py_new(_cls: &Py, elements: Self::Args, vm: &VirtualMachine) -> PyResult { - Self::from_iter(vm, elements) + fn py_new(_cls: &Py, iterable: Self::Args, vm: &VirtualMachine) -> PyResult { + let inner = match iterable { + OptionalArg::Present(iterable) => PySetInner::from_object(iterable, vm)?, + OptionalArg::Missing => PySetInner::default(), + }; + Ok(Self { + inner, + ..Default::default() + }) } } diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 3e75e6f27a6..9dda6194a0c 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -463,6 +463,24 @@ impl Dict { K: DictKey + ?Sized, { let hash = key.key_hash(vm)?; + self.insert_known_hash(vm, key, hash, value) + } + + /// Store a key whose hash the caller already knows. + /// + /// `hash` must equal `key.key_hash(vm)`; a wrong one lands the entry in a + /// bucket no lookup probes, silently losing the key. Only pass a hash from + /// [`Self::keys_with_hashes`] on a container holding this same key. + pub(crate) fn insert_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + value: T, + ) -> PyResult<()> + where + K: DictKey + ?Sized, + { let _removed = loop { let (entry_index, index_index) = self.lookup(vm, key, hash, None)?; let mut inner = self.write(); @@ -512,7 +530,18 @@ impl Dict { key: &K, ) -> PyResult { let key_hash = key.key_hash(vm)?; - let (entry, _) = self.lookup(vm, key, key_hash, None)?; + self.contains_known_hash(vm, key, key_hash) + } + + /// [`Self::contains`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn contains_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + ) -> PyResult { + let (entry, _) = self.lookup(vm, key, hash, None)?; Ok(entry.index().is_some()) } @@ -697,6 +726,21 @@ impl Dict { self.remove_if_exists(vm, key).map(|opt| opt.is_some()) } + /// [`Self::delete_if_exists`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn delete_if_exists_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + ) -> PyResult + where + K: DictKey + ?Sized, + { + self.remove_if_known_hash(vm, key, hash, |_| Ok(true)) + .map(|opt| opt.is_some()) + } + pub(crate) fn delete_if(&self, vm: &VirtualMachine, key: &K, pred: F) -> PyResult where K: DictKey + ?Sized, @@ -725,6 +769,22 @@ impl Dict { F: Fn(&T) -> PyResult, { let hash = key.key_hash(vm)?; + self.remove_if_known_hash(vm, key, hash, pred) + } + + /// [`Self::remove_if`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + fn remove_if_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + pred: F, + ) -> PyResult> + where + K: DictKey + ?Sized, + F: Fn(&T) -> PyResult, + { let removed = loop { let lookup = self.lookup(vm, key, hash, None)?; match self.pop_inner_if(lookup, &pred)? { @@ -742,6 +802,18 @@ impl Dict { value: T, ) -> PyResult<()> { let hash = key.key_hash(vm)?; + self.delete_or_insert_known_hash(vm, key, hash, value) + } + + /// [`Self::delete_or_insert`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn delete_or_insert_known_hash( + &self, + vm: &VirtualMachine, + key: &PyObject, + hash: HashValue, + value: T, + ) -> PyResult<()> { let _removed = loop { let lookup = self.lookup(vm, key, hash, None)?; let (entry, index_index) = lookup; @@ -888,6 +960,16 @@ impl Dict { .collect() } + /// All keys paired with the hash stored in their entry, for feeding + /// [`Self::insert_known_hash`] without re-calling `__hash__`. + pub(crate) fn keys_with_hashes(&self) -> Vec<(PyObjectRef, HashValue)> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| (v.key.clone(), v.hash))) + .collect() + } + pub(crate) fn values(&self) -> Vec { self.read() .entries diff --git a/crates/vm/src/function/protocol.rs b/crates/vm/src/function/protocol.rs index 25ef62b458d..d503fabaca8 100644 --- a/crates/vm/src/function/protocol.rs +++ b/crates/vm/src/function/protocol.rs @@ -86,6 +86,11 @@ unsafe impl Traverse for ArgIterable { } impl ArgIterable { + #[must_use] + pub(crate) fn as_object(&self) -> &PyObject { + &self.iterable + } + /// Returns an iterator over this sequence of objects. /// /// This operation may fail if an exception is raised while invoking the From 643039de9403ab44b922c38b837c052fb5c8f013 Mon Sep 17 00:00:00 2001 From: Jeaeun Kim <109906379+kyokuping@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:22:05 +0900 Subject: [PATCH 21/28] Fix __qualname__ of compiler-generated __annotate__ functions (#8498) Assisted-by: Codex:5.6-sol --- Lib/test/test_type_annotations.py | 1 - crates/codegen/src/compile.rs | 64 ++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3f056f2b753..c98b99e98e9 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -843,7 +843,6 @@ def test_complex_comprehension_inlining_exec(self): lamb = list(genexp)[0] self.assertEqual(lamb(), 42) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__annotate__' != 'f.__annotate__' def test_annotate_qualname(self): code = """ def f() -> None: diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index ba2d41f12b3..b16540762fd 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -2175,7 +2175,7 @@ impl<'warnings> Compiler<'warnings> { /// On success, returns the saved CompileContext to pass to exit_annotation_scope. fn enter_annotation_scope( &mut self, - _func_name: &str, + func_name: &str, loc: TextRange, ) -> CompileResult> { if !self.push_annotation_symbol_table() { @@ -2200,6 +2200,12 @@ impl<'warnings> Compiler<'warnings> { lineno.to_u32(), )?; + // enter_scope() qualified the scope by the enclosing scope only; redo it + // now that the annotated function is known. Only signature annotations + // get this treatment - deferred class and module annotations are + // compiled inside the scope they belong to and are already qualified. + self.set_annotation_qualname(func_name); + // Keep the internal ".format" name; exit_annotation_scope() // renames it to "format" on the final code object. self.current_code_info() @@ -2605,11 +2611,24 @@ impl<'warnings> Compiler<'warnings> { /// Set the qualified name for the current code object // = compiler_set_qualname fn set_qualname(&mut self) -> String { - let qualname = self.make_qualname(); + self.set_qualname_for_function(None) + } + + /// Set the qualname of an annotation scope, qualified by the function whose + /// signature it annotates. CPython records that name on the annotation + /// block's symbol table entry (`ste_function_name`) and folds it into the + /// qualname, so `f`'s annotation scope is named `f.__annotate__`. + fn set_annotation_qualname(&mut self, function_name: &str) { + self.set_qualname_for_function(Some(function_name)); + } + + fn set_qualname_for_function(&mut self, function_name: Option<&str>) -> String { + let qualname = self.make_qualname(function_name); self.current_code_info().metadata.qualname = Some(qualname.clone()); qualname } - fn make_qualname(&mut self) -> String { + + fn make_qualname(&mut self, function_name: Option<&str>) -> String { let stack_size = self.code_stack.len(); assert!(stack_size >= 1); @@ -2693,10 +2712,10 @@ impl<'warnings> Compiler<'warnings> { } } - // Build the qualified name - if force_global { + // Build the prefix the current name is qualified by, if any + let base = if force_global { // For global symbols, qualname is just the name - current_obj_name + None } else { // Check parent scope type let parent_obj_name = &parent.metadata.name; @@ -2709,23 +2728,32 @@ impl<'warnings> Compiler<'warnings> { ) ); + // Use parent's qualname if available, otherwise use parent_obj_name + let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); + if is_function_parent { // For functions, append . to parent qualname - // Use parent's qualname if available, otherwise use parent_obj_name - let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); - format!("{parent_qualname}..{current_obj_name}") + Some(format!("{parent_qualname}.")) + } else if parent_qualname == "" { + // Module level, nothing to qualify by + None } else { // For classes and other scopes, use parent's qualname directly - // Use parent's qualname if available, otherwise use parent_obj_name - let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); - if parent_qualname == "" { - // Module level, just use the name - current_obj_name - } else { - // Concatenate parent qualname with current name - format!("{parent_qualname}.{current_obj_name}") - } + Some(parent_qualname.clone()) } + }; + + // An annotation scope is compiled in the scope enclosing the function it + // annotates, so the function itself is missing from the prefix above. + let base = match (base, function_name) { + (Some(base), Some(function_name)) => Some(format!("{base}.{function_name}")), + (None, Some(function_name)) => Some(function_name.to_owned()), + (base, None) => base, + }; + + match base { + Some(base) => format!("{base}.{current_obj_name}"), + None => current_obj_name, } } From 9de06ccdb34d35b564e8dcc8e60d22f7fe01d0f1 Mon Sep 17 00:00:00 2001 From: Jeaeun Kim <109906379+kyokuping@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:24:44 +0900 Subject: [PATCH 22/28] jit: preserve return types for recursive calls (#8499) Assisted-by: Codex:5.6-sol --- crates/jit/src/instructions.rs | 18 ++++++++++++++++-- crates/jit/tests/bool_tests.rs | 14 ++++++++++++++ crates/jit/tests/float_tests.rs | 14 ++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/crates/jit/src/instructions.rs b/crates/jit/src/instructions.rs index 67cf07f6e7f..a3c4ca800c4 100644 --- a/crates/jit/src/instructions.rs +++ b/crates/jit/src/instructions.rs @@ -548,8 +548,22 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { JitValue::FuncRef(reference) => { let call = self.builder.ins().call(reference, &args); - let returns = self.builder.inst_results(call); - self.stack.push(JitValue::Int(returns[0])); + // The only callable reachable here is this function itself, + // so the result carries the declared return type - it is not + // always an Int. A function whose return type is still + // unknown has no return slot in the signature it was + // declared with, and there is nothing to type the result as. + let ret = match *self.builder.inst_results(call) { + [] => None, + [val] => Some(val), + _ => return Err(JitCompileError::NotSupported), + }; + let val = match (self.sig.ret.clone(), ret) { + (Some(JitType::None), None) => JitValue::None, + (Some(ty), Some(val)) => JitValue::from_type_and_value(ty, val), + _ => return Err(JitCompileError::NotSupported), + }; + self.stack.push(val); Ok(()) } diff --git a/crates/jit/tests/bool_tests.rs b/crates/jit/tests/bool_tests.rs index 8a5f4ea9db3..1874ee4d55d 100644 --- a/crates/jit/tests/bool_tests.rs +++ b/crates/jit/tests/bool_tests.rs @@ -202,4 +202,18 @@ mod tests { assert_eq!(lte(false, 1), Ok(1)); assert_eq!(lte(true, 0), Ok(0)); } + + #[test] + fn recursive_bool() { + let recursive_bool = jit_function! { recursive_bool(n: i64) -> bool => r##" + def recursive_bool(n: int) -> bool: + if n == 0: + return True + return not recursive_bool(n - 1) + "## }; + + assert_eq!(recursive_bool(0), Ok(true)); + assert_eq!(recursive_bool(1), Ok(false)); + assert_eq!(recursive_bool(4), Ok(true)); + } } diff --git a/crates/jit/tests/float_tests.rs b/crates/jit/tests/float_tests.rs index b9bbb3ea63c..f667b1e764a 100644 --- a/crates/jit/tests/float_tests.rs +++ b/crates/jit/tests/float_tests.rs @@ -379,4 +379,18 @@ mod tests { assert_eq!(float_lte(f64::NAN, f64::NAN), Ok(false)); assert_eq!(float_lte(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); } + + #[test] + fn recursive_float() { + let recursive_float = jit_function! { recursive_float(n: i64) -> f64 => r##" + def recursive_float(n: int) -> float: + if n == 0: + return 1.0 + return recursive_float(n - 1) / 2.0 + "## }; + + assert_eq!(recursive_float(0), Ok(1.0)); + assert_eq!(recursive_float(1), Ok(0.5)); + assert_eq!(recursive_float(4), Ok(0.0625)); + } } From d64cc2cff5ee611862ca9d2a27ccbd9a6f740e5d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:28:09 +0900 Subject: [PATCH 23/28] Fix marshal recursive reference loading (#8501) * Fix marshal recursive reference loading Create reference-tracked containers before reading their children so recursive list, dict, set, and tuple graphs can be unmarshaled. Preserve interned string markers through the runtime bag and add an initialization-only tuple construction path. Assisted-by: Codex:gpt-5 * Preserve marshal container insertion errors Keep Python exceptions raised while constructing unmarshaled sets, frozensets, and dictionaries instead of collapsing them into ValueError. This makes abnormal recursive hash-container streams report TypeError like CPython and removes the remaining test_marshal expected failure. Assisted-by: Codex:gpt-5 * Enable full abnormal marshal reference loop test * Test direct marshal tuple reference loop Assisted-by: Codex:gpt-5 --- Lib/test/test_marshal.py | 21 ++-- crates/compiler-core/src/marshal.rs | 168 ++++++++++++++++++++++++---- crates/vm/src/builtins/tuple.rs | 101 ++++++++++++++--- crates/vm/src/stdlib/marshal.rs | 165 +++++++++++++++++++++------ 4 files changed, 366 insertions(+), 89 deletions(-) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index ad4c6095abf..1f04a7f697e 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -319,7 +319,6 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_list(self): a = [] a.append(a) @@ -331,7 +330,6 @@ def test_reference_loop_list(self): self.assertIsInstance(b, list) self.assertIs(b[0], b) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_dict(self): a = {} a[None] = a @@ -343,7 +341,6 @@ def test_reference_loop_dict(self): self.assertIsInstance(b, dict) self.assertIs(b[None], b) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_tuple(self): a = ([],) a[0].append(a) @@ -387,21 +384,18 @@ def test_reference_loop_slice(self): for v in range(marshal.version + 1): self.assertRaises(ValueError, marshal.dumps, a, v) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_reference_loop_list(self): data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [] a = marshal.loads(data) self.assertIsInstance(a, list) self.assertIs(a[0], a) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_reference_loop_dict(self): data = b'\xfbNr\x00\x00\x00\x000' # {None: } a = marshal.loads(data) self.assertIsInstance(a, dict) self.assertIs(a[None], a) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_abnormal_reference_loops(self): # Indirect self-references of tuples. data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([],) @@ -416,13 +410,13 @@ def test_loads_abnormal_reference_loops(self): self.assertIsInstance(a[0], dict) self.assertIs(a[0][None], a) - # Direct self-reference which cannot be created in Python. - # This creates a reference loop which cannot be collected. - if False: - data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) - a = marshal.loads(data) - self.assertIsInstance(a, tuple) - self.assertIs(a[0], a) + # Direct self-reference which cannot be created in Python. CPython + # leaves this disabled because its reference counting cannot collect + # the resulting cycle; RustPython's tracing collector can. + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) # Direct self-references which cannot be created in Python # because of unhashability. @@ -748,7 +742,6 @@ class InterningTestCase(unittest.TestCase, HelperMixin): strobj = "this is an interned string" strobj = sys.intern(strobj) - @unittest.expectedFailure # TODO: RUSTPYTHON def testIntern(self): s = marshal.loads(marshal.dumps(self.strobj)) self.assertEqual(s, self.strobj) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index a2a23054e4b..dd5d4f2cddb 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -516,7 +516,7 @@ fn read_const_value( let code = deserialize_code_inner(rdr, bag, depth - 1, refs)?; bag.make_code(code) } else { - deserialize_value_typed(rdr, bag, depth, refs, typ)? + deserialize_value_typed(rdr, bag, depth, refs, typ, slot)? }; if let Some(idx) = slot { refs[idx] = Some(value.clone()); @@ -540,6 +540,10 @@ pub trait MarshalBag: Copy { fn make_str(&self, value: &Wtf8) -> Self::Value; + fn make_interned_str(&self, value: &Wtf8) -> Self::Value { + self.make_str(value) + } + fn make_bytes(&self, value: &[u8]) -> Self::Value; fn make_int(&self, value: BigInt) -> Self::Value; @@ -564,6 +568,51 @@ pub trait MarshalBag: Copy { it: impl Iterator, ) -> Result; + /// Install partially-built containers in the marshal reference table + /// before reading their children, as CPython's `r_object()` does. + /// Runtime bags can opt in; constant bags retain collect-then-construct. + fn make_tuple_placeholder(&self, _len: usize) -> Option { + None + } + + fn set_tuple_item( + &self, + _tuple: &Self::Value, + _index: usize, + _value: Self::Value, + ) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_list_placeholder(&self, _len: usize) -> Option { + None + } + + fn set_list_item(&self, _list: &Self::Value, _index: usize, _value: Self::Value) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_set_placeholder(&self) -> Option { + None + } + + fn insert_set_item(&self, _set: &Self::Value, _value: Self::Value) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_dict_placeholder(&self) -> Option { + None + } + + fn insert_dict_item( + &self, + _dict: &Self::Value, + _key: Self::Value, + _value: Self::Value, + ) -> Result<()> { + Err(MarshalError::BadType) + } + fn make_slice( &self, _start: Self::Value, @@ -755,7 +804,7 @@ fn deserialize_value_after_header( let code = deserialize_code_inner(rdr, bag.constant_bag(), depth - 1, &mut inner_refs)?; bag.make_code(code) } else { - deserialize_value_typed(rdr, bag, depth, refs, typ)? + deserialize_value_typed(rdr, bag, depth, refs, typ, slot)? }; if let Some(idx) = slot { @@ -770,6 +819,7 @@ fn deserialize_value_typed( depth: usize, refs: &mut Vec>, typ: Type, + slot: Option, ) -> Result { if depth == 0 { return Err(MarshalError::InvalidBytecode); @@ -806,21 +856,42 @@ fn deserialize_value_typed( let value = Complex64 { re, im }; bag.make_complex(value) } - Type::Ascii | Type::AsciiInterned | Type::Unicode | Type::Interned => { + Type::Ascii | Type::Unicode => { let len = rdr.read_u32()?; let value = rdr.read_wtf8(len)?; bag.make_str(value) } - Type::ShortAscii | Type::ShortAsciiInterned => { + Type::AsciiInterned | Type::Interned => { + let len = rdr.read_u32()?; + let value = rdr.read_wtf8(len)?; + bag.make_interned_str(value) + } + Type::ShortAscii => { let len = rdr.read_u8()? as u32; let value = rdr.read_wtf8(len)?; bag.make_str(value) } + Type::ShortAsciiInterned => { + let len = rdr.read_u8()? as u32; + let value = rdr.read_wtf8(len)?; + bag.make_interned_str(value) + } Type::SmallTuple => { let len = rdr.read_u8()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_tuple(it))? + if let Some(index) = slot + && let Some(tuple) = bag.make_tuple_placeholder(len) + { + refs[index] = Some(tuple.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_tuple_item(&tuple, item_index, item)?; + } + tuple + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_tuple(it))? + } } Type::Null => { return Err(MarshalError::BadType); @@ -830,22 +901,55 @@ fn deserialize_value_typed( return Err(MarshalError::BadType); } Type::Tuple => { - let len = rdr.read_u32()?; + let len = rdr.read_u32()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_tuple(it))? + if let Some(index) = slot + && let Some(tuple) = bag.make_tuple_placeholder(len) + { + refs[index] = Some(tuple.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_tuple_item(&tuple, item_index, item)?; + } + tuple + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_tuple(it))? + } } Type::List => { - let len = rdr.read_u32()?; + let len = rdr.read_u32()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_list(it))?? + if let Some(index) = slot + && let Some(list) = bag.make_list_placeholder(len) + { + refs[index] = Some(list.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_list_item(&list, item_index, item)?; + } + list + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_list(it))?? + } } Type::Set => { - let len = rdr.read_u32()?; + let len = rdr.read_u32()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_set(it))?? + if let Some(index) = slot + && let Some(set) = bag.make_set_placeholder() + { + refs[index] = Some(set.clone()); + for _ in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.insert_set_item(&set, item)?; + } + set + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_set(it))?? + } } Type::FrozenSet => { let len = rdr.read_u32()?; @@ -855,17 +959,33 @@ fn deserialize_value_typed( } Type::Dict => { let d = depth - 1; - let mut pairs = Vec::new(); - loop { - let raw = rdr.read_u8()?; - if raw & !FLAG_REF == b'0' { - break; + if let Some(index) = slot + && let Some(dict) = bag.make_dict_placeholder() + { + refs[index] = Some(dict.clone()); + loop { + let raw = rdr.read_u8()?; + if raw & !FLAG_REF == b'0' { + break; + } + let key = deserialize_value_after_header(rdr, bag, d, refs, raw)?; + let value = deserialize_value_depth(rdr, bag, d, refs)?; + bag.insert_dict_item(&dict, key, value)?; + } + dict + } else { + let mut pairs = Vec::new(); + loop { + let raw = rdr.read_u8()?; + if raw & !FLAG_REF == b'0' { + break; + } + let key = deserialize_value_after_header(rdr, bag, d, refs, raw)?; + let value = deserialize_value_depth(rdr, bag, d, refs)?; + pairs.push((key, value)); } - let k = deserialize_value_after_header(rdr, bag, d, refs, raw)?; - let v = deserialize_value_depth(rdr, bag, d, refs)?; - pairs.push((k, v)); + bag.make_dict(pairs.into_iter())? } - bag.make_dict(pairs.into_iter())? } Type::Bytes => { // After marshaling, byte arrays are converted into bytes. diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index d48639b2c11..06fa2519205 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -23,12 +23,60 @@ use crate::{ vm::VirtualMachine, }; use alloc::fmt; -use core::cell::Cell; +use core::cell::{Cell, UnsafeCell}; use core::ptr::NonNull; #[pyclass(module = false, name = "tuple", traverse = "manual")] pub struct PyTuple { - elements: Box<[R]>, + elements: TupleElements, +} + +/// Tuple storage is immutable after publication, but marshal must publish a +/// tuple in its reference table before recursively reading its children. +/// This mirrors CPython's `PyTuple_New` followed by `PyTuple_SET_ITEM`. +struct TupleElements(UnsafeCell>); + +unsafe impl Send for TupleElements {} +unsafe impl Sync for TupleElements {} + +impl TupleElements { + const fn new(elements: Box<[R]>) -> Self { + Self(UnsafeCell::new(elements)) + } + + fn as_slice(&self) -> &[R] { + // SAFETY: initialization writes happen only while the tuple is owned by + // the synchronous marshal decoder; afterwards the storage is immutable. + unsafe { &*self.0.get() } + } + + fn get_mut(&mut self) -> &mut Box<[R]> { + self.0.get_mut() + } + + /// # Safety + /// The tuple must still be in its private initialization phase, and each + /// placeholder index must be replaced at most once before it is observable. + unsafe fn set_initializing(&self, index: usize, value: R) { + unsafe { (*self.0.get())[index] = value }; + } +} + +impl core::ops::Deref for TupleElements { + type Target = [R]; + + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} + +impl<'a, R> IntoIterator for &'a TupleElements { + type Item = &'a R; + type IntoIter = core::slice::Iter<'a, R>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } } impl fmt::Debug for PyTuple { @@ -42,11 +90,11 @@ impl fmt::Debug for PyTuple { // Note: Only impl for PyTuple (the default) unsafe impl Traverse for PyTuple { fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { - self.elements.traverse(traverse_fn); + self.elements.as_slice().traverse(traverse_fn); } fn clear(&mut self, out: &mut Vec) { - let elements = core::mem::take(&mut self.elements); + let elements = core::mem::take(self.elements.get_mut()); out.extend(elements.into_vec()); } } @@ -206,7 +254,7 @@ impl Constructor for PyTuple { fn py_new(_cls: &Py, elements: Self::Args, _vm: &VirtualMachine) -> PyResult { Ok(Self { - elements: elements.into_boxed_slice(), + elements: TupleElements::new(elements.into_boxed_slice()), }) } } @@ -245,19 +293,19 @@ impl<'a, R> core::iter::IntoIterator for &'a Py> { impl PyTuple { #[must_use] - pub const fn as_slice(&self) -> &[R] { + pub fn as_slice(&self) -> &[R] { &self.elements } #[inline] #[must_use] - pub const fn len(&self) -> usize { + pub fn len(&self) -> usize { self.elements.len() } #[inline] #[must_use] - pub const fn is_empty(&self) -> bool { + pub fn is_empty(&self) -> bool { self.elements.is_empty() } @@ -274,7 +322,13 @@ impl PyTuple { ctx.empty_tuple.clone() } else { let elements = elements.into_boxed_slice(); - PyRef::new_ref(Self { elements }, ctx.types.tuple_type.to_owned(), None) + PyRef::new_ref( + Self { + elements: TupleElements::new(elements), + }, + ctx.types.tuple_type.to_owned(), + None, + ) } } @@ -283,7 +337,20 @@ impl PyTuple { /// Calling this function implies trying micro optimization for non-zero-sized tuple. #[must_use] pub const fn new_unchecked(elements: Box<[PyObjectRef]>) -> Self { - Self { elements } + Self { + elements: TupleElements::new(elements), + } + } + + pub(crate) fn new_marshal_placeholder(len: usize, ctx: &Context) -> PyRef { + Self::new_ref(vec![ctx.none(); len], ctx) + } + + /// # Safety + /// This tuple must be a marshal placeholder which has not escaped the + /// decoder, and `index` must not have been replaced previously. + pub(crate) unsafe fn set_marshal_item(&self, index: usize, value: PyObjectRef) { + unsafe { self.elements.set_initializing(index, value) }; } fn repeat(zelf: PyRef, value: isize, vm: &VirtualMachine) -> PyResult> { @@ -298,7 +365,10 @@ impl PyTuple { } else { let v = zelf.elements.mul(vm, value)?; let elements = v.into_boxed_slice(); - Self { elements }.into_ref(&vm.ctx) + Self { + elements: TupleElements::new(elements), + } + .into_ref(&vm.ctx) }) } @@ -341,7 +411,10 @@ impl PyTuple { .chain(other.as_slice()) .cloned() .collect::>(); - Self { elements }.into_ref(&vm.ctx) + Self { + elements: TupleElements::new(elements), + } + .into_ref(&vm.ctx) } }); PyArithmeticValue::from_option(added.ok()) @@ -360,7 +433,7 @@ impl PyTuple { #[inline] #[must_use] - pub const fn __len__(&self) -> usize { + pub fn __len__(&self) -> usize { self.elements.len() } @@ -425,7 +498,7 @@ impl PyTuple { let tup_arg = if zelf.class().is(vm.ctx.types.tuple_type) { zelf } else { - Self::new_ref(zelf.elements.clone().into_vec(), &vm.ctx) + Self::new_ref(zelf.elements.as_slice().to_vec(), &vm.ctx) }; (tup_arg,) } diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index ace3aff58f2..a4665d0f5b6 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -9,14 +9,16 @@ mod decl { use crate::{ PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{ - PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat, - PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple, + PyBaseExceptionRef, PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, + PyEllipsis, PyFloat, PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, + PyTuple, }, convert::ToPyObject, function::{ArgBytesLike, OptionalArg}, object::{AsObject, PyPayload}, protocol::PyBuffer, }; + use core::cell::RefCell; use malachite_bigint::BigInt; use num_traits::Zero; use rustpython_compiler_core::marshal::{self, DumpableValue}; @@ -386,79 +388,165 @@ mod decl { } #[derive(Copy, Clone)] - struct PyMarshalBag<'a>(&'a VirtualMachine); + struct PyMarshalBag<'a> { + vm: &'a VirtualMachine, + pending_error: &'a RefCell>, + } + + impl<'a> PyMarshalBag<'a> { + fn new( + vm: &'a VirtualMachine, + pending_error: &'a RefCell>, + ) -> Self { + Self { vm, pending_error } + } + + fn remember_python_error(&self, error: PyBaseExceptionRef) -> marshal::MarshalError { + let mut pending = self.pending_error.borrow_mut(); + if pending.is_none() { + *pending = Some(error); + } + marshal::MarshalError::BadType + } + } impl<'a> marshal::MarshalBag for PyMarshalBag<'a> { type Value = PyObjectRef; type ConstantBag = PyVmBag<'a>; fn make_bool(&self, value: bool) -> Self::Value { - self.0.ctx.new_bool(value).into() + self.vm.ctx.new_bool(value).into() } fn make_none(&self) -> Self::Value { - self.0.ctx.none() + self.vm.ctx.none() } fn make_ellipsis(&self) -> Self::Value { - self.0.ctx.ellipsis.clone().into() + self.vm.ctx.ellipsis.clone().into() } fn make_float(&self, value: f64) -> Self::Value { - self.0.ctx.new_float(value).into() + self.vm.ctx.new_float(value).into() } fn make_complex(&self, value: num_complex::Complex64) -> Self::Value { - self.0.ctx.new_complex(value).into() + self.vm.ctx.new_complex(value).into() } fn make_str(&self, value: &Wtf8) -> Self::Value { - self.0.ctx.new_str(value).into() + self.vm.ctx.new_str(value).into() + } + fn make_interned_str(&self, value: &Wtf8) -> Self::Value { + self.vm.ctx.intern_str(value).to_owned().into() } fn make_bytes(&self, value: &[u8]) -> Self::Value { - self.0.ctx.new_bytes(value.to_vec()).into() + self.vm.ctx.new_bytes(value.to_vec()).into() } fn make_int(&self, value: BigInt) -> Self::Value { - self.0.ctx.new_int(value).into() + self.vm.ctx.new_int(value).into() } fn make_tuple(&self, elements: impl Iterator) -> Self::Value { - self.0.ctx.new_tuple(elements.collect()).into() + self.vm.ctx.new_tuple(elements.collect()).into() + } + fn make_tuple_placeholder(&self, len: usize) -> Option { + Some(PyTuple::new_marshal_placeholder(len, &self.vm.ctx).into()) + } + fn set_tuple_item( + &self, + tuple: &Self::Value, + index: usize, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let tuple = tuple + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + // SAFETY: compiler-core calls this only on a fresh placeholder, + // once per index, before returning it to Python code. + unsafe { tuple.set_marshal_item(index, value) }; + Ok(()) } fn make_code(&self, code: CodeObject) -> Self::Value { - crate::builtins::PyCode::new_ref_with_bag(self.0, code).into() + crate::builtins::PyCode::new_ref_with_bag(self.vm, code).into() } fn make_stop_iter(&self) -> Result { - Ok(self.0.ctx.exceptions.stop_iteration.to_owned().into()) + Ok(self.vm.ctx.exceptions.stop_iteration.to_owned().into()) } fn make_list( &self, it: impl Iterator, ) -> Result { - Ok(self.0.ctx.new_list(it.collect()).into()) + Ok(self.vm.ctx.new_list(it.collect()).into()) + } + fn make_list_placeholder(&self, len: usize) -> Option { + Some(self.vm.ctx.new_list(vec![self.vm.ctx.none(); len]).into()) + } + fn set_list_item( + &self, + list: &Self::Value, + index: usize, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let list = list + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + list.borrow_vec_mut()[index] = value; + Ok(()) } fn make_set( &self, it: impl Iterator, ) -> Result { - let set = PySet::default().into_ref(&self.0.ctx); + let set = PySet::default().into_ref(&self.vm.ctx); for elem in it { - set.add(elem, self.0).unwrap() + set.add(elem, self.vm) + .map_err(|error| self.remember_python_error(error))?; } Ok(set.into()) } + fn make_set_placeholder(&self) -> Option { + Some(PySet::default().into_ref(&self.vm.ctx).into()) + } + fn insert_set_item( + &self, + set: &Self::Value, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let set = set + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + set.add(value, self.vm) + .map_err(|error| self.remember_python_error(error)) + } fn make_frozenset( &self, it: impl Iterator, ) -> Result { - Ok(PyFrozenSet::from_iter(self.0, it) - .unwrap() - .to_pyobject(self.0)) + PyFrozenSet::from_iter(self.vm, it) + .map(|set| set.to_pyobject(self.vm)) + .map_err(|error| self.remember_python_error(error)) } fn make_dict( &self, it: impl Iterator, ) -> Result { - let dict = self.0.ctx.new_dict(); + let dict = self.vm.ctx.new_dict(); for (k, v) in it { - dict.set_item(&*k, v, self.0).unwrap() + dict.set_item(&*k, v, self.vm) + .map_err(|error| self.remember_python_error(error))?; } Ok(dict.into()) } + fn make_dict_placeholder(&self) -> Option { + Some(self.vm.ctx.new_dict().into()) + } + fn insert_dict_item( + &self, + dict: &Self::Value, + key: Self::Value, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let dict = dict + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + dict.set_item(&*key, value, self.vm) + .map_err(|error| self.remember_python_error(error)) + } fn make_slice( &self, start: Self::Value, @@ -466,7 +554,7 @@ mod decl { step: Self::Value, ) -> Result { use crate::builtins::PySlice; - let vm = self.0; + let vm = self.vm; Ok(PySlice { start: if vm.is_none(&start) { None @@ -480,7 +568,21 @@ mod decl { .into()) } fn constant_bag(self) -> Self::ConstantBag { - PyVmBag(self.0) + PyVmBag(self.vm) + } + } + + fn deserialize_value( + rdr: &mut impl marshal::Read, + vm: &VirtualMachine, + ) -> PyResult { + let pending_error = RefCell::new(None); + match marshal::deserialize_value(rdr, PyMarshalBag::new(vm, &pending_error)) { + Ok(value) => Ok(value), + Err(error) => Err(pending_error.into_inner().unwrap_or_else(|| match error { + marshal::MarshalError::Eof => vm.new_eof_error("marshal data too short"), + _ => vm.new_value_error("bad marshal data"), + })), } } @@ -502,11 +604,7 @@ mod decl { vm.new_buffer_error("Buffer provided to marshal.loads() is not contiguous") })?; - let result = - marshal::deserialize_value(&mut &buf[..], PyMarshalBag(vm)).map_err(|e| match e { - marshal::MarshalError::Eof => vm.new_eof_error("marshal data too short"), - _ => vm.new_value_error("bad marshal data"), - })?; + let result = deserialize_value(&mut &buf[..], vm)?; if !allow_code { check_no_code(&result, vm)?; } @@ -534,14 +632,7 @@ mod decl { let mut rdr: &[u8] = &buf; let len_before = rdr.len(); - let result = - marshal::deserialize_value(&mut rdr, PyMarshalBag(vm)).map_err(|e| match e { - marshal::MarshalError::Eof => vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "marshal data too short".into(), - ), - _ => vm.new_value_error("bad marshal data"), - })?; + let result = deserialize_value(&mut rdr, vm)?; let consumed = len_before - rdr.len(); // Seek file to just after the consumed bytes From 24bd3b33f9c6d1a3d32ab297457f7a1b73984263 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:55:23 +0900 Subject: [PATCH 24/28] ssl: pass test_ssl with the rustls backend (#8502) * Fix rustls test_ssl compatibility Assisted-by: OpenAI Codex:GPT-5 * Keep urllib3 compatible SSL version prefix Assisted-by: OpenAI Codex:GPT-5 --- crates/stdlib/src/ssl.rs | 16 ++++--- crates/stdlib/src/ssl/cert.rs | 40 +++++++++++------ crates/vm/src/stdlib/_thread.rs | 76 ++++++++++++++++++++++++++++++++- 3 files changed, 109 insertions(+), 23 deletions(-) diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 04b905d544e..18d171a8583 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -323,15 +323,17 @@ mod _ssl { #[pyattr] const ALERT_DESCRIPTION_NO_APPLICATION_PROTOCOL: i32 = 120; - // Version info - reporting as OpenSSL 3.3.0 for compatibility + // `ssl.py` still requires OpenSSL-shaped numeric compatibility fields even + // for non-OpenSSL TLS providers. Keep them in the supported 3.x ABI range, + // but report the actual rustls/AWS-LC backend in the human-readable string. #[pyattr] - const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; // OpenSSL 3.3.0 (808452096) + const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; #[pyattr] - const OPENSSL_VERSION: &str = "OpenSSL 3.3.0 (rustls/0.23)"; + const OPENSSL_VERSION: &str = "OpenSSL 3.3.0-compatible (AWS-LC/rustls 0.23)"; #[pyattr] - const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release + const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); #[pyattr] - const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release + const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // Default cipher list for rustls - using modern secure ciphers #[pyattr] @@ -2816,8 +2818,8 @@ mod _ssl { super::compat::SslError::create_ssl_error_with_reason( vm, Some("SSL"), - "CALLBACK_FAILED", - "[SSL: CALLBACK_FAILED] callback failed", + "PARSE_TLSEXT", + "[SSL: PARSE_TLSEXT] SNI callback owner is no longer available", ) })?; let server_name_py: PyObjectRef = match sni_name { diff --git a/crates/stdlib/src/ssl/cert.rs b/crates/stdlib/src/ssl/cert.rs index f12f4307239..47d11f730b2 100644 --- a/crates/stdlib/src/ssl/cert.rs +++ b/crates/stdlib/src/ssl/cert.rs @@ -287,9 +287,11 @@ pub(super) fn is_ca_certificate(cert_der: &[u8]) -> bool { return ext.value.ca; } - // No Basic Constraints extension -> NOT a CA certificate - // (matches OpenSSL X509_check_ca() behavior) - false + // X509_check_ca() also retains OpenSSL's legacy trust-anchor rule: a + // self-issued X.509v1 certificate has no extensions at all, but is still + // classified as a CA. CPython's test CA at capath/4e1295a3.0 exercises + // precisely this case. + cert.version().0 == 0 && cert.subject() == cert.issuer() } /// Convert an X509Name to Python nested tuple format for SSL certificate dicts @@ -867,26 +869,36 @@ impl ServerCertVerifier for NoVerifier { fn verify_tls12_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, ) -> Result { - // Accept all signatures without verification - Ok(HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &CryptoExt::get_provider().signature_verification_algorithms, + ) } fn verify_tls13_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, ) -> Result { - // Accept all signatures without verification - Ok(HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &CryptoExt::get_provider().signature_verification_algorithms, + ) } fn supported_verify_schemes(&self) -> Vec { - ALL_SIGNATURE_SCHEMES.to_vec() + CryptoExt::get_provider() + .signature_verification_algorithms + .supported_schemes() } } diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 70304e63980..377c68dca74 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -605,14 +605,35 @@ pub(crate) mod _thread { vm.state.thread_count.fetch_sub(1); } + /// Default stack size for Python threads in **debug builds only**, where + /// Rust stack frames are substantially larger than in release. Rust's + /// `std::thread::Builder` otherwise defaults to 2 MB, which is too small + /// for the call chains the Python stdlib runs on helper threads in debug + /// (e.g. the SSL test server, see #7941). Release builds keep the prior + /// behavior — leave the stack size unset and let Rust's std default apply + /// — to avoid oversized virtual stack mappings when many threads spawn. + #[cfg(debug_assertions)] + const DEFAULT_THREAD_STACK_SIZE: usize = 8 * 1024 * 1024; + + /// Configure a `thread::Builder` with the stack size to use for a new + /// Python thread. Uses the value set via `threading.stack_size(N)` when + /// the user has provided one (non-zero). Otherwise, debug builds fall + /// back to [`DEFAULT_THREAD_STACK_SIZE`] and release builds leave the + /// builder unmodified (Rust's std default applies). fn apply_thread_stack_size( thread_builder: thread::Builder, vm: &VirtualMachine, ) -> thread::Builder { let configured = vm.state.stacksize.load(); if configured != 0 { - thread_builder.stack_size(configured) - } else { + return thread_builder.stack_size(configured); + } + #[cfg(debug_assertions)] + { + thread_builder.stack_size(DEFAULT_THREAD_STACK_SIZE) + } + #[cfg(not(debug_assertions))] + { thread_builder } } @@ -1996,4 +2017,55 @@ pub(crate) mod _thread { Ok(handle_clone) } + + #[cfg(test)] + mod tests { + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + use super::*; + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + use crate::Interpreter; + + /// Regression test for #7941: a Python thread started without an + /// explicit `threading.stack_size()` must not run on Rust's 2 MiB + /// std default in debug builds, where the call chains the stdlib + /// runs on helper threads (e.g. the SSL test server) overflowed it. + #[test] + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + fn default_python_thread_stack_size_debug() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + assert_eq!(vm.state.stacksize.load(), 0); + let builder = apply_thread_stack_size(thread::Builder::new(), vm); + let stack_size = builder + .spawn(current_thread_stack_size) + .expect("failed to spawn thread") + .join() + .expect("thread panicked"); + assert!( + stack_size >= DEFAULT_THREAD_STACK_SIZE, + "Python thread stack size is {stack_size} bytes, expected at least {DEFAULT_THREAD_STACK_SIZE}" + ); + }); + } + + #[cfg(all(debug_assertions, target_os = "linux"))] + fn current_thread_stack_size() -> usize { + use libc::{ + pthread_attr_destroy, pthread_attr_getstacksize, pthread_attr_t, + pthread_getattr_np, pthread_self, + }; + let mut attr: pthread_attr_t = unsafe { core::mem::zeroed() }; + unsafe { + assert_eq!(pthread_getattr_np(pthread_self(), &mut attr), 0); + let mut size = 0; + assert_eq!(pthread_attr_getstacksize(&attr, &mut size), 0); + pthread_attr_destroy(&mut attr); + size + } + } + + #[cfg(all(debug_assertions, target_os = "macos"))] + fn current_thread_stack_size() -> usize { + unsafe { libc::pthread_get_stacksize_np(libc::pthread_self()) } + } + } } From e02e215353494c52c3a95a6030b567794c0071d5 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Thu, 13 Aug 2026 04:29:13 +0900 Subject: [PATCH 25/28] ci: automate OSCCA pull request triage (#8505) Assisted-by: Codex:gpt-5.6-sol --- .github/workflows/oscca-pr.yml | 70 ++++++++++++++++++++++++++++++++++ .github/zizmor.yml | 5 +++ 2 files changed, 75 insertions(+) create mode 100644 .github/workflows/oscca-pr.yml diff --git a/.github/workflows/oscca-pr.yml b/.github/workflows/oscca-pr.yml new file mode 100644 index 00000000000..f96c18fd3c3 --- /dev/null +++ b/.github/workflows/oscca-pr.yml @@ -0,0 +1,70 @@ +name: Manage OSCCA pull requests + +on: + pull_request_target: + types: [opened] + +permissions: {} + +jobs: + label-and-assign: + name: Label and assign OSCCA pull request + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + issues: write + steps: + - name: Label and assign pull request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const osccaUsers = new Set( + [ + "2jiyong", + "chestnut1717", + "devyubin", + "fregataa", + "hyoinandout", + "HyoJongPark", + "JaceJung-dev", + "jinmay", + "jiwahn", + "kangdora", + "kim-jaedeok", + "kyokuping", + "leehanjeong", + "lms0806", + "lsahn-gh", + "moreal", + "name-of-okja", + "rlaisqls", + "seungje0612", + "shAn-kor", + "sigmaith", + "teddygood", + "widehyo1", + "YangSiJun528", + "zzarbttoo", + ].map((login) => login.toLowerCase()), + ); + const pullRequest = context.payload.pull_request; + const author = pullRequest.user.login; + + if (!osccaUsers.has(author.toLowerCase())) { + core.info(`${author} is not an OSCCA participant; skipping.`); + return; + } + + const issue = { + ...context.repo, + issue_number: pullRequest.number, + }; + + await github.rest.issues.addLabels({ + ...issue, + labels: ["z-ca-2026"], + }); + await github.rest.issues.addAssignees({ + ...issue, + assignees: [author], + }); diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 33ac61c6489..02ceb805c2c 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,4 +1,9 @@ rules: + dangerous-triggers: + ignore: + # pull_request_target is needed to label and assign PRs from forks with issues: write. + # The workflow does not check out or execute pull request code. + - oscca-pr.yml:3 excessive-permissions: ignore: # pull_request_target is needed to post PR comments with pull-requests: write. From 3a98ef746a3c050fafd59f0331c922c9bbd5a685 Mon Sep 17 00:00:00 2001 From: Joshua Megnauth <48846352+joshuamegnauth54@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:30:53 -0400 Subject: [PATCH 26/28] Allow and document clippy::drain_collect (#8500) RustPython's tail call machinery pre-allocates and reuses a vector. The code drains the vector into a new vector which is stored elsewhere. Clippy warns that this pattern causes a spurious location. Clippy is usually right that this pattern is suspect, but in this case the initial vector is reused so we want to keep the initial location. --- crates/vm/src/stdlib/_codecs.rs | 11 +++++------ crates/vm/src/vm/mod.rs | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs index 69d9e0e4fde..497d62fcc81 100644 --- a/crates/vm/src/stdlib/_codecs.rs +++ b/crates/vm/src/stdlib/_codecs.rs @@ -791,19 +791,18 @@ mod _codecs_windows { // Convert code point to UTF-16 let mut wchars = [0u16; 2]; - let wchar_len; let is_surrogate = (0xD800..=0xDFFF).contains(&ch); - if is_surrogate { - wchar_len = 0; // Can't encode surrogates normally + let wchar_len = if is_surrogate { + 0 // Can't encode surrogates normally } else if ch < 0x10000 { wchars[0] = ch as u16; - wchar_len = 1; + 1 } else { wchars[0] = ((ch - 0x10000) >> 10) as u16 + 0xD800; wchars[1] = ((ch - 0x10000) & 0x3FF) as u16 + 0xDC00; - wchar_len = 2; - } + 2 + }; if !is_surrogate { let mut buf = [0u8; 8]; diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 6009f421e12..7c7d017c1fd 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1467,6 +1467,10 @@ impl VirtualMachine { let initial_ptr = self.take_pending_tailcall(); // Drain the refs that keep the initial callee's raw pointers alive. + #[allow( + clippy::drain_collect, + reason = "`pending_tailcall_refs`'s allocation is intentionally reused" + )] let initial_refs = unsafe { &mut *self.pending_tailcall_refs.get() } .drain(..) .collect(); @@ -1498,6 +1502,10 @@ impl VirtualMachine { let result = crate::frame::run_iframe(callee, self); match result { Ok(ExecutionResult::TailCall) => { + #[allow( + clippy::drain_collect, + reason = "`pending_tailcall_refs`'s allocation is intentionally reused" + )] let refs = unsafe { &mut *self.pending_tailcall_refs.get() } .drain(..) .collect(); @@ -1548,6 +1556,10 @@ impl VirtualMachine { let result = crate::frame::run_iframe(caller_iframe, self); match result { Ok(ExecutionResult::TailCall) => { + #[allow( + clippy::drain_collect, + reason = "`pending_tailcall_refs`'s allocation is intentionally reused" + )] let refs = unsafe { &mut *self.pending_tailcall_refs.get() } .drain(..) .collect(); @@ -1609,6 +1621,10 @@ impl VirtualMachine { let result = crate::frame::run_iframe(caller_iframe, self); match result { Ok(ExecutionResult::TailCall) => { + #[allow( + clippy::drain_collect, + reason = "`pending_tailcall_refs`'s allocation is intentionally reused" + )] let refs = unsafe { &mut *self.pending_tailcall_refs.get() } .drain(..) .collect(); From db5de5e5e238f8b0e03b2cff17f8603bbf45ef84 Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Thu, 13 Aug 2026 04:31:53 +0900 Subject: [PATCH 27/28] Reuse stored hashes in dict.fromkeys() (#8503) * Reuse stored hashes in dict.fromkeys() Closes #8490. Co-Authored-By: Claude Opus 5 (1M context) * Make fromkeys_known_hashes a PyDict associated fn --------- Co-authored-by: Claude Opus 5 (1M context) --- Lib/test/test_set.py | 1 - crates/vm/src/builtins/dict.rs | 30 ++++++++++++++++++++++++++---- crates/vm/src/builtins/set.rs | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 4d062b42ded..42f11c9eb28 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -330,7 +330,6 @@ def test_cyclical_repr(self): name = repr(s).partition('(')[0] # strip class name self.assertEqual(repr(s), '%s({%s(...)})' % (name, name)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_do_not_rehash_dict_keys(self): n = 10 d = dict.fromkeys(map(HashCountingInt, range(n))) diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 5db071e1d8f..fbc23a0dde7 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -1,6 +1,6 @@ use super::{ IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, PyMappingProxy, PySet, - PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set::PySetInner, + PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set, set::PySetInner, }; use crate::common::lock::LazyLock; use crate::object::{Traverse, TraverseFn}; @@ -9,7 +9,7 @@ use crate::{ TryFromObject, atomic_func, builtins::{PyList, PyTuple, iter::builtins_iter, type_::PyAttributes}, class::{PyClassDef, PyClassImpl}, - common::ascii, + common::{ascii, hash::PyHash}, dict_inner::{self, DictKey}, function::{ArgIterable, FuncArgs, KwArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, iter::PyExactSizeIterator, @@ -354,6 +354,20 @@ impl PyDict { ) -> PyResult> { self.entries.get(vm, key) } + + /// Keys of `obj` with their stored hashes, or `None` if it must be iterated + /// generically. Only exact dicts and sets qualify, as in CPython's + /// `_PyDict_FromKeys`: a subclass may override `__iter__`. + fn fromkeys_known_hashes( + obj: &PyObject, + vm: &VirtualMachine, + ) -> Option> { + if let Some(dict) = obj.downcast_ref_if_exact::(vm) { + Some(dict.entries.keys_with_hashes()) + } else { + set::exact_set_keys_with_hashes(obj, vm) + } + } } // Python dict methods: @@ -384,8 +398,16 @@ impl PyDict { let d = PyType::call(&class, ().into(), vm)?; match d.downcast_exact::(vm) { Ok(pydict) => { - for key in iterable.iter(vm)? { - pydict.__setitem__(key?, value.clone(), vm)?; + if let Some(keys) = Self::fromkeys_known_hashes(iterable.as_object(), vm) { + for (key, hash) in keys { + pydict + .entries + .insert_known_hash(vm, &*key, hash, value.clone())?; + } + } else { + for key in iterable.iter(vm)? { + pydict.__setitem__(key?, value.clone(), vm)?; + } } Ok(pydict.into_pyref().into()) } diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index 62bdb0f0da5..6961040c792 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -605,6 +605,23 @@ fn extract_set(obj: &PyObject) -> Option<&PySetInner> { }) } +/// Elements of `obj` with their stored hashes, or `None` unless `obj` is exactly +/// a `set` or `frozenset` — `PyAnySet_CheckExact`, where [`extract_set`] is the +/// subclass-inclusive `PyAnySet_Check`. +pub(super) fn exact_set_keys_with_hashes( + obj: &PyObject, + vm: &VirtualMachine, +) -> Option> { + let inner = obj + .downcast_ref_if_exact::(vm) + .map(|set| &set.inner) + .or_else(|| { + obj.downcast_ref_if_exact::(vm) + .map(|frozen| &frozen.inner) + })?; + Some(inner.content.keys_with_hashes()) +} + fn reduce_set(zelf: &PyObject, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option) { ( zelf.class().to_owned(), From 670152068cc8624374b2e20c0ede0a10787e3270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9D=B4=EA=B4=91=ED=9A=A8?= <87641474+widehyo1@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:32:35 +0900 Subject: [PATCH 28/28] itertools: defer dropwhile predicate validation (#8504) Store the `dropwhile` predicate as a Python object and call it while advancing the iterator. This defers callable validation until the predicate is first needed, matching CPython for empty input while preserving exception propagation during iteration. Remove the now-passing `test_dropwhile` expected-failure marker. Assisted-by: Codex:gpt-5.6-sol --- Lib/test/test_itertools.py | 1 - crates/vm/src/stdlib/itertools.rs | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 585f6611ade..b91e3735d94 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1260,7 +1260,6 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 041620298e7..30a4d8773be 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -10,7 +10,7 @@ mod decl { rc::PyRc, }, convert::ToPyObject, - function::{ArgCallable, FuncArgs, OptionalArg, OptionalOption, PosArgs}, + function::{FuncArgs, OptionalArg, OptionalOption, PosArgs}, protocol::{PyIter, PyIterReturn, PyNumber}, raise_if_stop, stdlib::sys, @@ -477,7 +477,7 @@ mod decl { #[pyclass(name = "dropwhile")] #[derive(Debug, PyPayload)] struct PyItertoolsDropwhile { - predicate: ArgCallable, + predicate: PyObjectRef, iterable: PyIter, start_flag: AtomicCell, } @@ -485,7 +485,7 @@ mod decl { #[derive(FromArgs)] struct DropwhileNewArgs { #[pyarg(positional)] - predicate: ArgCallable, + predicate: PyObjectRef, #[pyarg(positional)] iterable: PyIter, } @@ -522,8 +522,7 @@ mod decl { if !zelf.start_flag.load() { loop { let obj = raise_if_stop!(iterable.next(vm)?); - let pred = predicate.clone(); - let pred_value = pred.invoke((obj.clone(),), vm)?; + let pred_value = predicate.call((obj.clone(),), vm)?; if !pred_value.try_to_bool(vm)? { zelf.start_flag.store(true); return Ok(PyIterReturn::Return(obj));