diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae82353c284..d832398ebf6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -526,7 +526,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: cargo-bins/cargo-binstall@ead08b90bd7b2e6d81963fb9cf0b7239f66d5db4 # v1.21.0 + - uses: cargo-bins/cargo-binstall@e00d2c94cc0067b77737821097a62d91c0301baa # v1.21.1 - name: cargo shear run: | @@ -557,7 +557,7 @@ jobs: uses: reviewdog/action-actionlint@50842263c20a7c46bd0065b9e624d3c569db061e # v1.73.0 - name: zizmor - uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1 + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 - name: restore prek cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -567,7 +567,7 @@ jobs: - name: install prek id: prek - uses: j178/prek-action@5337cb91e0fa35a7ff31b9ca345126d8bbbcdf16 # v2.0.6 + uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 with: cache: false show-verbose-logs: false diff --git a/.github/workflows/oscca-pr.yml b/.github/workflows/oscca-pr.yml new file mode 100644 index 00000000000..f96c18fd3c3 --- /dev/null +++ b/.github/workflows/oscca-pr.yml @@ -0,0 +1,70 @@ +name: Manage OSCCA pull requests + +on: + pull_request_target: + types: [opened] + +permissions: {} + +jobs: + label-and-assign: + name: Label and assign OSCCA pull request + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + issues: write + steps: + - name: Label and assign pull request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const osccaUsers = new Set( + [ + "2jiyong", + "chestnut1717", + "devyubin", + "fregataa", + "hyoinandout", + "HyoJongPark", + "JaceJung-dev", + "jinmay", + "jiwahn", + "kangdora", + "kim-jaedeok", + "kyokuping", + "leehanjeong", + "lms0806", + "lsahn-gh", + "moreal", + "name-of-okja", + "rlaisqls", + "seungje0612", + "shAn-kor", + "sigmaith", + "teddygood", + "widehyo1", + "YangSiJun528", + "zzarbttoo", + ].map((login) => login.toLowerCase()), + ); + const pullRequest = context.payload.pull_request; + const author = pullRequest.user.login; + + if (!osccaUsers.has(author.toLowerCase())) { + core.info(`${author} is not an OSCCA participant; skipping.`); + return; + } + + const issue = { + ...context.repo, + issue_number: pullRequest.number, + }; + + await github.rest.issues.addLabels({ + ...issue, + labels: ["z-ca-2026"], + }); + await github.rest.issues.addAssignees({ + ...issue, + assignees: [author], + }); diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml index 4f2944408f2..3ac44585943 100644 --- a/.github/workflows/upgrade-pylib.lock.yml +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -58,7 +58,7 @@ jobs: comment_repo: "" steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Check workflow file timestamps @@ -99,7 +99,7 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Checkout repository @@ -806,7 +806,7 @@ jobs: total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Download agent output artifact @@ -927,7 +927,7 @@ jobs: success: ${{ steps.parse_results.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Download agent artifacts @@ -1039,7 +1039,7 @@ jobs: process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@bbb8042878459948333b15b66f27113f4b5c1b9a # v0.83.4 + uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3 with: destination: /opt/gh-aw/actions - name: Download agent output artifact diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 33ac61c6489..02ceb805c2c 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,4 +1,9 @@ rules: + dangerous-triggers: + ignore: + # pull_request_target is needed to label and assign PRs from forks with issues: write. + # The workflow does not check out or execute pull request code. + - oscca-pr.yml:3 excessive-permissions: ignore: # pull_request_target is needed to post PR comments with pull-requests: write. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db9869bb3a5..9ffc4b8d4fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: priority: 0 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.0 + rev: v0.16.1 hooks: - id: ruff-format priority: 0 diff --git a/Cargo.lock b/Cargo.lock index 3acba335146..26015d39933 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1768,11 +1768,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.31" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", @@ -1784,12 +1785,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + [[package]] name = "jiff-static" -version = "0.2.31" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.119", @@ -2022,9 +2033,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50474818739ccab820cd57bca432d6b02d090b47f9e85501d963cd05851f82eb" +checksum = "03dcace986b149f29509af6ca70e6182bccce916b644424ecf484faa8ddc899a" dependencies = [ "zlib-rs", ] @@ -2779,9 +2790,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "libc", "once_cell", @@ -2793,18 +2804,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -2812,9 +2823,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2824,9 +2835,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", @@ -5070,9 +5081,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" [[package]] name = "zmij" diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index f5a0bc56958..582e5b6de6e 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -18,7 +18,6 @@ def test_EOF_single_quote(self): self.assertEqual(str(cm.exception), expect) self.assertEqual(cm.exception.offset, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_EOFS(self): expect = ("unterminated triple-quoted string literal (detected at line 3) (, line 1)") with self.assertRaises(SyntaxError) as cm: @@ -45,7 +44,6 @@ def test_EOFS(self): self.assertEqual(cm.exception.text, "ä = '''thîs is ") self.assertEqual(cm.exception.offset, 5) - @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_EOFS_with_file(self): expect = ("(, line 1)") @@ -86,7 +84,6 @@ def test_EOFS_with_file(self): ' ^', 'SyntaxError: unterminated triple-quoted string literal (detected at line 4)']) - @unittest.expectedFailure # TODO: RUSTPYTHON @warnings_helper.ignore_warnings(category=SyntaxWarning) def test_eof_with_line_continuation(self): expect = "unexpected EOF while parsing (, line 1)" @@ -94,7 +91,6 @@ def test_eof_with_line_continuation(self): compile('"\\Xhh" \\', '', 'exec') self.assertEqual(str(cm.exception), expect) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_line_continuation_EOF(self): """A continuation at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (, line 1)' @@ -127,7 +123,6 @@ def test_line_continuation_EOF(self): exec('\\') self.assertEqual(str(cm.exception), expect) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(not sys.executable, "sys.executable required") @force_not_colorized def test_line_continuation_EOF_from_file_bpo2180(self): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 7ab4c810a08..7c81c4b3905 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2145,7 +2145,6 @@ class AssertionErrorTests(unittest.TestCase): def tearDown(self): unlink(TESTFN) - @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_assertion_error_location(self): cases = [ diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index f6452341e1e..aa28108312e 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -558,7 +558,6 @@ def test_unicode_in_error_message(self): with self.assertRaisesRegex(ValueError, str_err): "{a:%ЫйЯЧ}".format(a='a') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_negative_zero(self): ## default behavior self.assertEqual(f"{-0.:.1f}", "-0.0") diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index e865c9bf059..b91e3735d94 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1136,7 +1136,6 @@ def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) @@ -1261,7 +1260,6 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index ad4c6095abf..1f04a7f697e 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -319,7 +319,6 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_list(self): a = [] a.append(a) @@ -331,7 +330,6 @@ def test_reference_loop_list(self): self.assertIsInstance(b, list) self.assertIs(b[0], b) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_dict(self): a = {} a[None] = a @@ -343,7 +341,6 @@ def test_reference_loop_dict(self): self.assertIsInstance(b, dict) self.assertIs(b[None], b) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_tuple(self): a = ([],) a[0].append(a) @@ -387,21 +384,18 @@ def test_reference_loop_slice(self): for v in range(marshal.version + 1): self.assertRaises(ValueError, marshal.dumps, a, v) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_reference_loop_list(self): data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [] a = marshal.loads(data) self.assertIsInstance(a, list) self.assertIs(a[0], a) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_reference_loop_dict(self): data = b'\xfbNr\x00\x00\x00\x000' # {None: } a = marshal.loads(data) self.assertIsInstance(a, dict) self.assertIs(a[None], a) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_abnormal_reference_loops(self): # Indirect self-references of tuples. data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([],) @@ -416,13 +410,13 @@ def test_loads_abnormal_reference_loops(self): self.assertIsInstance(a[0], dict) self.assertIs(a[0][None], a) - # Direct self-reference which cannot be created in Python. - # This creates a reference loop which cannot be collected. - if False: - data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) - a = marshal.loads(data) - self.assertIsInstance(a, tuple) - self.assertIs(a[0], a) + # Direct self-reference which cannot be created in Python. CPython + # leaves this disabled because its reference counting cannot collect + # the resulting cycle; RustPython's tracing collector can. + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) # Direct self-references which cannot be created in Python # because of unhashability. @@ -748,7 +742,6 @@ class InterningTestCase(unittest.TestCase, HelperMixin): strobj = "this is an interned string" strobj = sys.intern(strobj) - @unittest.expectedFailure # TODO: RUSTPYTHON def testIntern(self): s = marshal.loads(marshal.dumps(self.strobj)) self.assertEqual(s, self.strobj) diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py index 51f7fd257e0..b64e684f805 100644 --- a/Lib/test/test_modulefinder.py +++ b/Lib/test/test_modulefinder.py @@ -390,8 +390,6 @@ def test_bytecode(self): os.remove(source_path) self._do_test(bytecode_test) - # TODO: RUSTPYTHON; panics at code.rs with 'called Option::unwrap() on a None value' - @unittest.skip("TODO: RUSTPYTHON; panics in co_filename replacement") def test_replace_paths(self): old_path = os.path.join(self.test_dir, 'a', 'module.py') new_path = os.path.join(self.test_dir, 'a', 'spam.py') diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 4d062b42ded..42f11c9eb28 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -330,7 +330,6 @@ def test_cyclical_repr(self): name = repr(s).partition('(')[0] # strip class name self.assertEqual(repr(s), '%s({%s(...)})' % (name, name)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_do_not_rehash_dict_keys(self): n = 10 d = dict.fromkeys(map(HashCountingInt, range(n))) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 8dad1ba7382..e1759f1aa6b 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4556,7 +4556,6 @@ def test_sendfile(self): s.sendfile(file) self.assertEqual(s.recv(1024), TEST_DATA) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_session(self): client_context, server_context, hostname = testing_context() # TODO: sessions aren't compatible with TLSv1.3 yet @@ -4614,7 +4613,6 @@ def test_session(self): self.assertEqual(sess_stat['accept'], 4) self.assertEqual(sess_stat['hits'], 2) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False != True def test_session_handling(self): client_context, server_context, hostname = testing_context() client_context2, _, _ = testing_context() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 5ed844c34f0..0e81c6f6db2 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1922,7 +1922,6 @@ def test_newline_and_space_at_the_end_of_the_source_without_newline(self): tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline)) self.assertEqual(tokens, expected_tokens) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'SyntaxError' not found in b'OSError: stream did not contain valid UTF-8\n' def test_invalid_character_in_fstring_middle(self): # See gh-103824 script = b'''F""" diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3f056f2b753..c98b99e98e9 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -843,7 +843,6 @@ def test_complex_comprehension_inlining_exec(self): lamb = list(genexp)[0] self.assertEqual(lamb(), 42) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__annotate__' != 'f.__annotate__' def test_annotate_qualname(self): code = """ def f() -> None: diff --git a/README.md b/README.md index 37b577d0f0e..cb086687d8f 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ needed to prevent stack overflow on Windows): ```bash $ cd RustPython -$ cargo run --release demo_closures.py +$ cargo run --release -- -c 'print("Hello, RustPython!")' Hello, RustPython! ``` diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index ba2d41f12b3..b16540762fd 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -2175,7 +2175,7 @@ impl<'warnings> Compiler<'warnings> { /// On success, returns the saved CompileContext to pass to exit_annotation_scope. fn enter_annotation_scope( &mut self, - _func_name: &str, + func_name: &str, loc: TextRange, ) -> CompileResult> { if !self.push_annotation_symbol_table() { @@ -2200,6 +2200,12 @@ impl<'warnings> Compiler<'warnings> { lineno.to_u32(), )?; + // enter_scope() qualified the scope by the enclosing scope only; redo it + // now that the annotated function is known. Only signature annotations + // get this treatment - deferred class and module annotations are + // compiled inside the scope they belong to and are already qualified. + self.set_annotation_qualname(func_name); + // Keep the internal ".format" name; exit_annotation_scope() // renames it to "format" on the final code object. self.current_code_info() @@ -2605,11 +2611,24 @@ impl<'warnings> Compiler<'warnings> { /// Set the qualified name for the current code object // = compiler_set_qualname fn set_qualname(&mut self) -> String { - let qualname = self.make_qualname(); + self.set_qualname_for_function(None) + } + + /// Set the qualname of an annotation scope, qualified by the function whose + /// signature it annotates. CPython records that name on the annotation + /// block's symbol table entry (`ste_function_name`) and folds it into the + /// qualname, so `f`'s annotation scope is named `f.__annotate__`. + fn set_annotation_qualname(&mut self, function_name: &str) { + self.set_qualname_for_function(Some(function_name)); + } + + fn set_qualname_for_function(&mut self, function_name: Option<&str>) -> String { + let qualname = self.make_qualname(function_name); self.current_code_info().metadata.qualname = Some(qualname.clone()); qualname } - fn make_qualname(&mut self) -> String { + + fn make_qualname(&mut self, function_name: Option<&str>) -> String { let stack_size = self.code_stack.len(); assert!(stack_size >= 1); @@ -2693,10 +2712,10 @@ impl<'warnings> Compiler<'warnings> { } } - // Build the qualified name - if force_global { + // Build the prefix the current name is qualified by, if any + let base = if force_global { // For global symbols, qualname is just the name - current_obj_name + None } else { // Check parent scope type let parent_obj_name = &parent.metadata.name; @@ -2709,23 +2728,32 @@ impl<'warnings> Compiler<'warnings> { ) ); + // Use parent's qualname if available, otherwise use parent_obj_name + let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); + if is_function_parent { // For functions, append . to parent qualname - // Use parent's qualname if available, otherwise use parent_obj_name - let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); - format!("{parent_qualname}..{current_obj_name}") + Some(format!("{parent_qualname}.")) + } else if parent_qualname == "" { + // Module level, nothing to qualify by + None } else { // For classes and other scopes, use parent's qualname directly - // Use parent's qualname if available, otherwise use parent_obj_name - let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); - if parent_qualname == "" { - // Module level, just use the name - current_obj_name - } else { - // Concatenate parent qualname with current name - format!("{parent_qualname}.{current_obj_name}") - } + Some(parent_qualname.clone()) } + }; + + // An annotation scope is compiled in the scope enclosing the function it + // annotates, so the function itself is missing from the prefix above. + let base = match (base, function_name) { + (Some(base), Some(function_name)) => Some(format!("{base}.{function_name}")), + (None, Some(function_name)) => Some(function_name.to_owned()), + (base, None) => base, + }; + + match base { + Some(base) => format!("{base}.{current_obj_name}"), + None => current_obj_name, } } diff --git a/crates/common/src/format.rs b/crates/common/src/format.rs index b5f062f97a6..1c5c0a9c9de 100644 --- a/crates/common/src/format.rs +++ b/crates/common/src/format.rs @@ -229,6 +229,7 @@ pub struct FormatSpec { align: Option, align_specified: bool, sign: Option, + no_neg_0: bool, alternate_form: bool, width: Option, grouping_option: Option, @@ -285,6 +286,14 @@ fn parse_alternate_form(text: &Wtf8) -> (bool, &Wtf8) { } } +fn parse_no_negative_zero(text: &Wtf8) -> (bool, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('z') => (true, chars.as_wtf8()), + _ => (false, text), + } +} + fn parse_zero(text: &Wtf8) -> (bool, &Wtf8) { let mut chars = text.code_points(); match chars.next().and_then(CodePoint::to_char) { @@ -349,6 +358,7 @@ impl FormatSpec { let (mut fill, mut align, text) = parse_fill_and_align(text); let align_specified = align.is_some(); let (sign, text) = FormatSign::parse(text); + let (no_neg_0, text) = parse_no_negative_zero(text); let (alternate_form, text) = parse_alternate_form(text); let (zero, text) = parse_zero(text); let (width, text) = parse_number(text)?; @@ -378,6 +388,7 @@ impl FormatSpec { align, align_specified, sign, + no_neg_0, alternate_form, width, grouping_option, @@ -505,6 +516,25 @@ impl FormatSpec { Ok(()) } + fn formatted_magnitude_is_zero(magnitude: &str) -> bool { + let mut saw_digit = false; + for byte in magnitude.bytes() { + if byte.is_ascii_digit() { + saw_digit = true; + if byte != b'0' { + return false; + } + } + } + saw_digit + } + + fn is_negative_after_zero_coercion(&self, num: f64, magnitude: &str) -> bool { + num.is_sign_negative() + && !num.is_nan() + && !(self.no_neg_0 && Self::formatted_magnitude_is_zero(magnitude)) + } + fn validate_complex_padding_and_alignment(&self) -> Result<(), FormatSpecError> { match &self.fill.unwrap_or_else(|| ' '.into()).to_char() { Some('0') => Err(FormatSpecError::ZeroPadding), @@ -674,6 +704,9 @@ impl FormatSpec { Some(FormatType::Number(Case::Lower)) => self.format_int_radix(magnitude, 10), _ => return self.format_int(num), }?; + if self.no_neg_0 { + return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")); + } let magnitude_str = Self::apply_locale_formatting(raw_magnitude_str, locale); @@ -723,7 +756,7 @@ impl FormatSpec { let magnitude_str = Self::apply_locale_formatting(raw_magnitude_str, locale); let format_sign = self.sign.unwrap_or(FormatSign::Minus); - let sign_str = if num.is_sign_negative() && !num.is_nan() { + let sign_str = if self.is_negative_after_zero_coercion(num, &magnitude_str) { "-" } else { match format_sign { @@ -812,11 +845,16 @@ impl FormatSpec { self.format_float(x as f64) } None => { + if self.no_neg_0 { + return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")); + } let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase(); Ok(first_letter.collect::() + &input.to_string()[1..]) } - Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(*c, "int")), - _ => Err(FormatSpecError::InvalidFormatSpecifier), + Some(format_type) => { + let ch = char::from(format_type); + Err(FormatSpecError::UnknownFormatCode(ch, "bool")) + } } } @@ -925,8 +963,9 @@ impl FormatSpec { }, }, }; + let raw_magnitude_str = raw_magnitude_str?; let format_sign = self.sign.unwrap_or(FormatSign::Minus); - let sign_str = if num.is_sign_negative() && !num.is_nan() { + let sign_str = if self.is_negative_after_zero_coercion(num, &raw_magnitude_str) { "-" } else { match format_sign { @@ -935,7 +974,7 @@ impl FormatSpec { FormatSign::MinusOrSpace => " ", } }; - let magnitude_str = self.add_magnitude_separators(raw_magnitude_str?, sign_str); + let magnitude_str = self.add_magnitude_separators(raw_magnitude_str, sign_str); let magnitude_str = self.add_frac_separators(magnitude_str); Ok( self.format_sign_and_align( @@ -986,15 +1025,24 @@ impl FormatSpec { Err(FormatSpecError::UnknownFormatCode('N', "int")) } Some(FormatType::String) => Err(FormatSpecError::UnknownFormatCode('s', "int")), - Some(FormatType::Character) => match (self.precision, self.sign, self.alternate_form) { - (Some(_), _, _) => Err(FormatSpecError::PrecisionNotAllowed), - (_, Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), - (_, _, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), - (_, _, _) => match num.to_u32() { - Some(n) if n <= 0x10ffff => Ok(core::char::from_u32(n).unwrap().to_string()), - Some(_) | None => Err(FormatSpecError::CodeNotInRange), - }, - }, + Some(FormatType::Character) => { + if self.precision.is_some() { + Err(FormatSpecError::PrecisionNotAllowed) + } else if self.no_neg_0 { + Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")) + } else { + match (self.sign, self.alternate_form) { + (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), + (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), + _ => match num.to_u32() { + Some(n) if n <= 0x10ffff => { + Ok(core::char::from_u32(n).unwrap().to_string()) + } + Some(_) | None => Err(FormatSpecError::CodeNotInRange), + }, + } + } + } Some( FormatType::GeneralFormat(_) | FormatType::FixedPoint(_) @@ -1007,6 +1055,9 @@ impl FormatSpec { Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(c, "int")), None => self.format_int_radix(magnitude, 10), }?; + if self.no_neg_0 { + return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")); + } let format_sign = self.sign.unwrap_or(FormatSign::Minus); let sign_str = match num.sign() { Sign::Minus => "-", @@ -1032,6 +1083,9 @@ impl FormatSpec { self.validate_format(FormatType::String)?; match self.format_type { Some(FormatType::String) | None => { + if self.no_neg_0 { + return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("string")); + } if self.align == Some(FormatAlign::AfterSign) && self.align_specified { return Err(FormatSpecError::StringAlignmentFlag); } @@ -1074,7 +1128,8 @@ impl FormatSpec { // Format real part let formatted_re = if num.re != 0.0 || num.re.is_negative_zero() || self.format_type.is_some() { - let sign_re = if num.re.is_sign_negative() && !num.is_nan() { + let re = self.format_complex_float(num.re)?; + let sign_re = if self.is_negative_after_zero_coercion(num.re, &re) { "-" } else { match self.sign.unwrap_or(FormatSign::Minus) { @@ -1083,21 +1138,24 @@ impl FormatSpec { FormatSign::MinusOrSpace => " ", } }; - let re = self.format_complex_float(num.re)?; format!("{sign_re}{re}") } else { String::new() }; // Format imaginary part - let sign_im = if num.im.is_sign_negative() && !num.im.is_nan() { + let im = self.format_complex_float(num.im)?; + let sign_im = if self.is_negative_after_zero_coercion(num.im, &im) { "-" } else if formatted_re.is_empty() { - "" + match self.sign.unwrap_or(FormatSign::Minus) { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + } } else { "+" }; - let im = self.format_complex_float(num.im)?; Ok((formatted_re, format!("{sign_im}{im}j"))) } @@ -1274,6 +1332,7 @@ pub enum FormatSpecError { CodeNotInRange, ZeroPadding, AlignmentFlag, + NegativeZeroCoercionNotAllowed(&'static str), StringAlignmentFlag, NotImplemented(char, &'static str), } @@ -1625,6 +1684,7 @@ mod tests { align: None, align_specified: false, sign: None, + no_neg_0: false, alternate_form: false, width: Some(33), grouping_option: None, @@ -1643,6 +1703,7 @@ mod tests { align: Some(FormatAlign::Right), align_specified: true, sign: None, + no_neg_0: false, alternate_form: false, width: Some(33), grouping_option: None, @@ -1661,6 +1722,7 @@ mod tests { align: Some(FormatAlign::Right), align_specified: true, sign: Some(FormatSign::Minus), + no_neg_0: false, alternate_form: true, width: Some(23), grouping_option: Some(FormatGrouping::Comma), @@ -1779,6 +1841,92 @@ mod tests { ); } + #[test] + fn format_negative_zero_coercion() { + let int_spec = FormatSpec::parse("z8").unwrap(); + assert_eq!( + int_spec.format_int(&BigInt::from(-42)), + Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")) + ); + assert_eq!( + FormatSpec::parse("zs") + .unwrap() + .format_int(&BigInt::from(0)), + Err(FormatSpecError::UnknownFormatCode('s', "int")) + ); + assert_eq!( + FormatSpec::parse("z.1d") + .unwrap() + .format_int(&BigInt::from(0)), + Err(FormatSpecError::PrecisionNotAllowed) + ); + assert_eq!( + FormatSpec::parse("+zc") + .unwrap() + .format_int(&BigInt::from(0)), + Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer")) + ); + + let float_spec = FormatSpec::parse("z.2f").unwrap(); + assert_eq!(float_spec.format_float(-0.0001), Ok("0.00".to_owned())); + + let complex_spec = FormatSpec::parse("z").unwrap(); + assert_eq!( + complex_spec.format_complex(&Complex64::new(-0.0, -0.0)), + Ok("(0+0j)".to_owned()) + ); + let pure_imaginary = Complex64::new(0.0, -0.0); + assert_eq!( + FormatSpec::parse("+z") + .unwrap() + .format_complex(&pure_imaginary), + Ok("+0j".to_owned()) + ); + assert_eq!( + FormatSpec::parse(" z") + .unwrap() + .format_complex(&pure_imaginary), + Ok(" 0j".to_owned()) + ); + + let string_value = "value".to_owned(); + assert_eq!( + FormatSpec::parse("z").unwrap().format_string(&string_value), + Err(FormatSpecError::NegativeZeroCoercionNotAllowed("string")) + ); + assert_eq!( + FormatSpec::parse("zd") + .unwrap() + .format_string(&string_value), + Err(FormatSpecError::UnknownFormatCode('d', "str")) + ); + assert_eq!( + FormatSpec::parse("zs").unwrap().format_bool(false), + Err(FormatSpecError::UnknownFormatCode('s', "bool")) + ); + + let locale = LocaleInfo { + thousands_sep: ",".to_owned(), + decimal_point: ".".to_owned(), + grouping: vec![3, 0], + }; + let locale_spec = FormatSpec::parse("zn").unwrap(); + assert_eq!( + locale_spec.format_float_locale(-0.0, &locale), + Ok("0".to_owned()) + ); + assert_eq!( + locale_spec.format_complex_locale(&Complex64::new(-0.0, -0.0), &locale), + Ok("0+0j".to_owned()) + ); + assert_eq!( + FormatSpec::parse("z.1n") + .unwrap() + .format_int_locale(&BigInt::from(0), &locale), + Err(FormatSpecError::PrecisionNotAllowed) + ); + } + #[test] fn format_int() { assert_eq!( diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index a2a23054e4b..dd5d4f2cddb 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -516,7 +516,7 @@ fn read_const_value( let code = deserialize_code_inner(rdr, bag, depth - 1, refs)?; bag.make_code(code) } else { - deserialize_value_typed(rdr, bag, depth, refs, typ)? + deserialize_value_typed(rdr, bag, depth, refs, typ, slot)? }; if let Some(idx) = slot { refs[idx] = Some(value.clone()); @@ -540,6 +540,10 @@ pub trait MarshalBag: Copy { fn make_str(&self, value: &Wtf8) -> Self::Value; + fn make_interned_str(&self, value: &Wtf8) -> Self::Value { + self.make_str(value) + } + fn make_bytes(&self, value: &[u8]) -> Self::Value; fn make_int(&self, value: BigInt) -> Self::Value; @@ -564,6 +568,51 @@ pub trait MarshalBag: Copy { it: impl Iterator, ) -> Result; + /// Install partially-built containers in the marshal reference table + /// before reading their children, as CPython's `r_object()` does. + /// Runtime bags can opt in; constant bags retain collect-then-construct. + fn make_tuple_placeholder(&self, _len: usize) -> Option { + None + } + + fn set_tuple_item( + &self, + _tuple: &Self::Value, + _index: usize, + _value: Self::Value, + ) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_list_placeholder(&self, _len: usize) -> Option { + None + } + + fn set_list_item(&self, _list: &Self::Value, _index: usize, _value: Self::Value) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_set_placeholder(&self) -> Option { + None + } + + fn insert_set_item(&self, _set: &Self::Value, _value: Self::Value) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_dict_placeholder(&self) -> Option { + None + } + + fn insert_dict_item( + &self, + _dict: &Self::Value, + _key: Self::Value, + _value: Self::Value, + ) -> Result<()> { + Err(MarshalError::BadType) + } + fn make_slice( &self, _start: Self::Value, @@ -755,7 +804,7 @@ fn deserialize_value_after_header( let code = deserialize_code_inner(rdr, bag.constant_bag(), depth - 1, &mut inner_refs)?; bag.make_code(code) } else { - deserialize_value_typed(rdr, bag, depth, refs, typ)? + deserialize_value_typed(rdr, bag, depth, refs, typ, slot)? }; if let Some(idx) = slot { @@ -770,6 +819,7 @@ fn deserialize_value_typed( depth: usize, refs: &mut Vec>, typ: Type, + slot: Option, ) -> Result { if depth == 0 { return Err(MarshalError::InvalidBytecode); @@ -806,21 +856,42 @@ fn deserialize_value_typed( let value = Complex64 { re, im }; bag.make_complex(value) } - Type::Ascii | Type::AsciiInterned | Type::Unicode | Type::Interned => { + Type::Ascii | Type::Unicode => { let len = rdr.read_u32()?; let value = rdr.read_wtf8(len)?; bag.make_str(value) } - Type::ShortAscii | Type::ShortAsciiInterned => { + Type::AsciiInterned | Type::Interned => { + let len = rdr.read_u32()?; + let value = rdr.read_wtf8(len)?; + bag.make_interned_str(value) + } + Type::ShortAscii => { let len = rdr.read_u8()? as u32; let value = rdr.read_wtf8(len)?; bag.make_str(value) } + Type::ShortAsciiInterned => { + let len = rdr.read_u8()? as u32; + let value = rdr.read_wtf8(len)?; + bag.make_interned_str(value) + } Type::SmallTuple => { let len = rdr.read_u8()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_tuple(it))? + if let Some(index) = slot + && let Some(tuple) = bag.make_tuple_placeholder(len) + { + refs[index] = Some(tuple.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_tuple_item(&tuple, item_index, item)?; + } + tuple + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_tuple(it))? + } } Type::Null => { return Err(MarshalError::BadType); @@ -830,22 +901,55 @@ fn deserialize_value_typed( return Err(MarshalError::BadType); } Type::Tuple => { - let len = rdr.read_u32()?; + let len = rdr.read_u32()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_tuple(it))? + if let Some(index) = slot + && let Some(tuple) = bag.make_tuple_placeholder(len) + { + refs[index] = Some(tuple.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_tuple_item(&tuple, item_index, item)?; + } + tuple + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_tuple(it))? + } } Type::List => { - let len = rdr.read_u32()?; + let len = rdr.read_u32()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_list(it))?? + if let Some(index) = slot + && let Some(list) = bag.make_list_placeholder(len) + { + refs[index] = Some(list.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_list_item(&list, item_index, item)?; + } + list + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_list(it))?? + } } Type::Set => { - let len = rdr.read_u32()?; + let len = rdr.read_u32()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_set(it))?? + if let Some(index) = slot + && let Some(set) = bag.make_set_placeholder() + { + refs[index] = Some(set.clone()); + for _ in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.insert_set_item(&set, item)?; + } + set + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_set(it))?? + } } Type::FrozenSet => { let len = rdr.read_u32()?; @@ -855,17 +959,33 @@ fn deserialize_value_typed( } Type::Dict => { let d = depth - 1; - let mut pairs = Vec::new(); - loop { - let raw = rdr.read_u8()?; - if raw & !FLAG_REF == b'0' { - break; + if let Some(index) = slot + && let Some(dict) = bag.make_dict_placeholder() + { + refs[index] = Some(dict.clone()); + loop { + let raw = rdr.read_u8()?; + if raw & !FLAG_REF == b'0' { + break; + } + let key = deserialize_value_after_header(rdr, bag, d, refs, raw)?; + let value = deserialize_value_depth(rdr, bag, d, refs)?; + bag.insert_dict_item(&dict, key, value)?; + } + dict + } else { + let mut pairs = Vec::new(); + loop { + let raw = rdr.read_u8()?; + if raw & !FLAG_REF == b'0' { + break; + } + let key = deserialize_value_after_header(rdr, bag, d, refs, raw)?; + let value = deserialize_value_depth(rdr, bag, d, refs)?; + pairs.push((key, value)); } - let k = deserialize_value_after_header(rdr, bag, d, refs, raw)?; - let v = deserialize_value_depth(rdr, bag, d, refs)?; - pairs.push((k, v)); + bag.make_dict(pairs.into_iter())? } - bag.make_dict(pairs.into_iter())? } Type::Bytes => { // After marshaling, byte arrays are converted into bytes. diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 43f7174d804..7562e8939b9 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -50,9 +50,13 @@ pub enum CompileError { impl CompileError { #[must_use] - pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self { + pub fn from_ruff_parse_error( + error: parser::ParseError, + source_file: &SourceFile, + mode: Mode, + ) -> Self { let raw_location = error.location; - let diagnostic = match cpython_parse_diagnostic_override(&error, source_file) { + let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) { Some(diagnostic) => diagnostic, None => default_parse_diagnostic(error, source_file), }; @@ -129,6 +133,13 @@ fn source_location(source_file: &SourceFile, offset: TextSize) -> SourceLocation .source_location(offset, PositionEncoding::Utf8) } +// Call only with UTF-8 character boundaries for Python-facing offsets. +fn source_location_in_code_points(source_file: &SourceFile, offset: TextSize) -> SourceLocation { + source_file + .to_source_code() + .source_location(offset, PositionEncoding::Utf32) +} + fn source_locations( source_file: &SourceFile, start: TextSize, @@ -175,6 +186,21 @@ impl NormalizedParseDiagnostic { ) } + fn other_in_code_points( + source_file: &SourceFile, + message: String, + start: usize, + end: usize, + ) -> Self { + let start = TextSize::new(start as u32); + let end = TextSize::new(end as u32); + Self::new( + parser::ParseErrorType::OtherError(message), + source_location_in_code_points(source_file, start), + source_location_in_code_points(source_file, end), + ) + } + const fn with_unclosed_bracket(mut self, is_unclosed_bracket: bool) -> Self { self.is_unclosed_bracket = is_unclosed_bracket; self @@ -184,6 +210,7 @@ impl NormalizedParseDiagnostic { fn cpython_parse_diagnostic_override( error: &parser::ParseError, source_file: &SourceFile, + mode: Mode, ) -> Option { let source_text = source_file.source_text(); @@ -223,6 +250,18 @@ fn cpython_parse_diagnostic_override( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) ) { + // Only a backslash at the end of the source is an EOF error. + let terminal_backslash = source_text.len().checked_sub(1); + if !matches!(mode, Mode::Eval) + && terminal_backslash == Some(error.location.start().to_usize()) + { + let loc = source_line_end_location(source_file, error.location.start()); + return Some(NormalizedParseDiagnostic::new( + parser::ParseErrorType::OtherError("unexpected EOF while parsing".to_owned()), + loc, + loc, + )); + } let loc = source_location(source_file, error.location.start() + TextSize::from(1)); return Some(NormalizedParseDiagnostic::new( error.error.clone(), @@ -231,7 +270,15 @@ fn cpython_parse_diagnostic_override( )); } - source_error!(unterminated_string_error(source_text)); + if let Some((message, start, end)) = unterminated_string_error(source_text) { + // The scanner reports quote positions, which are UTF-8 character boundaries. + return Some(NormalizedParseDiagnostic::other_in_code_points( + source_file, + message, + start, + end, + )); + } source_error!(expected_indented_block_error(error, source_text)); if matches!( @@ -5176,7 +5223,7 @@ fn _compile_with_syntax_warning_handler<'a>( }; let parser_options = parser::ParseOptions::from(parser_mode); let parsed = parser::parse(source_file.source_text(), parser_options) - .map_err(|err| CompileError::from_ruff_parse_error(err, &source_file))?; + .map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?; if opts.dont_imply_dedent && matches!(mode, Mode::Single) && let Some(error) = dont_imply_dedent_source_error(&source_file) @@ -5235,7 +5282,7 @@ pub fn _compile_symtable( let res = match mode { Mode::Exec | Mode::Single | Mode::BlockExpr => { let ast = ruff_python_parser::parse_module(source_file.source_text()) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?; + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { @@ -5254,7 +5301,7 @@ pub fn _compile_symtable( source_file.source_text(), parser::Mode::Expression.into(), ) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?; + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { diff --git a/crates/host_env/src/fileutils.rs b/crates/host_env/src/fileutils.rs index a4922e7a2fe..a8e56bb1c0b 100644 --- a/crates/host_env/src/fileutils.rs +++ b/crates/host_env/src/fileutils.rs @@ -1,6 +1,8 @@ // Python/fileutils.c in CPython #![allow(non_snake_case)] +use alloc::ffi::CString; + #[cfg(not(windows))] pub use rustix::fs::Stat as StatStruct; @@ -16,9 +18,8 @@ pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result { pub mod windows { use crate::crt_fd; use crate::windows::ToWideString; - use alloc::ffi::CString; use libc::{S_IFCHR, S_IFDIR, S_IFMT}; - use std::ffi::{OsStr, OsString}; + use std::ffi::OsStr; use std::os::windows::io::AsRawHandle; use std::sync::OnceLock; use windows_sys::Win32::Foundation::{ @@ -33,6 +34,7 @@ pub mod windows { use windows_sys::Win32::System::LibraryLoader::{GetProcAddress, LoadLibraryW}; use windows_sys::Win32::System::SystemServices::IO_REPARSE_TAG_SYMLINK; use windows_sys::core::PCWSTR; + use windows_sys::w; pub const S_IFIFO: libc::c_int = 0o010000; pub const S_IFLNK: libc::c_int = 0o120000; @@ -302,16 +304,13 @@ pub mod windows { let GetFileInformationByName = GET_FILE_INFORMATION_BY_NAME .get_or_init(|| { - let library_name = - OsString::from("api-ms-win-core-file-l2-1-4.dll").to_wide_with_nul(); - let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; + let library_name = w!("api-ms-win-core-file-l2-1-4.dll"); + let module = unsafe { LoadLibraryW(library_name) }; if module.is_null() { return None; } - let name = CString::new("GetFileInformationByName").unwrap(); - if let Some(proc) = - unsafe { GetProcAddress(module, name.as_bytes_with_nul().as_ptr()) } - { + let name = c"GetFileInformationByName"; + if let Some(proc) = unsafe { GetProcAddress(module, name.as_ptr().cast()) } { Some(unsafe { core::mem::transmute::< unsafe extern "system" fn() -> isize, @@ -458,7 +457,6 @@ pub unsafe fn fclose(fp: *mut CFile) -> core::ffi::c_int { reason = "false positive: core::io::ErrorKind is unstable (core_io)" )] pub fn fopen(path: &std::path::Path, mode: &str) -> std::io::Result<*mut CFile> { - use alloc::ffi::CString; use std::fs::File; // Currently only supports read mode diff --git a/crates/host_env/src/nt.rs b/crates/host_env/src/nt.rs index 780a75910ea..7e0591600b1 100644 --- a/crates/host_env/src/nt.rs +++ b/crates/host_env/src/nt.rs @@ -22,19 +22,22 @@ use crate::{ windows::{CheckWin32Bool, CheckWin32Handle, CheckWin32Sentinel, HandleToOwned, ToWideString}, }; use libc::intptr_t; -use windows_sys::Win32::{ - Foundation::{ - CloseHandle, ERROR_INVALID_HANDLE, GetLastError, HANDLE, INVALID_HANDLE_VALUE, MAX_PATH, - }, - Globalization::{CP_UTF8, MultiByteToWideChar, WideCharToMultiByte}, - Storage::FileSystem::{ - CreateFileW, FILE_BASIC_INFO, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, - FILE_READ_ATTRIBUTES, FILE_TYPE_UNKNOWN, FileBasicInfo, FindClose, FindFirstFileW, - GetFileAttributesW, GetFileInformationByHandleEx, GetFileType, GetFullPathNameW, - INVALID_FILE_ATTRIBUTES, OPEN_EXISTING, SetFileAttributesW, SetFileInformationByHandle, - WIN32_FIND_DATAW, +use windows_sys::{ + Win32::{ + Foundation::{ + CloseHandle, ERROR_INVALID_HANDLE, GetLastError, HANDLE, INVALID_HANDLE_VALUE, MAX_PATH, + }, + Globalization::{CP_UTF8, MultiByteToWideChar, WideCharToMultiByte}, + Storage::FileSystem::{ + CreateFileW, FILE_BASIC_INFO, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, + FILE_READ_ATTRIBUTES, FILE_TYPE_UNKNOWN, FileBasicInfo, FindClose, FindFirstFileW, + GetFileAttributesW, GetFileInformationByHandleEx, GetFileType, GetFullPathNameW, + INVALID_FILE_ATTRIBUTES, OPEN_EXISTING, SetFileAttributesW, SetFileInformationByHandle, + WIN32_FIND_DATAW, + }, + System::{Console, Threading}, }, - System::{Console, Threading}, + w, }; pub type Handle = HANDLE; @@ -1172,12 +1175,10 @@ pub fn mkdir(path: &widestring::WideCStr, mode: i32) -> io::Result<()> { lpSecurityDescriptor: core::ptr::null_mut(), bInheritHandle: 0, }; - let sddl: Vec = "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)\0" - .encode_utf16() - .collect(); + let sddl = w!("D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"); unsafe { ConvertStringSecurityDescriptorToSecurityDescriptorW( - sddl.as_ptr(), + sddl, SDDL_REVISION_1, &mut sec_attr.lpSecurityDescriptor, core::ptr::null_mut(), @@ -1699,10 +1700,10 @@ pub fn get_terminal_size_handle(h: HANDLE) -> io::Result<(usize, usize)> { if err != windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED { return Err(io::Error::last_os_error()); } - let conout: Vec = "CONOUT$\0".encode_utf16().collect(); + let conout = w!("CONOUT$"); let console_handle = unsafe { CreateFileW( - conout.as_ptr(), + conout, windows_sys::Win32::Foundation::GENERIC_READ | windows_sys::Win32::Foundation::GENERIC_WRITE, windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ diff --git a/crates/host_env/src/os.rs b/crates/host_env/src/os.rs index 01711687c6c..7af8f586110 100644 --- a/crates/host_env/src/os.rs +++ b/crates/host_env/src/os.rs @@ -519,13 +519,14 @@ pub fn set_errno(value: i32) { #[cfg(not(any(unix, windows, target_os = "wasi")))] pub fn set_errno(_value: i32) {} -#[cfg(unix)] +// WASIp1, like Unix, provides byte-preserving OsStr conversions. +#[cfg(any(unix, all(target_os = "wasi", not(target_env = "p2"))))] pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { - use std::os::unix::ffi::OsStrExt; + use self::ffi::OsStrExt; Ok(std::ffi::OsStr::from_bytes(b)) } -#[cfg(not(unix))] +#[cfg(not(any(unix, all(target_os = "wasi", not(target_env = "p2")))))] pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { Ok(core::str::from_utf8(b)?.as_ref()) } diff --git a/crates/host_env/src/winapi.rs b/crates/host_env/src/winapi.rs index af53910089e..19e18f32d3f 100644 --- a/crates/host_env/src/winapi.rs +++ b/crates/host_env/src/winapi.rs @@ -65,6 +65,7 @@ pub use windows_sys::Win32::{ }, UI::WindowsAndMessaging::SW_HIDE, }; +use windows_sys::w; pub type Handle = HANDLE; pub type StdHandle = windows_sys::Win32::System::Console::STD_HANDLE; @@ -1093,14 +1094,14 @@ where return Err(MimeRegistryReadError::Os(err)); } - let content_type_key: Vec = "Content Type\0".encode_utf16().collect(); + let content_type_key = w!("Content Type"); let mut type_buf = [0u16; 256]; let mut cb_type = (type_buf.len() * 2) as u32; let mut reg_type = 0; let err = unsafe { RegQueryValueExW( subkey, - content_type_key.as_ptr(), + content_type_key, core::ptr::null_mut(), &mut reg_type, type_buf.as_mut_ptr().cast(), diff --git a/crates/host_env/src/windows.rs b/crates/host_env/src/windows.rs index bde8d679737..635f12f3f38 100644 --- a/crates/host_env/src/windows.rs +++ b/crates/host_env/src/windows.rs @@ -4,24 +4,27 @@ use std::{ io, os::windows::ffi::{OsStrExt, OsStringExt}, }; -use windows_sys::Win32::{ - Foundation::{ - E_POINTER, ERROR_INSUFFICIENT_BUFFER, ERROR_INVALID_FLAGS, ERROR_NO_UNICODE_TRANSLATION, - MAX_PATH, S_OK, - }, - Networking::WinSock::WSAStartup, - Storage::FileSystem::{ - GetFileVersionInfoSizeW, GetFileVersionInfoW, VS_FIXEDFILEINFO, VerQueryValueW, - }, - System::{ - Diagnostics::Debug::{ - FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM, - FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, +use windows_sys::{ + Win32::{ + Foundation::{ + E_POINTER, ERROR_INSUFFICIENT_BUFFER, ERROR_INVALID_FLAGS, + ERROR_NO_UNICODE_TRANSLATION, MAX_PATH, S_OK, + }, + Networking::WinSock::WSAStartup, + Storage::FileSystem::{ + GetFileVersionInfoSizeW, GetFileVersionInfoW, VS_FIXEDFILEINFO, VerQueryValueW, + }, + System::{ + Diagnostics::Debug::{ + FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM, + FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, + }, + LibraryLoader::{GetModuleFileNameW, GetModuleHandleW}, + SystemInformation::{GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW}, + Threading::{GetCurrentThreadStackLimits, SetThreadStackGuarantee}, }, - LibraryLoader::{GetModuleFileNameW, GetModuleHandleW}, - SystemInformation::{GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW}, - Threading::{GetCurrentThreadStackLimits, SetThreadStackGuarantee}, }, + w, }; /// _MAX_ENV from Windows CRT stdlib.h - maximum environment variable size @@ -154,8 +157,8 @@ pub struct WindowsVersionInfo { fn get_kernel32_version() -> io::Result<(u32, u32, u32)> { unsafe { - let module_name: Vec = OsStr::new("kernel32.dll").to_wide_with_nul(); - let h_kernel32 = GetModuleHandleW(module_name.as_ptr()).check_nonnull()?; + let module_name = w!("kernel32.dll"); + let h_kernel32 = GetModuleHandleW(module_name).check_nonnull()?; let mut kernel32_path = [0u16; MAX_PATH as usize]; let len = GetModuleFileNameW( @@ -181,13 +184,13 @@ fn get_kernel32_version() -> io::Result<(u32, u32, u32)> { ) .check_win32_bool()?; - let sub_block: Vec = OsStr::new("").to_wide_with_nul(); + let sub_block = w!(""); let mut ffi_ptr: *mut VS_FIXEDFILEINFO = core::ptr::null_mut(); let mut ffi_len: u32 = 0; VerQueryValueW( ver_block.as_ptr() as *const _, - sub_block.as_ptr(), + sub_block, &mut ffi_ptr as *mut *mut VS_FIXEDFILEINFO as *mut *mut _, &mut ffi_len as *mut u32, ) diff --git a/crates/host_env/src/wmi.rs b/crates/host_env/src/wmi.rs index 592ffd1dc23..2b46eebcbe5 100644 --- a/crates/host_env/src/wmi.rs +++ b/crates/host_env/src/wmi.rs @@ -7,15 +7,17 @@ use core::ffi::c_void; use core::ptr::{NonNull, null, null_mut}; +use widestring::WideCString; use windows_sys::Win32::Foundation::{ - CloseHandle, ERROR_BROKEN_PIPE, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY, GetLastError, HANDLE, - WAIT_OBJECT_0, WAIT_TIMEOUT, + CloseHandle, ERROR_BROKEN_PIPE, ERROR_INVALID_NAME, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY, + GetLastError, HANDLE, WAIT_OBJECT_0, WAIT_TIMEOUT, }; use windows_sys::Win32::Storage::FileSystem::{ReadFile, WriteFile}; use windows_sys::Win32::System::Pipes::CreatePipe; use windows_sys::Win32::System::Threading::{ CreateEventW, CreateThread, GetExitCodeThread, SetEvent, WaitForSingleObject, }; +use windows_sys::w; use crate::ctypes::wcslen; @@ -256,10 +258,6 @@ const fn failed(hr: HRESULT) -> bool { hr < 0 } -fn wide_str(s: &str) -> Vec { - s.encode_utf16().chain(core::iter::once(0)).collect() -} - unsafe fn wait_event(event: HANDLE, timeout: u32) -> u32 { match unsafe { WaitForSingleObject(event, timeout) } { WAIT_OBJECT_0 => 0, @@ -346,8 +344,8 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { } if succeeded(hr) { - let root_cimv2 = wide_str("ROOT\\CIMV2"); - let bstr_root = unsafe { SysAllocString(root_cimv2.as_ptr()) }; + let root_cimv2 = w!("ROOT\\CIMV2"); + let bstr_root = unsafe { SysAllocString(root_cimv2) }; hr = unsafe { locator_connect_server( locator, @@ -384,8 +382,8 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { }; } if succeeded(hr) { - let wql = wide_str("WQL"); - let bstr_wql = unsafe { SysAllocString(wql.as_ptr()) }; + let wql = w!("WQL"); + let bstr_wql = unsafe { SysAllocString(wql) }; hr = unsafe { services_exec_query( services, @@ -557,7 +555,9 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { } pub fn exec_query(query_str: &str) -> Result { - let query_wide = wide_str(query_str); + let query = WideCString::from_str(query_str) + .map_err(|_| ExecQueryError::Code(ERROR_INVALID_NAME))? + .into(); let mut h_thread: HANDLE = null_mut(); let mut err: u32 = 0; @@ -579,7 +579,7 @@ pub fn exec_query(query_str: &str) -> Result { err = GetLastError(); } else { let thread_data = Box::new(QueryThreadData { - query: query_wide, + query, write_pipe, init_event, connect_event, diff --git a/crates/jit/src/instructions.rs b/crates/jit/src/instructions.rs index 9b4656da5ad..a3c4ca800c4 100644 --- a/crates/jit/src/instructions.rs +++ b/crates/jit/src/instructions.rs @@ -39,6 +39,7 @@ impl JitValue { JitType::Int => Self::Int(val), JitType::Float => Self::Float(val), JitType::Bool => Self::Bool(val), + JitType::None => unreachable!("None cannot be used as an argument type"), } } @@ -47,7 +48,8 @@ impl JitValue { Self::Int(_) => Some(JitType::Int), Self::Float(_) => Some(JitType::Float), Self::Bool(_) => Some(JitType::Bool), - Self::None | Self::Null | Self::Tuple(_) | Self::FuncRef(_) => None, + Self::None => Some(JitType::None), + Self::Null | Self::Tuple(_) | Self::FuncRef(_) => None, } } @@ -112,8 +114,9 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { #[expect(clippy::mut_mut, reason = "This seems like a false positive")] let builder = &mut self.builder; let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; + let cranelift_ty = ty.to_cranelift().ok_or(JitCompileError::NotSupported)?; let local = self.variables[idx].get_or_insert_with(|| { - let var = builder.declare_var(ty.to_cranelift()); + let var = builder.declare_var(cranelift_ty); Local { var, ty: ty.clone(), @@ -328,27 +331,27 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { } fn return_value(&mut self, val: JitValue) -> Result<(), JitCompileError> { - if let Some(ref ty) = self.sig.ret { - // If the signature has a return type, enforce it - if val.to_jit_type().as_ref() != Some(ty) { + let val_type = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; + if let Some(ref ret_type) = self.sig.ret { + if ret_type != &val_type { return Err(JitCompileError::NotSupported); } } else { - // First time we see a return, define it in the signature - let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; - self.sig.ret = Some(ty.clone()); - self.builder - .func - .signature - .returns - .push(AbiParam::new(ty.to_cranelift())); + self.sig.ret = Some(val_type.clone()); + if let Some(val_type) = val_type.to_cranelift() { + self.builder + .func + .signature + .returns + .push(AbiParam::new(val_type)); + } } - // If this is e.g. an Int, Float, or Bool we have a Cranelift `Value`. - // If we have JitValue::None or .Tuple(...) but can't handle that, error out (or handle differently). - let cr_val = val.into_value().ok_or(JitCompileError::NotSupported)?; - - self.builder.ins().return_(&[cr_val]); + if let Some(cr_val) = val.into_value() { + self.builder.ins().return_(&[cr_val]); + } else { + self.builder.ins().return_(&[]); + } Ok(()) } @@ -545,8 +548,22 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { JitValue::FuncRef(reference) => { let call = self.builder.ins().call(reference, &args); - let returns = self.builder.inst_results(call); - self.stack.push(JitValue::Int(returns[0])); + // The only callable reachable here is this function itself, + // so the result carries the declared return type - it is not + // always an Int. A function whose return type is still + // unknown has no return slot in the signature it was + // declared with, and there is nothing to type the result as. + let ret = match *self.builder.inst_results(call) { + [] => None, + [val] => Some(val), + _ => return Err(JitCompileError::NotSupported), + }; + let val = match (self.sig.ret.clone(), ret) { + (Some(JitType::None), None) => JitValue::None, + (Some(ty), Some(val)) => JitValue::from_type_and_value(ty, val), + _ => return Err(JitCompileError::NotSupported), + }; + self.stack.push(val); Ok(()) } diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index dbaa4a3eb26..0c700e93cf8 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -61,19 +61,12 @@ impl Jit { ret: Option, ) -> Result<(FuncId, JitSig), JitCompileError> { for arg in args { - self.ctx - .func - .signature - .params - .push(AbiParam::new(arg.to_cranelift())); + let arg = arg.to_cranelift().ok_or(JitCompileError::NotSupported)?; + self.ctx.func.signature.params.push(AbiParam::new(arg)); } - if ret.is_some() { - self.ctx - .func - .signature - .returns - .push(AbiParam::new(ret.clone().unwrap().to_cranelift())); + if let Some(ret) = ret.as_ref().and_then(JitType::to_cranelift) { + self.ctx.func.signature.returns.push(AbiParam::new(ret)); } let id = self.module.declare_function( @@ -167,7 +160,10 @@ impl CompiledCode { libffi::middle::CodePtr::from_ptr(self.code as *const _), cif_args, ); - self.sig.ret.as_ref().map(|ty| value.to_typed(ty)) + match self.sig.ret.as_ref() { + Some(JitType::None) | None => None, + Some(ty) => Some(value.to_typed(ty)), + } } } } @@ -193,14 +189,16 @@ pub enum JitType { Int, Float, Bool, + None, } impl JitType { - fn to_cranelift(&self) -> types::Type { + fn to_cranelift(&self) -> Option { match self { - Self::Int => types::I64, - Self::Float => types::F64, - Self::Bool => types::I8, + Self::Int => Some(types::I64), + Self::Float => Some(types::F64), + Self::Bool => Some(types::I8), + Self::None => None, } } @@ -209,6 +207,7 @@ impl JitType { Self::Int => libffi::middle::Type::i64(), Self::Float => libffi::middle::Type::f64(), Self::Bool => libffi::middle::Type::u8(), + Self::None => libffi::middle::Type::void(), } } } @@ -306,6 +305,7 @@ impl UnTypedAbiValue { JitType::Int => AbiValue::Int(self.int), JitType::Float => AbiValue::Float(self.float), JitType::Bool => AbiValue::Bool(self.boolean != 0), + JitType::None => unreachable!("None has no ABI value"), } } } diff --git a/crates/jit/tests/bool_tests.rs b/crates/jit/tests/bool_tests.rs index 8a5f4ea9db3..1874ee4d55d 100644 --- a/crates/jit/tests/bool_tests.rs +++ b/crates/jit/tests/bool_tests.rs @@ -202,4 +202,18 @@ mod tests { assert_eq!(lte(false, 1), Ok(1)); assert_eq!(lte(true, 0), Ok(0)); } + + #[test] + fn recursive_bool() { + let recursive_bool = jit_function! { recursive_bool(n: i64) -> bool => r##" + def recursive_bool(n: int) -> bool: + if n == 0: + return True + return not recursive_bool(n - 1) + "## }; + + assert_eq!(recursive_bool(0), Ok(true)); + assert_eq!(recursive_bool(1), Ok(false)); + assert_eq!(recursive_bool(4), Ok(true)); + } } diff --git a/crates/jit/tests/float_tests.rs b/crates/jit/tests/float_tests.rs index b9bbb3ea63c..f667b1e764a 100644 --- a/crates/jit/tests/float_tests.rs +++ b/crates/jit/tests/float_tests.rs @@ -379,4 +379,18 @@ mod tests { assert_eq!(float_lte(f64::NAN, f64::NAN), Ok(false)); assert_eq!(float_lte(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); } + + #[test] + fn recursive_float() { + let recursive_float = jit_function! { recursive_float(n: i64) -> f64 => r##" + def recursive_float(n: int) -> float: + if n == 0: + return 1.0 + return recursive_float(n - 1) / 2.0 + "## }; + + assert_eq!(recursive_float(0), Ok(1.0)); + assert_eq!(recursive_float(1), Ok(0.5)); + assert_eq!(recursive_float(4), Ok(0.0625)); + } } diff --git a/crates/jit/tests/misc_tests.rs b/crates/jit/tests/misc_tests.rs index b73100ad6ec..5404df0a769 100644 --- a/crates/jit/tests/misc_tests.rs +++ b/crates/jit/tests/misc_tests.rs @@ -2,16 +2,15 @@ mod tests { use rustpython_jit::{AbiValue, JitArgumentError}; - // TODO currently broken - // #[test] - // fn test_no_return_value() { - // let func = jit_function! { func() => r##" - // def func(): - // pass - // "## }; - // - // assert_eq!(func(), Ok(())); - // } + #[test] + fn no_return_value() { + let func = jit_function! { func() => r##" + def func(): + pass + "## }; + + assert_eq!(func(), Ok(())); + } #[test] fn invoke() { diff --git a/crates/sre_engine/src/engine.rs b/crates/sre_engine/src/engine.rs index a110a75d65f..690801e0d9d 100644 --- a/crates/sre_engine/src/engine.rs +++ b/crates/sre_engine/src/engine.rs @@ -998,12 +998,12 @@ fn search_info_literal( return true; } + // `state.cursor` is `req.start + skip`, the position the + // tail match resumes from; advancing past the prefix + // instead would resume at `req.start + len` and only + // agree when the prefix ends at the skip boundary. let mut next_ctx = ctx; - if skip != 0 { - next_ctx.advance_char::(); - } else { - next_ctx.cursor = state.cursor; - } + next_ctx.cursor = state.cursor; if _match(req, state, next_ctx) { return true; diff --git a/crates/sre_engine/tests/tests.rs b/crates/sre_engine/tests/tests.rs index 53f5225d4ad..0a9f45c374e 100644 --- a/crates/sre_engine/tests/tests.rs +++ b/crates/sre_engine/tests/tests.rs @@ -252,4 +252,19 @@ mod tests { #[rustfmt::skip] let p = Pattern { pattern: "\u{e0}+", code: &[14, 4, 0, 1, 4294967295, 24, 6, 1, 4294967295, 16, 224, 1, 1] }; // END GENERATED } + + #[test] + fn search_literal_prefix_longer_than_skip() { + // The INFO block carries prefix_len=4 and prefix_skip=2, so the tail + // match has to resume at the skip boundary rather than past the whole + // prefix. + // pattern p = re.compile('ab(cd)') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "ab(cd)", code: &[14, 14, 1, 4, 4, 4, 2, 97, 98, 99, 100, 0, 0, 0, 0, 16, 97, 16, 98, 17, 0, 16, 99, 16, 100, 17, 1, 1] }; + // END GENERATED + let (req, mut state) = p.state("xabcdcd"); + assert!(state.search(req)); + assert_eq!(state.start, 1); + assert_eq!(state.cursor.position, 5); + } } diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 454fdcba899..18d171a8583 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -137,6 +137,8 @@ mod _ssl { #[pyattr] const PROTOCOL_TLSv1_3: i32 = 6; + static NEXT_SSL_SESSION_NONCE: AtomicUsize = AtomicUsize::new(1); + // Protocol version constants for TLSVersion enum #[pyattr] const PROTO_SSLv3: i32 = 0x0300; @@ -321,15 +323,17 @@ mod _ssl { #[pyattr] const ALERT_DESCRIPTION_NO_APPLICATION_PROTOCOL: i32 = 120; - // Version info - reporting as OpenSSL 3.3.0 for compatibility + // `ssl.py` still requires OpenSSL-shaped numeric compatibility fields even + // for non-OpenSSL TLS providers. Keep them in the supported 3.x ABI range, + // but report the actual rustls/AWS-LC backend in the human-readable string. #[pyattr] - const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; // OpenSSL 3.3.0 (808452096) + const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; #[pyattr] - const OPENSSL_VERSION: &str = "OpenSSL 3.3.0 (rustls/0.23)"; + const OPENSSL_VERSION: &str = "OpenSSL 3.3.0-compatible (AWS-LC/rustls 0.23)"; #[pyattr] - const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release + const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); #[pyattr] - const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release + const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // Default cipher list for rustls - using modern secure ciphers #[pyattr] @@ -439,6 +443,31 @@ mod _ssl { lifetime: u64, } + impl SessionData { + // NOTE: This is NOT the actual TLS session ID, just a unique identifier. + fn new(server_name: &str, lifetime: u64) -> Self { + let creation_time = SystemTime::now(); + let nonce = NEXT_SSL_SESSION_NONCE.fetch_add(1, Ordering::Relaxed); + let mut hasher = Sha256::new(); + hasher.update(server_name.as_bytes()); + hasher.update( + creation_time + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .to_le_bytes(), + ); + hasher.update(nonce.to_le_bytes()); + + Self { + _server_name: server_name.to_owned(), + session_id: hasher.finalize()[..16].to_vec(), + creation_time, + lifetime, + } + } + } + // Type alias to simplify complex session cache type type SessionCache = Arc, Arc>>>>; @@ -466,20 +495,6 @@ mod _ssl { // ✓ session_reused - tracked via handshake_kind() // ✗ Actual TLS session ID/ticket data - NOT ACCESSIBLE - // Generate a synthetic session ID from server name and timestamp - // NOTE: This is NOT the actual TLS session ID, just a unique identifier - fn generate_session_id_from_metadata(server_name: &str, time: SystemTime) -> Vec { - let mut hasher = Sha256::new(); - hasher.update(server_name.as_bytes()); - hasher.update( - time.duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() - .to_le_bytes(), - ); - hasher.finalize()[..16].to_vec() - } - // Custom ClientSessionStore that tracks session metadata for Python access // NOTE: This wraps ClientSessionMemoryCache and records metadata when sessions are stored #[derive(Debug)] @@ -488,6 +503,39 @@ mod _ssl { session_cache: SessionCache, } + impl PythonClientSessionStore { + fn new(session_cache: SessionCache) -> Self { + Self { + inner: Arc::new(ClientSessionMemoryCache::new(SSL_SESSION_CACHE_SIZE)), + session_cache, + } + } + + fn transfer_session( + &self, + target: &Self, + server_name: &ServerName<'static>, + kind: ClientSessionKind, + ) { + if let Some(group) = self.kx_hint(server_name) { + target.set_kx_hint(server_name.clone(), group); + } + + match kind { + ClientSessionKind::Tls12 => { + if let Some(session) = self.tls12_session(server_name) { + target.set_tls12_session(server_name.clone(), session); + } + } + ClientSessionKind::Tls13 => { + if let Some(ticket) = self.take_tls13_ticket(server_name) { + target.insert_tls13_ticket(server_name.clone(), ticket); + } + } + } + } + } + impl ClientSessionStore for PythonClientSessionStore { fn set_kx_hint(&self, server_name: ServerName<'static>, group: rustls::NamedGroup) { self.inner.set_kx_hint(server_name, group); @@ -508,17 +556,8 @@ mod _ssl { // Record metadata in Python-accessible cache // NOTE: We can't access value.session_id or value.ticket (private fields) // So we generate a synthetic ID from metadata - let creation_time = SystemTime::now(); let server_name_str = server_name.to_str(); - let session_data = SessionData { - _server_name: server_name_str.as_ref().to_string(), - session_id: generate_session_id_from_metadata( - server_name_str.as_ref(), - creation_time, - ), - creation_time, - lifetime: 7200, // TLS 1.2 default session lifetime - }; + let session_data = SessionData::new(server_name_str.as_ref(), 7200); let key = server_name_str.as_bytes().to_vec(); self.session_cache @@ -552,17 +591,8 @@ mod _ssl { // Record metadata in Python-accessible cache // NOTE: We can't access value.ticket or value.lifetime_secs (private fields) // So we use default values - let creation_time = SystemTime::now(); let server_name_str = server_name.to_str(); - let session_data = SessionData { - _server_name: server_name_str.to_string(), - session_id: generate_session_id_from_metadata( - server_name_str.as_ref(), - creation_time, - ), - creation_time, - lifetime: 7200, // Default TLS 1.3 ticket lifetime (Rustls uses this) - }; + let session_data = SessionData::new(server_name_str.as_ref(), 7200); let key = server_name_str.as_bytes().to_vec(); self.session_cache @@ -745,6 +775,8 @@ mod _ssl { #[pyclass(name = "_SSLContext", module = "ssl", traverse)] #[derive(Debug, PyPayload)] struct PySSLContext { + #[pytraverse(skip)] + context_identity: Arc<()>, #[pytraverse(skip)] protocol: i32, #[pytraverse(skip)] @@ -807,9 +839,6 @@ mod _ssl { // Session management #[pytraverse(skip)] client_session_cache: SessionCache, - // Rustls session store for actual TLS session resumption - #[pytraverse(skip)] - rustls_session_store: Arc, // Rustls server session store for server-side session resumption #[pytraverse(skip)] rustls_server_session_store: Arc, @@ -1926,8 +1955,9 @@ mod _ssl { .map(|o| o.downgrade(None, vm)) .transpose()?, ), - // Filter out Python None objects - only store actual SSLSession objects - session: PyRwLock::new(args.session.into_option().filter(|s| !vm.is_none(s))), + session: PyRwLock::new(None), + client_config: PyRwLock::new(None), + client_session_store: PyRwLock::new(None), incoming_bio: None, outgoing_bio: None, sni_state: PyRwLock::new(None), @@ -1945,6 +1975,12 @@ mod _ssl { .into_ref_with_type(vm, vm.class("_ssl", "_SSLSocket")) .map_err(|_| vm.new_type_error("Failed to create SSLSocket"))?; + if let Some(session) = args.session.into_option() + && !vm.is_none(&session) + { + ssl_socket_ref.set_session(session, vm)?; + } + Ok(ssl_socket_ref) } @@ -2008,8 +2044,9 @@ mod _ssl { .map(|o| o.downgrade(None, vm)) .transpose()?, ), - // Filter out Python None objects - only store actual SSLSession objects - session: PyRwLock::new(args.session.into_option().filter(|s| !vm.is_none(s))), + session: PyRwLock::new(None), + client_config: PyRwLock::new(None), + client_session_store: PyRwLock::new(None), incoming_bio: Some(args.incoming), outgoing_bio: Some(args.outgoing), sni_state: PyRwLock::new(None), @@ -2026,6 +2063,12 @@ mod _ssl { .into_ref_with_type(vm, vm.class("_ssl", "_SSLSocket")) .map_err(|_| vm.new_type_error("Failed to create SSLSocket"))?; + if let Some(session) = args.session.into_option() + && !vm.is_none(&session) + { + ssl_socket_ref.set_session(session, vm)?; + } + Ok(ssl_socket_ref) } @@ -2304,19 +2347,11 @@ mod _ssl { _ => (PROTO_MINIMUM_SUPPORTED, PROTO_MAXIMUM_SUPPORTED), // Auto-negotiate }; - // IMPORTANT: Create shared session cache BEFORE PySSLContext - // Both client_session_cache and PythonClientSessionStore.session_cache - // MUST point to the same HashMap to ensure Python-level and Rustls-level - // sessions are synchronized + // Session metadata is scoped to the SSLContext. Each per-connection + // rustls session store records into this shared cache. let shared_session_cache = Arc::new(ParkingRwLock::new(HashMap::new())); - let rustls_client_store = Arc::new(PythonClientSessionStore { - inner: Arc::new(rustls::client::ClientSessionMemoryCache::new( - SSL_SESSION_CACHE_SIZE, - )), - session_cache: shared_session_cache.clone(), - }); - Ok(Self { + context_identity: Arc::new(()), protocol, check_hostname: PyRwLock::new(protocol == PROTOCOL_TLS_CLIENT), verify_mode: PyRwLock::new(default_verify_mode), @@ -2340,7 +2375,6 @@ mod _ssl { x509_cert_count: PyRwLock::new(0), // Use the shared cache created above client_session_cache: shared_session_cache, - rustls_session_store: rustls_client_store, rustls_server_session_store: rustls::server::ServerSessionMemoryCache::new( SSL_SESSION_CACHE_SIZE, ), @@ -2390,6 +2424,13 @@ mod _ssl { owner: PyRwLock>>, // Session for resumption session: PyRwLock>, + // Client configuration used by this connection. Retained so the resulting + // SSLSession can reuse the same verifier and client credentials. + #[pytraverse(skip)] + client_config: PyRwLock>>, + // Per-connection store containing the session selected by this connection. + #[pytraverse(skip)] + client_session_store: PyRwLock>>, // MemoryBIO mode (optional) incoming_bio: Option>, outgoing_bio: Option>, @@ -2487,31 +2528,27 @@ mod _ssl { } // Create and store a session object after successful handshake - fn create_session_after_handshake(&self, vm: &VirtualMachine) { + fn create_session_after_handshake(&self, was_resumed: bool, vm: &VirtualMachine) { // Only create session for client-side connections if self.server_side { return; } - // Check if session already exists - let session_opt = self.session.read().clone(); - if let Some(ref s) = session_opt { - if vm.is_none(s) { - } else { - return; - } - } - // Get server hostname let server_name = self.server_hostname.read().clone(); + let previous_session = self.session.read().clone(); // Try to get session data from context's session cache // IMPORTANT: Acquire and release locks quickly to avoid deadlock - let context = self.context.read(); - let session_cache_arc = context.client_session_cache.clone(); - drop(context); // Release context lock ASAP + let (context_identity, session_cache_arc) = { + let context = self.context.read(); + ( + context.context_identity.clone(), + context.client_session_cache.clone(), + ) + }; - let (session_id, creation_time, lifetime) = if let Some(ref name) = server_name { + let cached_session_data = if let Some(ref name) = server_name { let key = name.as_bytes().to_vec(); // Clone the data we need while holding the lock, then immediately release @@ -2521,29 +2558,57 @@ mod _ssl { }; // Lock released here if let Some(session_data_arc) = session_data_opt { - let data = session_data_arc.lock(); - let result = (data.session_id.clone(), data.creation_time, data.lifetime); - drop(data); // Explicit unlock - result + session_data_arc.lock().clone() } else { - // Create new session ID if not in cache - let time = std::time::SystemTime::now(); - (generate_session_id_from_metadata(name, time), time, 7200) + SessionData::new(name, 7200) } } else { - // No server name, use defaults - let time = std::time::SystemTime::now(); - (vec![0; 16], time, 7200) + SessionData::new("", 7200) + }; + + let session_data = if was_resumed { + previous_session + .as_ref() + .and_then(|session| session.downcast_ref::()) + .map_or(cached_session_data, |session| SessionData { + _server_name: server_name.clone().unwrap_or_default(), + session_id: session.session_id.clone(), + creation_time: session.creation_time, + lifetime: session.lifetime, + }) + } else { + cached_session_data + }; + + let rustls_server_name = server_name.and_then(|name| ServerName::try_from(name).ok()); + let protocol_version = self + .connection + .lock() + .as_ref() + .and_then(|connection| connection.protocol_version()); + let session_kind = match protocol_version { + Some(rustls::ProtocolVersion::TLSv1_2) => ClientSessionKind::Tls12, + Some(rustls::ProtocolVersion::TLSv1_3) => ClientSessionKind::Tls13, + _ => return, + }; + + let Some(client_config) = self.client_config.write().take() else { + return; + }; + let Some(session_store) = self.client_session_store.write().take() else { + return; }; - // Create a new SSLSession object with real metadata let session = PySSLSession { - // Use dummy session data to indicate we have a ticket - // TLS 1.2+ always uses session tickets/resumption - session_data: vec![1], // Non-empty to indicate has_ticket=True - session_id, - creation_time, - lifetime, + context_identity, + client_config, + session_store, + server_name: rustls_server_name, + kind: session_kind, + session_id: session_data.session_id, + creation_time: session_data.creation_time, + lifetime: session_data.lifetime, + has_ticket: true, }; let py_session = session.into_pyobject(vm); @@ -2638,7 +2703,7 @@ mod _ssl { let _ = self.track_used_ca_from_capath(); } - self.create_session_after_handshake(vm); + self.create_session_after_handshake(was_resumed, vm); } // Internal implementation with timeout control @@ -2753,8 +2818,8 @@ mod _ssl { super::compat::SslError::create_ssl_error_with_reason( vm, Some("SSL"), - "CALLBACK_FAILED", - "[SSL: CALLBACK_FAILED] callback failed", + "PARSE_TLSEXT", + "[SSL: PARSE_TLSEXT] SNI callback owner is no longer available", ) })?; let server_name_py: PyObjectRef = match sni_name { @@ -3486,9 +3551,9 @@ mod _ssl { let check_hostname = *ctx.check_hostname.read(); let verify_flags = *ctx.verify_flags.read(); + let context_identity = ctx.context_identity.clone(); - // Get session store before dropping ctx - let session_store = ctx.rustls_session_store.clone(); + let session_cache = ctx.client_session_cache.clone(); // Get CRLs for revocation checking let crls_clone = ctx.crls.read().clone(); @@ -3496,31 +3561,6 @@ mod _ssl { // Drop ctx early to avoid borrow conflicts drop(ctx); - // Build client config using compat helper - let config_options = ClientConfigOptions { - protocol_settings, - root_store: if verify_mode != CERT_NONE { - Some(root_store_clone) - } else { - None - }, - ca_certs_der: ca_certs_der_clone, - cert_chain: if !cert_chain_clone.is_empty() { - Some(cert_chain_clone) - } else { - None - }, - private_key: private_key_opt, - verify_server_cert: verify_mode != CERT_NONE, - check_hostname, - verify_flags, - session_store: Some(session_store), - crls: crls_clone, - }; - - let config = - create_client_config(config_options).map_err(|e| vm.new_value_error(e))?; - // Parse server name for SNI // Convert to ServerName use rustls::pki_types::ServerName; @@ -3539,10 +3579,61 @@ mod _ssl { ) }; - let conn = ClientConnection::new(Arc::new(config), server_name.clone()) - .map_err(|e| { - vm.new_value_error(format!("Failed to create client connection: {e}")) - })?; + let explicit_session = self.session.read().clone(); + let session_store = Arc::new(PythonClientSessionStore::new(session_cache)); + let config = if let Some(session) = explicit_session { + let session = session + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?; + if !Arc::ptr_eq(&session.context_identity, &context_identity) { + return Err( + vm.new_value_error("Session refers to a different SSLContext.") + ); + } + if session.server_name.as_ref() == Some(&server_name) { + session.session_store.transfer_session( + &session_store, + &server_name, + session.kind, + ); + } + let mut config = (*session.client_config).clone(); + config.resumption = + rustls::client::Resumption::store(session_store.clone()); + Arc::new(config) + } else { + let config_options = ClientConfigOptions { + protocol_settings, + root_store: if verify_mode != CERT_NONE { + Some(root_store_clone) + } else { + None + }, + ca_certs_der: ca_certs_der_clone, + cert_chain: if !cert_chain_clone.is_empty() { + Some(cert_chain_clone) + } else { + None + }, + private_key: private_key_opt, + verify_server_cert: verify_mode != CERT_NONE, + check_hostname, + verify_flags, + session_store: Some(session_store.clone()), + crls: crls_clone, + }; + Arc::new( + create_client_config(config_options) + .map_err(|e| vm.new_value_error(e))?, + ) + }; + + *self.client_config.write() = Some(config.clone()); + *self.client_session_store.write() = Some(session_store); + + let conn = ClientConnection::new(config, server_name).map_err(|e| { + vm.new_value_error(format!("Failed to create client connection: {e}")) + })?; *conn_guard = Some(Connection::Client(conn)); } @@ -4046,12 +4137,15 @@ mod _ssl { #[pygetset(setter)] fn set_session(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // Validate that value is an SSLSession - if !value.is(vm.ctx.types.none_type) { - // Try to downcast to SSLSession to validate - let _ = value - .downcast_ref::() - .ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?; + let session = value + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?; + + if !Arc::ptr_eq( + &session.context_identity, + &self.context.read().context_identity, + ) { + return Err(vm.new_value_error("Session refers to a different SSLContext.")); } // Check if this is a client socket @@ -4065,11 +4159,7 @@ mod _ssl { } // Store the session for potential use during handshake - *self.session.write() = if value.is(vm.ctx.types.none_type) { - None - } else { - Some(value) - }; + *self.session.write() = Some(value); Ok(()) } @@ -4751,22 +4841,31 @@ mod _ssl { // SSLSession - represents a cached SSL session // NOTE: This is an EMULATION - actual session data is managed by Rustls internally + #[derive(Debug, Clone, Copy)] + enum ClientSessionKind { + Tls12, + Tls13, + } + #[pyattr] #[pyclass(name = "SSLSession", module = "ssl")] #[derive(Debug, PyPayload)] struct PySSLSession { - // Session data - serialized rustls session (EMULATED - kept empty) - session_data: Vec, + context_identity: Arc<()>, + client_config: Arc, + session_store: Arc, + server_name: Option>, + kind: ClientSessionKind, // Session ID - synthetic ID generated from metadata (NOT actual TLS session ID) - #[allow(dead_code)] session_id: Vec, // Session metadata creation_time: std::time::SystemTime, // Lifetime in seconds (default 7200 = 2 hours) lifetime: u64, + has_ticket: bool, } - #[pyclass(flags(BASETYPE))] + #[pyclass(flags(BASETYPE), with(Comparable))] impl PySSLSession { #[pygetset] fn time(&self) -> i64 { @@ -4791,20 +4890,29 @@ mod _ssl { #[pygetset] fn id(&self, vm: &VirtualMachine) -> PyBytesRef { - // Return session ID (hash of session data for uniqueness) - - let mut hasher = DefaultHasher::new(); - self.session_data.hash(&mut hasher); - let hash = hasher.finish(); - - // Convert hash to bytes - vm.ctx.new_bytes(hash.to_be_bytes().to_vec()) + vm.ctx.new_bytes(self.session_id.clone()) } #[pygetset] fn has_ticket(&self) -> bool { - // For rustls, if we have session data, we have a ticket - !self.session_data.is_empty() + self.has_ticket + } + } + + impl Comparable for PySSLSession { + fn cmp( + zelf: &Py, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult { + op.eq_only(|| { + if let Some(other_session) = other.downcast_ref::() { + Ok((zelf.session_id == other_session.session_id).into()) + } else { + Ok(PyComparisonValue::NotImplemented) + } + }) } } diff --git a/crates/stdlib/src/ssl/cert.rs b/crates/stdlib/src/ssl/cert.rs index f12f4307239..47d11f730b2 100644 --- a/crates/stdlib/src/ssl/cert.rs +++ b/crates/stdlib/src/ssl/cert.rs @@ -287,9 +287,11 @@ pub(super) fn is_ca_certificate(cert_der: &[u8]) -> bool { return ext.value.ca; } - // No Basic Constraints extension -> NOT a CA certificate - // (matches OpenSSL X509_check_ca() behavior) - false + // X509_check_ca() also retains OpenSSL's legacy trust-anchor rule: a + // self-issued X.509v1 certificate has no extensions at all, but is still + // classified as a CA. CPython's test CA at capath/4e1295a3.0 exercises + // precisely this case. + cert.version().0 == 0 && cert.subject() == cert.issuer() } /// Convert an X509Name to Python nested tuple format for SSL certificate dicts @@ -867,26 +869,36 @@ impl ServerCertVerifier for NoVerifier { fn verify_tls12_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, ) -> Result { - // Accept all signatures without verification - Ok(HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &CryptoExt::get_provider().signature_verification_algorithms, + ) } fn verify_tls13_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, ) -> Result { - // Accept all signatures without verification - Ok(HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &CryptoExt::get_provider().signature_verification_algorithms, + ) } fn supported_verify_schemes(&self) -> Vec { - ALL_SIGNATURE_SCHEMES.to_vec() + CryptoExt::get_provider() + .signature_verification_algorithms + .supported_schemes() } } diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index b1132a55a20..b25c9d5c499 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -1366,19 +1366,25 @@ impl PyCode { OptionalArg::Missing => self.code.instructions.clone(), }; + let intern_all = |objs: Vec, field: &str| -> PyResult> { + objs.into_iter() + .map(|o| { + let s = o.downcast_ref::().ok_or_else(|| { + vm.new_type_error(format!("{field} must be a tuple of strings")) + })?; + Ok(vm.ctx.intern_str(s.as_wtf8())) + }) + .collect::>>() + .map(Vec::into_boxed_slice) + }; + let cellvars = match co_cellvars { - OptionalArg::Present(cellvars) => cellvars - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), + OptionalArg::Present(cellvars) => intern_all(cellvars, "co_cellvars")?, OptionalArg::Missing => self.code.cellvars.clone(), }; let freevars = match co_freevars { - OptionalArg::Present(freevars) => freevars - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), + OptionalArg::Present(freevars) => intern_all(freevars, "co_freevars")?, OptionalArg::Missing => self.code.freevars.clone(), }; @@ -1411,10 +1417,10 @@ impl PyCode { posonlyarg_count, arg_count, kwonlyarg_count, - source_path: source_path.as_object().as_interned_str(vm).unwrap(), + source_path: vm.ctx.intern_str(source_path.as_wtf8()), first_line_number, - obj_name: obj_name.as_object().as_interned_str(vm).unwrap(), - qualname: qualname.as_object().as_interned_str(vm).unwrap(), + obj_name: vm.ctx.intern_str(obj_name.as_wtf8()), + qualname: vm.ctx.intern_str(qualname.as_wtf8()), max_stackdepth, instructions, @@ -1422,14 +1428,8 @@ impl PyCode { // It can be removed once we move every other code to use linetable only. locations: self.code.locations.clone(), constants: constants.into_iter().map(Literal).collect(), - names: names - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), - varnames: varnames - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), + names: intern_all(names, "co_names")?, + varnames: intern_all(varnames, "co_varnames")?, cellvars, freevars, localspluskinds: self.code.localspluskinds.clone(), diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 5db071e1d8f..fbc23a0dde7 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -1,6 +1,6 @@ use super::{ IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, PyMappingProxy, PySet, - PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set::PySetInner, + PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set, set::PySetInner, }; use crate::common::lock::LazyLock; use crate::object::{Traverse, TraverseFn}; @@ -9,7 +9,7 @@ use crate::{ TryFromObject, atomic_func, builtins::{PyList, PyTuple, iter::builtins_iter, type_::PyAttributes}, class::{PyClassDef, PyClassImpl}, - common::ascii, + common::{ascii, hash::PyHash}, dict_inner::{self, DictKey}, function::{ArgIterable, FuncArgs, KwArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, iter::PyExactSizeIterator, @@ -354,6 +354,20 @@ impl PyDict { ) -> PyResult> { self.entries.get(vm, key) } + + /// Keys of `obj` with their stored hashes, or `None` if it must be iterated + /// generically. Only exact dicts and sets qualify, as in CPython's + /// `_PyDict_FromKeys`: a subclass may override `__iter__`. + fn fromkeys_known_hashes( + obj: &PyObject, + vm: &VirtualMachine, + ) -> Option> { + if let Some(dict) = obj.downcast_ref_if_exact::(vm) { + Some(dict.entries.keys_with_hashes()) + } else { + set::exact_set_keys_with_hashes(obj, vm) + } + } } // Python dict methods: @@ -384,8 +398,16 @@ impl PyDict { let d = PyType::call(&class, ().into(), vm)?; match d.downcast_exact::(vm) { Ok(pydict) => { - for key in iterable.iter(vm)? { - pydict.__setitem__(key?, value.clone(), vm)?; + if let Some(keys) = Self::fromkeys_known_hashes(iterable.as_object(), vm) { + for (key, hash) in keys { + pydict + .entries + .insert_known_hash(vm, &*key, hash, value.clone())?; + } + } else { + for key in iterable.iter(vm)? { + pydict.__setitem__(key?, value.clone(), vm)?; + } } Ok(pydict.into_pyref().into()) } diff --git a/crates/vm/src/builtins/map.rs b/crates/vm/src/builtins/map.rs index 4dda9caf211..cb8db23e640 100644 --- a/crates/vm/src/builtins/map.rs +++ b/crates/vm/src/builtins/map.rs @@ -37,9 +37,12 @@ impl Constructor for PyMap { fn py_new( _cls: &Py, (mapper, iterators, args): Self::Args, - _vm: &VirtualMachine, + vm: &VirtualMachine, ) -> PyResult { let iterators = iterators.into_vec(); + if iterators.is_empty() { + return Err(vm.new_type_error("map() must have at least two arguments.")); + } let strict = Radium::new(args.strict.unwrap_or(false)); Ok(Self { mapper, diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index cd724abc5c1..6961040c792 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -195,6 +195,26 @@ impl PySetInner { Ok(set) } + /// Build a set from an arbitrary object, reusing stored hashes when the + /// source is a set/frozenset/dict. + fn from_object(iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let set = Self::default(); + set.update_internal(iterable, vm)?; + Ok(set) + } + + /// Elements of `obj` with their stored hashes, or `None` if `obj` keeps + /// none and must be iterated generically. Mirrors the `PyAnySet_Check` / + /// `PyDict_CheckExact` fast paths in CPython's `set_update_internal`. + fn cached_hashes(obj: &PyObject, vm: &VirtualMachine) -> Option> { + if let Some(set) = extract_set(obj) { + Some(set.content.keys_with_hashes()) + } else { + obj.downcast_ref_if_exact::(vm) + .map(|dict| dict._as_dict_inner().keys_with_hashes()) + } + } + fn fold_op( &self, others: impl core::iter::Iterator, @@ -228,6 +248,17 @@ impl PySetInner { Self::wrap_unhashable_error(result, needle, vm) } + /// [`Self::contains`] with a known hash. Such a needle came out of a + /// set/dict, so it is hashable and needs no frozenset retry. + fn contains_known_hash( + &self, + needle: &PyObject, + hash: PyHash, + vm: &VirtualMachine, + ) -> PyResult { + self.content.contains_known_hash(vm, needle, hash) + } + fn compare(&self, other: &Self, op: PyComparisonOp, vm: &VirtualMachine) -> PyResult { if op == PyComparisonOp::Ne { return self.compare(other, PyComparisonOp::Eq, vm).map(|eq| !eq); @@ -251,6 +282,12 @@ impl PySetInner { pub(super) fn union(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = self.clone(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (item, hash) in elements { + set.add_known_hash(item, hash, vm)?; + } + return Ok(set); + } for item in other.iter(vm)? { set.add(item?, vm)?; } @@ -260,6 +297,14 @@ impl PySetInner { pub(super) fn intersection(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = Self::default(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (obj, hash) in elements { + if self.contains_known_hash(&obj, hash, vm)? { + set.add_known_hash(obj, hash, vm)?; + } + } + return Ok(set); + } for item in other.iter(vm)? { let obj = item?; if self.contains(&obj, vm)? { @@ -271,6 +316,12 @@ impl PySetInner { pub(super) fn difference(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = self.copy(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (item, hash) in elements { + set.content.delete_if_exists_known_hash(vm, &*item, hash)?; + } + return Ok(set); + } for item in other.iter(vm)? { set.content.delete_if_exists(vm, &*item?)?; } @@ -284,6 +335,16 @@ impl PySetInner { ) -> PyResult { let new_inner = self.clone(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + // the source is already duplicate-free + for (item, hash) in elements { + new_inner + .content + .delete_or_insert_known_hash(vm, &item, hash, ())?; + } + return Ok(new_inner); + } + // We want to remove duplicates in other let other_set = Self::from_iter(other.iter(vm)?, vm)?; @@ -333,6 +394,12 @@ impl PySetInner { Self::wrap_unhashable_error(result, &item, vm) } + /// [`Self::add`] with a known hash. + fn add_known_hash(&self, item: PyObjectRef, hash: PyHash, vm: &VirtualMachine) -> PyResult<()> { + let result = self.content.insert_known_hash(vm, &*item, hash, ()); + Self::wrap_unhashable_error(result, &item, vm) + } + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let result = self.retry_op_with_frozenset(&item, vm, |item, vm| self.content.delete(vm, item)); @@ -393,15 +460,15 @@ impl PySetInner { } fn merge_set(&self, any_set: AnySet, vm: &VirtualMachine) -> PyResult<()> { - for item in any_set.as_inner().elements() { - self.add(item, vm)?; + for (item, hash) in any_set.as_inner().content.keys_with_hashes() { + self.add_known_hash(item, hash, vm)?; } Ok(()) } fn merge_dict(&self, dict: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { - for (key, _value) in dict { - self.add(key, vm)?; + for (key, hash) in dict._as_dict_inner().keys_with_hashes() { + self.add_known_hash(key, hash, vm)?; } Ok(()) } @@ -413,8 +480,8 @@ impl PySetInner { ) -> PyResult<()> { let temp_inner = self.fold_op(others, Self::intersection, vm)?; self.clear(); - for obj in temp_inner.elements() { - self.add(obj, vm)?; + for (obj, hash) in temp_inner.content.keys_with_hashes() { + self.add_known_hash(obj, hash, vm)?; } Ok(()) } @@ -425,6 +492,12 @@ impl PySetInner { vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { + if let Some(elements) = Self::cached_hashes(iterable.as_object(), vm) { + for (item, hash) in elements { + self.content.delete_if_exists_known_hash(vm, &*item, hash)?; + } + continue; + } let items = iterable.iter(vm)?.collect::, _>>()?; for item in items { self.content.delete_if_exists(vm, &*item)?; @@ -439,6 +512,14 @@ impl PySetInner { vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { + if let Some(elements) = Self::cached_hashes(iterable.as_object(), vm) { + // the source is already duplicate-free + for (item, hash) in elements { + self.content + .delete_or_insert_known_hash(vm, &item, hash, ())?; + } + continue; + } // We want to remove duplicates in iterable let iterable_set = Self::from_iter(iterable.iter(vm)?, vm)?; for item in iterable_set.elements() { @@ -524,6 +605,23 @@ fn extract_set(obj: &PyObject) -> Option<&PySetInner> { }) } +/// Elements of `obj` with their stored hashes, or `None` unless `obj` is exactly +/// a `set` or `frozenset` — `PyAnySet_CheckExact`, where [`extract_set`] is the +/// subclass-inclusive `PyAnySet_Check`. +pub(super) fn exact_set_keys_with_hashes( + obj: &PyObject, + vm: &VirtualMachine, +) -> Option> { + let inner = obj + .downcast_ref_if_exact::(vm) + .map(|set| &set.inner) + .or_else(|| { + obj.downcast_ref_if_exact::(vm) + .map(|frozen| &frozen.inner) + })?; + Some(inner.content.keys_with_hashes()) +} + fn reduce_set(zelf: &PyObject, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option) { ( zelf.class().to_owned(), @@ -955,7 +1053,7 @@ impl Representable for PySet { } impl Constructor for PyFrozenSet { - type Args = Vec; + type Args = OptionalArg; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { let is_exact_frozenset = cls.is(vm.ctx.types.frozenset_type); @@ -988,11 +1086,11 @@ impl Constructor for PyFrozenSet { return Ok(input.clone()); } - iterable.into_option() + iterable } else { match &args.args[..] { - [] => None, - [iterable] => Some(iterable.clone()), + [] => OptionalArg::Missing, + [iterable] => OptionalArg::Present(iterable.clone()), slice => { return Err(vm.new_type_error(format!( "frozenset expected at most 1 argument, got {}", @@ -1002,23 +1100,25 @@ impl Constructor for PyFrozenSet { } }; - let elements = if let Some(iterable) = iterable_opt { - iterable.try_to_value(vm)? - } else { - vec![] - }; + let payload = Self::py_new(&cls, iterable_opt, vm)?; // Return empty frozenset singleton - if is_exact_frozenset && elements.is_empty() { + if is_exact_frozenset && payload.inner.len() == 0 { return Ok(vm.ctx.empty_frozenset.clone().into()); } - let payload = Self::py_new(&cls, elements, vm)?; payload.into_ref_with_type(vm, cls).map(Into::into) } - fn py_new(_cls: &Py, elements: Self::Args, vm: &VirtualMachine) -> PyResult { - Self::from_iter(vm, elements) + fn py_new(_cls: &Py, iterable: Self::Args, vm: &VirtualMachine) -> PyResult { + let inner = match iterable { + OptionalArg::Present(iterable) => PySetInner::from_object(iterable, vm)?, + OptionalArg::Missing => PySetInner::default(), + }; + Ok(Self { + inner, + ..Default::default() + }) } } diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index d48639b2c11..06fa2519205 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -23,12 +23,60 @@ use crate::{ vm::VirtualMachine, }; use alloc::fmt; -use core::cell::Cell; +use core::cell::{Cell, UnsafeCell}; use core::ptr::NonNull; #[pyclass(module = false, name = "tuple", traverse = "manual")] pub struct PyTuple { - elements: Box<[R]>, + elements: TupleElements, +} + +/// Tuple storage is immutable after publication, but marshal must publish a +/// tuple in its reference table before recursively reading its children. +/// This mirrors CPython's `PyTuple_New` followed by `PyTuple_SET_ITEM`. +struct TupleElements(UnsafeCell>); + +unsafe impl Send for TupleElements {} +unsafe impl Sync for TupleElements {} + +impl TupleElements { + const fn new(elements: Box<[R]>) -> Self { + Self(UnsafeCell::new(elements)) + } + + fn as_slice(&self) -> &[R] { + // SAFETY: initialization writes happen only while the tuple is owned by + // the synchronous marshal decoder; afterwards the storage is immutable. + unsafe { &*self.0.get() } + } + + fn get_mut(&mut self) -> &mut Box<[R]> { + self.0.get_mut() + } + + /// # Safety + /// The tuple must still be in its private initialization phase, and each + /// placeholder index must be replaced at most once before it is observable. + unsafe fn set_initializing(&self, index: usize, value: R) { + unsafe { (*self.0.get())[index] = value }; + } +} + +impl core::ops::Deref for TupleElements { + type Target = [R]; + + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} + +impl<'a, R> IntoIterator for &'a TupleElements { + type Item = &'a R; + type IntoIter = core::slice::Iter<'a, R>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } } impl fmt::Debug for PyTuple { @@ -42,11 +90,11 @@ impl fmt::Debug for PyTuple { // Note: Only impl for PyTuple (the default) unsafe impl Traverse for PyTuple { fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { - self.elements.traverse(traverse_fn); + self.elements.as_slice().traverse(traverse_fn); } fn clear(&mut self, out: &mut Vec) { - let elements = core::mem::take(&mut self.elements); + let elements = core::mem::take(self.elements.get_mut()); out.extend(elements.into_vec()); } } @@ -206,7 +254,7 @@ impl Constructor for PyTuple { fn py_new(_cls: &Py, elements: Self::Args, _vm: &VirtualMachine) -> PyResult { Ok(Self { - elements: elements.into_boxed_slice(), + elements: TupleElements::new(elements.into_boxed_slice()), }) } } @@ -245,19 +293,19 @@ impl<'a, R> core::iter::IntoIterator for &'a Py> { impl PyTuple { #[must_use] - pub const fn as_slice(&self) -> &[R] { + pub fn as_slice(&self) -> &[R] { &self.elements } #[inline] #[must_use] - pub const fn len(&self) -> usize { + pub fn len(&self) -> usize { self.elements.len() } #[inline] #[must_use] - pub const fn is_empty(&self) -> bool { + pub fn is_empty(&self) -> bool { self.elements.is_empty() } @@ -274,7 +322,13 @@ impl PyTuple { ctx.empty_tuple.clone() } else { let elements = elements.into_boxed_slice(); - PyRef::new_ref(Self { elements }, ctx.types.tuple_type.to_owned(), None) + PyRef::new_ref( + Self { + elements: TupleElements::new(elements), + }, + ctx.types.tuple_type.to_owned(), + None, + ) } } @@ -283,7 +337,20 @@ impl PyTuple { /// Calling this function implies trying micro optimization for non-zero-sized tuple. #[must_use] pub const fn new_unchecked(elements: Box<[PyObjectRef]>) -> Self { - Self { elements } + Self { + elements: TupleElements::new(elements), + } + } + + pub(crate) fn new_marshal_placeholder(len: usize, ctx: &Context) -> PyRef { + Self::new_ref(vec![ctx.none(); len], ctx) + } + + /// # Safety + /// This tuple must be a marshal placeholder which has not escaped the + /// decoder, and `index` must not have been replaced previously. + pub(crate) unsafe fn set_marshal_item(&self, index: usize, value: PyObjectRef) { + unsafe { self.elements.set_initializing(index, value) }; } fn repeat(zelf: PyRef, value: isize, vm: &VirtualMachine) -> PyResult> { @@ -298,7 +365,10 @@ impl PyTuple { } else { let v = zelf.elements.mul(vm, value)?; let elements = v.into_boxed_slice(); - Self { elements }.into_ref(&vm.ctx) + Self { + elements: TupleElements::new(elements), + } + .into_ref(&vm.ctx) }) } @@ -341,7 +411,10 @@ impl PyTuple { .chain(other.as_slice()) .cloned() .collect::>(); - Self { elements }.into_ref(&vm.ctx) + Self { + elements: TupleElements::new(elements), + } + .into_ref(&vm.ctx) } }); PyArithmeticValue::from_option(added.ok()) @@ -360,7 +433,7 @@ impl PyTuple { #[inline] #[must_use] - pub const fn __len__(&self) -> usize { + pub fn __len__(&self) -> usize { self.elements.len() } @@ -425,7 +498,7 @@ impl PyTuple { let tup_arg = if zelf.class().is(vm.ctx.types.tuple_type) { zelf } else { - Self::new_ref(zelf.elements.clone().into_vec(), &vm.ctx) + Self::new_ref(zelf.elements.as_slice().to_vec(), &vm.ctx) }; (tup_arg,) } diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 3e75e6f27a6..9dda6194a0c 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -463,6 +463,24 @@ impl Dict { K: DictKey + ?Sized, { let hash = key.key_hash(vm)?; + self.insert_known_hash(vm, key, hash, value) + } + + /// Store a key whose hash the caller already knows. + /// + /// `hash` must equal `key.key_hash(vm)`; a wrong one lands the entry in a + /// bucket no lookup probes, silently losing the key. Only pass a hash from + /// [`Self::keys_with_hashes`] on a container holding this same key. + pub(crate) fn insert_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + value: T, + ) -> PyResult<()> + where + K: DictKey + ?Sized, + { let _removed = loop { let (entry_index, index_index) = self.lookup(vm, key, hash, None)?; let mut inner = self.write(); @@ -512,7 +530,18 @@ impl Dict { key: &K, ) -> PyResult { let key_hash = key.key_hash(vm)?; - let (entry, _) = self.lookup(vm, key, key_hash, None)?; + self.contains_known_hash(vm, key, key_hash) + } + + /// [`Self::contains`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn contains_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + ) -> PyResult { + let (entry, _) = self.lookup(vm, key, hash, None)?; Ok(entry.index().is_some()) } @@ -697,6 +726,21 @@ impl Dict { self.remove_if_exists(vm, key).map(|opt| opt.is_some()) } + /// [`Self::delete_if_exists`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn delete_if_exists_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + ) -> PyResult + where + K: DictKey + ?Sized, + { + self.remove_if_known_hash(vm, key, hash, |_| Ok(true)) + .map(|opt| opt.is_some()) + } + pub(crate) fn delete_if(&self, vm: &VirtualMachine, key: &K, pred: F) -> PyResult where K: DictKey + ?Sized, @@ -725,6 +769,22 @@ impl Dict { F: Fn(&T) -> PyResult, { let hash = key.key_hash(vm)?; + self.remove_if_known_hash(vm, key, hash, pred) + } + + /// [`Self::remove_if`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + fn remove_if_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + pred: F, + ) -> PyResult> + where + K: DictKey + ?Sized, + F: Fn(&T) -> PyResult, + { let removed = loop { let lookup = self.lookup(vm, key, hash, None)?; match self.pop_inner_if(lookup, &pred)? { @@ -742,6 +802,18 @@ impl Dict { value: T, ) -> PyResult<()> { let hash = key.key_hash(vm)?; + self.delete_or_insert_known_hash(vm, key, hash, value) + } + + /// [`Self::delete_or_insert`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn delete_or_insert_known_hash( + &self, + vm: &VirtualMachine, + key: &PyObject, + hash: HashValue, + value: T, + ) -> PyResult<()> { let _removed = loop { let lookup = self.lookup(vm, key, hash, None)?; let (entry, index_index) = lookup; @@ -888,6 +960,16 @@ impl Dict { .collect() } + /// All keys paired with the hash stored in their entry, for feeding + /// [`Self::insert_known_hash`] without re-calling `__hash__`. + pub(crate) fn keys_with_hashes(&self) -> Vec<(PyObjectRef, HashValue)> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| (v.key.clone(), v.hash))) + .collect() + } + pub(crate) fn values(&self) -> Vec { self.read() .entries diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 4facffee15d..9c42df966ea 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -245,7 +245,11 @@ impl VirtualMachine { _ => true, }; - if same_line { + // A lone continuation at EOF has no highlighted source span. + let lone_line_continuation = + maybe_end_offset == Some(-1) && l_text.to_string_lossy() == "\\"; + + if same_line && !lone_line_continuation { let mut end_offset = match maybe_end_offset { Some(0) | None => offset, Some(end_offset) => end_offset, @@ -1385,14 +1389,8 @@ impl OSErrorBuilder { vec![strerror.to_pyobject(vm)] }; - let payload = PyOSError::py_new(&exc_type, args.clone().into(), vm) - .expect("new_os_error usage error"); - let os_error = payload - .into_ref_with_type_lazy_dict(vm, exc_type) - .expect("new_os_error usage error"); - PyOSError::slot_init(os_error.as_object().to_owned(), args.into(), vm) - .expect("new_os_error usage error"); - os_error + vm.new_payload_exception::(exc_type, args.into()) + .expect("new_os_error usage error") } } @@ -2144,7 +2142,10 @@ pub(super) mod types { .downcast_ref::() .and_then(|errno| errno.try_to_primitive::(vm).ok()) .and_then(|errno| super::errno_to_exc_type(errno, vm)) - .and_then(|typ| vm.invoke_exception(typ, args_vec).ok()) + .and_then(|typ| { + vm.new_payload_exception::(typ.to_owned(), args_vec.into()) + .ok() + }) { return error.to_pyresult(vm); } diff --git a/crates/vm/src/format.rs b/crates/vm/src/format.rs index 80b906505bf..657601e1470 100644 --- a/crates/vm/src/format.rs +++ b/crates/vm/src/format.rs @@ -77,6 +77,12 @@ impl IntoPyException for FormatSpecError { Self::AlignmentFlag => { vm.new_value_error("'=' alignment flag is not allowed in complex format specifier") } + Self::NegativeZeroCoercionNotAllowed(type_name) => { + let msg = format!( + "Negative zero coercion (z) not allowed in {type_name} format specifier" + ); + vm.new_value_error(msg) + } Self::StringAlignmentFlag => { vm.new_value_error("'=' alignment not allowed in string format specifier") } diff --git a/crates/vm/src/function/protocol.rs b/crates/vm/src/function/protocol.rs index 25ef62b458d..d503fabaca8 100644 --- a/crates/vm/src/function/protocol.rs +++ b/crates/vm/src/function/protocol.rs @@ -86,6 +86,11 @@ unsafe impl Traverse for ArgIterable { } impl ArgIterable { + #[must_use] + pub(crate) fn as_object(&self) -> &PyObject { + &self.iterable + } + /// Returns an iterator over this sequence of objects. /// /// This operation may fail if an exception is raised while invoking the diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs index 69d9e0e4fde..497d62fcc81 100644 --- a/crates/vm/src/stdlib/_codecs.rs +++ b/crates/vm/src/stdlib/_codecs.rs @@ -791,19 +791,18 @@ mod _codecs_windows { // Convert code point to UTF-16 let mut wchars = [0u16; 2]; - let wchar_len; let is_surrogate = (0xD800..=0xDFFF).contains(&ch); - if is_surrogate { - wchar_len = 0; // Can't encode surrogates normally + let wchar_len = if is_surrogate { + 0 // Can't encode surrogates normally } else if ch < 0x10000 { wchars[0] = ch as u16; - wchar_len = 1; + 1 } else { wchars[0] = ((ch - 0x10000) >> 10) as u16 + 0xD800; wchars[1] = ((ch - 0x10000) & 0x3FF) as u16 + 0xDC00; - wchar_len = 2; - } + 2 + }; if !is_surrogate { let mut buf = [0u8; 8]; diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 43a869fbecf..ab1be4297ec 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -20,7 +20,8 @@ cfg_select! { } use crate::{ - AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyModule, + AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyModule, PyOSError}, }; pub use _io::{OpenArgs, io_open as open}; use rustpython_host_env::io as host_io; @@ -943,14 +944,17 @@ mod _io { Some(n) => n, None => { // BlockingIOError(errno, msg, characters_written=0) - return Err(vm.invoke_exception( - vm.ctx.exceptions.blocking_io_error, - vec![ - vm.new_pyobj(EAGAIN), - vm.new_pyobj("write could not complete without blocking"), - vm.new_pyobj(0), - ], - )?); + return Err(vm + .new_payload_exception::( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(0), + ] + .into(), + )? + .upcast()); } }; self.write_pos += n as Offset; @@ -1154,14 +1158,17 @@ mod _io { self.buffer[self.write_end as usize..][..avail].copy_from_slice(&buf[..avail]); self.write_end += avail as Offset; self.pos += avail as Offset; - return Err(vm.invoke_exception( - vm.ctx.exceptions.blocking_io_error, - vec![ - vm.new_pyobj(EAGAIN), - vm.new_pyobj("write could not complete without blocking"), - vm.new_pyobj(avail), - ], - )?); + return Err(vm + .new_payload_exception::( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(avail), + ] + .into(), + )? + .upcast()); } Err(e) => return Err(e), } @@ -1200,14 +1207,17 @@ mod _io { self.write_end = buffer_size; // BlockingIOError(errno, msg, characters_written) let chars_written = written + buffer_len; - return Err(vm.invoke_exception( - vm.ctx.exceptions.blocking_io_error, - vec![ - vm.new_pyobj(EAGAIN), - vm.new_pyobj("write could not complete without blocking"), - vm.new_pyobj(chars_written), - ], - )?); + return Err(vm + .new_payload_exception::( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(chars_written), + ] + .into(), + )? + .upcast()); } None => break, } diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 0a80285dbe3..377c68dca74 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -605,14 +605,35 @@ pub(crate) mod _thread { vm.state.thread_count.fetch_sub(1); } + /// Default stack size for Python threads in **debug builds only**, where + /// Rust stack frames are substantially larger than in release. Rust's + /// `std::thread::Builder` otherwise defaults to 2 MB, which is too small + /// for the call chains the Python stdlib runs on helper threads in debug + /// (e.g. the SSL test server, see #7941). Release builds keep the prior + /// behavior — leave the stack size unset and let Rust's std default apply + /// — to avoid oversized virtual stack mappings when many threads spawn. + #[cfg(debug_assertions)] + const DEFAULT_THREAD_STACK_SIZE: usize = 8 * 1024 * 1024; + + /// Configure a `thread::Builder` with the stack size to use for a new + /// Python thread. Uses the value set via `threading.stack_size(N)` when + /// the user has provided one (non-zero). Otherwise, debug builds fall + /// back to [`DEFAULT_THREAD_STACK_SIZE`] and release builds leave the + /// builder unmodified (Rust's std default applies). fn apply_thread_stack_size( thread_builder: thread::Builder, vm: &VirtualMachine, ) -> thread::Builder { let configured = vm.state.stacksize.load(); if configured != 0 { - thread_builder.stack_size(configured) - } else { + return thread_builder.stack_size(configured); + } + #[cfg(debug_assertions)] + { + thread_builder.stack_size(DEFAULT_THREAD_STACK_SIZE) + } + #[cfg(not(debug_assertions))] + { thread_builder } } @@ -635,7 +656,7 @@ pub(crate) mod _thread { #[pyfunction] fn exit(vm: &VirtualMachine) -> PyResult { - Err(vm.invoke_exception(vm.ctx.exceptions.system_exit, vec![])?) + Err(vm.new_system_exit(vec![].into())) } thread_local!(static SENTINELS: RefCell>> = const { RefCell::new(Vec::new()) }); @@ -1996,4 +2017,55 @@ pub(crate) mod _thread { Ok(handle_clone) } + + #[cfg(test)] + mod tests { + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + use super::*; + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + use crate::Interpreter; + + /// Regression test for #7941: a Python thread started without an + /// explicit `threading.stack_size()` must not run on Rust's 2 MiB + /// std default in debug builds, where the call chains the stdlib + /// runs on helper threads (e.g. the SSL test server) overflowed it. + #[test] + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + fn default_python_thread_stack_size_debug() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + assert_eq!(vm.state.stacksize.load(), 0); + let builder = apply_thread_stack_size(thread::Builder::new(), vm); + let stack_size = builder + .spawn(current_thread_stack_size) + .expect("failed to spawn thread") + .join() + .expect("thread panicked"); + assert!( + stack_size >= DEFAULT_THREAD_STACK_SIZE, + "Python thread stack size is {stack_size} bytes, expected at least {DEFAULT_THREAD_STACK_SIZE}" + ); + }); + } + + #[cfg(all(debug_assertions, target_os = "linux"))] + fn current_thread_stack_size() -> usize { + use libc::{ + pthread_attr_destroy, pthread_attr_getstacksize, pthread_attr_t, + pthread_getattr_np, pthread_self, + }; + let mut attr: pthread_attr_t = unsafe { core::mem::zeroed() }; + unsafe { + assert_eq!(pthread_getattr_np(pthread_self(), &mut attr), 0); + let mut size = 0; + assert_eq!(pthread_attr_getstacksize(&attr, &mut size), 0); + pthread_attr_destroy(&mut attr); + size + } + } + + #[cfg(all(debug_assertions, target_os = "macos"))] + fn current_thread_stack_size() -> usize { + unsafe { libc::pthread_get_stacksize_np(libc::pthread_self()) } + } + } } diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 7817d6ecb97..35f404f0f3b 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -1041,7 +1041,7 @@ mod builtins { #[pyfunction] pub(super) fn exit(exit_code_arg: OptionalArg, vm: &VirtualMachine) -> PyResult { let code = exit_code_arg.unwrap_or_else(|| vm.ctx.new_int(0).into()); - Err(vm.invoke_exception(vm.ctx.exceptions.system_exit, vec![code])?) + Err(vm.new_system_exit(vec![code].into())) } #[derive(Debug, Default, FromArgs)] diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 041620298e7..30a4d8773be 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -10,7 +10,7 @@ mod decl { rc::PyRc, }, convert::ToPyObject, - function::{ArgCallable, FuncArgs, OptionalArg, OptionalOption, PosArgs}, + function::{FuncArgs, OptionalArg, OptionalOption, PosArgs}, protocol::{PyIter, PyIterReturn, PyNumber}, raise_if_stop, stdlib::sys, @@ -477,7 +477,7 @@ mod decl { #[pyclass(name = "dropwhile")] #[derive(Debug, PyPayload)] struct PyItertoolsDropwhile { - predicate: ArgCallable, + predicate: PyObjectRef, iterable: PyIter, start_flag: AtomicCell, } @@ -485,7 +485,7 @@ mod decl { #[derive(FromArgs)] struct DropwhileNewArgs { #[pyarg(positional)] - predicate: ArgCallable, + predicate: PyObjectRef, #[pyarg(positional)] iterable: PyIter, } @@ -522,8 +522,7 @@ mod decl { if !zelf.start_flag.load() { loop { let obj = raise_if_stop!(iterable.next(vm)?); - let pred = predicate.clone(); - let pred_value = pred.invoke((obj.clone(),), vm)?; + let pred_value = predicate.call((obj.clone(),), vm)?; if !pred_value.try_to_bool(vm)? { zelf.start_flag.store(true); return Ok(PyIterReturn::Return(obj)); diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index ace3aff58f2..a4665d0f5b6 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -9,14 +9,16 @@ mod decl { use crate::{ PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{ - PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat, - PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple, + PyBaseExceptionRef, PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, + PyEllipsis, PyFloat, PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, + PyTuple, }, convert::ToPyObject, function::{ArgBytesLike, OptionalArg}, object::{AsObject, PyPayload}, protocol::PyBuffer, }; + use core::cell::RefCell; use malachite_bigint::BigInt; use num_traits::Zero; use rustpython_compiler_core::marshal::{self, DumpableValue}; @@ -386,79 +388,165 @@ mod decl { } #[derive(Copy, Clone)] - struct PyMarshalBag<'a>(&'a VirtualMachine); + struct PyMarshalBag<'a> { + vm: &'a VirtualMachine, + pending_error: &'a RefCell>, + } + + impl<'a> PyMarshalBag<'a> { + fn new( + vm: &'a VirtualMachine, + pending_error: &'a RefCell>, + ) -> Self { + Self { vm, pending_error } + } + + fn remember_python_error(&self, error: PyBaseExceptionRef) -> marshal::MarshalError { + let mut pending = self.pending_error.borrow_mut(); + if pending.is_none() { + *pending = Some(error); + } + marshal::MarshalError::BadType + } + } impl<'a> marshal::MarshalBag for PyMarshalBag<'a> { type Value = PyObjectRef; type ConstantBag = PyVmBag<'a>; fn make_bool(&self, value: bool) -> Self::Value { - self.0.ctx.new_bool(value).into() + self.vm.ctx.new_bool(value).into() } fn make_none(&self) -> Self::Value { - self.0.ctx.none() + self.vm.ctx.none() } fn make_ellipsis(&self) -> Self::Value { - self.0.ctx.ellipsis.clone().into() + self.vm.ctx.ellipsis.clone().into() } fn make_float(&self, value: f64) -> Self::Value { - self.0.ctx.new_float(value).into() + self.vm.ctx.new_float(value).into() } fn make_complex(&self, value: num_complex::Complex64) -> Self::Value { - self.0.ctx.new_complex(value).into() + self.vm.ctx.new_complex(value).into() } fn make_str(&self, value: &Wtf8) -> Self::Value { - self.0.ctx.new_str(value).into() + self.vm.ctx.new_str(value).into() + } + fn make_interned_str(&self, value: &Wtf8) -> Self::Value { + self.vm.ctx.intern_str(value).to_owned().into() } fn make_bytes(&self, value: &[u8]) -> Self::Value { - self.0.ctx.new_bytes(value.to_vec()).into() + self.vm.ctx.new_bytes(value.to_vec()).into() } fn make_int(&self, value: BigInt) -> Self::Value { - self.0.ctx.new_int(value).into() + self.vm.ctx.new_int(value).into() } fn make_tuple(&self, elements: impl Iterator) -> Self::Value { - self.0.ctx.new_tuple(elements.collect()).into() + self.vm.ctx.new_tuple(elements.collect()).into() + } + fn make_tuple_placeholder(&self, len: usize) -> Option { + Some(PyTuple::new_marshal_placeholder(len, &self.vm.ctx).into()) + } + fn set_tuple_item( + &self, + tuple: &Self::Value, + index: usize, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let tuple = tuple + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + // SAFETY: compiler-core calls this only on a fresh placeholder, + // once per index, before returning it to Python code. + unsafe { tuple.set_marshal_item(index, value) }; + Ok(()) } fn make_code(&self, code: CodeObject) -> Self::Value { - crate::builtins::PyCode::new_ref_with_bag(self.0, code).into() + crate::builtins::PyCode::new_ref_with_bag(self.vm, code).into() } fn make_stop_iter(&self) -> Result { - Ok(self.0.ctx.exceptions.stop_iteration.to_owned().into()) + Ok(self.vm.ctx.exceptions.stop_iteration.to_owned().into()) } fn make_list( &self, it: impl Iterator, ) -> Result { - Ok(self.0.ctx.new_list(it.collect()).into()) + Ok(self.vm.ctx.new_list(it.collect()).into()) + } + fn make_list_placeholder(&self, len: usize) -> Option { + Some(self.vm.ctx.new_list(vec![self.vm.ctx.none(); len]).into()) + } + fn set_list_item( + &self, + list: &Self::Value, + index: usize, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let list = list + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + list.borrow_vec_mut()[index] = value; + Ok(()) } fn make_set( &self, it: impl Iterator, ) -> Result { - let set = PySet::default().into_ref(&self.0.ctx); + let set = PySet::default().into_ref(&self.vm.ctx); for elem in it { - set.add(elem, self.0).unwrap() + set.add(elem, self.vm) + .map_err(|error| self.remember_python_error(error))?; } Ok(set.into()) } + fn make_set_placeholder(&self) -> Option { + Some(PySet::default().into_ref(&self.vm.ctx).into()) + } + fn insert_set_item( + &self, + set: &Self::Value, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let set = set + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + set.add(value, self.vm) + .map_err(|error| self.remember_python_error(error)) + } fn make_frozenset( &self, it: impl Iterator, ) -> Result { - Ok(PyFrozenSet::from_iter(self.0, it) - .unwrap() - .to_pyobject(self.0)) + PyFrozenSet::from_iter(self.vm, it) + .map(|set| set.to_pyobject(self.vm)) + .map_err(|error| self.remember_python_error(error)) } fn make_dict( &self, it: impl Iterator, ) -> Result { - let dict = self.0.ctx.new_dict(); + let dict = self.vm.ctx.new_dict(); for (k, v) in it { - dict.set_item(&*k, v, self.0).unwrap() + dict.set_item(&*k, v, self.vm) + .map_err(|error| self.remember_python_error(error))?; } Ok(dict.into()) } + fn make_dict_placeholder(&self) -> Option { + Some(self.vm.ctx.new_dict().into()) + } + fn insert_dict_item( + &self, + dict: &Self::Value, + key: Self::Value, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let dict = dict + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + dict.set_item(&*key, value, self.vm) + .map_err(|error| self.remember_python_error(error)) + } fn make_slice( &self, start: Self::Value, @@ -466,7 +554,7 @@ mod decl { step: Self::Value, ) -> Result { use crate::builtins::PySlice; - let vm = self.0; + let vm = self.vm; Ok(PySlice { start: if vm.is_none(&start) { None @@ -480,7 +568,21 @@ mod decl { .into()) } fn constant_bag(self) -> Self::ConstantBag { - PyVmBag(self.0) + PyVmBag(self.vm) + } + } + + fn deserialize_value( + rdr: &mut impl marshal::Read, + vm: &VirtualMachine, + ) -> PyResult { + let pending_error = RefCell::new(None); + match marshal::deserialize_value(rdr, PyMarshalBag::new(vm, &pending_error)) { + Ok(value) => Ok(value), + Err(error) => Err(pending_error.into_inner().unwrap_or_else(|| match error { + marshal::MarshalError::Eof => vm.new_eof_error("marshal data too short"), + _ => vm.new_value_error("bad marshal data"), + })), } } @@ -502,11 +604,7 @@ mod decl { vm.new_buffer_error("Buffer provided to marshal.loads() is not contiguous") })?; - let result = - marshal::deserialize_value(&mut &buf[..], PyMarshalBag(vm)).map_err(|e| match e { - marshal::MarshalError::Eof => vm.new_eof_error("marshal data too short"), - _ => vm.new_value_error("bad marshal data"), - })?; + let result = deserialize_value(&mut &buf[..], vm)?; if !allow_code { check_no_code(&result, vm)?; } @@ -534,14 +632,7 @@ mod decl { let mut rdr: &[u8] = &buf; let len_before = rdr.len(); - let result = - marshal::deserialize_value(&mut rdr, PyMarshalBag(vm)).map_err(|e| match e { - marshal::MarshalError::Eof => vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "marshal data too short".into(), - ), - _ => vm.new_value_error("bad marshal data"), - })?; + let result = deserialize_value(&mut rdr, vm)?; let consumed = len_before - rdr.len(); // Seek file to just after the consumed bytes diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 8f264d7a739..c7cc2fd298a 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -776,8 +776,7 @@ pub mod sys { } else { vec![status] }; - let exc = vm.invoke_exception(vm.ctx.exceptions.system_exit, args)?; - Err(exc) + Err(vm.new_system_exit(args.into())) } #[pyfunction] @@ -818,8 +817,27 @@ pub mod sys { vm: &VirtualMachine, ) -> PyResult<()> { let stderr = super::get_stderr(vm)?; + // Keep runtime SyntaxErrors on the normal traceback path. + let has_traceback = !vm.is_none(&exc_tb); match vm.normalize_exception(exc_type, exc_val.clone(), exc_tb) { Ok(exc) => { + let native_syntax_error_display = !has_traceback + && exc.fast_isinstance(vm.ctx.exceptions.syntax_error) + && exc + .as_object() + .get_attr("msg", vm) + .ok() + .and_then(|msg| msg.downcast::().ok()) + .is_some_and(|msg| msg.to_string_lossy() == "unexpected EOF while parsing") + && exc + .as_object() + .get_attr("text", vm) + .ok() + .and_then(|text| text.downcast::().ok()) + .is_some_and(|text| text.to_string_lossy().trim_end() == "\\"); + if native_syntax_error_display { + return vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc); + } // PyErr_Display: try traceback._print_exception_bltin first if let Ok(tb_mod) = vm.import("traceback", 0) && let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm) diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index e0a086c10db..7c7d017c1fd 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1467,6 +1467,10 @@ impl VirtualMachine { let initial_ptr = self.take_pending_tailcall(); // Drain the refs that keep the initial callee's raw pointers alive. + #[allow( + clippy::drain_collect, + reason = "`pending_tailcall_refs`'s allocation is intentionally reused" + )] let initial_refs = unsafe { &mut *self.pending_tailcall_refs.get() } .drain(..) .collect(); @@ -1498,6 +1502,10 @@ impl VirtualMachine { let result = crate::frame::run_iframe(callee, self); match result { Ok(ExecutionResult::TailCall) => { + #[allow( + clippy::drain_collect, + reason = "`pending_tailcall_refs`'s allocation is intentionally reused" + )] let refs = unsafe { &mut *self.pending_tailcall_refs.get() } .drain(..) .collect(); @@ -1548,6 +1556,10 @@ impl VirtualMachine { let result = crate::frame::run_iframe(caller_iframe, self); match result { Ok(ExecutionResult::TailCall) => { + #[allow( + clippy::drain_collect, + reason = "`pending_tailcall_refs`'s allocation is intentionally reused" + )] let refs = unsafe { &mut *self.pending_tailcall_refs.get() } .drain(..) .collect(); @@ -1609,6 +1621,10 @@ impl VirtualMachine { let result = crate::frame::run_iframe(caller_iframe, self); match result { Ok(ExecutionResult::TailCall) => { + #[allow( + clippy::drain_collect, + reason = "`pending_tailcall_refs`'s allocation is intentionally reused" + )] let refs = unsafe { &mut *self.pending_tailcall_refs.get() } .drain(..) .collect(); @@ -2737,7 +2753,7 @@ impl VirtualMachine { if self.state.finalizing.load(Ordering::Acquire) && !self.is_main_thread() { // once finalization starts, // non-main Python threads should stop running bytecode. - return Err(self.invoke_exception(self.ctx.exceptions.system_exit, vec![])?); + return Err(self.new_system_exit(vec![].into())); } // Suspend this thread if stop-the-world is in progress diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index 91d5885e740..a1c2552cef4 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -113,6 +113,10 @@ mod file_run { "source code cannot contain null bytes".into(), )); } + #[cfg(feature = "parser")] + // Match compile() by honoring BOMs and encoding cookies in files. + let source = self.decode_source_bytes(&source_bytes, path, false)?; + #[cfg(not(feature = "parser"))] let source = String::from_utf8(source_bytes) .map_err(|err| self.new_os_error(err.to_string()))?; let code_obj = self diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 48909c1a41e..6110a3d5b1a 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -12,17 +12,18 @@ use rustpython_compiler::{CompileError, ParseError}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ - PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyOSError, PyStrRef, - PyType, PyTypeRef, + PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyOSError, + PyStopIteration, PyStrRef, PySystemExit, PyType, PyTypeRef, builtin_func::PyNativeFunction, descriptor::PyMethodDescriptor, tuple::{IntoPyTuple, PyTupleRef}, }, convert::{ToPyException, ToPyObject}, exceptions::OSErrorBuilder, - function::{IntoPyNativeFn, PyMethodFlags}, + function::{FuncArgs, IntoPyNativeFn, PyMethodFlags}, scope::Scope, set_attrs, + types::{Constructor, Initializer}, vm::VirtualMachine, }; @@ -353,6 +354,26 @@ impl VirtualMachine { .expect("vm.new_exception() called with an invalid exception type") } + /// Construct a built-in exception type that carries a payload, directly + /// (`py_new` + `slot_init`), without routing through `PyType::call`. + /// Only valid for a built-in `T` whose exact type is known at compile time. + pub fn new_payload_exception(&self, cls: PyTypeRef, args: FuncArgs) -> PyResult> + where + T: Constructor + Initializer, + { + debug_assert_eq!( + cls.slots.basicsize, + size_of::(), + "vm.new_payload_exception::<{}>() called with mismatched type '{}'", + core::any::type_name::(), + cls.name() + ); + let payload = T::py_new(&cls, args.clone(), self)?; + let exc = payload.into_ref_with_type_lazy_dict(self, cls)?; + T::slot_init(exc.as_object().to_owned(), args, self)?; + Ok(exc) + } + pub fn new_os_error(&self, msg: impl ToPyObject) -> PyRef { self.new_os_subtype_error(self.ctx.exceptions.os_error.to_owned(), None, msg) .upcast() @@ -752,7 +773,7 @@ impl VirtualMachine { Some(line + "\n") } - let statement = source.and_then(|src| get_statement(src, error.location())); + let mut statement = source.and_then(|src| get_statement(src, error.location())); let mut msg = error.to_string(); if !msg.starts_with("Exceeds the limit ") @@ -799,6 +820,16 @@ impl VirtualMachine { } let SyntaxErrorInfo { msg, narrow_caret } = syntax_error_info; + let unterminated_triple_quoted_string = + msg.starts_with("unterminated triple-quoted string literal"); + let unexpected_eof_error = msg == "unexpected EOF while parsing"; + if unterminated_triple_quoted_string + && let Some(statement) = statement.as_mut() + && statement.ends_with('\n') + { + // CPython omits the parser-added final newline from SyntaxError.text. + statement.pop(); + } let check_version_suite_error = msg.starts_with("Async functions are") || msg.starts_with("Async for loops are") || msg.starts_with("Async with statements are") @@ -820,12 +851,14 @@ impl VirtualMachine { // Set end_lineno and end_offset if available if let Some((end_lineno, end_offset)) = error.python_end_location() { - let (end_lineno, end_offset) = if check_version_suite_error - && statement - .as_deref() - .and_then(|line| line.chars().next()) - .is_some_and(|ch| ch.is_ascii_whitespace()) - { + // EOF errors have no source span in CPython. + let no_end_offset = unexpected_eof_error + || (check_version_suite_error + && statement + .as_deref() + .and_then(|line| line.chars().next()) + .is_some_and(|ch| ch.is_ascii_whitespace())); + let (end_lineno, end_offset) = if no_end_offset { (end_lineno, -1) } else if line_end_binary_operator_error && end_offset == offset_raw { (end_lineno, (end_offset + 1) as isize) @@ -893,16 +926,20 @@ impl VirtualMachine { exc } - pub fn new_stop_iteration(&self, value: Option) -> PyBaseExceptionRef { - let stop_iteration_error = self.ctx.exceptions.stop_iteration; - let args = if let Some(value) = value { - vec![value] - } else { - Vec::new() - }; - let exc = self.invoke_exception(stop_iteration_error, args); + pub fn new_system_exit(&self, args: FuncArgs) -> PyBaseExceptionRef { + self.new_payload_exception::(self.ctx.exceptions.system_exit.to_owned(), args) + .expect("SystemExit construction from internal args is infallible") + .upcast() + } - exc.expect("StopIteration is a BaseException Subclass.") + pub fn new_stop_iteration(&self, value: Option) -> PyBaseExceptionRef { + let args: FuncArgs = value.map(|v| vec![v]).unwrap_or_default().into(); + self.new_payload_exception::( + self.ctx.exceptions.stop_iteration.to_owned(), + args, + ) + .expect("StopIteration construction from internal args is infallible") + .upcast() } fn new_downcast_error(