diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e9e2041d482..de9c49e7224 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -99,6 +99,9 @@ updates: result-like: patterns: - "result-like*" + ruff: + patterns: + - "ruff_*" security-framework: patterns: - "security-framework*" @@ -149,11 +152,6 @@ updates: zeroize: patterns: - "zeroize*" - ignore: - # TODO: Remove when we use ruff from crates.io - # for some reason dependabot only updates the Cargo.lock file when dealing - # with git dependencies. i.e. not updating the version in Cargo.toml - - dependency-name: "ruff_*" - package-ecosystem: github-actions directory: / schedule: diff --git a/Cargo.lock b/Cargo.lock index 743e293f161..5921279f61d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -100,7 +100,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -111,7 +111,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -144,6 +144,12 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "arrayvec" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" + [[package]] name = "ascii" version = "1.1.0" @@ -1145,7 +1151,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1237,7 +1243,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1385,9 +1391,9 @@ dependencies = [ [[package]] name = "get-size-derive2" -version = "0.7.4" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b6d1e2f75c16bfbcd0f95d84f99858a6e2f885c2287d1f5c3a96e8444a34b4" +checksum = "1da24fbda09ec01bca7cfa1797c0e520e75123bccb01dcdf9041f8aa65183bc2" dependencies = [ "attribute-derive", "quote", @@ -1396,15 +1402,16 @@ dependencies = [ [[package]] name = "get-size2" -version = "0.7.4" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cf31a6d70300cf81461098f7797571362387ef4bf85d32ac47eaa59b3a5a1a" +checksum = "823645bc6404ae2915707777061a47d3a031a9ee0bff51b34ec973df3d8d2990" dependencies = [ "compact_str", "get-size-derive2", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "ordermap", "smallvec", + "thin-vec", ] [[package]] @@ -1946,7 +1953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "160f2eade097f30263b548aae5deb12ad349c909baa710fa24b92c9090b2e006" dependencies = [ "scopeguard", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3285,6 +3292,80 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2fa174f449b262fd212584e349d4e12ededd67e0ddb9a1b8fba7f6dbae14fa7" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags 2.13.0", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thin-vec", + "thiserror", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b0f43d161dc1d99fdb3b937725a25f218e5de916a6ae246ffe77ec71708afd" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "compact_str", + "get-size2", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "rustc-hash", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", + "unicode_names2 1.3.0", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57bb4ae7e2c54211ae60155dede845fc42e283e7e81ad8eb7419548dbed12be" +dependencies = [ + "itertools 0.14.0", + "ruff_source_file", + "ruff_text_size", + "unicode-ident", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca531110ce8b14a2a78bd58068e2830b9bb5f0cd6ec324a2582e9651c75d0ae" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad5aa3adafaae7def1c0ec6a62f80368c3f710dc89adfe25e313d50ed5bad358" +dependencies = [ + "get-size2", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -3319,7 +3400,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3395,7 +3476,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3429,12 +3510,12 @@ dependencies = [ "libc", "log", "pyo3", + "ruff_python_parser", "rustls", "rustls-graviola", "rustpython-capi", "rustpython-compiler", "rustpython-pylib", - "rustpython-ruff_python_parser", "rustpython-stdlib", "rustpython-vm", "rustyline", @@ -3467,11 +3548,11 @@ dependencies = [ "num-complex", "num-traits", "rapidhash", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "rustpython-compiler-core", "rustpython-literal", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_parser", - "rustpython-ruff_text_size", "rustpython-wtf8", "thiserror", "unicode_names2 2.0.0", @@ -3504,12 +3585,12 @@ dependencies = [ name = "rustpython-compiler" version = "0.5.0" dependencies = [ + "ruff_python_ast", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_parser", - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", "thiserror", ] @@ -3524,7 +3605,7 @@ dependencies = [ "malachite-bigint", "num-complex", "num-traits", - "rustpython-ruff_source_file", + "ruff_source_file", "rustpython-wtf8", ] @@ -3532,8 +3613,8 @@ dependencies = [ name = "rustpython-compiler-source" version = "0.4.1+deprecated" dependencies = [ - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", + "ruff_source_file", + "ruff_text_size", ] [[package]] @@ -3628,77 +3709,6 @@ dependencies = [ "rustpython-derive", ] -[[package]] -name = "rustpython-ruff_python_ast" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f021ff72cabf5e2cd6d8ec8813d376a8445a228dc610ab56c27bd9054cda70d4" -dependencies = [ - "aho-corasick", - "bitflags 2.13.0", - "compact_str", - "get-size2", - "is-macro", - "memchr", - "rustc-hash", - "rustpython-ruff_python_trivia", - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", - "thiserror", -] - -[[package]] -name = "rustpython-ruff_python_parser" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01e6ee78bd9671fb5766664b2695fe1f2a92a961f4d9101646c570d8acdb1e0b" -dependencies = [ - "bitflags 2.13.0", - "bstr", - "compact_str", - "get-size2", - "memchr", - "rustc-hash", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_trivia", - "rustpython-ruff_text_size", - "static_assertions", - "unicode-ident", - "unicode-normalization", - "unicode_names2 1.3.0", -] - -[[package]] -name = "rustpython-ruff_python_trivia" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e7cfd1056f3a02ff0d2d0e4474286ca963260782f878b7b81c1dd87432e682" -dependencies = [ - "itertools 0.14.0", - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", - "unicode-ident", -] - -[[package]] -name = "rustpython-ruff_source_file" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "948107aad62ddb12a11fc7bf68a49e52a0b0a3737d415a2505e54f5a9edac737" -dependencies = [ - "memchr", - "rustpython-ruff_text_size", -] - -[[package]] -name = "rustpython-ruff_text_size" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8291ee0f5a779e54ccd4e0151a0c426f8b49a123f99b5b6545db17ccdd4277aa" -dependencies = [ - "get-size2", -] - [[package]] name = "rustpython-sre_engine" version = "0.5.0" @@ -3764,6 +3774,10 @@ dependencies = [ "pymath", "rand 0.10.1", "rapidhash", + "ruff_python_ast", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", "rustls", "rustls-native-certs", "rustls-pemfile", @@ -3772,10 +3786,6 @@ dependencies = [ "rustpython-derive", "rustpython-host_env", "rustpython-pylib", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_parser", - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", "rustpython-vm", "sha1 0.11.0", "sha2", @@ -3836,6 +3846,9 @@ dependencies = [ "psm", "rapidhash", "result-like", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "rustpython-codegen", "rustpython-common", "rustpython-compiler", @@ -3844,9 +3857,6 @@ dependencies = [ "rustpython-host_env", "rustpython-jit", "rustpython-literal", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_parser", - "rustpython-ruff_text_size", "rustpython-sre_engine", "rustyline", "scopeguard", @@ -3854,6 +3864,7 @@ dependencies = [ "static_assertions", "strum", "strum_macros", + "thin-vec", "thiserror", "timsort", "wasm-bindgen", @@ -4200,7 +4211,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4341,10 +4352,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4362,6 +4373,12 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +[[package]] +name = "thin-vec" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f7e269b48f0a7dd0146680fa24b50cc67fc0373f086a5b2f99bd084639b482" + [[package]] name = "thiserror" version = "2.0.18" @@ -4841,7 +4858,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 677de28ef1c..1ce809f1d58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -183,19 +183,10 @@ rustpython-sre_engine = { path = "crates/sre_engine", version = "0.5.0" } rustpython-wtf8 = { path = "crates/wtf8", version = "0.5.0" } rustpython-doc = { path = "crates/doc", version = "0.5.0" } -# Use RustPython-packaged Ruff crates from the published fork while keeping -# existing crate names in the codebase. -ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.15.8" } -ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.15.8" } -ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.15.8" } -ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.15.8" } -# To update ruff crates, comment out the above lines and uncomment the following lines to pull directly from the Ruff repository at the specified commit hash. -# Ruff tag 0.15.8 is based on commit c2a8815842f9dc5d24ec19385eae0f1a7188b0d9 -# at the time of this capture. We use the commit hash to ensure reproducible builds. -# ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" } -# ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" } -# ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" } -# ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" } +ruff_python_parser = { version = "0.0.2" } +ruff_python_ast = { version = "0.0.2" } +ruff_text_size = { version = "0.0.2" } +ruff_source_file = { version = "0.0.2" } der = { version = "0.8", features = ["alloc", "oid", "pem", "zeroize"] } phf = { version = "0.13.1", default-features = false, features = ["macros"]} @@ -330,6 +321,7 @@ x509-cert = "0.2.5" x509-parser = "0.18" xml = "1.3" writeable = "0.6" +thin-vec = "0.2" # Lints diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4d117be1b88..bb0b5e21739 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -283,7 +283,6 @@ def test_none_assignment(self): self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile def test_import(self): succeed = [ 'import sys', diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index cf90de7b115..8e9c9bb2b16 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1165,7 +1165,6 @@ def continue_in_finally_after_return2(x): """, True) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_yield(self): # Allowed as standalone statement def g(): yield 1 diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 8d359a646d9..bc8afa2d555 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2955,7 +2955,6 @@ def test_invalid_syntax_2(self): pass """) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_syntax_3(self): self.assert_syntax_error(""" match ...: diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 0934f22d470..8f8d6748507 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -376,7 +376,7 @@ Traceback (most recent call last): SyntaxError: invalid syntax ->>> match ...: +>>> match ...: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE ... case {**_}: ... ... Traceback (most recent call last): @@ -2239,7 +2239,7 @@ Invalid pattern matching constructs: - >>> match ...: # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> match ...: ... case 42 as _: ... ... Traceback (most recent call last): diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index 13b789f52dc..68d522edfe3 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -168,7 +168,7 @@ ... SyntaxError: iterable unpacking cannot be used in comprehension - >>> {**{} for a in [1]} # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + >>> {**{} for a in [1]} Traceback (most recent call last): ... SyntaxError: dict unpacking cannot be used in dict comprehension diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index a178a23bd2f..e23a16a168d 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -8268,11 +8268,17 @@ impl Compiler { generators, &|compiler, collection_add_i| { // changed evaluation order for Py38 named expression PEP 572 - compiler.compile_expression(key)?; + compiler.compile_expression( + key.as_ref() + .expect("RustPython does not support PEP798 yet"), + )?; compiler.compile_expression(value)?; compiler.set_source_range(TextRange::new( - key.range().start(), + key.as_ref() + .expect("RustPython does not support PEP798 yet") + .range() + .start(), value.range().end(), )); emit!( @@ -8285,12 +8291,22 @@ impl Compiler { Ok(()) }, ComprehensionType::Dict, - Self::contains_await(key) - || Self::contains_await(value) + Self::contains_await( + key.as_ref() + .expect("RustPython does not support PEP798 yet"), + ) || Self::contains_await(value) || Self::generators_contain_await(generators), *range, - TextRange::new(key.range().start(), value.range().end()), - key.range(), + TextRange::new( + key.as_ref() + .expect("RustPython does not support PEP798 yet") + .range() + .start(), + value.range().end(), + ), + key.as_ref() + .expect("RustPython does not support PEP798 yet") + .range(), )?; } ast::Expr::Generator(ast::ExprGenerator { @@ -11542,7 +11558,9 @@ impl Compiler { ast::ConversionFlag::Ascii => ConvertValueOparg::Ascii, }; - if let Some(ast::DebugText { leading, trailing }) = &fstring_expr.debug_text { + if let Some(debug_text) = &fstring_expr.debug_text { + let leading = debug_text.leading(); + let trailing = debug_text.trailing(); let range = fstring_expr.expression.range(); let leading = strip_fstring_debug_comments(leading); let trailing = strip_fstring_debug_comments(trailing); @@ -11672,7 +11690,9 @@ impl Compiler { } } ast::InterpolatedStringElement::Interpolation(fstring_expr) => { - if let Some(ast::DebugText { leading, trailing }) = &fstring_expr.debug_text { + if let Some(debug_text) = &fstring_expr.debug_text { + let leading = debug_text.leading(); + let trailing = debug_text.trailing(); let range = fstring_expr.expression.range(); let source = self.source_file.slice(range); let text = [ @@ -11759,7 +11779,9 @@ impl Compiler { .push_wtf8(&self.compile_tstring_literal_value(lit, tstring.flags)); } ast::InterpolatedStringElement::Interpolation(interp) => { - if let Some(ast::DebugText { leading, trailing }) = &interp.debug_text { + if let Some(debug_text) = &interp.debug_text { + let leading = debug_text.leading(); + let trailing = debug_text.trailing(); let range = interp.expression.range(); let source = self.source_file.slice(range); let text = [ diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 27ba10ebadb..cc0edb9cc15 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -2163,6 +2163,19 @@ impl SymbolTableBuilder { self.in_iter_def_exp = true; } // Dict comprehension - is_generator = false (can be inlined) + + let Some(key) = key else { + // RustPython does not support PEP 798 yet + return Err(SymbolTableError { + error: "dict unpacking cannot be used in dict comprehension".into(), + location: Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ), + }); + }; + self.scan_comprehension("", key, Some(value), generators, *range, false)?; self.in_iter_def_exp = was_in_iter_def_exp; } diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index d7f754e2f9d..539a56c17ec 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -253,7 +253,11 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { range: _range, }) => { self.p("{")?; - self.unparse_expr(key, precedence::TEST)?; + self.unparse_expr( + key.as_ref() + .expect("RustPython does not support PEP798 yet"), + precedence::TEST, + )?; self.p(": ")?; self.unparse_expr(value, precedence::TEST)?; self.unparse_comp(generators)?; @@ -554,7 +558,9 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { let buffered = fmt::from_fn(|f| Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1)) .to_string(); - if let Some(ast::DebugText { leading, trailing }) = debug_text { + if let Some(debug_text) = debug_text { + let leading = debug_text.leading(); + let trailing = debug_text.trailing(); self.p(leading)?; self.p(self.source.slice(val.range()))?; self.p(trailing)?; diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml index 83e41fa1f5f..5b8c0164ed5 100644 --- a/crates/vm/Cargo.toml +++ b/crates/vm/Cargo.toml @@ -78,6 +78,7 @@ psm = { workspace = true } optional = { workspace = true } result-like = { workspace = true } timsort = { workspace = true } +thin-vec = { workspace = true } ## unicode stuff icu_casemap = { workspace = true } diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index 6bbbbefe504..f79fce88e81 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -4,6 +4,8 @@ //! This module makes use of the parser logic, and translates all ast nodes //! into python ast.AST objects. +use thin_vec::ThinVec; + pub(crate) use python::_ast::module_def; mod pyast; @@ -559,7 +561,7 @@ fn strip_docstrings(top: &mut ast::Mod) { } #[cfg(feature = "parser")] -fn strip_docstring_in_body(body: &mut Vec) { +fn strip_docstring_in_body(body: &mut ThinVec) { if let Some(range) = take_docstring(body) && body.is_empty() { @@ -581,7 +583,7 @@ fn strip_docstring_in_body(body: &mut Vec) { } #[cfg(feature = "parser")] -fn take_docstring(body: &mut Vec) -> Option { +fn take_docstring(body: &mut ThinVec) -> Option { let ast::Stmt::Expr(expr_stmt) = body.first()? else { return None; }; diff --git a/crates/vm/src/stdlib/_ast/argument.rs b/crates/vm/src/stdlib/_ast/argument.rs index 626024f5bd6..13924f57999 100644 --- a/crates/vm/src/stdlib/_ast/argument.rs +++ b/crates/vm/src/stdlib/_ast/argument.rs @@ -1,15 +1,17 @@ +use thin_vec::ThinVec; + use super::*; use rustpython_compiler_core::SourceFile; pub(super) struct PositionalArguments { pub range: TextRange, - pub args: Box<[ast::Expr]>, + pub args: ThinVec, } impl Node for PositionalArguments { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { args, range: _ } = self; - BoxedSlice(args).ast_to_object(vm, source_file) + args.ast_to_object(vm, source_file) } fn ast_from_object( @@ -17,9 +19,9 @@ impl Node for PositionalArguments { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult { - let args: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; + let args: ThinVec<_> = Node::ast_from_object(vm, source_file, object)?; Ok(Self { - args: args.0, + args, range: TextRange::default(), // TODO }) } @@ -27,14 +29,14 @@ impl Node for PositionalArguments { pub(super) struct KeywordArguments { pub range: TextRange, - pub keywords: Box<[ast::Keyword]>, + pub keywords: ThinVec, } impl Node for KeywordArguments { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { keywords, range: _ } = self; // TODO: use range - BoxedSlice(keywords).ast_to_object(vm, source_file) + keywords.ast_to_object(vm, source_file) } fn ast_from_object( @@ -42,9 +44,9 @@ impl Node for KeywordArguments { source_file: &SourceFile, object: PyObjectRef, ) -> PyResult { - let keywords: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; + let keywords: ThinVec<_> = Node::ast_from_object(vm, source_file, object)?; Ok(Self { - keywords: keywords.0, + keywords, range: TextRange::default(), // TODO }) } @@ -59,7 +61,7 @@ pub(super) fn merge_function_call_arguments( ast::Arguments { node_index: Default::default(), range, - args: pos_args.args, + args: pos_args.args.into(), keywords: key_args.keywords, } } @@ -82,7 +84,7 @@ pub(super) fn split_function_call_arguments( // debug_assert!(range.contains_range(positional_arguments_range)); let positional_arguments = PositionalArguments { range: positional_arguments_range, - args, + args: args.into(), }; let keyword_arguments_range = keywords @@ -121,7 +123,7 @@ pub(super) fn split_class_def_args( // debug_assert!(range.contains_range(positional_arguments_range)); let positional_arguments = PositionalArguments { range: positional_arguments_range, - args, + args: args.into(), }; let keyword_arguments_range = keywords @@ -149,18 +151,18 @@ pub(super) fn merge_class_def_args( let args = if let Some(positional_arguments) = positional_arguments { positional_arguments.args } else { - vec![].into_boxed_slice() + ThinVec::new() }; let keywords = if let Some(keyword_arguments) = keyword_arguments { keyword_arguments.keywords } else { - vec![].into_boxed_slice() + ThinVec::new() }; Some(Box::new(ast::Arguments { node_index: Default::default(), range: Default::default(), // TODO - args, + args: args.into(), keywords, })) } diff --git a/crates/vm/src/stdlib/_ast/constant.rs b/crates/vm/src/stdlib/_ast/constant.rs index b1a8a015689..23cc453280a 100644 --- a/crates/vm/src/stdlib/_ast/constant.rs +++ b/crates/vm/src/stdlib/_ast/constant.rs @@ -332,7 +332,7 @@ fn constant_to_ruff_expr(value: Constant) -> ast::Expr { node_index: Default::default(), range, args: args.into(), - keywords: Box::default(), + keywords: Default::default(), }, }) } diff --git a/crates/vm/src/stdlib/_ast/elif_else_clause.rs b/crates/vm/src/stdlib/_ast/elif_else_clause.rs index 0afdbc02ac1..38f98f9007d 100644 --- a/crates/vm/src/stdlib/_ast/elif_else_clause.rs +++ b/crates/vm/src/stdlib/_ast/elif_else_clause.rs @@ -1,3 +1,5 @@ +use thin_vec::ThinVec; + use super::*; use rustpython_compiler_core::SourceFile; @@ -55,7 +57,7 @@ pub(super) fn ast_from_object( ) -> PyResult { let test = Node::ast_from_object(vm, source_file, get_node_field(vm, &object, "test", "If")?)?; let body = Node::ast_from_object(vm, source_file, get_node_field(vm, &object, "body", "If")?)?; - let orelse: Vec = Node::ast_from_object( + let orelse: ThinVec = Node::ast_from_object( vm, source_file, get_node_field(vm, &object, "orelse", "If")?, diff --git a/crates/vm/src/stdlib/_ast/module.rs b/crates/vm/src/stdlib/_ast/module.rs index b4c2468d33b..d814be69472 100644 --- a/crates/vm/src/stdlib/_ast/module.rs +++ b/crates/vm/src/stdlib/_ast/module.rs @@ -1,3 +1,5 @@ +use thin_vec::ThinVec; + use super::*; use crate::stdlib::_ast::type_ignore::TypeIgnore; use rustpython_compiler_core::SourceFile; @@ -113,7 +115,7 @@ impl Node for ast::ModModule { pub(super) struct ModInteractive { pub(crate) range: TextRange, - pub(crate) body: Vec, + pub(crate) body: ThinVec, } // constructor diff --git a/crates/vm/src/stdlib/_ast/node.rs b/crates/vm/src/stdlib/_ast/node.rs index 4ee3893b665..6e46e7ebe98 100644 --- a/crates/vm/src/stdlib/_ast/node.rs +++ b/crates/vm/src/stdlib/_ast/node.rs @@ -1,3 +1,5 @@ +use thin_vec::ThinVec; + use crate::{PyObjectRef, PyResult, VirtualMachine}; use rustpython_compiler_core::SourceFile; @@ -42,6 +44,34 @@ impl Node for Vec { } } +impl Node for ThinVec { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + vm.ctx + .new_list( + self.into_iter() + .map(|node| node.ast_to_object(vm, source_file)) + .collect(), + ) + .into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult { + // Recursion guard for each element: prevents stack overflow when a + // sequence element transitively references the sequence itself + // (e.g. `l = ast.List(...); l.elts = [l]`). See issue #4862. + vm.extract_elements_with(&object, |obj| { + vm.with_recursion("while traversing AST node", || { + Node::ast_from_object(vm, source_file, obj) + }) + }) + .map(Into::into) + } +} + impl Node for Box { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { (*self).ast_to_object(vm, source_file) diff --git a/crates/vm/src/stdlib/_ast/parameter.rs b/crates/vm/src/stdlib/_ast/parameter.rs index b0c807a2922..6e183c58396 100644 --- a/crates/vm/src/stdlib/_ast/parameter.rs +++ b/crates/vm/src/stdlib/_ast/parameter.rs @@ -1,3 +1,5 @@ +use thin_vec::ThinVec; + use super::*; use rustpython_compiler_core::SourceFile; @@ -266,8 +268,8 @@ impl Node for ParameterDefaults { } fn extract_positional_parameter_defaults( - pos_only_args: Vec, - args: Vec, + pos_only_args: ThinVec, + args: ThinVec, ) -> ( PositionalParameters, PositionalParameters, @@ -327,29 +329,30 @@ fn merge_positional_parameter_defaults( args: PositionalParameters, defaults: ParameterDefaults, ) -> PyResult<( - Vec, - Vec, + ThinVec, + ThinVec, )> { let posonlyargs = posonlyargs.args; let args = args.args; let defaults = defaults.defaults; - let mut posonlyargs: Vec<_> = as IntoIterator>::into_iter(posonlyargs) + let mut posonlyargs = as IntoIterator>::into_iter(posonlyargs) .map(|parameter| ast::ParameterWithDefault { node_index: Default::default(), range: Default::default(), parameter, default: None, }) - .collect(); - let mut args: Vec<_> = as IntoIterator>::into_iter(args) + .collect::>(); + + let mut args = as IntoIterator>::into_iter(args) .map(|parameter| ast::ParameterWithDefault { node_index: Default::default(), range: Default::default(), parameter, default: None, }) - .collect(); + .collect::>(); // If an argument has a default value, insert it // Note that "defaults" will only contain default values for the last "n" parameters @@ -372,7 +375,7 @@ fn merge_positional_parameter_defaults( } fn extract_keyword_parameter_defaults( - kw_only_args: Vec, + kw_only_args: ThinVec, ) -> (KeywordParameters, ParameterDefaults) { let mut defaults = vec![]; defaults.extend(kw_only_args.iter().map(|item| item.default.clone())); @@ -409,7 +412,7 @@ fn merge_keyword_parameter_defaults( vm: &VirtualMachine, kw_only_args: KeywordParameters, defaults: ParameterDefaults, -) -> PyResult> { +) -> PyResult> { if kw_only_args.keywords.len() != defaults.defaults.len() { return Err( vm.new_value_error("length of kwonlyargs is not the same as kw_defaults on arguments") diff --git a/crates/vm/src/stdlib/_ast/pattern.rs b/crates/vm/src/stdlib/_ast/pattern.rs index a383c25a6b7..dc48d02e502 100644 --- a/crates/vm/src/stdlib/_ast/pattern.rs +++ b/crates/vm/src/stdlib/_ast/pattern.rs @@ -1,3 +1,5 @@ +use thin_vec::ThinVec; + use super::*; use rustpython_compiler_core::SourceFile; @@ -387,7 +389,7 @@ impl Node for ast::PatternMatchClass { } } -struct PatternMatchClassPatterns(Vec); +struct PatternMatchClassPatterns(ThinVec); impl Node for PatternMatchClassPatterns { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -403,7 +405,7 @@ impl Node for PatternMatchClassPatterns { } } -struct PatternMatchClassKeywordAttributes(Vec); +struct PatternMatchClassKeywordAttributes(ThinVec); impl Node for PatternMatchClassKeywordAttributes { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -419,7 +421,7 @@ impl Node for PatternMatchClassKeywordAttributes { } } -struct PatternMatchClassKeywordPatterns(Vec); +struct PatternMatchClassKeywordPatterns(ThinVec); impl Node for PatternMatchClassKeywordPatterns { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -562,7 +564,7 @@ fn merge_pattern_match_class( patterns: PatternMatchClassPatterns, kwd_attrs: PatternMatchClassKeywordAttributes, kwd_patterns: PatternMatchClassKeywordPatterns, -) -> (Vec, Vec) { +) -> (ThinVec, Vec) { let keywords = kwd_attrs .0 .into_iter() diff --git a/crates/vm/src/stdlib/_ast/string.rs b/crates/vm/src/stdlib/_ast/string.rs index 24cae476694..e77851663a1 100644 --- a/crates/vm/src/stdlib/_ast/string.rs +++ b/crates/vm/src/stdlib/_ast/string.rs @@ -595,11 +595,11 @@ fn ruff_tstring_element_to_template_str_part( let expr_str = if let Some(debug_text) = debug_text { let expr_source = source_file.slice(expr_range); let mut expr_with_debug = String::with_capacity( - debug_text.leading.len() + expr_source.len() + debug_text.trailing.len(), + debug_text.leading().len() + expr_source.len() + debug_text.trailing().len(), ); - expr_with_debug.push_str(&debug_text.leading); + expr_with_debug.push_str(debug_text.leading()); expr_with_debug.push_str(expr_source); - expr_with_debug.push_str(&debug_text.trailing); + expr_with_debug.push_str(debug_text.trailing()); strip_interpolation_expr(&expr_with_debug) } else { tstring_interpolation_expr_str(source_file, range, expr_range) diff --git a/crates/vm/src/stdlib/_ast/validate.rs b/crates/vm/src/stdlib/_ast/validate.rs index cad37d38610..6bf16390914 100644 --- a/crates/vm/src/stdlib/_ast/validate.rs +++ b/crates/vm/src/stdlib/_ast/validate.rs @@ -381,7 +381,13 @@ fn validate_expr(vm: &VirtualMachine, expr: &ast::Expr, ctx: ast::ExprContext) - } ast::Expr::DictComp(dict) => { validate_comprehension(vm, &dict.generators)?; - validate_expr(vm, &dict.key, ast::ExprContext::Load)?; + validate_expr( + vm, + dict.key + .as_ref() + .expect("RustPython does not support PEP798 yet"), + ast::ExprContext::Load, + )?; validate_expr(vm, &dict.value, ast::ExprContext::Load) } ast::Expr::Generator(generator) => { diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index 440bcdb9b5f..9056215e8eb 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -75,7 +75,7 @@ fn parse_python_file(filename: &Path) -> ParsedFile { Ok(source) => { let num_lines = source.lines().count(); let result = parse_module(&source) - .map(|x| x.into_suite()) + .map(|x| x.into_suite().into()) .map_err(|e| e.to_string()); ParsedFile { num_lines, result } }