Skip to content

Commit 1aa52eb

Browse files
ShaharNavehyouknowone
ShaharNaveh
authored and
youknowone
committed
Base implementation of _tokenize module
Port from PR RustPython#6240 by ShaharNaveh, adapted to current codebase. Uses ruff_python_parser for tokenization via TokenizerIter.
1 parent ae8d8c7 commit 1aa52eb

4 files changed

Lines changed: 402 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/stdlib/Cargo.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,11 @@ rustpython-derive = { workspace = true }
3131
rustpython-vm = { workspace = true, default-features = false, features = ["compiler"]}
3232
rustpython-common = { workspace = true }
3333

34+
ruff_python_parser = { workspace = true }
35+
ruff_python_ast = { workspace = true }
36+
ruff_text_size = { workspace = true }
37+
ruff_source_file = { workspace = true }
38+
3439
ahash = { workspace = true }
3540
ascii = { workspace = true }
3641
cfg-if = { workspace = true }

‎crates/stdlib/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ mod pystruct;
4949
mod random;
5050
mod statistics;
5151
mod suggestions;
52+
mod tokenize;
5253
// TODO: maybe make this an extension module, if we ever get those
5354
// mod re;
5455
#[cfg(all(feature = "host_env", not(target_arch = "wasm32")))]
@@ -225,6 +226,7 @@ pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef>
225226
ssl::module_def(ctx),
226227
statistics::module_def(ctx),
227228
suggestions::module_def(ctx),
229+
tokenize::module_def(ctx),
228230
#[cfg(all(feature = "host_env", unix, not(target_os = "redox")))]
229231
syslog::module_def(ctx),
230232
#[cfg(all(

0 commit comments

Comments
 (0)