Skip to content

Multi phase module init #6740

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 5 commits into from
Jan 22, 2026
Merged

Multi phase module init #6740

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3ccff6d
os.stat remove winodws-only attrs
youknowone Jan 21, 2026
fee1a4b
multiphase
youknowone Jan 16, 2026
32d2406
module_def
youknowone Jan 16, 2026
bc02b23
module_exec
youknowone Jan 16, 2026
2037645
Implement Py_mod_create slot support in multi-phase init
youknowone Jan 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
module_exec
  • Loading branch information
@youknowone
youknowone committed Jan 21, 2026
commit bc02b2318ccaef18f49f0be1b49846f36d8db50c
46 changes: 0 additions & 46 deletions Lib/test/test_pickle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,46 +136,6 @@ def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes
def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_buffers_error()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_builtin_functions()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_bytearray_memoization()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_bytes_memoization()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_in_band_buffers()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_oob_buffers()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_oob_buffers_writable_to_readonly()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_buffers_error()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_builtin_functions()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes
Expand All @@ -201,7 +161,6 @@ def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes
def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_oob_buffers_writable_to_readonly()


class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests,
BigmemPickleTests, unittest.TestCase):

Expand Down Expand Up @@ -250,11 +209,6 @@ def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this
def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_buffers_error()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_builtin_functions()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes
Expand Down
5 changes: 0 additions & 5 deletions Lib/test/test_pickletools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,6 @@ def test_optimize_binput_and_memoize(self):
def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_buffers_error()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_builtin_functions()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes
Expand Down
4 changes: 1 addition & 3 deletions benches/microbenchmarks.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,7 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenc
settings.user_site_directory = false;

Interpreter::with_init(settings, |vm| {
for (name, init) in rustpython_stdlib::get_module_inits() {
vm.add_native_module(name, init);
}
vm.add_native_module_defs(rustpython_stdlib::get_module_defs());
})
.enter(|vm| {
let setup_code = vm
Expand Down
20 changes: 13 additions & 7 deletions crates/derive-impl/src/pymodule.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ struct ModuleContext {
name: String,
function_items: FunctionNursery,
attribute_items: ItemNursery,
has_extend_module: bool, // TODO: check if `fn extend_module` exists
has_module_exec: bool,
errors: Vec<syn::Error>,
}

Expand All @@ -82,6 +82,12 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To

// collect to context
for item in items.iter_mut() {
// Check if module_exec function is already defined
if let Item::Fn(func) = item
&& func.sig.ident == "module_exec"
{
context.has_module_exec = true;
}
Comment on lines +85 to +90
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot Jan 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

🧩 Analysis chain

🏁 Script executed:

# First, let's explore the file structure and locate the file
find . -name "pymodule.rs" -type f

Repository: RustPython/RustPython

Length of output: 101

🏁 Script executed:

# Read the context around lines 85-90 in the file to understand the code
cat -n crates/derive-impl/src/pymodule.rs | sed -n '75,100p'

Repository: RustPython/RustPython

Length of output: 1226

🏁 Script executed:

# Search for any cfg-gated module_exec definitions in the entire codebase
rg --type=rust -B5 'fn\s+module_exec' --multiline-dotted

Repository: RustPython/RustPython

Length of output: 519

🏁 Script executed:

# Search for where def.slots.exec is set or assigned
rg --type=rust 'def\.slots\.exec|slots\.exec\s*=' -B2 -A2

Repository: RustPython/RustPython

Length of output: 752

🏁 Script executed:

# Search for module_exec with surrounding context, checking for #[cfg(...)]
rg --type=rust -B10 'fn\s+module_exec' | head -50

Repository: RustPython/RustPython

Length of output: 2866

🏁 Script executed:

# Search specifically for cfg-gated module_exec
rg --type=rust '#\[cfg' -A10 | rg -A10 'module_exec'

Repository: RustPython/RustPython

Length of output: 2829

🏁 Script executed:

# Find where context.has_module_exec is used
rg --type=rust 'has_module_exec' -B2 -A2

Repository: RustPython/RustPython

Length of output: 1176

🏁 Script executed:

# Find where def.slots.exec = Some(module_exec) is generated with more context
rg --type=rust 'def\.slots\.exec\s*=\s*Some' -B10 -A5 crates/derive-impl/src/pymodule.rs

Repository: RustPython/RustPython

Length of output: 769

🏁 Script executed:

# Also search for where the module definition is generated to understand full flow
rg --type=rust 'gen_module_def|ModuleDef' crates/derive-impl/src/pymodule.rs | head -20

Repository: RustPython/RustPython

Length of output: 293

🏁 Script executed:

# Search for any actual cfg-gated module_exec definitions that might have caused issues
rg --type=rust -B3 '#\[cfg' -A3 | rg -B3 -A3 'fn\s+module_exec'

Repository: RustPython/RustPython

Length of output: 1309

🏁 Script executed:

# Read the full context around where has_module_exec is used to understand the code generation
cat -n crates/derive-impl/src/pymodule.rs | sed -n '110,150p'

Repository: RustPython/RustPython

Length of output: 1913

🏁 Script executed:

# Check if there's any conditional generation based on has_module_exec
rg --type=rust 'has_module_exec' -B5 -A10 crates/derive-impl/src/pymodule.rs

Repository: RustPython/RustPython

Length of output: 1806

Reject cfg‑gated module_exec to prevent build failures.

When a user defines module_exec with a #[cfg(...)] attribute, the code checks for its presence (line 87) and sets context.has_module_exec = true. However, the code generation at line 142 unconditionally generates def.slots.exec = Some(module_exec); regardless of whether module_exec is cfg-gated. In builds where the cfg condition is false, the function won't exist but the code still references it, causing a hard compile error.

The suggested diagnostic check is appropriate: detect cfg-gated module_exec and emit an error, directing users to either move the cfg to the containing module or provide an explicit fallback.

🤖 Prompt for AI Agents 
In `@crates/derive-impl/src/pymodule.rs` around lines 85 - 90, The derive
currently marks context.has_module_exec = true when it sees an Item::Fn named
module_exec but doesn't handle cfg-gated declarations, which leads to emitting
def.slots.exec = Some(module_exec) unconditionally; update the visitor in
pymodule.rs to detect if the function/item named module_exec has any #[cfg(...)]
attributes (inspect func.attrs for a Meta path "cfg") and, if so, emit a
compile_error (or return a syn::Error) rejecting cfg‑gated module_exec with a
clear message that the cfg must be moved to the containing module or a non-gated
fallback provided, instead of setting context.has_module_exec = true and
generating def.slots.exec = Some(module_exec).

if matches!(item, Item::Impl(_) | Item::Trait(_)) {
// #[pyclass] implementations
continue;
Expand Down Expand Up @@ -133,7 +139,7 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
methods: METHOD_DEFS,
slots: Default::default(),
};
def.slots.exec = Some(extend_module);
def.slots.exec = Some(module_exec);
def
})
}
Expand All @@ -146,16 +152,16 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
use ::rustpython_vm::PyPayload;
let module = ::rustpython_vm::builtins::PyModule::from_def(module_def(&vm.ctx)).into_ref(&vm.ctx);
__init_dict(vm, &module);
extend_module(vm, &module).unwrap();
module_exec(vm, &module).unwrap();
module
}
},
]);
}
if !is_submodule && !context.has_extend_module {
if !is_submodule && !context.has_module_exec {
items.push(parse_quote! {
pub(crate) fn extend_module(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> {
__extend_module(vm, module);
pub(crate) fn module_exec(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> {
__module_exec(vm, module);
Ok(())
}
});
Expand Down Expand Up @@ -192,7 +198,7 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
}
},
parse_quote! {
pub(crate) fn __extend_module(
pub(crate) fn __module_exec(
vm: &::rustpython_vm::VirtualMachine,
module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>,
) {
Expand Down
4 changes: 2 additions & 2 deletions crates/derive/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream {
}

/// This attribute must be applied to an inline module.
/// It defines a Python module in the form a `make_module` function in the module;
/// this has to be used in a `get_module_inits` to properly register the module.
/// It defines a Python module in the form of a `module_def` function in the module;
/// this has to be used in a `get_module_defs` to properly register the module.
/// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module.
/// # Arguments
/// - `name`: the name of the python module,
Expand Down
50 changes: 22 additions & 28 deletions crates/stdlib/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,33 +1,6 @@
// spell-checker:ignore typecode tofile tolist fromfile

use rustpython_vm::{PyRef, VirtualMachine, builtins::PyModule};

pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = array::make_module(vm);

let array = module
.get_attr("array", vm)
.expect("Expect array has array type.");

let collections_abc = vm
.import("collections.abc", 0)
.expect("Expect collections exist.");
let abc = collections_abc
.get_attr("abc", vm)
.expect("Expect collections has abc submodule.");
let mutable_sequence = abc
.get_attr("MutableSequence", vm)
.expect("Expect collections.abc has MutableSequence type.");

let register = &mutable_sequence
.get_attr("register", vm)
.expect("Expect collections.abc.MutableSequence has register method.");
register
.call((array,), vm)
.expect("Expect collections.abc.MutableSequence.register(array.array) not fail.");

module
}
pub(crate) use array::module_def;

#[pymodule(name = "array")]
mod array {
Expand Down Expand Up @@ -1658,4 +1631,25 @@ mod array {
};
PyArray::from(array).into_ref_with_type(vm, cls)
}

// Register array.array as collections.abc.MutableSequence
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::vm::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);

let array_type = module
.get_attr("array", vm)
.expect("array module has array type");

// vm.import returns the top-level module, so we need to get abc submodule
let collections_abc = vm.import("collections.abc", 0)?;
let abc = collections_abc.get_attr("abc", vm)?;
let mutable_sequence = abc.get_attr("MutableSequence", vm)?;
let register = mutable_sequence.get_attr("register", vm)?;
register.call((array_type,), vm)?;

Ok(())
}
}
30 changes: 16 additions & 14 deletions crates/stdlib/src/contextvars.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,9 @@
use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, class::StaticType};
pub(crate) use _contextvars::module_def;

use crate::vm::PyRef;
use _contextvars::PyContext;
use core::cell::RefCell;

pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = _contextvars::make_module(vm);
let token_type = module.get_attr("Token", vm).unwrap();
token_type
.set_attr(
"MISSING",
_contextvars::ContextTokenMissing::static_type().to_owned(),
vm,
)
.unwrap();
module
}

thread_local! {
// TODO: Vec doesn't seem to match copy behavior
static CONTEXTS: RefCell<Vec<PyRef<PyContext>>> = RefCell::default();
Expand Down Expand Up @@ -643,4 +632,17 @@ mod _contextvars {
fn copy_context(vm: &VirtualMachine) -> PyContext {
PyContext::current(vm).copy()
}

// Set Token.MISSING attribute
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::vm::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);

let token_type = module.get_attr("Token", vm)?;
token_type.set_attr("MISSING", ContextTokenMissing::static_type().to_owned(), vm)?;

Ok(())
}
}
12 changes: 5 additions & 7 deletions crates/stdlib/src/gc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ pub(crate) use gc::module_def;

#[pymodule]
mod gc {
use crate::vm::{PyObjectRef, PyResult, VirtualMachine, function::FuncArgs};
use crate::vm::{PyResult, VirtualMachine, function::FuncArgs};

#[pyfunction]
fn collect(_args: FuncArgs, _vm: &VirtualMachine) -> i32 {
Expand Down Expand Up @@ -40,15 +40,13 @@ mod gc {
}

#[pyfunction]
fn get_referents(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef {
// RustPython does not track object references.
vm.ctx.new_tuple(vec![]).into()
fn get_referents(_args: FuncArgs, vm: &VirtualMachine) -> PyResult {
Err(vm.new_not_implemented_error(""))
}

#[pyfunction]
fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef {
// RustPython does not track object references.
vm.ctx.new_list(vec![]).into()
fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyResult {
Err(vm.new_not_implemented_error(""))
}

#[pyfunction]
Expand Down
Loading