Skip to content

Commit 7db6dba

Browse files
committed
rewrite finalize_modules with phased algorithm
Replace the absence of module finalization during interpreter shutdown with a 5-phase algorithm matching pylifecycle.c finalize_modules(): 1. Set special sys attributes to None, restore stdio 2. Set all sys.modules values to None, collect module dicts 3. Clear sys.modules dict 4. Clear module dicts in reverse import order (2-pass _PyModule_ClearDict) 5. Clear sys and builtins dicts last This ensures __del__ methods are called during shutdown and modules are cleaned up in reverse import order without hardcoded module names.
1 parent 0b9c90f commit 7db6dba

5 files changed

Lines changed: 166 additions & 3 deletions

File tree

.cspell.dict/cpython.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ pybuilddir
137137
pycore
138138
pydecimal
139139
Pyfunc
140+
pylifecycle
140141
pymain
141142
pyrepl
142143
PYTHONTRACEMALLOC

Lib/test/test_builtin.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2330,8 +2330,6 @@ def test_baddecorator(self):
23302330

23312331
class ShutdownTest(unittest.TestCase):
23322332

2333-
# TODO: RUSTPYTHON
2334-
@unittest.expectedFailure
23352333
def test_cleanup(self):
23362334
# Issue #19255: builtins are still available at shutdown
23372335
code = """if 1:

Lib/test/test_sys.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1172,7 +1172,6 @@ def test_is_gil_enabled(self):
11721172
else:
11731173
self.assertTrue(sys._is_gil_enabled())
11741174

1175-
@unittest.expectedFailure # TODO: RUSTPYTHON; AtExit.__del__ is not invoked because module destruction is missing.
11761175
def test_is_finalizing(self):
11771176
self.assertIs(sys.is_finalizing(), False)
11781177
# Don't use the atexit module because _Py_Finalizing is only set

crates/vm/src/vm/interpreter.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,7 @@ impl Interpreter {
391391
/// 1. Wait for thread shutdown (call threading._shutdown).
392392
/// 1. Mark vm as finalizing.
393393
/// 1. Run atexit exit functions.
394+
/// 1. Finalize modules (clear module dicts in reverse import order).
394395
/// 1. Mark vm as finalized.
395396
///
396397
/// Note that calling `finalize` is not necessary by purpose though.
@@ -425,6 +426,9 @@ impl Interpreter {
425426
// Run atexit exit functions
426427
atexit::_run_exitfuncs(vm);
427428

429+
// Finalize modules: clear module dicts in reverse import order
430+
vm.finalize_modules();
431+
428432
vm.flush_std();
429433

430434
exit_code

crates/vm/src/vm/mod.rs

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -620,6 +620,167 @@ impl VirtualMachine {
620620
}
621621
}
622622

623+
/// Clear module references during shutdown.
624+
/// Follows the same phased algorithm as pylifecycle.c finalize_modules():
625+
/// no hardcoded module names, reverse import order, only builtins/sys last.
626+
pub fn finalize_modules(&self) {
627+
// Phase 1: Set special sys/builtins attributes to None, restore stdio
628+
self.finalize_modules_delete_special();
629+
630+
// Phase 2: Remove all modules from sys.modules (set values to None),
631+
// and collect module dicts preserving import order.
632+
let module_dicts = self.finalize_remove_modules();
633+
634+
// Phase 3: Clear sys.modules dict
635+
self.finalize_clear_modules_dict();
636+
637+
// Phase 4: Clear module dicts in reverse import order using 2-pass algorithm.
638+
// This drops references to objects in module namespaces, triggering __del__.
639+
self.finalize_clear_module_dicts(&module_dicts);
640+
641+
// Phase 5: Clear sys and builtins dicts last
642+
self.finalize_clear_sys_builtins_dict();
643+
}
644+
645+
/// Phase 1: Set special sys attributes to None and restore stdio.
646+
fn finalize_modules_delete_special(&self) {
647+
let none = self.ctx.none();
648+
let sys_dict = self.sys_module.dict();
649+
650+
// Set special sys attributes to None
651+
for attr in &[
652+
"path",
653+
"argv",
654+
"ps1",
655+
"ps2",
656+
"last_exc",
657+
"last_type",
658+
"last_value",
659+
"last_traceback",
660+
"path_importer_cache",
661+
"meta_path",
662+
"path_hooks",
663+
] {
664+
let _ = sys_dict.set_item(*attr, none.clone(), self);
665+
}
666+
667+
// Restore stdin/stdout/stderr from __stdin__/__stdout__/__stderr__
668+
for (std_name, dunder_name) in &[
669+
("stdin", "__stdin__"),
670+
("stdout", "__stdout__"),
671+
("stderr", "__stderr__"),
672+
] {
673+
let restored = sys_dict
674+
.get_item_opt(*dunder_name, self)
675+
.ok()
676+
.flatten()
677+
.unwrap_or_else(|| none.clone());
678+
let _ = sys_dict.set_item(*std_name, restored, self);
679+
}
680+
681+
// builtins._ = None
682+
let _ = self.builtins.dict().set_item("_", none, self);
683+
}
684+
685+
/// Phase 2: Set all sys.modules values to None and collect module dicts.
686+
/// Returns a list of (name, dict) preserving import order.
687+
fn finalize_remove_modules(&self) -> Vec<(String, PyDictRef)> {
688+
let mut module_dicts = Vec::new();
689+
690+
let Ok(modules) = self.sys_module.get_attr(identifier!(self, modules), self) else {
691+
return module_dicts;
692+
};
693+
let Some(modules_dict) = modules.downcast_ref::<PyDict>() else {
694+
return module_dicts;
695+
};
696+
697+
let none = self.ctx.none();
698+
let items: Vec<_> = modules_dict.into_iter().collect();
699+
700+
for (key, value) in items {
701+
let name = key
702+
.downcast_ref::<PyStr>()
703+
.map(|s| s.as_str().to_owned())
704+
.unwrap_or_default();
705+
706+
// Save dict reference for later 2-pass clearing
707+
if let Some(module) = value.downcast_ref::<PyModule>() {
708+
module_dicts.push((name, module.dict()));
709+
}
710+
711+
// Set the value to None in sys.modules
712+
let _ = modules_dict.set_item(&*key, none.clone(), self);
713+
}
714+
715+
module_dicts
716+
}
717+
718+
/// Phase 3: Clear sys.modules dict.
719+
fn finalize_clear_modules_dict(&self) {
720+
if let Ok(modules) = self.sys_module.get_attr(identifier!(self, modules), self)
721+
&& let Some(modules_dict) = modules.downcast_ref::<PyDict>()
722+
{
723+
modules_dict.clear();
724+
}
725+
}
726+
727+
/// Phase 4: Clear module dicts in reverse import order.
728+
/// Skips builtins and sys (they are cleared last in phase 5).
729+
fn finalize_clear_module_dicts(&self, module_dicts: &[(String, PyDictRef)]) {
730+
let sys_dict = self.sys_module.dict();
731+
let builtins_dict = self.builtins.dict();
732+
733+
// Iterate in reverse (last imported → first cleared)
734+
for (_name, module_dict) in module_dicts.iter().rev() {
735+
// Skip builtins and sys dicts (cleared last in phase 5)
736+
if module_dict.is(&sys_dict) || module_dict.is(&builtins_dict) {
737+
continue;
738+
}
739+
740+
// 2-pass clearing
741+
Self::module_clear_dict(module_dict, self);
742+
}
743+
}
744+
745+
/// 2-pass module dict clearing (_PyModule_ClearDict algorithm).
746+
/// Pass 1: Set names starting with '_' (except __builtins__) to None.
747+
/// Pass 2: Set all remaining names (except __builtins__) to None.
748+
fn module_clear_dict(dict: &Py<PyDict>, vm: &VirtualMachine) {
749+
let none = vm.ctx.none();
750+
751+
// Pass 1: names starting with '_' (except __builtins__)
752+
for (key, value) in dict.into_iter().collect::<Vec<_>>() {
753+
if vm.is_none(&value) {
754+
continue;
755+
}
756+
if let Some(key_str) = key.downcast_ref::<PyStr>() {
757+
let name = key_str.as_str();
758+
if name.starts_with('_') && name != "__builtins__" && name != "__spec__" {
759+
let _ = dict.set_item(name, none.clone(), vm);
760+
}
761+
}
762+
}
763+
764+
// Pass 2: all remaining (except __builtins__)
765+
for (key, value) in dict.into_iter().collect::<Vec<_>>() {
766+
if vm.is_none(&value) {
767+
continue;
768+
}
769+
if let Some(key_str) = key.downcast_ref::<PyStr>()
770+
&& key_str.as_str() != "__builtins__"
771+
&& key_str.as_str() != "__spec__"
772+
{
773+
let _ = dict.set_item(key_str.as_str(), none.clone(), vm);
774+
}
775+
}
776+
}
777+
778+
/// Phase 5: Clear sys and builtins dicts last.
779+
fn finalize_clear_sys_builtins_dict(&self) {
780+
Self::module_clear_dict(&self.sys_module.dict(), self);
781+
Self::module_clear_dict(&self.builtins.dict(), self);
782+
}
783+
623784
pub fn current_recursion_depth(&self) -> usize {
624785
self.recursion_depth.get()
625786
}

0 commit comments

Comments
 (0)