Add minimal capi lifecycle support (#7648)

bschoenmaeckers · youknowone · web-flow · commit 22d4f43ad467 · 2026-05-06T02:02:26.000+09:00
* Add minimal capi lifecycle support

* Force enable `threading` on `stdlib`

---------

Co-authored-by: Jeong, YunWon &lt;jeong@youknowone.org&gt;
diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt
@@ -161,6 +161,7 @@ Pyfunc
 pylifecycle
 pymain
 pyrepl
+pystate
 PYTHONTRACEMALLOC
 PYTHONUTF8
 pythonw
diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt
@@ -189,6 +189,7 @@ pycodecs
 pycs
 pydatetime
 pyexpat
+PYGILSTATE
 pyio
 pymain
 PYTHONAPI
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -10,6 +10,7 @@ repository.workspace = true
 license.workspace = true
 
 [features]
+capi = ["dep:rustpython-capi", "threading"]
 default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls", "host_env"]
 host_env = ["rustpython-vm/host_env", "rustpython-stdlib?/host_env"]
 importlib = ["rustpython-vm/importlib"]
@@ -31,6 +32,7 @@ tkinter = ["rustpython-stdlib/tkinter"]
 winresource = "0.1"
 
 [dependencies]
+rustpython-capi = { workspace = true, optional = true }
 rustpython-compiler = { workspace = true }
 rustpython-pylib = { workspace = true, optional = true }
 rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
@@ -140,6 +142,7 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [workspace.dependencies]
+rustpython-capi = { path = "crates/capi", version = "0.5.0" }
 rustpython-compiler-core = { path = "crates/compiler-core", version = "0.5.0" }
 rustpython-compiler = { path = "crates/compiler", version = "0.5.0" }
 rustpython-codegen = { path = "crates/codegen", version = "0.5.0" }
@@ -257,7 +260,6 @@ rustls-platform-verifier = "0.7"
 rustyline = "18"
 serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] }
 schannel = "0.1.29"
-scoped-tls = "1"
 scopeguard = "1"
 sha-1 = "0.10.0"
 sha2 = "0.10.2"
diff --git a/build.rs b/build.rs
@@ -1,17 +1,29 @@
 fn main() {
-    if std::env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows" {
-        println!("cargo:rerun-if-changed=logo.ico");
-        let mut res = winresource::WindowsResource::new();
-        if std::path::Path::new("logo.ico").exists() {
-            res.set_icon("logo.ico");
-        } else {
-            println!("cargo:warning=logo.ico not found, skipping icon embedding");
-            return;
+    let target = std::env::var("CARGO_CFG_TARGET_OS").unwrap();
+    let capi_enabled = std::env::var_os("CARGO_FEATURE_CAPI").is_some();
+
+    match target.as_str() {
+        "linux" if capi_enabled => {
+            println!("cargo:rustc-link-arg-bin=rustpython=-Wl,--export-dynamic");
         }
-        res.compile()
-            .map_err(|e| {
-                println!("cargo:warning=Failed to compile Windows resources: {e}");
-            })
-            .ok();
+        "macos" if capi_enabled => {
+            println!("cargo:rustc-link-arg-bin=rustpython=-Wl,-export_dynamic");
+        }
+        "windows" => {
+            println!("cargo:rerun-if-changed=logo.ico");
+            let mut res = winresource::WindowsResource::new();
+            if std::path::Path::new("logo.ico").exists() {
+                res.set_icon("logo.ico");
+            } else {
+                println!("cargo:warning=logo.ico not found, skipping icon embedding");
+                return;
+            }
+            res.compile()
+                .map_err(|e| {
+                    println!("cargo:warning=Failed to compile Windows resources: {e}");
+                })
+                .ok();
+        }
+        _ => {}
     }
 }
diff --git a/crates/capi/.cargo/config.toml b/crates/capi/.cargo/config.toml
@@ -0,0 +1,3 @@
+[env]
+PYO3_CONFIG_FILE = { value = "pyo3-rustpython.config", relative = true }
+PYO3_NO_PYTHON = { value = "1" }
diff --git a/crates/capi/Cargo.toml b/crates/capi/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "rustpython-capi"
+description = "Minimal CPython C-API compatibility exports for RustPython"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license.workspace = true
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+rustpython-vm = { workspace = true, features = ["threading"] }
+rustpython-stdlib = {workspace = true, features = ["threading"] }
+
+[dev-dependencies]
+pyo3 = { version = "0.28", features = ["auto-initialize", "abi3"] }
+
+[lints]
+workspace = true
+
+[package.metadata.cargo-shear]
+# Not a direct dependency (yet), but we need to enable threading support in the stdlib.
+ignored = ["rustpython-stdlib"]
diff --git a/crates/capi/pyo3-rustpython.config b/crates/capi/pyo3-rustpython.config
@@ -0,0 +1,5 @@
+implementation=CPython
+version=3.14
+shared=true
+abi3=true
+suppress_build_script_link_lines=true
diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs
@@ -0,0 +1,19 @@
+#![allow(clippy::missing_safety_doc)]
+
+use crate::pylifecycle::MAIN_INTERP;
+use rustpython_vm::Interpreter;
+pub use rustpython_vm::PyObject;
+use std::sync::MutexGuard;
+
+extern crate alloc;
+
+pub mod pylifecycle;
+pub mod pystate;
+pub mod refcount;
+
+/// Get main interpreter of this process. Will be None if it has not been initialized yet.
+pub fn get_main_interpreter() -> MutexGuard<'static, Option<Interpreter>> {
+    MAIN_INTERP
+        .lock()
+        .expect("Failed to lock interpreter mutex")
+}
diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs
@@ -0,0 +1,51 @@
+use crate::get_main_interpreter;
+use crate::pystate::ensure_thread_has_vm_attached;
+use core::ffi::c_int;
+use rustpython_vm::Interpreter;
+use rustpython_vm::vm::thread::ThreadedVirtualMachine;
+use std::sync::Mutex;
+
+pub(crate) static MAIN_INTERP: Mutex<Option<Interpreter>> = Mutex::new(None);
+
+/// Request a thread local vm from the main interpreter
+pub(crate) fn request_vm_from_interpreter() -> ThreadedVirtualMachine {
+    get_main_interpreter()
+        .as_ref()
+        .expect("Interpreter not initialized")
+        .enter(|vm| vm.new_thread())
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_IsInitialized() -> c_int {
+    get_main_interpreter().is_some() as c_int
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_Initialize() {
+    Py_InitializeEx(0);
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_InitializeEx(_initsigs: c_int) {
+    let mut interp = get_main_interpreter();
+    if interp.is_none() {
+        *interp = Interpreter::with_init(Default::default(), |_vm| {}).into();
+        drop(interp);
+        ensure_thread_has_vm_attached();
+    }
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_Finalize() {
+    let _ = Py_FinalizeEx();
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_FinalizeEx() -> c_int {
+    0
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_IsFinalizing() -> c_int {
+    0
+}
diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs
@@ -0,0 +1,111 @@
+use crate::pylifecycle::request_vm_from_interpreter;
+use core::ffi::c_int;
+use core::ptr;
+use rustpython_vm::vm::thread::{
+    CurrentVmAttachState, attach_current_thread, release_current_thread,
+};
+
+#[allow(non_camel_case_types)]
+type PyGILState_STATE = c_int;
+const PYGILSTATE_LOCKED: PyGILState_STATE = 0;
+const PYGILSTATE_UNLOCKED: PyGILState_STATE = 1;
+
+#[repr(C)]
+pub struct PyThreadState {
+    _interp: *mut core::ffi::c_void,
+}
+
+/// Make sure this thread has a running vm attached. This only creates a new vm if we don't already
+/// have one. So this will only create a new vm when we are in a new thread created outside RustPython.
+pub(crate) fn ensure_thread_has_vm_attached() -> CurrentVmAttachState {
+    attach_current_thread(request_vm_from_interpreter)
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE {
+    match ensure_thread_has_vm_attached() {
+        CurrentVmAttachState::AlreadyAttached => PYGILSTATE_LOCKED,
+        CurrentVmAttachState::Attached => PYGILSTATE_UNLOCKED,
+    }
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyGILState_Release(state: PyGILState_STATE) {
+    if state == PYGILSTATE_UNLOCKED {
+        release_current_thread(CurrentVmAttachState::Attached);
+    }
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState {
+    ptr::null_mut()
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::get_main_interpreter;
+    use crate::pystate::{PyGILState_Ensure, PyGILState_Release};
+    use pyo3::prelude::*;
+    use rustpython_vm::vm::thread::{current_vm_is_set, with_current_vm};
+
+    #[test]
+    fn test_new_thread() {
+        Python::attach(|_py| {
+            with_current_vm(|_vm| {
+                assert!(
+                    current_vm_is_set(),
+                    "This thread did not have a vm attached"
+                )
+            });
+
+            std::thread::spawn(move || {
+                Python::attach(|_py| {
+                    with_current_vm(|_vm| {
+                        assert!(
+                            current_vm_is_set(),
+                            "This thread did not have a vm attached"
+                        )
+                    });
+                });
+            })
+            .join()
+            .unwrap();
+        })
+    }
+
+    #[test]
+    fn test_current_vm_main_thread() {
+        Python::initialize();
+
+        // let RustPython create a vm for this thread.
+        let vm = get_main_interpreter()
+            .as_ref()
+            .unwrap()
+            .enter(|vm| vm.new_thread());
+
+        // Attach the vm using RustPython
+        vm.run(|_vm| {
+            assert!(current_vm_is_set(), "This thread should have a vm attached");
+
+            Python::attach(|_py| {
+                with_current_vm(|_vm| {
+                    assert!(current_vm_is_set());
+                })
+            })
+        });
+    }
+
+    #[test]
+    fn test_gilstate_release_detaches_external_thread() {
+        Python::initialize();
+
+        std::thread::spawn(|| {
+            let state = PyGILState_Ensure();
+            assert!(current_vm_is_set());
+            PyGILState_Release(state);
+            assert!(!current_vm_is_set());
+        })
+        .join()
+        .unwrap();
+    }
+}
diff --git a/crates/capi/src/refcount.rs b/crates/capi/src/refcount.rs
@@ -0,0 +1,15 @@
+use crate::PyObject;
+use core::ptr::NonNull;
+use rustpython_vm::PyObjectRef;
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn _Py_DecRef(op: *mut PyObject) {
+    // By dropping PyObjectRef, we will decrement the reference count.
+    unsafe { drop(PyObjectRef::from_raw(NonNull::new_unchecked(op))) };
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn _Py_IncRef(op: *mut PyObject) {
+    // Don't drop the owned value, as we just want to increment the refcount.
+    core::mem::forget(unsafe { (*op).to_owned() });
+}
diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml
@@ -65,7 +65,6 @@ num-traits = { workspace = true }
 num_enum = { workspace = true }
 parking_lot = { workspace = true }
 paste = { workspace = true }
-scoped-tls = { workspace = true }
 scopeguard = { workspace = true }
 serde = { workspace = true, optional = true }
 static_assertions = { workspace = true }
diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs
@@ -158,7 +158,7 @@ impl CodecsRegistry {
     /// # Safety
     /// Must only be called after fork() in the child process when no other
     /// threads exist.
-    #[cfg(all(unix, feature = "threading"))]
+    #[cfg(all(unix, feature = "threading", feature = "host_env"))]
     pub(crate) unsafe fn reinit_after_fork(&self) {
         unsafe { crate::common::lock::reinit_rwlock_after_fork(&self.inner) };
     }
diff --git a/crates/vm/src/intern.rs b/crates/vm/src/intern.rs
@@ -36,7 +36,7 @@ impl StringPool {
     /// # Safety
     /// Must only be called after fork() in the child process when no other
     /// threads exist.
-    #[cfg(all(unix, feature = "threading"))]
+    #[cfg(all(unix, feature = "threading", feature = "host_env"))]
     pub(crate) unsafe fn reinit_after_fork(&self) {
         unsafe { crate::common::lock::reinit_rwlock_after_fork(&self.inner) };
     }
diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs
diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs
diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs
diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs
diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs
diff --git a/src/lib.rs b/src/lib.rs

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[env]`
	`2`	`+PYO3_CONFIG_FILE = { value = "pyo3-rustpython.config", relative = true }`
	`3`	`+PYO3_NO_PYTHON = { value = "1" }`