From 54156f638f7f6c3599751c6db9c06ecde8ee13c6 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sat, 4 Apr 2026 17:21:44 +0200 Subject: [PATCH 01/24] export c api --- Cargo.lock | 27 ++++-- crates/capi/Cargo.toml | 18 ++++ crates/capi/src/bytesobject.rs | 16 ++++ crates/capi/src/import.rs | 9 ++ crates/capi/src/lib.rs | 33 +++++++ crates/capi/src/longobject.rs | 40 ++++++++ crates/capi/src/object.rs | 94 +++++++++++++++++++ crates/capi/src/pyerrors.rs | 58 ++++++++++++ crates/capi/src/pylifecycle.rs | 49 ++++++++++ crates/capi/src/pystate.rs | 28 ++++++ crates/capi/src/refcount.rs | 35 +++++++ crates/capi/src/traceback.rs | 9 ++ crates/capi/src/tupleobject.rs | 15 +++ crates/capi/src/unicodeobject.rs | 34 +++++++ .../pyo3_embed/.cargo/config.toml | 2 + example_projects/pyo3_embed/Cargo.toml | 12 +++ example_projects/pyo3_embed/README.md | 11 +++ example_projects/pyo3_embed/build.rs | 6 ++ .../pyo3_embed/pyo3-rustpython.config | 6 ++ example_projects/pyo3_embed/src/main.rs | 10 ++ 20 files changed, 502 insertions(+), 10 deletions(-) create mode 100644 crates/capi/Cargo.toml create mode 100644 crates/capi/src/bytesobject.rs create mode 100644 crates/capi/src/import.rs create mode 100644 crates/capi/src/lib.rs create mode 100644 crates/capi/src/longobject.rs create mode 100644 crates/capi/src/object.rs create mode 100644 crates/capi/src/pyerrors.rs create mode 100644 crates/capi/src/pylifecycle.rs create mode 100644 crates/capi/src/pystate.rs create mode 100644 crates/capi/src/refcount.rs create mode 100644 crates/capi/src/traceback.rs create mode 100644 crates/capi/src/tupleobject.rs create mode 100644 crates/capi/src/unicodeobject.rs create mode 100644 example_projects/pyo3_embed/.cargo/config.toml create mode 100644 example_projects/pyo3_embed/Cargo.toml create mode 100644 example_projects/pyo3_embed/README.md create mode 100644 example_projects/pyo3_embed/build.rs create mode 100644 example_projects/pyo3_embed/pyo3-rustpython.config create mode 100644 example_projects/pyo3_embed/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 5fd981fd135..1eca00b9fbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2643,9 +2643,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "libc", "once_cell", @@ -2657,18 +2657,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -2676,9 +2676,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2688,9 +2688,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", @@ -3104,6 +3104,13 @@ dependencies = [ "winresource", ] +[[package]] +name = "rustpython-capi" +version = "0.5.0" +dependencies = [ + "rustpython-vm", +] + [[package]] name = "rustpython-codegen" version = "0.5.0" diff --git a/crates/capi/Cargo.toml b/crates/capi/Cargo.toml new file mode 100644 index 00000000000..bf3bc8ca285 --- /dev/null +++ b/crates/capi/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "rustpython-capi" +description = "Minimal CPython C-API compatibility exports for RustPython" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[lib] +crate-type = ["staticlib"] + +[dependencies] +rustpython-vm = { workspace = true } + +[lints] +workspace = true diff --git a/crates/capi/src/bytesobject.rs b/crates/capi/src/bytesobject.rs new file mode 100644 index 00000000000..a5164bc8487 --- /dev/null +++ b/crates/capi/src/bytesobject.rs @@ -0,0 +1,16 @@ +use core::ffi::c_char; +use core::ptr; + +use crate::PyObject; + +#[unsafe(no_mangle)] +pub extern "C" fn PyBytes_Size(_bytes: *mut PyObject) -> isize { + crate::log_stub("PyBytes_Size"); + 0 +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyBytes_AsString(_bytes: *mut PyObject) -> *mut c_char { + crate::log_stub("PyBytes_AsString"); + ptr::null_mut() +} diff --git a/crates/capi/src/import.rs b/crates/capi/src/import.rs new file mode 100644 index 00000000000..29ea7996687 --- /dev/null +++ b/crates/capi/src/import.rs @@ -0,0 +1,9 @@ +use core::ptr; + +use crate::PyObject; + +#[unsafe(no_mangle)] +pub extern "C" fn PyImport_Import(_name: *mut PyObject) -> *mut PyObject { + crate::log_stub("PyImport_Import"); + ptr::null_mut() +} diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs new file mode 100644 index 00000000000..6da5d0e04fc --- /dev/null +++ b/crates/capi/src/lib.rs @@ -0,0 +1,33 @@ +use core::ffi::c_long; +pub use rustpython_vm::{PyObject}; + +extern crate alloc; + +pub mod bytesobject; +pub mod import; +pub mod longobject; +pub mod object; +pub mod pyerrors; +pub mod pylifecycle; +pub mod pystate; +pub mod refcount; +pub mod traceback; +pub mod tupleobject; +pub mod unicodeobject; + + +#[repr(C)] +pub struct PyThreadState { + _private: [u8; 0], +} + +#[repr(C)] +pub struct PyLongObject { + ob_base: PyObject, + value: c_long, +} + +#[inline] +pub(crate) fn log_stub(name: &str) { + eprintln!("[rustpython-capi stub] {name} called"); +} diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs new file mode 100644 index 00000000000..586b8c4497a --- /dev/null +++ b/crates/capi/src/longobject.rs @@ -0,0 +1,40 @@ +use core::ffi::{c_long, c_longlong, c_ulong, c_ulonglong}; +use core::ptr; + +use crate::PyObject; + +#[unsafe(no_mangle)] +pub extern "C" fn PyLong_FromLong(_value: c_long) -> *mut PyObject { + crate::log_stub("PyLong_FromLong"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyLong_FromLongLong(_value: c_longlong) -> *mut PyObject { + crate::log_stub("PyLong_FromLongLong"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyLong_FromSsize_t(_value: isize) -> *mut PyObject { + crate::log_stub("PyLong_FromSsize_t"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyLong_FromSize_t(_value: usize) -> *mut PyObject { + crate::log_stub("PyLong_FromSize_t"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyLong_FromUnsignedLong(_value: c_ulong) -> *mut PyObject { + crate::log_stub("PyLong_FromUnsignedLong"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyLong_FromUnsignedLongLong(_value: c_ulonglong) -> *mut PyObject { + crate::log_stub("PyLong_FromUnsignedLongLong"); + ptr::null_mut() +} diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs new file mode 100644 index 00000000000..f457606bac2 --- /dev/null +++ b/crates/capi/src/object.rs @@ -0,0 +1,94 @@ +use std::mem::{transmute, MaybeUninit}; +use std::ptr::NonNull; +use rustpython_vm::{PyObjectRef, VirtualMachine, Context}; +use rustpython_vm::builtins::PyType; +use crate::{PyObject}; + +type PyTypeObject = MaybeUninit<&'static PyType>; + +#[unsafe(no_mangle)] +pub static mut PyType_Type: PyTypeObject = MaybeUninit::uninit(); + +#[unsafe(no_mangle)] +pub static mut PyLong_Type: PyTypeObject = MaybeUninit::uninit(); + +#[unsafe(no_mangle)] +pub static mut PyTuple_Type: PyTypeObject = MaybeUninit::uninit(); + +#[unsafe(no_mangle)] +pub static mut PyUnicode_Type: PyTypeObject = MaybeUninit::uninit(); + +unsafe fn setup_type_pointers(ctx: &Context) { + let zoo = &ctx.types; + + unsafe { + PyType_Type.write(zoo.type_type.payload()); + PyLong_Type.write(zoo.int_type.payload()); + PyTuple_Type.write(zoo.tuple_type.payload()); + PyUnicode_Type.write(zoo.str_type.payload()); + } +} + +#[unsafe(no_mangle)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub extern "C" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { + if op.is_null() { + return std::ptr::null_mut(); + } + + // SAFETY: op is non-null and expected to be a valid pointer for this shim. + unsafe { transmute((*op).class()) } +} + +#[unsafe(no_mangle)] +pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { + _Py_TYPE(op) +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyType_GetFlags(_ty: *mut PyTypeObject) -> usize { + crate::log_stub("PyType_GetFlags"); + 0 +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyType_GetName(_ty: *mut PyTypeObject) -> *mut PyObject { + crate::log_stub("PyType_GetName"); + std::ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyType_GetQualName(_ty: *mut PyTypeObject) -> *mut PyObject { + crate::log_stub("PyType_GetQualName"); + std::ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyObject_CallNoArgs(_callable: *mut PyObject) -> *mut PyObject { + crate::log_stub("PyObject_CallNoArgs"); + std::ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyObject_GetAttr(_obj: *mut PyObject, _name: *mut PyObject) -> *mut PyObject { + crate::log_stub("PyObject_GetAttr"); + std::ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyObject_Repr(_obj: *mut PyObject) -> *mut PyObject { + crate::log_stub("PyObject_Repr"); + std::ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyObject_Str(_obj: *mut PyObject) -> *mut PyObject { + crate::log_stub("PyObject_Str"); + std::ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn Py_GetConstantBorrowed(_constant_id: core::ffi::c_uint) -> *mut PyObject { + crate::log_stub("Py_GetConstantBorrowed"); + std::ptr::null_mut() +} diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs new file mode 100644 index 00000000000..fba1e4a14b4 --- /dev/null +++ b/crates/capi/src/pyerrors.rs @@ -0,0 +1,58 @@ +use core::ffi::{c_char, c_int}; +use core::ptr; + +use crate::PyObject; + +#[unsafe(no_mangle)] +pub static mut PyExc_BaseException: *mut PyObject = ptr::null_mut(); + +#[unsafe(no_mangle)] +pub static mut PyExc_TypeError: *mut PyObject = ptr::null_mut(); + +#[unsafe(no_mangle)] +pub extern "C" fn PyErr_GetRaisedException() -> *mut PyObject { + crate::log_stub("PyErr_GetRaisedException"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyErr_SetRaisedException(_exc: *mut PyObject) { + crate::log_stub("PyErr_SetRaisedException"); +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyErr_SetObject(_exception: *mut PyObject, _value: *mut PyObject) { + crate::log_stub("PyErr_SetObject"); +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyErr_SetString(_exception: *mut PyObject, _message: *const c_char) { + crate::log_stub("PyErr_SetString"); +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyErr_PrintEx(_set_sys_last_vars: c_int) { + crate::log_stub("PyErr_PrintEx"); +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyErr_WriteUnraisable(_obj: *mut PyObject) { + crate::log_stub("PyErr_WriteUnraisable"); +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyErr_NewExceptionWithDoc( + _name: *const c_char, + _doc: *const c_char, + _base: *mut PyObject, + _dict: *mut PyObject, +) -> *mut PyObject { + crate::log_stub("PyErr_NewExceptionWithDoc"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyException_GetTraceback(_exc: *mut PyObject) -> *mut PyObject { + crate::log_stub("PyException_GetTraceback"); + ptr::null_mut() +} diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs new file mode 100644 index 00000000000..96fbc7b81d5 --- /dev/null +++ b/crates/capi/src/pylifecycle.rs @@ -0,0 +1,49 @@ +use core::ffi::c_int; +use std::cell::RefCell; +use std::mem::ManuallyDrop; +use rustpython_vm::Interpreter; + +thread_local! { + pub static INTERP: RefCell>> = const { RefCell::new(None) }; +} + +#[unsafe(no_mangle)] +pub extern "C-unwind" fn Py_IsInitialized() -> c_int { + INTERP.with(|interp| interp.borrow().is_some() as c_int) +} + +#[unsafe(no_mangle)] +pub extern "C-unwind" fn Py_Initialize() { + Py_InitializeEx(0); +} + +#[unsafe(no_mangle)] +pub extern "C-unwind" fn Py_InitializeEx(_initsigs: c_int) { + if INTERP.with(|interp| interp.borrow().is_none()) { + let interp = Interpreter::with_init(Default::default(), |_vm| { + }); + + INTERP.with(|interp_ref| { + *interp_ref.borrow_mut() = Some(ManuallyDrop::new(interp)); + }); + } +} + +#[unsafe(no_mangle)] +pub extern "C-unwind" fn Py_Finalize() { + let _ = Py_FinalizeEx(); +} + +#[unsafe(no_mangle)] +pub extern "C-unwind" fn Py_FinalizeEx() -> c_int { + INTERP.with(|interp_ref| { + let interp = ManuallyDrop::into_inner(interp_ref.borrow_mut().take() + .expect("Py_FinalizeEx called without an active interpreter")); + interp.finalize(None) + }) as _ +} + +#[unsafe(no_mangle)] +pub extern "C-unwind" fn Py_IsFinalizing() -> c_int { + 0 +} diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs new file mode 100644 index 00000000000..24ce27f6f43 --- /dev/null +++ b/crates/capi/src/pystate.rs @@ -0,0 +1,28 @@ +use crate::{PyThreadState, log_stub}; +use core::ffi::c_int; +use core::ptr; + +#[allow(non_camel_case_types)] +type PyGILState_STATE = c_int; + +#[unsafe(no_mangle)] +pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE { + log_stub("PyGILState_Ensure"); + 0 +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyGILState_Release(_state: PyGILState_STATE) { + log_stub("PyGILState_Release"); +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState { + log_stub("PyEval_SaveThread"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyEval_RestoreThread(_tstate: *mut PyThreadState) { + log_stub("PyEval_RestoreThread"); +} diff --git a/crates/capi/src/refcount.rs b/crates/capi/src/refcount.rs new file mode 100644 index 00000000000..1f26304b871 --- /dev/null +++ b/crates/capi/src/refcount.rs @@ -0,0 +1,35 @@ +use alloc::boxed::Box; +use std::ptr::NonNull; +use rustpython_vm::PyObjectRef; +use crate::object::PyLong_Type; +use crate::{PyLongObject, PyObject}; + +#[unsafe(no_mangle)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub extern "C" fn _Py_DecRef(op: *mut PyObject) { + let Some(ptr) = NonNull::new(op) else { + return; + }; + + let owned = unsafe { + PyObjectRef::from_raw(ptr) + }; + + // Dropping so we decrement the refcount + drop(owned); +} + +#[unsafe(no_mangle)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub extern "C" fn _Py_IncRef(op: *mut PyObject) { + if op.is_null() { + return; + } + + // SAFETY: op is non-null and expected to be a valid pointer for this shim. + let owned = unsafe { + (*op).to_owned() + }; + + std::mem::forget(owned); +} diff --git a/crates/capi/src/traceback.rs b/crates/capi/src/traceback.rs new file mode 100644 index 00000000000..c73c952d5e7 --- /dev/null +++ b/crates/capi/src/traceback.rs @@ -0,0 +1,9 @@ +use core::ffi::c_int; + +use crate::PyObject; + +#[unsafe(no_mangle)] +pub extern "C" fn PyTraceBack_Print(_tb: *mut PyObject, _file: *mut PyObject) -> c_int { + crate::log_stub("PyTraceBack_Print"); + -1 +} diff --git a/crates/capi/src/tupleobject.rs b/crates/capi/src/tupleobject.rs new file mode 100644 index 00000000000..996a835bb78 --- /dev/null +++ b/crates/capi/src/tupleobject.rs @@ -0,0 +1,15 @@ +use core::ptr; + +use crate::PyObject; + +#[unsafe(no_mangle)] +pub extern "C" fn PyTuple_Size(_tuple: *mut PyObject) -> isize { + crate::log_stub("PyTuple_Size"); + 0 +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyTuple_GetItem(_tuple: *mut PyObject, _pos: isize) -> *mut PyObject { + crate::log_stub("PyTuple_GetItem"); + ptr::null_mut() +} diff --git a/crates/capi/src/unicodeobject.rs b/crates/capi/src/unicodeobject.rs new file mode 100644 index 00000000000..2b4fb081d66 --- /dev/null +++ b/crates/capi/src/unicodeobject.rs @@ -0,0 +1,34 @@ +use core::ffi::c_char; +use core::ptr; + +use crate::PyObject; + +#[unsafe(no_mangle)] +pub extern "C" fn PyUnicode_FromStringAndSize(_s: *const c_char, _len: isize) -> *mut PyObject { + crate::log_stub("PyUnicode_FromStringAndSize"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyUnicode_AsUTF8AndSize( + _unicode: *mut PyObject, + _size: *mut isize, +) -> *const c_char { + crate::log_stub("PyUnicode_AsUTF8AndSize"); + ptr::null() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyUnicode_AsEncodedString( + _unicode: *mut PyObject, + _encoding: *const c_char, + _errors: *const c_char, +) -> *mut PyObject { + crate::log_stub("PyUnicode_AsEncodedString"); + ptr::null_mut() +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyUnicode_InternInPlace(_string: *mut *mut PyObject) { + crate::log_stub("PyUnicode_InternInPlace"); +} diff --git a/example_projects/pyo3_embed/.cargo/config.toml b/example_projects/pyo3_embed/.cargo/config.toml new file mode 100644 index 00000000000..54a884e2458 --- /dev/null +++ b/example_projects/pyo3_embed/.cargo/config.toml @@ -0,0 +1,2 @@ +[env] +PYO3_CONFIG_FILE = { value = "pyo3-rustpython.config", relative = true } diff --git a/example_projects/pyo3_embed/Cargo.toml b/example_projects/pyo3_embed/Cargo.toml new file mode 100644 index 00000000000..6174170c143 --- /dev/null +++ b/example_projects/pyo3_embed/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "example_pyo3_embed" +version = "0.1.0" +edition = "2021" + +[dependencies] +pyo3 = { path = "../../../pyo3", default-features = false, features = ["abi3-py314"] } +rustpython-capi = { path = "../../crates/capi" } + +[workspace] + +[patch.crates-io] diff --git a/example_projects/pyo3_embed/README.md b/example_projects/pyo3_embed/README.md new file mode 100644 index 00000000000..7d9cc2cf98b --- /dev/null +++ b/example_projects/pyo3_embed/README.md @@ -0,0 +1,11 @@ +# PyO3 embed against RustPython C-API + +This example demonstrates linking `pyo3` against RustPython's minimal C-API shim (`rustpython-capi`) instead of a system CPython library. + +From this directory, run: + +```shell +cargo run +``` + +The local `.cargo/config.toml` sets `PYO3_CONFIG_FILE` automatically. diff --git a/example_projects/pyo3_embed/build.rs b/example_projects/pyo3_embed/build.rs new file mode 100644 index 00000000000..8d379be42c0 --- /dev/null +++ b/example_projects/pyo3_embed/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!( + "cargo:rustc-link-arg=/Users/basschoenmaeckers/repo/RustPython/target/debug/librustpython_capi.a" + ); + println!("cargo:rustc-link-lib=framework=CoreFoundation"); +} diff --git a/example_projects/pyo3_embed/pyo3-rustpython.config b/example_projects/pyo3_embed/pyo3-rustpython.config new file mode 100644 index 00000000000..2cf06cd886c --- /dev/null +++ b/example_projects/pyo3_embed/pyo3-rustpython.config @@ -0,0 +1,6 @@ +implementation=CPython +version=3.14 +shared=false +abi3=true +build_flags= +suppress_build_script_link_lines=true diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs new file mode 100644 index 00000000000..72ba168a907 --- /dev/null +++ b/example_projects/pyo3_embed/src/main.rs @@ -0,0 +1,10 @@ +use pyo3::prelude::*; +use pyo3::types::PyInt; + +fn main() { + Python::initialize(); + + Python::attach(|py| { + // let _x = PyInt::new(py, 123); + }); +} From 2250a1c50923847f737eb15df8b73737616f5c48 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 10:36:24 +0200 Subject: [PATCH 02/24] type mapping v1 --- crates/capi/src/object.rs | 51 +++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index f457606bac2..6a382bb7fa2 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -1,47 +1,56 @@ use std::mem::{transmute, MaybeUninit}; use std::ptr::NonNull; -use rustpython_vm::{PyObjectRef, VirtualMachine, Context}; +use rustpython_vm::{PyObjectRef, VirtualMachine, Context, AsObject}; use rustpython_vm::builtins::PyType; use crate::{PyObject}; +use crate::object::PyTypeObject::*; -type PyTypeObject = MaybeUninit<&'static PyType>; -#[unsafe(no_mangle)] -pub static mut PyType_Type: PyTypeObject = MaybeUninit::uninit(); -#[unsafe(no_mangle)] -pub static mut PyLong_Type: PyTypeObject = MaybeUninit::uninit(); +pub enum PyTypeObject { + Type, + Long, + Tuple, + Unicode, +} #[unsafe(no_mangle)] -pub static mut PyTuple_Type: PyTypeObject = MaybeUninit::uninit(); +pub static mut PyType_Type: PyTypeObject = Type; #[unsafe(no_mangle)] -pub static mut PyUnicode_Type: PyTypeObject = MaybeUninit::uninit(); +pub static mut PyLong_Type: PyTypeObject = Long; -unsafe fn setup_type_pointers(ctx: &Context) { - let zoo = &ctx.types; +#[unsafe(no_mangle)] +pub static mut PyTuple_Type: PyTypeObject = Tuple; - unsafe { - PyType_Type.write(zoo.type_type.payload()); - PyLong_Type.write(zoo.int_type.payload()); - PyTuple_Type.write(zoo.tuple_type.payload()); - PyUnicode_Type.write(zoo.str_type.payload()); - } -} +#[unsafe(no_mangle)] +pub static mut PyUnicode_Type: PyTypeObject = Unicode; #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { +pub extern "C-unwind" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { if op.is_null() { return std::ptr::null_mut(); } - // SAFETY: op is non-null and expected to be a valid pointer for this shim. - unsafe { transmute((*op).class()) } + let zoo = &Context::genesis().types; + let ty = unsafe { (*op).class()}; + + if ty.is(zoo.type_type) { + unsafe { &raw mut PyType_Type } + } else if ty.is(zoo.int_type){ + unsafe { &raw mut PyLong_Type } + } else if ty.is(zoo.tuple_type){ + unsafe { &raw mut PyTuple_Type } + } else if ty.is(zoo.str_type){ + unsafe { &raw mut PyUnicode_Type } + } else { + todo!("Unsupported type: {:?}", ty.name()); + } } #[unsafe(no_mangle)] -pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { +pub extern "C-unwind" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { _Py_TYPE(op) } From bc96211cab0abeead51ba565ed11cd6951788154 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 10:59:08 +0200 Subject: [PATCH 03/24] Type mapping v2 --- crates/capi/src/object.rs | 69 +++++++++++++++++++++++-------------- crates/capi/src/refcount.rs | 4 +-- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index 6a382bb7fa2..7f60a8753aa 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -1,30 +1,46 @@ -use std::mem::{transmute, MaybeUninit}; -use std::ptr::NonNull; -use rustpython_vm::{PyObjectRef, VirtualMachine, Context, AsObject}; +use std::sync::LazyLock; +use rustpython_vm::{Context, AsObject, Py}; use rustpython_vm::builtins::PyType; use crate::{PyObject}; -use crate::object::PyTypeObject::*; -pub enum PyTypeObject { - Type, - Long, - Tuple, - Unicode, +pub struct PyTypeObject { + ty: LazyLock<&'static Py>, +} + +impl PyTypeObject { + const fn new(f: fn() -> &'static Py) -> PyTypeObject + { + PyTypeObject { + ty: LazyLock::new(f), + } + } } #[unsafe(no_mangle)] -pub static mut PyType_Type: PyTypeObject = Type; +pub static mut PyType_Type: PyTypeObject = PyTypeObject::new(|| { + let zoo = &Context::genesis().types; + zoo.type_type +}); #[unsafe(no_mangle)] -pub static mut PyLong_Type: PyTypeObject = Long; +pub static mut PyLong_Type: PyTypeObject = PyTypeObject::new(|| { + let zoo = &Context::genesis().types; + zoo.int_type +}); #[unsafe(no_mangle)] -pub static mut PyTuple_Type: PyTypeObject = Tuple; +pub static mut PyTuple_Type: PyTypeObject = PyTypeObject::new(|| { + let zoo = &Context::genesis().types; + zoo.tuple_type +}); #[unsafe(no_mangle)] -pub static mut PyUnicode_Type: PyTypeObject = Unicode; +pub static mut PyUnicode_Type: PyTypeObject = PyTypeObject::new(|| { + let zoo = &Context::genesis().types; + zoo.union_type +}); #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] @@ -33,20 +49,21 @@ pub extern "C-unwind" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { return std::ptr::null_mut(); } - let zoo = &Context::genesis().types; - let ty = unsafe { (*op).class()}; - - if ty.is(zoo.type_type) { - unsafe { &raw mut PyType_Type } - } else if ty.is(zoo.int_type){ - unsafe { &raw mut PyLong_Type } - } else if ty.is(zoo.tuple_type){ - unsafe { &raw mut PyTuple_Type } - } else if ty.is(zoo.str_type){ - unsafe { &raw mut PyUnicode_Type } - } else { - todo!("Unsupported type: {:?}", ty.name()); + unsafe { + let ty = (*op).class(); + if ty.is(*PyTuple_Type.ty) { + &raw mut PyType_Type + } else if ty.is(*PyTuple_Type.ty){ + &raw mut PyLong_Type + } else if ty.is(*PyTuple_Type.ty){ + &raw mut PyTuple_Type + } else if ty.is(*PyTuple_Type.ty){ + &raw mut PyUnicode_Type + } else { + todo!("Unsupported type: {:?}", ty.name()); + } } + } #[unsafe(no_mangle)] diff --git a/crates/capi/src/refcount.rs b/crates/capi/src/refcount.rs index 1f26304b871..76652175617 100644 --- a/crates/capi/src/refcount.rs +++ b/crates/capi/src/refcount.rs @@ -1,8 +1,6 @@ -use alloc::boxed::Box; use std::ptr::NonNull; use rustpython_vm::PyObjectRef; -use crate::object::PyLong_Type; -use crate::{PyLongObject, PyObject}; +use crate::{PyObject}; #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] From 46307f6cebc83d6b1cdf850e551b9f6f8ae4d161 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 11:18:11 +0200 Subject: [PATCH 04/24] Implement type flags --- crates/capi/src/longobject.rs | 17 ++++- crates/capi/src/object.rs | 97 ++++++++++++++++--------- example_projects/pyo3_embed/src/main.rs | 5 +- 3 files changed, 81 insertions(+), 38 deletions(-) diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs index 586b8c4497a..a7d620908b7 100644 --- a/crates/capi/src/longobject.rs +++ b/crates/capi/src/longobject.rs @@ -2,11 +2,22 @@ use core::ffi::{c_long, c_longlong, c_ulong, c_ulonglong}; use core::ptr; use crate::PyObject; +use crate::pylifecycle::INTERP; +use rustpython_vm::PyObjectRef; #[unsafe(no_mangle)] -pub extern "C" fn PyLong_FromLong(_value: c_long) -> *mut PyObject { - crate::log_stub("PyLong_FromLong"); - ptr::null_mut() +pub extern "C" fn PyLong_FromLong(value: c_long) -> *mut PyObject { + INTERP.with(|interp_ref| { + let interp = interp_ref.borrow(); + let interp = interp + .as_ref() + .expect("PyLong_FromLong called before Py_InitializeEx"); + + interp.enter(|vm| { + let obj: PyObjectRef = vm.ctx.new_int(value).into(); + obj.into_raw().as_ptr() + }) + }) } #[unsafe(no_mangle)] diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index 7f60a8753aa..1e14f4c3f1b 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -1,46 +1,73 @@ -use std::sync::LazyLock; -use rustpython_vm::{Context, AsObject, Py}; -use rustpython_vm::builtins::PyType; -use crate::{PyObject}; - +use core::ffi::c_ulong; +use crate::PyObject; +use rustpython_vm::builtins::PyType; +use rustpython_vm::{AsObject, Context, Py}; +use std::sync::LazyLock; -pub struct PyTypeObject { +pub struct PyTypeObject { ty: LazyLock<&'static Py>, + flags: c_ulong, } impl PyTypeObject { - const fn new(f: fn() -> &'static Py) -> PyTypeObject - { + const fn new(f: fn() -> &'static Py, flags: c_ulong) -> PyTypeObject { PyTypeObject { ty: LazyLock::new(f), + flags, } } } +const PY_TPFLAGS_HAVE_STACKLESS_EXTENSION: c_ulong = 0; +const PY_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; +const PY_TPFLAGS_DEFAULT: c_ulong = + PY_TPFLAGS_HAVE_STACKLESS_EXTENSION | PY_TPFLAGS_HAVE_VERSION_TAG; +const PY_TPFLAGS_IMMUTABLETYPE: c_ulong = 1 << 8; +const PY_TPFLAGS_BASETYPE: c_ulong = 1 << 10; +const PY_TPFLAGS_LONG_SUBCLASS: c_ulong = 1 << 24; +const PY_TPFLAGS_TUPLE_SUBCLASS: c_ulong = 1 << 26; +const PY_TPFLAGS_UNICODE_SUBCLASS: c_ulong = 1 << 28; +const PY_TPFLAGS_TYPE_SUBCLASS: c_ulong = 1 << 31; + #[unsafe(no_mangle)] -pub static mut PyType_Type: PyTypeObject = PyTypeObject::new(|| { - let zoo = &Context::genesis().types; - zoo.type_type -}); +pub static mut PyType_Type: PyTypeObject = PyTypeObject::new( + || { + let zoo = &Context::genesis().types; + zoo.type_type + }, + PY_TPFLAGS_DEFAULT | PY_TPFLAGS_IMMUTABLETYPE | PY_TPFLAGS_BASETYPE | PY_TPFLAGS_TYPE_SUBCLASS, +); #[unsafe(no_mangle)] -pub static mut PyLong_Type: PyTypeObject = PyTypeObject::new(|| { - let zoo = &Context::genesis().types; - zoo.int_type -}); +pub static mut PyLong_Type: PyTypeObject = PyTypeObject::new( + || { + let zoo = &Context::genesis().types; + zoo.int_type + }, + PY_TPFLAGS_DEFAULT | PY_TPFLAGS_IMMUTABLETYPE | PY_TPFLAGS_BASETYPE | PY_TPFLAGS_LONG_SUBCLASS, +); #[unsafe(no_mangle)] -pub static mut PyTuple_Type: PyTypeObject = PyTypeObject::new(|| { - let zoo = &Context::genesis().types; - zoo.tuple_type -}); +pub static mut PyTuple_Type: PyTypeObject = PyTypeObject::new( + || { + let zoo = &Context::genesis().types; + zoo.tuple_type + }, + PY_TPFLAGS_DEFAULT | PY_TPFLAGS_IMMUTABLETYPE | PY_TPFLAGS_BASETYPE | PY_TPFLAGS_TUPLE_SUBCLASS, +); #[unsafe(no_mangle)] -pub static mut PyUnicode_Type: PyTypeObject = PyTypeObject::new(|| { - let zoo = &Context::genesis().types; - zoo.union_type -}); +pub static mut PyUnicode_Type: PyTypeObject = PyTypeObject::new( + || { + let zoo = &Context::genesis().types; + zoo.str_type + }, + PY_TPFLAGS_DEFAULT + | PY_TPFLAGS_IMMUTABLETYPE + | PY_TPFLAGS_BASETYPE + | PY_TPFLAGS_UNICODE_SUBCLASS, +); #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] @@ -51,19 +78,18 @@ pub extern "C-unwind" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { unsafe { let ty = (*op).class(); - if ty.is(*PyTuple_Type.ty) { - &raw mut PyType_Type - } else if ty.is(*PyTuple_Type.ty){ + if ty.is(*PyType_Type.ty) { + &raw mut PyType_Type + } else if ty.is(*PyLong_Type.ty) { &raw mut PyLong_Type - } else if ty.is(*PyTuple_Type.ty){ + } else if ty.is(*PyTuple_Type.ty) { &raw mut PyTuple_Type - } else if ty.is(*PyTuple_Type.ty){ + } else if ty.is(*PyUnicode_Type.ty) { &raw mut PyUnicode_Type } else { todo!("Unsupported type: {:?}", ty.name()); } } - } #[unsafe(no_mangle)] @@ -72,9 +98,14 @@ pub extern "C-unwind" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { } #[unsafe(no_mangle)] -pub extern "C" fn PyType_GetFlags(_ty: *mut PyTypeObject) -> usize { - crate::log_stub("PyType_GetFlags"); - 0 +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub extern "C-unwind" fn PyType_GetFlags(ty: *mut PyTypeObject) -> c_ulong { + if ty.is_null() { + panic!("PyType_GetFlags called with null type pointer"); + } + + // SAFETY: caller guarantees this is a valid exported type object pointer. + unsafe { (*ty).flags } } #[unsafe(no_mangle)] diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index 72ba168a907..8c24df57817 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -1,10 +1,11 @@ use pyo3::prelude::*; -use pyo3::types::PyInt; +use pyo3::types::{PyInt, PyString}; fn main() { Python::initialize(); Python::attach(|py| { - // let _x = PyInt::new(py, 123); + let number = PyInt::new(py, 123); + assert!(number.is_instance_of::()); }); } From b08d41ab2703917e9a67b7240a0d6fc5f42413bf Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 11:22:46 +0200 Subject: [PATCH 05/24] Implement `PyUnicode_FromStringAndSize` --- crates/capi/src/unicodeobject.rs | 31 ++++++++++++++++++++++--- example_projects/pyo3_embed/src/main.rs | 3 +++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/crates/capi/src/unicodeobject.rs b/crates/capi/src/unicodeobject.rs index 2b4fb081d66..0d61b4da465 100644 --- a/crates/capi/src/unicodeobject.rs +++ b/crates/capi/src/unicodeobject.rs @@ -1,12 +1,37 @@ use core::ffi::c_char; use core::ptr; +use core::slice; +use core::str; use crate::PyObject; +use crate::pylifecycle::INTERP; +use rustpython_vm::PyObjectRef; #[unsafe(no_mangle)] -pub extern "C" fn PyUnicode_FromStringAndSize(_s: *const c_char, _len: isize) -> *mut PyObject { - crate::log_stub("PyUnicode_FromStringAndSize"); - ptr::null_mut() +pub extern "C" fn PyUnicode_FromStringAndSize(s: *const c_char, len: isize) -> *mut PyObject { + let len = usize::try_from(len).expect("PyUnicode_FromStringAndSize called with negative len"); + let text = if s.is_null() { + if len != 0 { + panic!("PyUnicode_FromStringAndSize called with null data and non-zero len"); + } + "" + } else { + // SAFETY: caller passes a valid C buffer of length `len`. + let bytes = unsafe { slice::from_raw_parts(s.cast::(), len) }; + str::from_utf8(bytes).expect("PyUnicode_FromStringAndSize got non-UTF8 data") + }; + + INTERP.with(|interp_ref| { + let interp = interp_ref.borrow(); + let interp = interp + .as_ref() + .expect("PyUnicode_FromStringAndSize called before Py_InitializeEx"); + + interp.enter(|vm| { + let obj: PyObjectRef = vm.ctx.new_str(text).into(); + obj.into_raw().as_ptr() + }) + }) } #[unsafe(no_mangle)] diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index 8c24df57817..892ea65b94e 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -7,5 +7,8 @@ fn main() { Python::attach(|py| { let number = PyInt::new(py, 123); assert!(number.is_instance_of::()); + + let string = PyString::new(py, "Hello, World!"); + assert!(string.is_instance_of::()); }); } From 34c13f496a134128f153ca5bb60316231e8537ae Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 11:26:02 +0200 Subject: [PATCH 06/24] Do not use `C-unwind` --- crates/capi/src/object.rs | 6 +++--- crates/capi/src/pylifecycle.rs | 25 ++++++++++++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index 1e14f4c3f1b..e09ecdcd74f 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -71,7 +71,7 @@ pub static mut PyUnicode_Type: PyTypeObject = PyTypeObject::new( #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C-unwind" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { +pub extern "C" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { if op.is_null() { return std::ptr::null_mut(); } @@ -93,13 +93,13 @@ pub extern "C-unwind" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { } #[unsafe(no_mangle)] -pub extern "C-unwind" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { +pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { _Py_TYPE(op) } #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C-unwind" fn PyType_GetFlags(ty: *mut PyTypeObject) -> c_ulong { +pub extern "C" fn PyType_GetFlags(ty: *mut PyTypeObject) -> c_ulong { if ty.is_null() { panic!("PyType_GetFlags called with null type pointer"); } diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index 96fbc7b81d5..b70fe05fba9 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -1,27 +1,26 @@ use core::ffi::c_int; +use rustpython_vm::Interpreter; use std::cell::RefCell; use std::mem::ManuallyDrop; -use rustpython_vm::Interpreter; thread_local! { pub static INTERP: RefCell>> = const { RefCell::new(None) }; } #[unsafe(no_mangle)] -pub extern "C-unwind" fn Py_IsInitialized() -> c_int { +pub extern "C" fn Py_IsInitialized() -> c_int { INTERP.with(|interp| interp.borrow().is_some() as c_int) } #[unsafe(no_mangle)] -pub extern "C-unwind" fn Py_Initialize() { +pub extern "C" fn Py_Initialize() { Py_InitializeEx(0); } #[unsafe(no_mangle)] -pub extern "C-unwind" fn Py_InitializeEx(_initsigs: c_int) { +pub extern "C" fn Py_InitializeEx(_initsigs: c_int) { if INTERP.with(|interp| interp.borrow().is_none()) { - let interp = Interpreter::with_init(Default::default(), |_vm| { - }); + let interp = Interpreter::with_init(Default::default(), |_vm| {}); INTERP.with(|interp_ref| { *interp_ref.borrow_mut() = Some(ManuallyDrop::new(interp)); @@ -30,20 +29,24 @@ pub extern "C-unwind" fn Py_InitializeEx(_initsigs: c_int) { } #[unsafe(no_mangle)] -pub extern "C-unwind" fn Py_Finalize() { +pub extern "C" fn Py_Finalize() { let _ = Py_FinalizeEx(); } #[unsafe(no_mangle)] -pub extern "C-unwind" fn Py_FinalizeEx() -> c_int { +pub extern "C" fn Py_FinalizeEx() -> c_int { INTERP.with(|interp_ref| { - let interp = ManuallyDrop::into_inner(interp_ref.borrow_mut().take() - .expect("Py_FinalizeEx called without an active interpreter")); + let interp = ManuallyDrop::into_inner( + interp_ref + .borrow_mut() + .take() + .expect("Py_FinalizeEx called without an active interpreter"), + ); interp.finalize(None) }) as _ } #[unsafe(no_mangle)] -pub extern "C-unwind" fn Py_IsFinalizing() -> c_int { +pub extern "C" fn Py_IsFinalizing() -> c_int { 0 } From 24b2c6384b82aa3ad5ac54154f854c7330e953c2 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 11:41:57 +0200 Subject: [PATCH 07/24] implement `PyLong_AsLong` --- Cargo.lock | 1 + crates/capi/Cargo.toml | 1 + crates/capi/src/longobject.rs | 28 +++++++++++++++++++++++++ crates/capi/src/object.rs | 8 +------ crates/capi/src/pyerrors.rs | 3 +++ example_projects/pyo3_embed/src/main.rs | 1 + 6 files changed, 35 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1eca00b9fbb..5d534fb6651 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3108,6 +3108,7 @@ dependencies = [ name = "rustpython-capi" version = "0.5.0" dependencies = [ + "num-traits", "rustpython-vm", ] diff --git a/crates/capi/Cargo.toml b/crates/capi/Cargo.toml index bf3bc8ca285..422b47aaa31 100644 --- a/crates/capi/Cargo.toml +++ b/crates/capi/Cargo.toml @@ -12,6 +12,7 @@ license.workspace = true crate-type = ["staticlib"] [dependencies] +num-traits = { workspace = true } rustpython-vm = { workspace = true } [lints] diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs index a7d620908b7..8c4cb3df1fb 100644 --- a/crates/capi/src/longobject.rs +++ b/crates/capi/src/longobject.rs @@ -4,6 +4,7 @@ use core::ptr; use crate::PyObject; use crate::pylifecycle::INTERP; use rustpython_vm::PyObjectRef; +use rustpython_vm::builtins::PyInt; #[unsafe(no_mangle)] pub extern "C" fn PyLong_FromLong(value: c_long) -> *mut PyObject { @@ -49,3 +50,30 @@ pub extern "C" fn PyLong_FromUnsignedLongLong(_value: c_ulonglong) -> *mut PyObj crate::log_stub("PyLong_FromUnsignedLongLong"); ptr::null_mut() } + +#[unsafe(no_mangle)] +pub extern "C" fn PyLong_AsLong(obj: *mut PyObject) -> c_long { + if obj.is_null() { + panic!("PyLong_AsLong called with null object"); + } + + INTERP.with(|interp_ref| { + let interp = interp_ref.borrow(); + let interp = interp + .as_ref() + .expect("PyLong_AsLong called before Py_InitializeEx"); + + interp.enter(|_vm| { + // SAFETY: non-null checked above; caller promises a valid PyObject pointer. + let obj_ref = unsafe { &*obj }; + let int_obj = obj_ref + .downcast_ref::() + .expect("PyLong_AsLong currently only accepts int instances"); + + int_obj + .as_bigint() + .try_into() + .expect("PyLong_AsLong: value out of range for c_long") + }) + }) +} diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index e09ecdcd74f..f041d2774e3 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -70,8 +70,7 @@ pub static mut PyUnicode_Type: PyTypeObject = PyTypeObject::new( ); #[unsafe(no_mangle)] -#[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { +pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { if op.is_null() { return std::ptr::null_mut(); } @@ -92,11 +91,6 @@ pub extern "C" fn _Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { } } -#[unsafe(no_mangle)] -pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { - _Py_TYPE(op) -} - #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn PyType_GetFlags(ty: *mut PyTypeObject) -> c_ulong { diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs index fba1e4a14b4..3a567bcfece 100644 --- a/crates/capi/src/pyerrors.rs +++ b/crates/capi/src/pyerrors.rs @@ -9,6 +9,9 @@ pub static mut PyExc_BaseException: *mut PyObject = ptr::null_mut(); #[unsafe(no_mangle)] pub static mut PyExc_TypeError: *mut PyObject = ptr::null_mut(); +#[unsafe(no_mangle)] +pub static mut PyExc_OverflowError: *mut PyObject = ptr::null_mut(); + #[unsafe(no_mangle)] pub extern "C" fn PyErr_GetRaisedException() -> *mut PyObject { crate::log_stub("PyErr_GetRaisedException"); diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index 892ea65b94e..ca79a80340a 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -7,6 +7,7 @@ fn main() { Python::attach(|py| { let number = PyInt::new(py, 123); assert!(number.is_instance_of::()); + assert_eq!(number.extract::().unwrap(), 123); let string = PyString::new(py, "Hello, World!"); assert!(string.is_instance_of::()); From 384b378a63d30c360c6b19efafd77b3c7ec8928a Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 11:48:52 +0200 Subject: [PATCH 08/24] Add `with_vm` helper --- crates/capi/src/longobject.rs | 42 +++++++++++--------------------- crates/capi/src/pylifecycle.rs | 9 +++++++ crates/capi/src/unicodeobject.rs | 15 +++--------- 3 files changed, 27 insertions(+), 39 deletions(-) diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs index 8c4cb3df1fb..8af4ed9e51e 100644 --- a/crates/capi/src/longobject.rs +++ b/crates/capi/src/longobject.rs @@ -2,22 +2,15 @@ use core::ffi::{c_long, c_longlong, c_ulong, c_ulonglong}; use core::ptr; use crate::PyObject; -use crate::pylifecycle::INTERP; +use crate::pylifecycle::with_vm; use rustpython_vm::PyObjectRef; use rustpython_vm::builtins::PyInt; #[unsafe(no_mangle)] pub extern "C" fn PyLong_FromLong(value: c_long) -> *mut PyObject { - INTERP.with(|interp_ref| { - let interp = interp_ref.borrow(); - let interp = interp - .as_ref() - .expect("PyLong_FromLong called before Py_InitializeEx"); - - interp.enter(|vm| { - let obj: PyObjectRef = vm.ctx.new_int(value).into(); - obj.into_raw().as_ptr() - }) + with_vm(|vm| { + let obj: PyObjectRef = vm.ctx.new_int(value).into(); + obj.into_raw().as_ptr() }) } @@ -57,23 +50,16 @@ pub extern "C" fn PyLong_AsLong(obj: *mut PyObject) -> c_long { panic!("PyLong_AsLong called with null object"); } - INTERP.with(|interp_ref| { - let interp = interp_ref.borrow(); - let interp = interp - .as_ref() - .expect("PyLong_AsLong called before Py_InitializeEx"); - - interp.enter(|_vm| { - // SAFETY: non-null checked above; caller promises a valid PyObject pointer. - let obj_ref = unsafe { &*obj }; - let int_obj = obj_ref - .downcast_ref::() - .expect("PyLong_AsLong currently only accepts int instances"); + with_vm(|_vm| { + // SAFETY: non-null checked above; caller promises a valid PyObject pointer. + let obj_ref = unsafe { &*obj }; + let int_obj = obj_ref + .downcast_ref::() + .expect("PyLong_AsLong currently only accepts int instances"); - int_obj - .as_bigint() - .try_into() - .expect("PyLong_AsLong: value out of range for c_long") - }) + int_obj + .as_bigint() + .try_into() + .expect("PyLong_AsLong: value out of range for c_long") }) } diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index b70fe05fba9..d1a27ec4381 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -1,5 +1,6 @@ use core::ffi::c_int; use rustpython_vm::Interpreter; +use rustpython_vm::VirtualMachine; use std::cell::RefCell; use std::mem::ManuallyDrop; @@ -7,6 +8,14 @@ thread_local! { pub static INTERP: RefCell>> = const { RefCell::new(None) }; } +pub(crate) fn with_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { + INTERP.with(|interp_ref| { + let interp = interp_ref.borrow(); + let interp = interp.as_ref().expect("VM access before Py_InitializeEx"); + interp.enter(f) + }) +} + #[unsafe(no_mangle)] pub extern "C" fn Py_IsInitialized() -> c_int { INTERP.with(|interp| interp.borrow().is_some() as c_int) diff --git a/crates/capi/src/unicodeobject.rs b/crates/capi/src/unicodeobject.rs index 0d61b4da465..4a241f6c91f 100644 --- a/crates/capi/src/unicodeobject.rs +++ b/crates/capi/src/unicodeobject.rs @@ -4,7 +4,7 @@ use core::slice; use core::str; use crate::PyObject; -use crate::pylifecycle::INTERP; +use crate::pylifecycle::with_vm; use rustpython_vm::PyObjectRef; #[unsafe(no_mangle)] @@ -21,16 +21,9 @@ pub extern "C" fn PyUnicode_FromStringAndSize(s: *const c_char, len: isize) -> * str::from_utf8(bytes).expect("PyUnicode_FromStringAndSize got non-UTF8 data") }; - INTERP.with(|interp_ref| { - let interp = interp_ref.borrow(); - let interp = interp - .as_ref() - .expect("PyUnicode_FromStringAndSize called before Py_InitializeEx"); - - interp.enter(|vm| { - let obj: PyObjectRef = vm.ctx.new_str(text).into(); - obj.into_raw().as_ptr() - }) + with_vm(|vm| { + let obj: PyObjectRef = vm.ctx.new_str(text).into(); + obj.into_raw().as_ptr() }) } From f49e8592f9c1c415be54d62b7054b61f8852d8f8 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 12:03:44 +0200 Subject: [PATCH 09/24] Move `PyThreadState` to `pystate.rs` --- crates/capi/src/lib.rs | 15 +-------------- crates/capi/src/pystate.rs | 7 ++++++- crates/capi/src/refcount.rs | 12 ++++-------- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs index 6da5d0e04fc..7080bdbae49 100644 --- a/crates/capi/src/lib.rs +++ b/crates/capi/src/lib.rs @@ -1,5 +1,4 @@ -use core::ffi::c_long; -pub use rustpython_vm::{PyObject}; +pub use rustpython_vm::PyObject; extern crate alloc; @@ -15,18 +14,6 @@ pub mod traceback; pub mod tupleobject; pub mod unicodeobject; - -#[repr(C)] -pub struct PyThreadState { - _private: [u8; 0], -} - -#[repr(C)] -pub struct PyLongObject { - ob_base: PyObject, - value: c_long, -} - #[inline] pub(crate) fn log_stub(name: &str) { eprintln!("[rustpython-capi stub] {name} called"); diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs index 24ce27f6f43..9879348f8a7 100644 --- a/crates/capi/src/pystate.rs +++ b/crates/capi/src/pystate.rs @@ -1,10 +1,15 @@ -use crate::{PyThreadState, log_stub}; +use crate::log_stub; use core::ffi::c_int; use core::ptr; #[allow(non_camel_case_types)] type PyGILState_STATE = c_int; +#[repr(C)] +pub struct PyThreadState { + _interp: *mut std::ffi::c_void, +} + #[unsafe(no_mangle)] pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE { log_stub("PyGILState_Ensure"); diff --git a/crates/capi/src/refcount.rs b/crates/capi/src/refcount.rs index 76652175617..5e6f55f0460 100644 --- a/crates/capi/src/refcount.rs +++ b/crates/capi/src/refcount.rs @@ -1,6 +1,6 @@ -use std::ptr::NonNull; +use crate::PyObject; use rustpython_vm::PyObjectRef; -use crate::{PyObject}; +use std::ptr::NonNull; #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] @@ -9,9 +9,7 @@ pub extern "C" fn _Py_DecRef(op: *mut PyObject) { return; }; - let owned = unsafe { - PyObjectRef::from_raw(ptr) - }; + let owned = unsafe { PyObjectRef::from_raw(ptr) }; // Dropping so we decrement the refcount drop(owned); @@ -25,9 +23,7 @@ pub extern "C" fn _Py_IncRef(op: *mut PyObject) { } // SAFETY: op is non-null and expected to be a valid pointer for this shim. - let owned = unsafe { - (*op).to_owned() - }; + let owned = unsafe { (*op).to_owned() }; std::mem::forget(owned); } From e18eef7684987995c8fa0b5991de3cb4f7f9233b Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 13:33:05 +0200 Subject: [PATCH 10/24] Basic multi-threading support --- crates/capi/Cargo.toml | 2 +- crates/capi/src/longobject.rs | 2 +- crates/capi/src/pylifecycle.rs | 61 ++++++++++++++----------- crates/capi/src/pystate.rs | 28 ++++++++++-- crates/capi/src/unicodeobject.rs | 2 +- example_projects/pyo3_embed/src/main.rs | 8 ++++ 6 files changed, 68 insertions(+), 35 deletions(-) diff --git a/crates/capi/Cargo.toml b/crates/capi/Cargo.toml index 422b47aaa31..5a55536902b 100644 --- a/crates/capi/Cargo.toml +++ b/crates/capi/Cargo.toml @@ -13,7 +13,7 @@ crate-type = ["staticlib"] [dependencies] num-traits = { workspace = true } -rustpython-vm = { workspace = true } +rustpython-vm = { workspace = true, features = ["threading"]} [lints] workspace = true diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs index 8af4ed9e51e..efaa0fc2cfc 100644 --- a/crates/capi/src/longobject.rs +++ b/crates/capi/src/longobject.rs @@ -2,7 +2,7 @@ use core::ffi::{c_long, c_longlong, c_ulong, c_ulonglong}; use core::ptr; use crate::PyObject; -use crate::pylifecycle::with_vm; +use crate::pystate::with_vm; use rustpython_vm::PyObjectRef; use rustpython_vm::builtins::PyInt; diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index d1a27ec4381..2a6277ef95a 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -1,24 +1,26 @@ +use crate::log_stub; use core::ffi::c_int; use rustpython_vm::Interpreter; -use rustpython_vm::VirtualMachine; -use std::cell::RefCell; -use std::mem::ManuallyDrop; +use rustpython_vm::vm::thread::ThreadedVirtualMachine; +use std::sync::{Once, OnceLock, mpsc}; -thread_local! { - pub static INTERP: RefCell>> = const { RefCell::new(None) }; -} +static VM_REQUEST_TX: OnceLock>> = + OnceLock::new(); +static INITIALIZED: Once = Once::new(); -pub(crate) fn with_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { - INTERP.with(|interp_ref| { - let interp = interp_ref.borrow(); - let interp = interp.as_ref().expect("VM access before Py_InitializeEx"); - interp.enter(f) - }) +/// Request a vm from the main interpreter +pub(crate) fn request_vm_from_interpreter() -> ThreadedVirtualMachine { + let tx = VM_REQUEST_TX + .get() + .expect("VM request channel not initialized"); + let (response_tx, response_rx) = mpsc::channel(); + tx.send(response_tx).expect("Failed to send VM request"); + response_rx.recv().expect("Failed to receive VM response") } #[unsafe(no_mangle)] pub extern "C" fn Py_IsInitialized() -> c_int { - INTERP.with(|interp| interp.borrow().is_some() as c_int) + INITIALIZED.is_completed() as _ } #[unsafe(no_mangle)] @@ -28,13 +30,25 @@ pub extern "C" fn Py_Initialize() { #[unsafe(no_mangle)] pub extern "C" fn Py_InitializeEx(_initsigs: c_int) { - if INTERP.with(|interp| interp.borrow().is_none()) { - let interp = Interpreter::with_init(Default::default(), |_vm| {}); + if INITIALIZED.is_completed() { + panic!("Initialize called multiple times"); + } + + INITIALIZED.call_once(|| { + let (tx, rx) = mpsc::channel(); + VM_REQUEST_TX.set(tx).expect("VM request channel was already initialized"); - INTERP.with(|interp_ref| { - *interp_ref.borrow_mut() = Some(ManuallyDrop::new(interp)); + std::thread::spawn(move || { + let interp = Interpreter::with_init(Default::default(), |_vm| {}); + interp.enter(|vm| { + while let Ok(request) = rx.recv() { + request + .send(vm.new_thread()) + .expect("Failed to send VM response"); + } + }) }); - } + }); } #[unsafe(no_mangle)] @@ -44,15 +58,8 @@ pub extern "C" fn Py_Finalize() { #[unsafe(no_mangle)] pub extern "C" fn Py_FinalizeEx() -> c_int { - INTERP.with(|interp_ref| { - let interp = ManuallyDrop::into_inner( - interp_ref - .borrow_mut() - .take() - .expect("Py_FinalizeEx called without an active interpreter"), - ); - interp.finalize(None) - }) as _ + log_stub("Py_FinalizeEx"); + 0 } #[unsafe(no_mangle)] diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs index 9879348f8a7..bedcc610dd8 100644 --- a/crates/capi/src/pystate.rs +++ b/crates/capi/src/pystate.rs @@ -1,6 +1,23 @@ -use crate::log_stub; +use crate::pylifecycle::request_vm_from_interpreter; use core::ffi::c_int; use core::ptr; +use rustpython_vm::VirtualMachine; +use rustpython_vm::vm::thread::ThreadedVirtualMachine; +use std::cell::RefCell; + +thread_local! { + static VM: RefCell> = const { RefCell::new(None) }; +} + +pub fn with_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { + VM.with(|vm_ref| { + let vm = vm_ref.borrow(); + let vm = vm + .as_ref() + .expect("Thread was not attached to an interpreter"); + vm.run(f) + }) +} #[allow(non_camel_case_types)] type PyGILState_STATE = c_int; @@ -12,22 +29,23 @@ pub struct PyThreadState { #[unsafe(no_mangle)] pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE { - log_stub("PyGILState_Ensure"); + VM.with(|vm| { + vm.borrow_mut() + .get_or_insert_with(|| request_vm_from_interpreter()); + }); + 0 } #[unsafe(no_mangle)] pub extern "C" fn PyGILState_Release(_state: PyGILState_STATE) { - log_stub("PyGILState_Release"); } #[unsafe(no_mangle)] pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState { - log_stub("PyEval_SaveThread"); ptr::null_mut() } #[unsafe(no_mangle)] pub extern "C" fn PyEval_RestoreThread(_tstate: *mut PyThreadState) { - log_stub("PyEval_RestoreThread"); } diff --git a/crates/capi/src/unicodeobject.rs b/crates/capi/src/unicodeobject.rs index 4a241f6c91f..27240521110 100644 --- a/crates/capi/src/unicodeobject.rs +++ b/crates/capi/src/unicodeobject.rs @@ -4,7 +4,7 @@ use core::slice; use core::str; use crate::PyObject; -use crate::pylifecycle::with_vm; +use crate::pystate::with_vm; use rustpython_vm::PyObjectRef; #[unsafe(no_mangle)] diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index ca79a80340a..3ba2c549411 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -11,5 +11,13 @@ fn main() { let string = PyString::new(py, "Hello, World!"); assert!(string.is_instance_of::()); + + let number = number.unbind(); + std::thread::spawn(move || { + Python::attach(|py| { + let number = number.bind(py); + assert!(number.is_instance_of::()); + }); + }).join().unwrap(); }); } From 565bf5ebfbc8c56ad5a450832fdbd85b997c41ac Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 14:07:16 +0200 Subject: [PATCH 11/24] Cleanup --- Cargo.lock | 1 - crates/capi/Cargo.toml | 1 - example_projects/pyo3_embed/Cargo.toml | 2 +- example_projects/pyo3_embed/build.rs | 6 ------ example_projects/pyo3_embed/pyo3-rustpython.config | 3 +++ 5 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 example_projects/pyo3_embed/build.rs diff --git a/Cargo.lock b/Cargo.lock index 5d534fb6651..1eca00b9fbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3108,7 +3108,6 @@ dependencies = [ name = "rustpython-capi" version = "0.5.0" dependencies = [ - "num-traits", "rustpython-vm", ] diff --git a/crates/capi/Cargo.toml b/crates/capi/Cargo.toml index 5a55536902b..964a996143a 100644 --- a/crates/capi/Cargo.toml +++ b/crates/capi/Cargo.toml @@ -12,7 +12,6 @@ license.workspace = true crate-type = ["staticlib"] [dependencies] -num-traits = { workspace = true } rustpython-vm = { workspace = true, features = ["threading"]} [lints] diff --git a/example_projects/pyo3_embed/Cargo.toml b/example_projects/pyo3_embed/Cargo.toml index 6174170c143..d424a486184 100644 --- a/example_projects/pyo3_embed/Cargo.toml +++ b/example_projects/pyo3_embed/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -pyo3 = { path = "../../../pyo3", default-features = false, features = ["abi3-py314"] } +pyo3 = { version = "0.28.3", features = ["abi3-py314"] } rustpython-capi = { path = "../../crates/capi" } [workspace] diff --git a/example_projects/pyo3_embed/build.rs b/example_projects/pyo3_embed/build.rs deleted file mode 100644 index 8d379be42c0..00000000000 --- a/example_projects/pyo3_embed/build.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn main() { - println!( - "cargo:rustc-link-arg=/Users/basschoenmaeckers/repo/RustPython/target/debug/librustpython_capi.a" - ); - println!("cargo:rustc-link-lib=framework=CoreFoundation"); -} diff --git a/example_projects/pyo3_embed/pyo3-rustpython.config b/example_projects/pyo3_embed/pyo3-rustpython.config index 2cf06cd886c..2b097508e4f 100644 --- a/example_projects/pyo3_embed/pyo3-rustpython.config +++ b/example_projects/pyo3_embed/pyo3-rustpython.config @@ -4,3 +4,6 @@ shared=false abi3=true build_flags= suppress_build_script_link_lines=true +extra_build_script_line=cargo:rustc-link-search=native=/Users/basschoenmaeckers/repo/RustPython/target/debug +extra_build_script_line=cargo:rustc-link-lib=static=rustpython_capi +extra_build_script_line=cargo:rustc-link-lib=framework=CoreFoundation From 85b292811971ee6bbda46049d219ff4b374d99d5 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Sun, 5 Apr 2026 14:53:57 +0200 Subject: [PATCH 12/24] Attach thread in `Py_InitializeEx` --- crates/capi/src/pylifecycle.rs | 7 ++++++- crates/capi/src/pystate.rs | 14 ++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index 2a6277ef95a..d8b3fc2ae41 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -1,4 +1,5 @@ use crate::log_stub; +use crate::pystate::attach_vm_to_thread; use core::ffi::c_int; use rustpython_vm::Interpreter; use rustpython_vm::vm::thread::ThreadedVirtualMachine; @@ -36,7 +37,9 @@ pub extern "C" fn Py_InitializeEx(_initsigs: c_int) { INITIALIZED.call_once(|| { let (tx, rx) = mpsc::channel(); - VM_REQUEST_TX.set(tx).expect("VM request channel was already initialized"); + VM_REQUEST_TX + .set(tx) + .expect("VM request channel was already initialized"); std::thread::spawn(move || { let interp = Interpreter::with_init(Default::default(), |_vm| {}); @@ -49,6 +52,8 @@ pub extern "C" fn Py_InitializeEx(_initsigs: c_int) { }) }); }); + + attach_vm_to_thread(); } #[unsafe(no_mangle)] diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs index bedcc610dd8..de25dba790a 100644 --- a/crates/capi/src/pystate.rs +++ b/crates/capi/src/pystate.rs @@ -27,19 +27,22 @@ pub struct PyThreadState { _interp: *mut std::ffi::c_void, } -#[unsafe(no_mangle)] -pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE { +pub(crate) fn attach_vm_to_thread() { VM.with(|vm| { vm.borrow_mut() .get_or_insert_with(|| request_vm_from_interpreter()); }); +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE { + attach_vm_to_thread(); 0 } #[unsafe(no_mangle)] -pub extern "C" fn PyGILState_Release(_state: PyGILState_STATE) { -} +pub extern "C" fn PyGILState_Release(_state: PyGILState_STATE) {} #[unsafe(no_mangle)] pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState { @@ -47,5 +50,4 @@ pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState { } #[unsafe(no_mangle)] -pub extern "C" fn PyEval_RestoreThread(_tstate: *mut PyThreadState) { -} +pub extern "C" fn PyEval_RestoreThread(_tstate: *mut PyThreadState) {} From c603d37cd849691f8432f7a123170989a1a4c3fa Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Tue, 7 Apr 2026 10:55:29 +0200 Subject: [PATCH 13/24] Map RustPython type flags to CPython type flags --- crates/capi/src/object.rs | 98 +++++++++++++++++++--------------- crates/vm/src/builtins/type.rs | 2 +- 2 files changed, 55 insertions(+), 45 deletions(-) diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index f041d2774e3..5a765fa409e 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -1,73 +1,53 @@ -use core::ffi::c_ulong; - use crate::PyObject; +use core::ffi::c_ulong; use rustpython_vm::builtins::PyType; use rustpython_vm::{AsObject, Context, Py}; use std::sync::LazyLock; pub struct PyTypeObject { ty: LazyLock<&'static Py>, - flags: c_ulong, } impl PyTypeObject { - const fn new(f: fn() -> &'static Py, flags: c_ulong) -> PyTypeObject { + const fn new(f: fn() -> &'static Py) -> PyTypeObject { PyTypeObject { ty: LazyLock::new(f), - flags, } } } -const PY_TPFLAGS_HAVE_STACKLESS_EXTENSION: c_ulong = 0; -const PY_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; -const PY_TPFLAGS_DEFAULT: c_ulong = - PY_TPFLAGS_HAVE_STACKLESS_EXTENSION | PY_TPFLAGS_HAVE_VERSION_TAG; -const PY_TPFLAGS_IMMUTABLETYPE: c_ulong = 1 << 8; -const PY_TPFLAGS_BASETYPE: c_ulong = 1 << 10; const PY_TPFLAGS_LONG_SUBCLASS: c_ulong = 1 << 24; +const PY_TPFLAGS_LIST_SUBCLASS: c_ulong = 1 << 25; const PY_TPFLAGS_TUPLE_SUBCLASS: c_ulong = 1 << 26; +const PY_TPFLAGS_BYTES_SUBCLASS: c_ulong = 1 << 27; const PY_TPFLAGS_UNICODE_SUBCLASS: c_ulong = 1 << 28; +const PY_TPFLAGS_DICT_SUBCLASS: c_ulong = 1 << 29; +const PY_TPFLAGS_BASE_EXC_SUBCLASS: c_ulong = 1 << 30; const PY_TPFLAGS_TYPE_SUBCLASS: c_ulong = 1 << 31; #[unsafe(no_mangle)] -pub static mut PyType_Type: PyTypeObject = PyTypeObject::new( - || { - let zoo = &Context::genesis().types; - zoo.type_type - }, - PY_TPFLAGS_DEFAULT | PY_TPFLAGS_IMMUTABLETYPE | PY_TPFLAGS_BASETYPE | PY_TPFLAGS_TYPE_SUBCLASS, -); +pub static mut PyType_Type: PyTypeObject = PyTypeObject::new(|| { + let zoo = &Context::genesis().types; + zoo.type_type +}); #[unsafe(no_mangle)] -pub static mut PyLong_Type: PyTypeObject = PyTypeObject::new( - || { - let zoo = &Context::genesis().types; - zoo.int_type - }, - PY_TPFLAGS_DEFAULT | PY_TPFLAGS_IMMUTABLETYPE | PY_TPFLAGS_BASETYPE | PY_TPFLAGS_LONG_SUBCLASS, -); +pub static mut PyLong_Type: PyTypeObject = PyTypeObject::new(|| { + let zoo = &Context::genesis().types; + zoo.int_type +}); #[unsafe(no_mangle)] -pub static mut PyTuple_Type: PyTypeObject = PyTypeObject::new( - || { - let zoo = &Context::genesis().types; - zoo.tuple_type - }, - PY_TPFLAGS_DEFAULT | PY_TPFLAGS_IMMUTABLETYPE | PY_TPFLAGS_BASETYPE | PY_TPFLAGS_TUPLE_SUBCLASS, -); +pub static mut PyTuple_Type: PyTypeObject = PyTypeObject::new(|| { + let zoo = &Context::genesis().types; + zoo.tuple_type +}); #[unsafe(no_mangle)] -pub static mut PyUnicode_Type: PyTypeObject = PyTypeObject::new( - || { - let zoo = &Context::genesis().types; - zoo.str_type - }, - PY_TPFLAGS_DEFAULT - | PY_TPFLAGS_IMMUTABLETYPE - | PY_TPFLAGS_BASETYPE - | PY_TPFLAGS_UNICODE_SUBCLASS, -); +pub static mut PyUnicode_Type: PyTypeObject = PyTypeObject::new(|| { + let zoo = &Context::genesis().types; + zoo.str_type +}); #[unsafe(no_mangle)] pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { @@ -98,8 +78,38 @@ pub extern "C" fn PyType_GetFlags(ty: *mut PyTypeObject) -> c_ulong { panic!("PyType_GetFlags called with null type pointer"); } - // SAFETY: caller guarantees this is a valid exported type object pointer. - unsafe { (*ty).flags } + let ctx =Context::genesis(); + let zoo = &ctx.types; + let exp_zoo = &ctx.exceptions; + let ty_inner = unsafe { *(*ty).ty }; + let mut flags = ty_inner.slots.flags.bits(); + + if ty_inner.is_subtype(zoo.int_type) { + flags |= PY_TPFLAGS_LONG_SUBCLASS; + } + if ty_inner.is_subtype(zoo.list_type) { + flags |= PY_TPFLAGS_LIST_SUBCLASS + } + if ty_inner.is_subtype(zoo.tuple_type) { + flags |= PY_TPFLAGS_TUPLE_SUBCLASS; + } + if ty_inner.is_subtype(zoo.bytes_type) { + flags |= PY_TPFLAGS_BYTES_SUBCLASS; + } + if ty_inner.is_subtype(zoo.str_type) { + flags |= PY_TPFLAGS_UNICODE_SUBCLASS; + } + if ty_inner.is_subtype(zoo.dict_type) { + flags |= PY_TPFLAGS_DICT_SUBCLASS; + } + if ty_inner.is_subtype(exp_zoo.base_exception_type) { + flags |= PY_TPFLAGS_BASE_EXC_SUBCLASS; + } + if ty_inner.is_subtype(zoo.type_type) { + flags |= PY_TPFLAGS_TYPE_SUBCLASS; + } + + flags } #[unsafe(no_mangle)] diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 64801a1de5c..652c4db1d8b 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -1240,7 +1240,7 @@ impl PyType { } impl Py { - pub(crate) fn is_subtype(&self, other: &Self) -> bool { + pub fn is_subtype(&self, other: &Self) -> bool { is_subtype_with_mro(&self.mro.read(), self, other) } From cee9403330ac2cf1706f910dbdc2b1b4988c6e8a Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Tue, 7 Apr 2026 15:47:04 +0200 Subject: [PATCH 14/24] Use `*const Py` instead of `*mut PyTypeObject` --- crates/capi/src/object.rs | 107 +++++++++++++-------------------- crates/capi/src/pylifecycle.rs | 5 +- 2 files changed, 46 insertions(+), 66 deletions(-) diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index 5a765fa409e..fdf5d4caefc 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -1,20 +1,9 @@ use crate::PyObject; use core::ffi::c_ulong; +use std::mem::MaybeUninit; use rustpython_vm::builtins::PyType; -use rustpython_vm::{AsObject, Context, Py}; -use std::sync::LazyLock; - -pub struct PyTypeObject { - ty: LazyLock<&'static Py>, -} - -impl PyTypeObject { - const fn new(f: fn() -> &'static Py) -> PyTypeObject { - PyTypeObject { - ty: LazyLock::new(f), - } - } -} +use rustpython_vm::{Context, Py}; +use crate::pylifecycle::INITIALIZED; const PY_TPFLAGS_LONG_SUBCLASS: c_ulong = 1 << 24; const PY_TPFLAGS_LIST_SUBCLASS: c_ulong = 1 << 25; @@ -26,86 +15,74 @@ const PY_TPFLAGS_BASE_EXC_SUBCLASS: c_ulong = 1 << 30; const PY_TPFLAGS_TYPE_SUBCLASS: c_ulong = 1 << 31; #[unsafe(no_mangle)] -pub static mut PyType_Type: PyTypeObject = PyTypeObject::new(|| { - let zoo = &Context::genesis().types; - zoo.type_type -}); +pub static mut PyType_Type: MaybeUninit<&'static Py> = MaybeUninit::uninit(); #[unsafe(no_mangle)] -pub static mut PyLong_Type: PyTypeObject = PyTypeObject::new(|| { - let zoo = &Context::genesis().types; - zoo.int_type -}); +pub static mut PyLong_Type: MaybeUninit<&'static Py> = MaybeUninit::uninit(); #[unsafe(no_mangle)] -pub static mut PyTuple_Type: PyTypeObject = PyTypeObject::new(|| { - let zoo = &Context::genesis().types; - zoo.tuple_type -}); +pub static mut PyTuple_Type: MaybeUninit<&'static Py> = MaybeUninit::uninit(); #[unsafe(no_mangle)] -pub static mut PyUnicode_Type: PyTypeObject = PyTypeObject::new(|| { - let zoo = &Context::genesis().types; - zoo.str_type -}); +pub static mut PyUnicode_Type: MaybeUninit<&'static Py> = MaybeUninit::uninit(); -#[unsafe(no_mangle)] -pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *mut PyTypeObject { - if op.is_null() { - return std::ptr::null_mut(); - } +/// Initialize the static type pointers. This should be called once during interpreter initialization, +/// and before any of the static type pointers are used. +/// +/// Panics: +/// Panics when the interpreter is already initialized. +#[allow(static_mut_refs)] +pub(crate) fn init_static_type_pointers() { + assert!(!INITIALIZED.is_completed(), "Python already initialized, we should not touch the static type pointers"); + let zoo = &Context::genesis().types; unsafe { - let ty = (*op).class(); - if ty.is(*PyType_Type.ty) { - &raw mut PyType_Type - } else if ty.is(*PyLong_Type.ty) { - &raw mut PyLong_Type - } else if ty.is(*PyTuple_Type.ty) { - &raw mut PyTuple_Type - } else if ty.is(*PyUnicode_Type.ty) { - &raw mut PyUnicode_Type - } else { - todo!("Unsupported type: {:?}", ty.name()); - } - } + PyType_Type.write(zoo.type_type); + PyLong_Type.write(zoo.int_type); + PyTuple_Type.write(zoo.tuple_type); + PyUnicode_Type.write(zoo.str_type); + }; } #[unsafe(no_mangle)] -#[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn PyType_GetFlags(ty: *mut PyTypeObject) -> c_ulong { - if ty.is_null() { - panic!("PyType_GetFlags called with null type pointer"); - } +pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *const Py { + // SAFETY: The caller must guarantee that `op` is a valid pointer to a `PyObject`. + unsafe { (*op).class() } +} +#[unsafe(no_mangle)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub extern "C" fn PyType_GetFlags(ptr: *const Py) -> c_ulong { let ctx =Context::genesis(); let zoo = &ctx.types; let exp_zoo = &ctx.exceptions; - let ty_inner = unsafe { *(*ty).ty }; - let mut flags = ty_inner.slots.flags.bits(); - if ty_inner.is_subtype(zoo.int_type) { + // SAFETY: The caller must guarantee that `ptr` is a valid pointer to a `PyType` object. + let ty = unsafe { &*ptr}; + let mut flags = ty.slots.flags.bits(); + + if ty.is_subtype(zoo.int_type) { flags |= PY_TPFLAGS_LONG_SUBCLASS; } - if ty_inner.is_subtype(zoo.list_type) { + if ty.is_subtype(zoo.list_type) { flags |= PY_TPFLAGS_LIST_SUBCLASS } - if ty_inner.is_subtype(zoo.tuple_type) { + if ty.is_subtype(zoo.tuple_type) { flags |= PY_TPFLAGS_TUPLE_SUBCLASS; } - if ty_inner.is_subtype(zoo.bytes_type) { + if ty.is_subtype(zoo.bytes_type) { flags |= PY_TPFLAGS_BYTES_SUBCLASS; } - if ty_inner.is_subtype(zoo.str_type) { + if ty.is_subtype(zoo.str_type) { flags |= PY_TPFLAGS_UNICODE_SUBCLASS; } - if ty_inner.is_subtype(zoo.dict_type) { + if ty.is_subtype(zoo.dict_type) { flags |= PY_TPFLAGS_DICT_SUBCLASS; } - if ty_inner.is_subtype(exp_zoo.base_exception_type) { + if ty.is_subtype(exp_zoo.base_exception_type) { flags |= PY_TPFLAGS_BASE_EXC_SUBCLASS; } - if ty_inner.is_subtype(zoo.type_type) { + if ty.is_subtype(zoo.type_type) { flags |= PY_TPFLAGS_TYPE_SUBCLASS; } @@ -113,13 +90,13 @@ pub extern "C" fn PyType_GetFlags(ty: *mut PyTypeObject) -> c_ulong { } #[unsafe(no_mangle)] -pub extern "C" fn PyType_GetName(_ty: *mut PyTypeObject) -> *mut PyObject { +pub extern "C" fn PyType_GetName(_ptr: *const Py) -> *mut PyObject { crate::log_stub("PyType_GetName"); std::ptr::null_mut() } #[unsafe(no_mangle)] -pub extern "C" fn PyType_GetQualName(_ty: *mut PyTypeObject) -> *mut PyObject { +pub extern "C" fn PyType_GetQualName(_ptr: *const Py) -> *mut PyObject { crate::log_stub("PyType_GetQualName"); std::ptr::null_mut() } diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index d8b3fc2ae41..5b3c25a0e8c 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -4,10 +4,11 @@ use core::ffi::c_int; use rustpython_vm::Interpreter; use rustpython_vm::vm::thread::ThreadedVirtualMachine; use std::sync::{Once, OnceLock, mpsc}; +use crate::object::init_static_type_pointers; static VM_REQUEST_TX: OnceLock>> = OnceLock::new(); -static INITIALIZED: Once = Once::new(); +pub(crate) static INITIALIZED: Once = Once::new(); /// Request a vm from the main interpreter pub(crate) fn request_vm_from_interpreter() -> ThreadedVirtualMachine { @@ -36,6 +37,8 @@ pub extern "C" fn Py_InitializeEx(_initsigs: c_int) { } INITIALIZED.call_once(|| { + init_static_type_pointers(); + let (tx, rx) = mpsc::channel(); VM_REQUEST_TX .set(tx) From 0a50fccdafea8b3004e690d1e4960fd0da17ce7e Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Tue, 7 Apr 2026 16:18:43 +0200 Subject: [PATCH 15/24] Implement `PyUnicode_AsUTF8AndSize` --- crates/capi/src/unicodeobject.rs | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/crates/capi/src/unicodeobject.rs b/crates/capi/src/unicodeobject.rs index 27240521110..ee27e9d505c 100644 --- a/crates/capi/src/unicodeobject.rs +++ b/crates/capi/src/unicodeobject.rs @@ -1,11 +1,11 @@ +use crate::PyObject; +use crate::pystate::with_vm; use core::ffi::c_char; use core::ptr; use core::slice; use core::str; - -use crate::PyObject; -use crate::pystate::with_vm; use rustpython_vm::PyObjectRef; +use rustpython_vm::builtins::PyStr; #[unsafe(no_mangle)] pub extern "C" fn PyUnicode_FromStringAndSize(s: *const c_char, len: isize) -> *mut PyObject { @@ -28,12 +28,26 @@ pub extern "C" fn PyUnicode_FromStringAndSize(s: *const c_char, len: isize) -> * } #[unsafe(no_mangle)] -pub extern "C" fn PyUnicode_AsUTF8AndSize( - _unicode: *mut PyObject, - _size: *mut isize, -) -> *const c_char { - crate::log_stub("PyUnicode_AsUTF8AndSize"); - ptr::null() +pub extern "C" fn PyUnicode_AsUTF8AndSize(obj: *mut PyObject, size: *mut isize) -> *const c_char { + with_vm(|vm| { + let obj = unsafe { + obj.as_ref() + .expect("PyUnicode_AsUTF8AndSize called with null pointer") + }; + + let unicode = obj + .downcast_ref::() + .expect("PyUnicode_AsUTF8AndSize called with non-unicode object"); + + let str = unicode + .to_str() + .expect("only utf8 or ascii is currently supported in PyUnicode_AsUTF8AndSize"); + + if !size.is_null() { + unsafe { *size = str.len() as isize }; + } + str.as_ptr() as _ + }) } #[unsafe(no_mangle)] From f588979932a93d104825bb1eb1dcab249fb62f05 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Tue, 7 Apr 2026 16:19:11 +0200 Subject: [PATCH 16/24] Implement `PyType_GetName` --- crates/capi/src/object.rs | 28 ++++++++++++++----------- crates/capi/src/pylifecycle.rs | 2 +- example_projects/pyo3_embed/src/main.rs | 14 ++++++++++--- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index fdf5d4caefc..73d0005365e 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -1,9 +1,11 @@ use crate::PyObject; +use crate::pylifecycle::INITIALIZED; +use crate::pystate::with_vm; use core::ffi::c_ulong; -use std::mem::MaybeUninit; use rustpython_vm::builtins::PyType; +use rustpython_vm::convert::IntoObject; use rustpython_vm::{Context, Py}; -use crate::pylifecycle::INITIALIZED; +use std::mem::MaybeUninit; const PY_TPFLAGS_LONG_SUBCLASS: c_ulong = 1 << 24; const PY_TPFLAGS_LIST_SUBCLASS: c_ulong = 1 << 25; @@ -26,7 +28,6 @@ pub static mut PyTuple_Type: MaybeUninit<&'static Py> = MaybeUninit::uni #[unsafe(no_mangle)] pub static mut PyUnicode_Type: MaybeUninit<&'static Py> = MaybeUninit::uninit(); - /// Initialize the static type pointers. This should be called once during interpreter initialization, /// and before any of the static type pointers are used. /// @@ -34,7 +35,10 @@ pub static mut PyUnicode_Type: MaybeUninit<&'static Py> = MaybeUninit::u /// Panics when the interpreter is already initialized. #[allow(static_mut_refs)] pub(crate) fn init_static_type_pointers() { - assert!(!INITIALIZED.is_completed(), "Python already initialized, we should not touch the static type pointers"); + assert!( + !INITIALIZED.is_completed(), + "Python already initialized, we should not touch the static type pointers" + ); let zoo = &Context::genesis().types; unsafe { PyType_Type.write(zoo.type_type); @@ -53,12 +57,12 @@ pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *const Py { #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn PyType_GetFlags(ptr: *const Py) -> c_ulong { - let ctx =Context::genesis(); + let ctx = Context::genesis(); let zoo = &ctx.types; let exp_zoo = &ctx.exceptions; // SAFETY: The caller must guarantee that `ptr` is a valid pointer to a `PyType` object. - let ty = unsafe { &*ptr}; + let ty = unsafe { &*ptr }; let mut flags = ty.slots.flags.bits(); if ty.is_subtype(zoo.int_type) { @@ -90,15 +94,15 @@ pub extern "C" fn PyType_GetFlags(ptr: *const Py) -> c_ulong { } #[unsafe(no_mangle)] -pub extern "C" fn PyType_GetName(_ptr: *const Py) -> *mut PyObject { - crate::log_stub("PyType_GetName"); - std::ptr::null_mut() +pub extern "C" fn PyType_GetName(ptr: *const Py) -> *mut PyObject { + let ty = unsafe { &*ptr }; + with_vm(move |vm| ty.__name__(vm).into_object().into_raw().as_ptr()) } #[unsafe(no_mangle)] -pub extern "C" fn PyType_GetQualName(_ptr: *const Py) -> *mut PyObject { - crate::log_stub("PyType_GetQualName"); - std::ptr::null_mut() +pub extern "C" fn PyType_GetQualName(ptr: *const Py) -> *mut PyObject { + let ty = unsafe { &*ptr }; + with_vm(move |vm| ty.__qualname__(vm).into_object().into_raw().as_ptr()) } #[unsafe(no_mangle)] diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index 5b3c25a0e8c..a25f8dc0e27 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -1,10 +1,10 @@ use crate::log_stub; +use crate::object::init_static_type_pointers; use crate::pystate::attach_vm_to_thread; use core::ffi::c_int; use rustpython_vm::Interpreter; use rustpython_vm::vm::thread::ThreadedVirtualMachine; use std::sync::{Once, OnceLock, mpsc}; -use crate::object::init_static_type_pointers; static VM_REQUEST_TX: OnceLock>> = OnceLock::new(); diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index 3ba2c549411..cc9eef902fc 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -7,10 +7,13 @@ fn main() { Python::attach(|py| { let number = PyInt::new(py, 123); assert!(number.is_instance_of::()); - assert_eq!(number.extract::().unwrap(), 123); + assert_eq!(number.extract::()?, 123); let string = PyString::new(py, "Hello, World!"); assert!(string.is_instance_of::()); + assert_eq!(string.to_str()?, "Hello, World!"); + + assert_eq!(string.get_type().name()?.to_str()?, "str"); let number = number.unbind(); std::thread::spawn(move || { @@ -18,6 +21,11 @@ fn main() { let number = number.bind(py); assert!(number.is_instance_of::()); }); - }).join().unwrap(); - }); + }) + .join() + .unwrap(); + + PyResult::Ok(()) + }) + .unwrap(); } From edaca0ca799f1549a9411f4a5ab60671e9a5816f Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Tue, 7 Apr 2026 16:44:54 +0200 Subject: [PATCH 17/24] Implement alternative `PyLong` constructors --- crates/capi/src/longobject.rs | 44 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs index efaa0fc2cfc..a7a799d1b9e 100644 --- a/crates/capi/src/longobject.rs +++ b/crates/capi/src/longobject.rs @@ -2,46 +2,39 @@ use core::ffi::{c_long, c_longlong, c_ulong, c_ulonglong}; use core::ptr; use crate::PyObject; +use crate::pyerrors::{PyErr_SetString, PyExc_OverflowError}; use crate::pystate::with_vm; -use rustpython_vm::PyObjectRef; use rustpython_vm::builtins::PyInt; +use rustpython_vm::convert::IntoObject; #[unsafe(no_mangle)] pub extern "C" fn PyLong_FromLong(value: c_long) -> *mut PyObject { - with_vm(|vm| { - let obj: PyObjectRef = vm.ctx.new_int(value).into(); - obj.into_raw().as_ptr() - }) + with_vm(|vm| vm.ctx.new_int(value).into_object().into_raw().as_ptr()) } #[unsafe(no_mangle)] -pub extern "C" fn PyLong_FromLongLong(_value: c_longlong) -> *mut PyObject { - crate::log_stub("PyLong_FromLongLong"); - ptr::null_mut() +pub extern "C" fn PyLong_FromLongLong(value: c_longlong) -> *mut PyObject { + with_vm(|vm| vm.ctx.new_int(value).into_object().into_raw().as_ptr()) } #[unsafe(no_mangle)] -pub extern "C" fn PyLong_FromSsize_t(_value: isize) -> *mut PyObject { - crate::log_stub("PyLong_FromSsize_t"); - ptr::null_mut() +pub extern "C" fn PyLong_FromSsize_t(value: isize) -> *mut PyObject { + with_vm(|vm| vm.ctx.new_int(value).into_object().into_raw().as_ptr()) } #[unsafe(no_mangle)] -pub extern "C" fn PyLong_FromSize_t(_value: usize) -> *mut PyObject { - crate::log_stub("PyLong_FromSize_t"); - ptr::null_mut() +pub extern "C" fn PyLong_FromSize_t(value: usize) -> *mut PyObject { + with_vm(|vm| vm.ctx.new_int(value).into_object().into_raw().as_ptr()) } #[unsafe(no_mangle)] -pub extern "C" fn PyLong_FromUnsignedLong(_value: c_ulong) -> *mut PyObject { - crate::log_stub("PyLong_FromUnsignedLong"); - ptr::null_mut() +pub extern "C" fn PyLong_FromUnsignedLong(value: c_ulong) -> *mut PyObject { + with_vm(|vm| vm.ctx.new_int(value).into_object().into_raw().as_ptr()) } #[unsafe(no_mangle)] -pub extern "C" fn PyLong_FromUnsignedLongLong(_value: c_ulonglong) -> *mut PyObject { - crate::log_stub("PyLong_FromUnsignedLongLong"); - ptr::null_mut() +pub extern "C" fn PyLong_FromUnsignedLongLong(value: c_ulonglong) -> *mut PyObject { + with_vm(|vm| vm.ctx.new_int(value).into_object().into_raw().as_ptr()) } #[unsafe(no_mangle)] @@ -57,9 +50,12 @@ pub extern "C" fn PyLong_AsLong(obj: *mut PyObject) -> c_long { .downcast_ref::() .expect("PyLong_AsLong currently only accepts int instances"); - int_obj - .as_bigint() - .try_into() - .expect("PyLong_AsLong: value out of range for c_long") + int_obj.as_bigint().try_into().unwrap_or_else(|_| unsafe { + PyErr_SetString( + PyExc_OverflowError, + c"Python int too large to convert to C long".as_ptr(), + ); + -1 + }) }) } From e4c8bd74f2fbdc19e32c0061cfe219a4056b2c87 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Tue, 7 Apr 2026 19:37:04 +0200 Subject: [PATCH 18/24] Move `init_static_type_pointers` to `pylifecycle.rs` --- crates/capi/src/longobject.rs | 1 - crates/capi/src/object.rs | 21 --------------------- crates/capi/src/pylifecycle.rs | 26 ++++++++++++++++++++++++-- crates/capi/src/unicodeobject.rs | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs index a7a799d1b9e..1a876712452 100644 --- a/crates/capi/src/longobject.rs +++ b/crates/capi/src/longobject.rs @@ -1,5 +1,4 @@ use core::ffi::{c_long, c_longlong, c_ulong, c_ulonglong}; -use core::ptr; use crate::PyObject; use crate::pyerrors::{PyErr_SetString, PyExc_OverflowError}; diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index 73d0005365e..c3dfa1cc268 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -1,5 +1,4 @@ use crate::PyObject; -use crate::pylifecycle::INITIALIZED; use crate::pystate::with_vm; use core::ffi::c_ulong; use rustpython_vm::builtins::PyType; @@ -28,26 +27,6 @@ pub static mut PyTuple_Type: MaybeUninit<&'static Py> = MaybeUninit::uni #[unsafe(no_mangle)] pub static mut PyUnicode_Type: MaybeUninit<&'static Py> = MaybeUninit::uninit(); -/// Initialize the static type pointers. This should be called once during interpreter initialization, -/// and before any of the static type pointers are used. -/// -/// Panics: -/// Panics when the interpreter is already initialized. -#[allow(static_mut_refs)] -pub(crate) fn init_static_type_pointers() { - assert!( - !INITIALIZED.is_completed(), - "Python already initialized, we should not touch the static type pointers" - ); - let zoo = &Context::genesis().types; - unsafe { - PyType_Type.write(zoo.type_type); - PyLong_Type.write(zoo.int_type); - PyTuple_Type.write(zoo.tuple_type); - PyUnicode_Type.write(zoo.str_type); - }; -} - #[unsafe(no_mangle)] pub extern "C" fn Py_TYPE(op: *mut PyObject) -> *const Py { // SAFETY: The caller must guarantee that `op` is a valid pointer to a `PyObject`. diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index a25f8dc0e27..b7918751635 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -1,9 +1,9 @@ use crate::log_stub; -use crate::object::init_static_type_pointers; +use crate::object::{PyLong_Type, PyTuple_Type, PyType_Type, PyUnicode_Type}; use crate::pystate::attach_vm_to_thread; use core::ffi::c_int; -use rustpython_vm::Interpreter; use rustpython_vm::vm::thread::ThreadedVirtualMachine; +use rustpython_vm::{Context, Interpreter}; use std::sync::{Once, OnceLock, mpsc}; static VM_REQUEST_TX: OnceLock>> = @@ -20,6 +20,28 @@ pub(crate) fn request_vm_from_interpreter() -> ThreadedVirtualMachine { response_rx.recv().expect("Failed to receive VM response") } +/// Initialize the static type pointers. This should be called once during interpreter initialization, +/// and before any of the static type pointers are used. +/// +/// Panics: +/// Panics when the interpreter is already initialized. +#[allow(static_mut_refs)] +pub(crate) fn init_static_type_pointers() { + assert!( + !INITIALIZED.is_completed(), + "Python already initialized, we should not touch the static type pointers" + ); + let context = Context::genesis(); + let types = &context.types; + + unsafe { + PyType_Type.write(types.type_type); + PyLong_Type.write(types.int_type); + PyTuple_Type.write(types.tuple_type); + PyUnicode_Type.write(types.str_type); + }; +} + #[unsafe(no_mangle)] pub extern "C" fn Py_IsInitialized() -> c_int { INITIALIZED.is_completed() as _ diff --git a/crates/capi/src/unicodeobject.rs b/crates/capi/src/unicodeobject.rs index ee27e9d505c..837e6e2370a 100644 --- a/crates/capi/src/unicodeobject.rs +++ b/crates/capi/src/unicodeobject.rs @@ -29,7 +29,7 @@ pub extern "C" fn PyUnicode_FromStringAndSize(s: *const c_char, len: isize) -> * #[unsafe(no_mangle)] pub extern "C" fn PyUnicode_AsUTF8AndSize(obj: *mut PyObject, size: *mut isize) -> *const c_char { - with_vm(|vm| { + with_vm(|_vm| { let obj = unsafe { obj.as_ref() .expect("PyUnicode_AsUTF8AndSize called with null pointer") From 26c277744004c8034a854cddf48527b612686074 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Tue, 7 Apr 2026 19:50:44 +0200 Subject: [PATCH 19/24] Add support for `None`, `True`, `False`, `Ellipsis` & `NotImplemented` --- crates/capi/src/object.rs | 20 ++++++++++++++++---- example_projects/pyo3_embed/src/main.rs | 10 +++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index c3dfa1cc268..c5d8b58f14f 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -3,7 +3,8 @@ use crate::pystate::with_vm; use core::ffi::c_ulong; use rustpython_vm::builtins::PyType; use rustpython_vm::convert::IntoObject; -use rustpython_vm::{Context, Py}; +use rustpython_vm::{AsObject, Context, Py}; +use std::ffi::c_uint; use std::mem::MaybeUninit; const PY_TPFLAGS_LONG_SUBCLASS: c_ulong = 1 << 24; @@ -109,7 +110,18 @@ pub extern "C" fn PyObject_Str(_obj: *mut PyObject) -> *mut PyObject { } #[unsafe(no_mangle)] -pub extern "C" fn Py_GetConstantBorrowed(_constant_id: core::ffi::c_uint) -> *mut PyObject { - crate::log_stub("Py_GetConstantBorrowed"); - std::ptr::null_mut() +pub extern "C" fn Py_GetConstantBorrowed(constant_id: c_uint) -> *mut PyObject { + with_vm(|vm| { + let ctx = &vm.ctx; + match constant_id { + 0 => ctx.none.as_object(), + 1 => ctx.true_value.as_object(), + 2 => ctx.true_value.as_object(), + 3 => ctx.ellipsis.as_object(), + 4 => ctx.not_implemented.as_object(), + _ => panic!("Invalid constant_id passed to Py_GetConstantBorrowed"), + } + .as_raw() + .cast_mut() + }) } diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index cc9eef902fc..eef9bd7425f 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -1,5 +1,11 @@ use pyo3::prelude::*; -use pyo3::types::{PyInt, PyString}; +use pyo3::types::{PyInt, PyNone, PyString}; + + +#[pyfunction] +fn python_function(py: Python<'_>) -> Borrowed<'_, '_, PyNone> { + PyNone::get(py) +} fn main() { Python::initialize(); @@ -25,6 +31,8 @@ fn main() { .join() .unwrap(); + assert!(python_function(py).is_none()); + PyResult::Ok(()) }) .unwrap(); From 09466e2faf23e848008aa9f6ccf31146045e3efa Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Wed, 8 Apr 2026 11:51:25 +0200 Subject: [PATCH 20/24] Add support for PyBytes --- crates/capi/src/bytesobject.rs | 42 ++++++++++++++++++++----- example_projects/pyo3_embed/src/main.rs | 5 ++- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/crates/capi/src/bytesobject.rs b/crates/capi/src/bytesobject.rs index a5164bc8487..ebff4fd158f 100644 --- a/crates/capi/src/bytesobject.rs +++ b/crates/capi/src/bytesobject.rs @@ -1,16 +1,42 @@ +use crate::PyObject; +use crate::pystate::with_vm; use core::ffi::c_char; -use core::ptr; +use rustpython_vm::builtins::PyBytes; +use rustpython_vm::convert::IntoObject; -use crate::PyObject; +#[unsafe(no_mangle)] +pub extern "C" fn PyBytes_FromStringAndSize(bytes: *mut c_char, len: isize) -> *mut PyObject { + with_vm(|vm| { + if bytes.is_null() { + std::ptr::null_mut() + } else { + let bytes_slice = + unsafe { core::slice::from_raw_parts(bytes as *const u8, len as usize) }; + vm.ctx + .new_bytes(bytes_slice.to_vec()) + .into_object() + .into_raw() + .as_ptr() + } + }) +} #[unsafe(no_mangle)] -pub extern "C" fn PyBytes_Size(_bytes: *mut PyObject) -> isize { - crate::log_stub("PyBytes_Size"); - 0 +pub extern "C" fn PyBytes_Size(bytes: *mut PyObject) -> isize { + with_vm(|_vm| { + let bytes = unsafe { &*bytes } + .downcast_ref::() + .expect("PyBytes_Size argument must be a bytes object"); + bytes.as_bytes().len() as _ + }) } #[unsafe(no_mangle)] -pub extern "C" fn PyBytes_AsString(_bytes: *mut PyObject) -> *mut c_char { - crate::log_stub("PyBytes_AsString"); - ptr::null_mut() +pub extern "C" fn PyBytes_AsString(bytes: *mut PyObject) -> *mut c_char { + with_vm(|_vm| { + let bytes = unsafe { &*bytes } + .downcast_ref::() + .expect("PyBytes_AsString argument must be a bytes object"); + bytes.as_bytes().as_ptr() as _ + }) } diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index eef9bd7425f..1d4990655e4 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -1,5 +1,5 @@ use pyo3::prelude::*; -use pyo3::types::{PyInt, PyNone, PyString}; +use pyo3::types::{PyBytes, PyInt, PyNone, PyString}; #[pyfunction] @@ -33,6 +33,9 @@ fn main() { assert!(python_function(py).is_none()); + let bytes = PyBytes::new(py, b"Hello, World!"); + assert_eq!(bytes.as_bytes(), b"Hello, World!"); + PyResult::Ok(()) }) .unwrap(); From 94cc39bf0d5072cb93f3900744928293d5e47ac7 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Wed, 8 Apr 2026 15:22:59 +0200 Subject: [PATCH 21/24] Add basic exception support --- crates/capi/src/longobject.rs | 2 +- crates/capi/src/pyerrors.rs | 118 +++++++++++++++++++----- crates/capi/src/pylifecycle.rs | 8 +- crates/vm/src/vm/mod.rs | 15 ++- example_projects/pyo3_embed/src/main.rs | 5 +- 5 files changed, 123 insertions(+), 25 deletions(-) diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs index 1a876712452..a99e790e557 100644 --- a/crates/capi/src/longobject.rs +++ b/crates/capi/src/longobject.rs @@ -51,7 +51,7 @@ pub extern "C" fn PyLong_AsLong(obj: *mut PyObject) -> c_long { int_obj.as_bigint().try_into().unwrap_or_else(|_| unsafe { PyErr_SetString( - PyExc_OverflowError, + PyExc_OverflowError.assume_init(), c"Python int too large to convert to C long".as_ptr(), ); -1 diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs index 3a567bcfece..112e1caf591 100644 --- a/crates/capi/src/pyerrors.rs +++ b/crates/capi/src/pyerrors.rs @@ -1,31 +1,52 @@ -use core::ffi::{c_char, c_int}; -use core::ptr; - use crate::PyObject; +use crate::pystate::with_vm; +use core::ffi::{c_char, c_int}; +use rustpython_vm::builtins::{PyTuple, PyType}; +use rustpython_vm::convert::ToPyObject; +use std::ffi::CStr; +use std::mem::MaybeUninit; #[unsafe(no_mangle)] -pub static mut PyExc_BaseException: *mut PyObject = ptr::null_mut(); +pub static mut PyExc_BaseException: MaybeUninit<*mut PyObject> = MaybeUninit::uninit(); #[unsafe(no_mangle)] -pub static mut PyExc_TypeError: *mut PyObject = ptr::null_mut(); +pub static mut PyExc_TypeError: MaybeUninit<*mut PyObject> = MaybeUninit::uninit(); #[unsafe(no_mangle)] -pub static mut PyExc_OverflowError: *mut PyObject = ptr::null_mut(); +pub static mut PyExc_OverflowError: MaybeUninit<*mut PyObject> = MaybeUninit::uninit(); #[unsafe(no_mangle)] pub extern "C" fn PyErr_GetRaisedException() -> *mut PyObject { - crate::log_stub("PyErr_GetRaisedException"); - ptr::null_mut() + with_vm(|vm| { + vm.take_raised_exception().map_or_else( + || std::ptr::null_mut(), + |exc| exc.to_pyobject(vm).into_raw().as_ptr(), + ) + }) } #[unsafe(no_mangle)] -pub extern "C" fn PyErr_SetRaisedException(_exc: *mut PyObject) { - crate::log_stub("PyErr_SetRaisedException"); +pub extern "C" fn PyErr_SetRaisedException(exc: *mut PyObject) { + with_vm(|vm| { + let exception = unsafe { (&*exc).to_owned().downcast_unchecked() }; + vm.push_exception(Some(exception)); + }); } #[unsafe(no_mangle)] -pub extern "C" fn PyErr_SetObject(_exception: *mut PyObject, _value: *mut PyObject) { - crate::log_stub("PyErr_SetObject"); +pub extern "C" fn PyErr_SetObject(exception: *mut PyObject, value: *mut PyObject) { + with_vm(|vm| { + let exc_type = unsafe { (&*exception).to_owned() }; + let exc_val = unsafe { (&*value).to_owned() }; + + let normalized = vm + .normalize_exception(exc_type, exc_val, vm.ctx.none()) + .unwrap_or_else(|_| { + vm.new_type_error("exceptions must derive from BaseException".to_owned()) + }); + + vm.push_exception(Some(normalized)); + }); } #[unsafe(no_mangle)] @@ -35,27 +56,82 @@ pub extern "C" fn PyErr_SetString(_exception: *mut PyObject, _message: *const c_ #[unsafe(no_mangle)] pub extern "C" fn PyErr_PrintEx(_set_sys_last_vars: c_int) { - crate::log_stub("PyErr_PrintEx"); + with_vm(|vm| { + let exception = vm + .take_raised_exception() + .expect("No exception set in PyErr_PrintEx"); + + vm.print_exception(exception); + }); } #[unsafe(no_mangle)] -pub extern "C" fn PyErr_WriteUnraisable(_obj: *mut PyObject) { - crate::log_stub("PyErr_WriteUnraisable"); +pub extern "C" fn PyErr_WriteUnraisable(obj: *mut PyObject) { + with_vm(|vm| { + let exception = vm + .take_raised_exception() + .expect("No exception set in PyErr_WriteUnraisable"); + + let object = unsafe { vm.unwrap_or_none(obj.as_ref().map(|obj| obj.to_owned())) }; + + vm.run_unraisable(exception, None, object) + }); +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyErr_NewException( + name: *const c_char, + base: *mut PyObject, + dict: *mut PyObject, +) -> *mut PyObject { + with_vm(|vm| { + let (module, name) = unsafe { + CStr::from_ptr(name) + .to_str() + .expect("Exception name is not valid UTF-8") + .rsplit_once('.') + .expect("Exception name must be of the form 'module.ExceptionName'") + }; + + let bases = unsafe { base.as_ref() }.map(|bases| { + if let Some(ty) = bases.downcast_ref::() { + vec![ty.to_owned()] + } else if let Some(tuple) = bases.downcast_ref::() { + tuple + .iter() + .map(|item| item.to_owned().downcast()) + .collect::, _>>() + .expect("PyErr_NewException base tuple must contain only types") + } else { + panic!("PyErr_NewException base must be a type or a tuple of types"); + } + }); + + assert!( + dict.is_null(), + "PyErr_NewException with non-null dict is not supported yet" + ); + + vm.ctx + .new_exception_type(module, name, bases) + .to_pyobject(vm) + .into_raw() + .as_ptr() + }) } #[unsafe(no_mangle)] pub extern "C" fn PyErr_NewExceptionWithDoc( - _name: *const c_char, + name: *const c_char, _doc: *const c_char, - _base: *mut PyObject, - _dict: *mut PyObject, + base: *mut PyObject, + dict: *mut PyObject, ) -> *mut PyObject { - crate::log_stub("PyErr_NewExceptionWithDoc"); - ptr::null_mut() + PyErr_NewException(name, base, dict) } #[unsafe(no_mangle)] pub extern "C" fn PyException_GetTraceback(_exc: *mut PyObject) -> *mut PyObject { crate::log_stub("PyException_GetTraceback"); - ptr::null_mut() + std::ptr::null_mut() } diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs index b7918751635..644639814cb 100644 --- a/crates/capi/src/pylifecycle.rs +++ b/crates/capi/src/pylifecycle.rs @@ -1,9 +1,10 @@ use crate::log_stub; use crate::object::{PyLong_Type, PyTuple_Type, PyType_Type, PyUnicode_Type}; +use crate::pyerrors::{PyExc_BaseException, PyExc_OverflowError, PyExc_TypeError}; use crate::pystate::attach_vm_to_thread; use core::ffi::c_int; use rustpython_vm::vm::thread::ThreadedVirtualMachine; -use rustpython_vm::{Context, Interpreter}; +use rustpython_vm::{AsObject, Context, Interpreter}; use std::sync::{Once, OnceLock, mpsc}; static VM_REQUEST_TX: OnceLock>> = @@ -39,6 +40,11 @@ pub(crate) fn init_static_type_pointers() { PyLong_Type.write(types.int_type); PyTuple_Type.write(types.tuple_type); PyUnicode_Type.write(types.str_type); + + let exc = &context.exceptions; + PyExc_BaseException.write(exc.base_exception_type.as_object().as_raw().cast_mut()); + PyExc_TypeError.write(exc.type_error.as_object().as_raw().cast_mut()); + PyExc_OverflowError.write(exc.overflow_error.as_object().as_raw().cast_mut()); }; } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 231ba5cde38..e8f3b771a42 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -2037,7 +2037,7 @@ impl VirtualMachine { } } - pub(crate) fn push_exception(&self, exc: Option) { + pub fn push_exception(&self, exc: Option) { self.exceptions.borrow_mut().stack.push(exc); #[cfg(feature = "threading")] thread::update_thread_exception(self.topmost_exception()); @@ -2078,6 +2078,19 @@ impl VirtualMachine { thread::update_thread_exception(self.topmost_exception()); } + pub fn take_raised_exception(&self) -> Option { + let mut excs = self.exceptions.borrow_mut(); + if let Some(top) = excs.stack.last_mut() { + let exc = top.take(); + drop(excs); + #[cfg(feature = "threading")] + thread::update_thread_exception(self.topmost_exception()); + exc + } else { + None + } + } + pub(crate) fn contextualize_exception(&self, exception: &Py) { if let Some(context_exc) = self.topmost_exception() && !context_exc.is(exception) diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index 1d4990655e4..b4acdc97a92 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -1,7 +1,7 @@ +use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyInt, PyNone, PyString}; - #[pyfunction] fn python_function(py: Python<'_>) -> Borrowed<'_, '_, PyNone> { PyNone::get(py) @@ -36,6 +36,9 @@ fn main() { let bytes = PyBytes::new(py, b"Hello, World!"); assert_eq!(bytes.as_bytes(), b"Hello, World!"); + PyTypeError::new_err("This is a type error").restore(py); + assert!(PyErr::take(py).is_some()); + PyResult::Ok(()) }) .unwrap(); From 2ef64d2578e2a6f6df337cfd51de6909fb9350bc Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Wed, 8 Apr 2026 15:48:31 +0200 Subject: [PATCH 22/24] Add import support --- crates/capi/src/import.rs | 18 +++++++++++++----- example_projects/pyo3_embed/src/main.rs | 2 ++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/capi/src/import.rs b/crates/capi/src/import.rs index 29ea7996687..fd5b5d668b7 100644 --- a/crates/capi/src/import.rs +++ b/crates/capi/src/import.rs @@ -1,9 +1,17 @@ -use core::ptr; - use crate::PyObject; +use crate::pystate::with_vm; +use rustpython_vm::builtins::PyStr; #[unsafe(no_mangle)] -pub extern "C" fn PyImport_Import(_name: *mut PyObject) -> *mut PyObject { - crate::log_stub("PyImport_Import"); - ptr::null_mut() +pub extern "C" fn PyImport_Import(name: *mut PyObject) -> *mut PyObject { + with_vm(|vm| { + let name = unsafe { (&*name).downcast_unchecked_ref::() }; + vm.import(name, 0).map_or_else( + |err| { + vm.push_exception(Some(err)); + std::ptr::null_mut() + }, + |module| module.into_raw().as_ptr(), + ) + }) } diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index b4acdc97a92..b0728fc698e 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -39,6 +39,8 @@ fn main() { PyTypeError::new_err("This is a type error").restore(py); assert!(PyErr::take(py).is_some()); + py.import("sys")?; + PyResult::Ok(()) }) .unwrap(); From 55b1e14e5146508e5c795181a632849c2aca29fb Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Wed, 8 Apr 2026 17:18:50 +0200 Subject: [PATCH 23/24] Add basic call support --- crates/capi/src/abstract_.rs | 77 +++++++++++++++++++++++++ crates/capi/src/lib.rs | 1 + crates/capi/src/object.rs | 22 ++++--- example_projects/pyo3_embed/src/main.rs | 2 + 4 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 crates/capi/src/abstract_.rs diff --git a/crates/capi/src/abstract_.rs b/crates/capi/src/abstract_.rs new file mode 100644 index 00000000000..82fad911e9a --- /dev/null +++ b/crates/capi/src/abstract_.rs @@ -0,0 +1,77 @@ +use crate::pystate::with_vm; +use rustpython_vm::PyObject; +use rustpython_vm::builtins::{PyStr, PyTuple}; +use std::slice; + +const PY_VECTORCALL_ARGUMENTS_OFFSET: usize = 1usize << (usize::BITS as usize - 1); + +#[unsafe(no_mangle)] +pub extern "C" fn PyObject_CallNoArgs(callable: *mut PyObject) -> *mut PyObject { + with_vm(|vm| { + if callable.is_null() { + vm.push_exception(Some(vm.new_system_error( + "PyObject_CallNoArgs called with null callable".to_owned(), + ))); + return std::ptr::null_mut(); + } + + let callable = unsafe { &*callable }; + callable.call((), vm).map_or_else( + |err| { + vm.push_exception(Some(err)); + std::ptr::null_mut() + }, + |result| result.into_raw().as_ptr(), + ) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn PyObject_VectorcallMethod( + name: *mut PyObject, + args: *const *mut PyObject, + nargsf: usize, + kwnames: *mut PyObject, +) -> *mut PyObject { + with_vm(|vm| { + let args_len = nargsf & !PY_VECTORCALL_ARGUMENTS_OFFSET; + let num_positional_args = args_len - 1; + + let (receiver, args) = unsafe { slice::from_raw_parts(args, args_len) } + .split_first() + .expect("PyObject_VectorcallMethod should always have at least one argument"); + + let method_name = unsafe { (&*name).downcast_unchecked_ref::() }; + let callable = match unsafe { + (&**receiver) + .get_attr(method_name, vm) + } { + Ok(obj) => obj, + Err(err) => { + vm.push_exception(Some(err)); + return std::ptr::null_mut() + }, + }; + + let args = args + .iter() + .map(|arg| unsafe { &**arg }.to_owned()) + .collect::>(); + + let kwnames = unsafe { + kwnames + .as_ref() + .map(|tuple| &***tuple.downcast_unchecked_ref::()) + }; + + callable + .vectorcall(args, num_positional_args, kwnames, vm) + .map_or_else( + |err| { + vm.push_exception(Some(err)); + std::ptr::null_mut() + }, + |obj| obj.into_raw().as_ptr(), + ) + }) +} diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs index 7080bdbae49..35b3f276bda 100644 --- a/crates/capi/src/lib.rs +++ b/crates/capi/src/lib.rs @@ -2,6 +2,7 @@ pub use rustpython_vm::PyObject; extern crate alloc; +pub mod abstract_; pub mod bytesobject; pub mod import; pub mod longobject; diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index c5d8b58f14f..c2ee6d8ff99 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -4,7 +4,7 @@ use core::ffi::c_ulong; use rustpython_vm::builtins::PyType; use rustpython_vm::convert::IntoObject; use rustpython_vm::{AsObject, Context, Py}; -use std::ffi::c_uint; +use std::ffi::{c_int, c_uint}; use std::mem::MaybeUninit; const PY_TPFLAGS_LONG_SUBCLASS: c_ulong = 1 << 24; @@ -85,12 +85,6 @@ pub extern "C" fn PyType_GetQualName(ptr: *const Py) -> *mut PyObject { with_vm(move |vm| ty.__qualname__(vm).into_object().into_raw().as_ptr()) } -#[unsafe(no_mangle)] -pub extern "C" fn PyObject_CallNoArgs(_callable: *mut PyObject) -> *mut PyObject { - crate::log_stub("PyObject_CallNoArgs"); - std::ptr::null_mut() -} - #[unsafe(no_mangle)] pub extern "C" fn PyObject_GetAttr(_obj: *mut PyObject, _name: *mut PyObject) -> *mut PyObject { crate::log_stub("PyObject_GetAttr"); @@ -125,3 +119,17 @@ pub extern "C" fn Py_GetConstantBorrowed(constant_id: c_uint) -> *mut PyObject { .cast_mut() }) } + +#[unsafe(no_mangle)] +pub extern "C" fn PyObject_IsTrue(obj: *mut PyObject) -> c_int { + with_vm(|vm| { + let obj = unsafe { &*obj }; + obj.to_owned().is_true(vm).map_or_else( + |err| { + vm.push_exception(Some(err)); + -1 + }, + |is_true| is_true.into(), + ) + }) +} diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs index b0728fc698e..f93604230e5 100644 --- a/example_projects/pyo3_embed/src/main.rs +++ b/example_projects/pyo3_embed/src/main.rs @@ -19,6 +19,8 @@ fn main() { assert!(string.is_instance_of::()); assert_eq!(string.to_str()?, "Hello, World!"); + assert!(string.call_method1("endswith", ("!",))?.is_truthy()?); + assert_eq!(string.get_type().name()?.to_str()?, "str"); let number = number.unbind(); From 85b1ed437fc31810bc78977184dc8c55a1824636 Mon Sep 17 00:00:00 2001 From: Bas Schoenmaeckers Date: Wed, 8 Apr 2026 20:38:53 +0200 Subject: [PATCH 24/24] Use pyo3 as test harness --- Cargo.lock | 1 + .../capi}/.cargo/config.toml | 4 +- crates/capi/Cargo.toml | 3 ++ crates/capi/pyo3-rustpython.config | 6 +++ crates/capi/src/abstract_.rs | 14 ++++++ crates/capi/src/bytesobject.rs | 14 ++++++ crates/capi/src/import.rs | 12 +++++ crates/capi/src/longobject.rs | 15 ++++++ crates/capi/src/object.rs | 37 ++++++++++++++ crates/capi/src/pyerrors.rs | 14 ++++++ crates/capi/src/pystate.rs | 22 +++++++++ crates/capi/src/refcount.rs | 27 ++++++++++ crates/capi/src/unicodeobject.rs | 15 ++++++ example_projects/pyo3_embed/Cargo.toml | 12 ----- example_projects/pyo3_embed/README.md | 11 ----- .../pyo3_embed/pyo3-rustpython.config | 9 ---- example_projects/pyo3_embed/src/main.rs | 49 ------------------- 17 files changed, 182 insertions(+), 83 deletions(-) rename {example_projects/pyo3_embed => crates/capi}/.cargo/config.toml (70%) create mode 100644 crates/capi/pyo3-rustpython.config delete mode 100644 example_projects/pyo3_embed/Cargo.toml delete mode 100644 example_projects/pyo3_embed/README.md delete mode 100644 example_projects/pyo3_embed/pyo3-rustpython.config delete mode 100644 example_projects/pyo3_embed/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 1eca00b9fbb..5d70087432a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3108,6 +3108,7 @@ dependencies = [ name = "rustpython-capi" version = "0.5.0" dependencies = [ + "pyo3", "rustpython-vm", ] diff --git a/example_projects/pyo3_embed/.cargo/config.toml b/crates/capi/.cargo/config.toml similarity index 70% rename from example_projects/pyo3_embed/.cargo/config.toml rename to crates/capi/.cargo/config.toml index 54a884e2458..49ff5a492e6 100644 --- a/example_projects/pyo3_embed/.cargo/config.toml +++ b/crates/capi/.cargo/config.toml @@ -1,2 +1,2 @@ -[env] -PYO3_CONFIG_FILE = { value = "pyo3-rustpython.config", relative = true } +[profile.test.env] +PYO3_CONFIG_FILE = { value = "pyo3-rustpython.config", relative = true } \ No newline at end of file diff --git a/crates/capi/Cargo.toml b/crates/capi/Cargo.toml index 964a996143a..b0106ae223f 100644 --- a/crates/capi/Cargo.toml +++ b/crates/capi/Cargo.toml @@ -14,5 +14,8 @@ crate-type = ["staticlib"] [dependencies] rustpython-vm = { workspace = true, features = ["threading"]} +[dev-dependencies] +pyo3 = { version = "0.28.3", features = ["auto-initialize", "abi3-py314"] } + [lints] workspace = true diff --git a/crates/capi/pyo3-rustpython.config b/crates/capi/pyo3-rustpython.config new file mode 100644 index 00000000000..e039bd6d6bc --- /dev/null +++ b/crates/capi/pyo3-rustpython.config @@ -0,0 +1,6 @@ +implementation=CPython +version=3.14 +shared=true +abi3=true +build_flags= +suppress_build_script_link_lines=true diff --git a/crates/capi/src/abstract_.rs b/crates/capi/src/abstract_.rs index 82fad911e9a..915dc6236a4 100644 --- a/crates/capi/src/abstract_.rs +++ b/crates/capi/src/abstract_.rs @@ -75,3 +75,17 @@ pub extern "C" fn PyObject_VectorcallMethod( ) }) } + +#[cfg(test)] +mod tests { + use pyo3::prelude::*; + use pyo3::types::PyString; + + #[test] + fn test_call_method1() { + Python::attach(|py| { + let string = PyString::new(py, "Hello, World!"); + assert!(string.call_method1("endswith", ("!",)).unwrap().is_truthy().unwrap()); + }) + } +} diff --git a/crates/capi/src/bytesobject.rs b/crates/capi/src/bytesobject.rs index ebff4fd158f..8a9bb15d7dd 100644 --- a/crates/capi/src/bytesobject.rs +++ b/crates/capi/src/bytesobject.rs @@ -40,3 +40,17 @@ pub extern "C" fn PyBytes_AsString(bytes: *mut PyObject) -> *mut c_char { bytes.as_bytes().as_ptr() as _ }) } + +#[cfg(test)] +mod tests { + use pyo3::prelude::*; + use pyo3::types::PyBytes; + + #[test] + fn test_bytes() { + Python::attach(|py| { + let bytes = PyBytes::new(py, b"Hello, World!"); + assert_eq!(bytes.as_bytes(), b"Hello, World!"); + }) + } +} \ No newline at end of file diff --git a/crates/capi/src/import.rs b/crates/capi/src/import.rs index fd5b5d668b7..9d546e1de79 100644 --- a/crates/capi/src/import.rs +++ b/crates/capi/src/import.rs @@ -15,3 +15,15 @@ pub extern "C" fn PyImport_Import(name: *mut PyObject) -> *mut PyObject { ) }) } + +#[cfg(test)] +mod tests { + use pyo3::prelude::*; + + #[test] + fn test_import() { + Python::attach(|py| { + let _module = py.import("sys").unwrap(); + }) + } +} \ No newline at end of file diff --git a/crates/capi/src/longobject.rs b/crates/capi/src/longobject.rs index a99e790e557..b8d0ef740b5 100644 --- a/crates/capi/src/longobject.rs +++ b/crates/capi/src/longobject.rs @@ -58,3 +58,18 @@ pub extern "C" fn PyLong_AsLong(obj: *mut PyObject) -> c_long { }) }) } + +#[cfg(test)] +mod tests { + use pyo3::prelude::*; + use pyo3::types::PyInt; + + #[test] + fn test_py_int() { + Python::attach(|py| { + let number = PyInt::new(py, 123); + assert!(number.is_instance_of::()); + assert_eq!(number.extract::().unwrap(), 123); + }) + } +} diff --git a/crates/capi/src/object.rs b/crates/capi/src/object.rs index c2ee6d8ff99..4610e4b17a5 100644 --- a/crates/capi/src/object.rs +++ b/crates/capi/src/object.rs @@ -133,3 +133,40 @@ pub extern "C" fn PyObject_IsTrue(obj: *mut PyObject) -> c_int { ) }) } + +#[cfg(test)] +mod tests { + use pyo3::prelude::*; + use pyo3::types::{PyBool, PyString}; + + #[test] + fn test_is_truthy() { + Python::attach(|py| { + assert!(!py.None().is_truthy(py).unwrap()); + }) + } + + #[test] + fn test_is_none() { + Python::attach(|py| { + assert!(py.None().is_none(py)); + }) + } + + #[test] + #[cfg(false)] + fn test_bool() { + Python::attach(|py| { + assert!(PyBool::new(py, true).extract::().unwrap()); + assert!(!PyBool::new(py, false).extract::().unwrap()); + }) + } + + #[test] + fn test_type_name() { + Python::attach(|py| { + let string = PyString::new(py, "Hello, World!"); + assert_eq!(string.get_type().name().unwrap().to_str().unwrap(), "str"); + }) + } +} diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs index 112e1caf591..20c27cf6ce2 100644 --- a/crates/capi/src/pyerrors.rs +++ b/crates/capi/src/pyerrors.rs @@ -135,3 +135,17 @@ pub extern "C" fn PyException_GetTraceback(_exc: *mut PyObject) -> *mut PyObject crate::log_stub("PyException_GetTraceback"); std::ptr::null_mut() } + +#[cfg(test)] +mod tests { + use pyo3::exceptions::PyTypeError; + use pyo3::prelude::*; + + #[test] + fn test_raised_exception() { + Python::attach(|py| { + PyTypeError::new_err("This is a type error").restore(py); + assert!(PyErr::take(py).is_some()); + }) + } +} diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs index de25dba790a..06aae217985 100644 --- a/crates/capi/src/pystate.rs +++ b/crates/capi/src/pystate.rs @@ -51,3 +51,25 @@ pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState { #[unsafe(no_mangle)] pub extern "C" fn PyEval_RestoreThread(_tstate: *mut PyThreadState) {} + +#[cfg(test)] +mod tests { + use pyo3::prelude::*; + use pyo3::types::PyInt; + + #[test] + fn test_new_thread() { + Python::attach(|py| { + let number = PyInt::new(py, 123).unbind(); + + std::thread::spawn(move || { + Python::attach(|py| { + let number = number.bind(py); + assert!(number.is_instance_of::()); + }); + }) + .join() + .unwrap(); + }) + } +} diff --git a/crates/capi/src/refcount.rs b/crates/capi/src/refcount.rs index 5e6f55f0460..abdcd5cd54d 100644 --- a/crates/capi/src/refcount.rs +++ b/crates/capi/src/refcount.rs @@ -1,6 +1,7 @@ use crate::PyObject; use rustpython_vm::PyObjectRef; use std::ptr::NonNull; +use crate::pystate::with_vm; #[unsafe(no_mangle)] #[allow(clippy::not_unsafe_ptr_arg_deref)] @@ -27,3 +28,29 @@ pub extern "C" fn _Py_IncRef(op: *mut PyObject) { std::mem::forget(owned); } + +#[unsafe(no_mangle)] +pub extern "C" fn Py_REFCNT(op: *mut PyObject) -> isize { + with_vm(|vm| unsafe { &*op } + .strong_count() as _) +} + +#[cfg(test)] +mod tests { + use pyo3::prelude::*; + use pyo3::PyTypeInfo; + use pyo3::types::PyInt; + + #[test] + fn test_refcount() { + Python::attach(|py| { + let obj = PyInt::type_object(py); + unsafe { pyo3::ffi::Py_REFCNT(obj.as_ptr())}; + let ref_count = obj.get_refcnt(); + let obj_clone = obj.clone(); + assert_eq!(obj.get_refcnt(), ref_count + 1); + drop(obj_clone); + assert_eq!(obj.get_refcnt(), ref_count); + }); + } +} diff --git a/crates/capi/src/unicodeobject.rs b/crates/capi/src/unicodeobject.rs index 837e6e2370a..42a71ece01b 100644 --- a/crates/capi/src/unicodeobject.rs +++ b/crates/capi/src/unicodeobject.rs @@ -64,3 +64,18 @@ pub extern "C" fn PyUnicode_AsEncodedString( pub extern "C" fn PyUnicode_InternInPlace(_string: *mut *mut PyObject) { crate::log_stub("PyUnicode_InternInPlace"); } + +#[cfg(test)] +mod tests { + use pyo3::prelude::*; + use pyo3::types::PyString; + + #[test] + fn test_unicode() { + Python::attach(|py| { + let string = PyString::new(py, "Hello, World!"); + assert!(string.is_instance_of::()); + assert_eq!(string.to_str().unwrap(), "Hello, World!"); + }) + } +} diff --git a/example_projects/pyo3_embed/Cargo.toml b/example_projects/pyo3_embed/Cargo.toml deleted file mode 100644 index d424a486184..00000000000 --- a/example_projects/pyo3_embed/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "example_pyo3_embed" -version = "0.1.0" -edition = "2021" - -[dependencies] -pyo3 = { version = "0.28.3", features = ["abi3-py314"] } -rustpython-capi = { path = "../../crates/capi" } - -[workspace] - -[patch.crates-io] diff --git a/example_projects/pyo3_embed/README.md b/example_projects/pyo3_embed/README.md deleted file mode 100644 index 7d9cc2cf98b..00000000000 --- a/example_projects/pyo3_embed/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# PyO3 embed against RustPython C-API - -This example demonstrates linking `pyo3` against RustPython's minimal C-API shim (`rustpython-capi`) instead of a system CPython library. - -From this directory, run: - -```shell -cargo run -``` - -The local `.cargo/config.toml` sets `PYO3_CONFIG_FILE` automatically. diff --git a/example_projects/pyo3_embed/pyo3-rustpython.config b/example_projects/pyo3_embed/pyo3-rustpython.config deleted file mode 100644 index 2b097508e4f..00000000000 --- a/example_projects/pyo3_embed/pyo3-rustpython.config +++ /dev/null @@ -1,9 +0,0 @@ -implementation=CPython -version=3.14 -shared=false -abi3=true -build_flags= -suppress_build_script_link_lines=true -extra_build_script_line=cargo:rustc-link-search=native=/Users/basschoenmaeckers/repo/RustPython/target/debug -extra_build_script_line=cargo:rustc-link-lib=static=rustpython_capi -extra_build_script_line=cargo:rustc-link-lib=framework=CoreFoundation diff --git a/example_projects/pyo3_embed/src/main.rs b/example_projects/pyo3_embed/src/main.rs deleted file mode 100644 index f93604230e5..00000000000 --- a/example_projects/pyo3_embed/src/main.rs +++ /dev/null @@ -1,49 +0,0 @@ -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::*; -use pyo3::types::{PyBytes, PyInt, PyNone, PyString}; - -#[pyfunction] -fn python_function(py: Python<'_>) -> Borrowed<'_, '_, PyNone> { - PyNone::get(py) -} - -fn main() { - Python::initialize(); - - Python::attach(|py| { - let number = PyInt::new(py, 123); - assert!(number.is_instance_of::()); - assert_eq!(number.extract::()?, 123); - - let string = PyString::new(py, "Hello, World!"); - assert!(string.is_instance_of::()); - assert_eq!(string.to_str()?, "Hello, World!"); - - assert!(string.call_method1("endswith", ("!",))?.is_truthy()?); - - assert_eq!(string.get_type().name()?.to_str()?, "str"); - - let number = number.unbind(); - std::thread::spawn(move || { - Python::attach(|py| { - let number = number.bind(py); - assert!(number.is_instance_of::()); - }); - }) - .join() - .unwrap(); - - assert!(python_function(py).is_none()); - - let bytes = PyBytes::new(py, b"Hello, World!"); - assert_eq!(bytes.as_bytes(), b"Hello, World!"); - - PyTypeError::new_err("This is a type error").restore(py); - assert!(PyErr::take(py).is_some()); - - py.import("sys")?; - - PyResult::Ok(()) - }) - .unwrap(); -}