Skip to content
Prev Previous commit
Next Next commit
Set ThreadHandle state to Running in parent thread after spawn
  • Loading branch information
youknowone committed Mar 8, 2026
commit e37b15fedd1df5d6027ca2ffc2d181715e691072
155 changes: 115 additions & 40 deletions crates/vm/src/stdlib/posix.rs
Original file line number Diff line number Diff line change
Expand Up @@ -872,57 +872,129 @@ pub mod module {
run_at_forkers(after_forkers_parent, false, vm);
}

/// Warn if forking from a multi-threaded process
fn warn_if_multi_threaded(name: &str, vm: &VirtualMachine) {
// Only check threading if it was already imported
// Avoid vm.import() which can execute arbitrary Python code in the fork path
let threading = match vm
.sys_module
.get_attr("modules", vm)
.and_then(|m| m.get_item("threading", vm))
/// Best-effort number of OS threads in this process.
/// Returns <= 0 when unavailable, mirroring CPython fallback behavior.
fn get_number_of_os_threads() -> isize {
#[cfg(target_os = "macos")]
{
Ok(m) => m,
Err(_) => return,
};
let active = threading.get_attr("_active", vm).ok();
let limbo = threading.get_attr("_limbo", vm).ok();
type MachPortT = libc::c_uint;
type KernReturnT = libc::c_int;
type MachMsgTypeNumberT = libc::c_uint;
type ThreadActArrayT = *mut MachPortT;
const KERN_SUCCESS: KernReturnT = 0;
unsafe extern "C" {
fn mach_task_self() -> MachPortT;
fn task_for_pid(
task: MachPortT,
pid: libc::c_int,
target_task: *mut MachPortT,
) -> KernReturnT;
fn task_threads(
target_task: MachPortT,
act_list: *mut ThreadActArrayT,
act_list_cnt: *mut MachMsgTypeNumberT,
) -> KernReturnT;
fn vm_deallocate(
target_task: MachPortT,
address: libc::uintptr_t,
size: libc::uintptr_t,
) -> KernReturnT;
}

let count_dict = |obj: Option<crate::PyObjectRef>| -> usize {
obj.and_then(|o| o.length_opt(vm))
.and_then(|r| r.ok())
.unwrap_or(0)
};
let self_task = unsafe { mach_task_self() };
let mut proc_task: MachPortT = 0;
if unsafe { task_for_pid(self_task, libc::getpid(), &mut proc_task) } == KERN_SUCCESS {
let mut threads: ThreadActArrayT = core::ptr::null_mut();
let mut n_threads: MachMsgTypeNumberT = 0;
if unsafe { task_threads(proc_task, &mut threads, &mut n_threads) } == KERN_SUCCESS
{
if !threads.is_null() {
let _ = unsafe {
vm_deallocate(
self_task,
threads as libc::uintptr_t,
(n_threads as usize * core::mem::size_of::<MachPortT>())
as libc::uintptr_t,
)
};
}
return n_threads as isize;
}
}
0
}
#[cfg(target_os = "linux")]
{
use std::io::Read as _;
let mut file = match std::fs::File::open("/proc/self/stat") {
Ok(f) => f,
Err(_) => return 0,
};
let mut buf = [0u8; 160];
let n = match file.read(&mut buf) {
Ok(n) => n,
Err(_) => return 0,
};
let line = match std::str::from_utf8(&buf[..n]) {
Ok(s) => s,
Err(_) => return 0,
};
if let Some(field) = line.split_whitespace().nth(19) {
return field.parse::<isize>().unwrap_or(0);
}
return 0;
}
Comment on lines +926 to +946
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Linux: /proc/self/stat parsing is fragile when comm contains spaces.

The comm field (field 2 in /proc/self/stat) is enclosed in parentheses and can contain spaces, parentheses, or newlines. Using split_whitespace().nth(19) will yield incorrect results if the process name contains spaces.

For example, a process named "my app" produces:

123 (my app) S 456 ...

split_whitespace() splits this into ["123", "(my", "app)", "S", ...], making nth(19) point to the wrong field.

CPython handles this by finding the last ) to locate the end of the comm field, then counting fields from there.

🛡️ Suggested fix to handle comm field correctly
         #[cfg(target_os = "linux")]
         {
             use std::io::Read as _;
             let mut file = match std::fs::File::open("/proc/self/stat") {
                 Ok(f) => f,
                 Err(_) => return 0,
             };
             let mut buf = [0u8; 160];
             let n = match file.read(&mut buf) {
                 Ok(n) => n,
                 Err(_) => return 0,
             };
             let line = match core::str::from_utf8(&buf[..n]) {
                 Ok(s) => s,
                 Err(_) => return 0,
             };
-            if let Some(field) = line.split_whitespace().nth(19) {
-                return field.parse::<isize>().unwrap_or(0);
+            // Find end of comm field (last ')') to handle spaces in process name
+            let after_comm = match line.rfind(')') {
+                Some(idx) => &line[idx + 1..],
+                None => return 0,
+            };
+            // Field 20 (num_threads) is the 17th field after comm (fields 3-19 are 17 fields)
+            if let Some(field) = after_comm.split_whitespace().nth(17) {
+                return field.parse::<isize>().unwrap_or(0);
             }
             0
         }
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@crates/vm/src/stdlib/posix.rs` around lines 926 - 946, The current parsing of
/proc/self/stat is fragile because split_whitespace() will break when the comm
field (inside parentheses) contains spaces; instead, after reading the file and
converting to string, find the index of the last ')' in the line, take the
substring after that character, then split_whitespace() on that remainder and
parse the desired field (the same field you previously accessed with nth(19))
from the resultant iterator (preserve the existing return behavior using
parse::<isize>().unwrap_or(0)); replace the direct
line.split_whitespace().nth(19) usage with this "find last ')', take substring
after it, then split" approach to reliably locate the field even when comm
contains spaces.

#[cfg(not(any(target_os = "macos", target_os = "linux")))]
{
0
}
}

let num_threads = count_dict(active) + count_dict(limbo);
if num_threads > 1 {
// Use Python warnings module to ensure filters are applied correctly
let Ok(warnings) = vm.import("warnings", 0) else {
return;
/// Warn if forking from a multi-threaded process.
/// `num_os_threads` should be captured before parent after-fork hooks run.
fn warn_if_multi_threaded(name: &str, num_os_threads: isize, vm: &VirtualMachine) {
let num_threads = if num_os_threads > 0 {
num_os_threads as usize
} else {
// CPython fallback: if OS-level count isn't available, use the
// threading module's active+limbo view.
// Only check threading if it was already imported. Avoid vm.import()
// which can execute arbitrary Python code in the fork path.
let threading = match vm
.sys_module
.get_attr("modules", vm)
.and_then(|m| m.get_item("threading", vm))
{
Ok(m) => m,
Err(_) => return,
};
let Ok(warn_fn) = warnings.get_attr("warn", vm) else {
return;
let active = threading.get_attr("_active", vm).ok();
let limbo = threading.get_attr("_limbo", vm).ok();

// Match threading module internals and avoid sequence overcounting:
// count only dict-backed _active/_limbo containers.
let count_dict = |obj: Option<crate::PyObjectRef>| -> usize {
obj.and_then(|o| {
o.downcast_ref::<crate::builtins::PyDict>()
.map(|d| d.__len__())
})
.unwrap_or(0)
};

count_dict(active) + count_dict(limbo)
};

if num_threads > 1 {
let pid = unsafe { libc::getpid() };
let msg = format!(
"This process (pid={}) is multi-threaded, use of {}() may lead to deadlocks in the child.",
pid, name
);

// Call warnings.warn(message, DeprecationWarning, stacklevel=2)
// stacklevel=2 to point to the caller of fork()
let args = crate::function::FuncArgs::new(
vec![
vm.ctx.new_str(msg).into(),
vm.ctx.exceptions.deprecation_warning.as_object().to_owned(),
],
crate::function::KwArgs::new(
[("stacklevel".to_owned(), vm.ctx.new_int(2).into())]
.into_iter()
.collect(),
),
);
let _ = warn_fn.call(args, vm);
// Match PyErr_WarnFormat(..., stacklevel=1) in CPython.
// Best effort: ignore failures like CPython does in this path.
let _ =
crate::stdlib::warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm);
Comment on lines +994 to +997
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

🧩 Analysis chain

🏁 Script executed:

# First, let's find and read the relevant section of posix.rs
wc -l crates/vm/src/stdlib/posix.rs

Repository: RustPython/RustPython

Length of output: 99


🏁 Script executed:

# Read the specific lines mentioned in the review
sed -n '990,1010p' crates/vm/src/stdlib/posix.rs

Repository: RustPython/RustPython

Length of output: 866


🏁 Script executed:

# Find the warn_if_multi_threaded function definition to see its current signature
rg -n "fn warn_if_multi_threaded" crates/vm/src/stdlib/posix.rs -A 10

Repository: RustPython/RustPython

Length of output: 689


🏁 Script executed:

# Find the warnings::warn function to verify its return type
rg -n "fn warn" crates/vm/src/stdlib/warnings.rs -A 5

Repository: RustPython/RustPython

Length of output: 820


🏁 Script executed:

# Check where warn_if_multi_threaded is called to understand the context
rg -n "warn_if_multi_threaded" crates/vm/src/stdlib/posix.rs

Repository: RustPython/RustPython

Length of output: 225


🏁 Script executed:

# Search for any related discussions or TODOs about fork warning handling
rg -i "cpython|fork" crates/vm/src/stdlib/posix.rs -B 2 -A 2 | grep -A 5 -B 5 "warn\|error"

Repository: RustPython/RustPython

Length of output: 1656


🏁 Script executed:

# Check if there are any tests for the fork warning behavior
fd -e rs | xargs rg "test.*fork\|fork.*test" -l | head -5

Repository: RustPython/RustPython

Length of output: 47


🏁 Script executed:

# Let's check the broader context of the fork function to understand the error handling strategy
sed -n '1025,1060p' crates/vm/src/stdlib/posix.rs

Repository: RustPython/RustPython

Length of output: 1241


🏁 Script executed:

# Check if there are any other similar patterns where PyResult is dropped in critical functions
rg "let _ = .*warn" crates/vm/src/stdlib/posix.rs -B 3 -A 3

Repository: RustPython/RustPython

Length of output: 47


🌐 Web query:

CPython fork() warning error handling deprecation posixmodule.c

💡 Result:

In CPython, the “fork() in a multi-threaded process” DeprecationWarning is implemented in Modules/posixmodule.c (helper often named like warn_about_fork_with_threads()), and it’s emitted when CPython can detect more than 1 thread in the current process:

  • It tries to detect the thread count via OS-specific means (e.g., Linux /proc/self/stat thread field), and if that fails it falls back to counting what the threading module knows about (threading._active + threading._limbo). (raw.githubusercontent.com)
  • If num_threads > 1, it calls PyErr_WarnFormat(PyExc_DeprecationWarning, ...) with the message:
    “This process (pid=...) is multi-threaded, use of fork() may lead to deadlocks in the child.” (raw.githubusercontent.com)
  • This behavior is documented as a Python 3.12 change (“os.fork() can now raise a DeprecationWarning…; use a different multiprocessing start method like spawn or forkserver”). (docs.python.org)

The “warning vs error” handling difference (why it sometimes doesn’t fail under -Werror)

There has been a real behavioral difference across versions/branches in how the C code handles the return/exception state from PyErr_WarnFormat:

  • In the 3.12.0 code, after issuing the warning, CPython clears the warning/error indicator unconditionally (PyErr_Clear()), meaning that even if your warnings filter turns DeprecationWarning into an exception (e.g. -W error), the exception state can get swallowed inside posixmodule.c rather than propagating to Python code. (raw.githubusercontent.com)
  • In current main, the helper returns the result of PyErr_WarnFormat(...) to the caller (instead of clearing it), allowing “warnings as errors” configurations to actually behave like errors. (raw.githubusercontent.com)

Practical takeaway

  • If you want this to be fatal in CI: be aware that some released versions (notably early 3.12 code paths) may clear the exception state in C and thus won’t reliably fail with -W error in all scenarios. (raw.githubusercontent.com)
  • If you want to fix the root cause: don’t fork() from a multi-threaded process—switch to multiprocessing start methods (spawn/forkserver) or use subprocess/posix_spawn style process creation where applicable. (docs.python.org)

Citations:


Propagate warning errors instead of silently dropping them.

The current code uses let _ = to drop the PyResult from warnings::warn(), which prevents warning-as-error filters from working. If a user configures DeprecationWarning as an error via warning filters, os.fork() will still succeed instead of raising an exception. This diverges from CPython's current behavior, which propagates the warning result.

Suggested changes:

  1. Remove the error-dropping pattern and propagate the result:
crate::stdlib::warnings::warn(
    vm.ctx.exceptions.deprecation_warning,
    msg,
    1,
    vm,
)?;
  1. Update warn_if_multi_threaded to return PyResult<()> and propagate at the call site:
fn warn_if_multi_threaded(
    name: &str,
    num_os_threads: isize,
    vm: &VirtualMachine,
) -> PyResult<()> {
    // ...
}

warn_if_multi_threaded("fork", num_os_threads, vm)?;

Update the comment to accurately reflect the current CPython behavior (propagating rather than ignoring).

🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@crates/vm/src/stdlib/posix.rs` around lines 994 - 997, The code currently
drops the result of crate::stdlib::warnings::warn (using let _ = ...) which
prevents warning-as-error filters from taking effect; change the call site to
propagate the PyResult by removing the discard and using the try operator (i.e.
return the warn() result), change warn_if_multi_threaded to return PyResult<()>
(update its signature and return points) and propagate that result at each call
site (e.g., where warn_if_multi_threaded("fork", num_os_threads, vm) is
invoked), and update the surrounding comment to state that CPython propagates
warning errors rather than silently ignoring them.

}
}

Expand Down Expand Up @@ -953,9 +1025,12 @@ pub mod module {
if pid == 0 {
py_os_after_fork_child(vm);
} else {
// Match CPython timing: capture this before parent after-fork hooks
// in case those hooks start threads.
let num_os_threads = get_number_of_os_threads();
py_os_after_fork_parent(vm);
// Match CPython timing: warn only after parent callback path resumes world.
warn_if_multi_threaded("fork", vm);
warn_if_multi_threaded("fork", num_os_threads, vm);
}
if pid == -1 {
Err(nix::Error::from_raw(saved_errno).into_pyexception(vm))
Expand Down
Loading