Skip to content
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
67 commits
Select commit Hold shift + click to select a range
c5ff3f5
add support for os fork
Apr 15, 2023
c2dd77d
WIP: add stub functions for registring callbacks
Apr 16, 2023
bd25a54
fill stubs
Apr 16, 2023
2efe9bd
make things functional
Apr 16, 2023
3ee4da8
make args as kwargs for register at fork
Apr 16, 2023
c6da379
change error text as per as cpython
Apr 16, 2023
d51daaf
add skip test for os.fork on windows
Apr 17, 2023
9fc8952
chore! fix clippy errors
Apr 17, 2023
137fd29
skip fork functions in windows
Apr 17, 2023
f98c005
Merge branch 'main' of github.com:itsankitkp/RustPython into add-fork…
Apr 17, 2023
a58e429
skip unsupported function calls
Apr 17, 2023
cb5166f
remove un-needed skip decorators
Apr 17, 2023
ddcae23
Update Lib/test/test_socketserver.py
itsankitkp Apr 17, 2023
63a881c
remove un-needed decorator
Apr 17, 2023
abce797
fix lint
Apr 17, 2023
7aa1311
add todo items
Apr 17, 2023
5779c89
remove un-needed comment
Apr 17, 2023
37b73f3
fix test case getting stuck
Apr 18, 2023
eaa3e63
Update Lib/test/test_httpservers.py
itsankitkp Apr 18, 2023
e041cbd
fix test case getting stuck
itsankitkp Apr 18, 2023
ae634d2
skip failures due to missing module
itsankitkp Apr 19, 2023
e0a2228
Merge branch 'add-fork-in-os' of github.com:itsankitkp/RustPython int…
itsankitkp Apr 19, 2023
f54f715
Update Lib/test/test_threading.py
itsankitkp Apr 20, 2023
e27e9e9
Update Lib/test/test_threading.py
itsankitkp Apr 20, 2023
6199f76
Update Lib/test/test_threading.py
itsankitkp Apr 20, 2023
9ebdf0b
Update Lib/test/test_threading.py
itsankitkp Apr 20, 2023
9c525af
Update Lib/test/test_threading.py
itsankitkp Apr 20, 2023
84fda43
Update Lib/test/test_threading.py
itsankitkp Apr 20, 2023
8918617
Update Lib/test/test_threading.py
itsankitkp Apr 20, 2023
123bc2d
Update Lib/test/test_threading.py
itsankitkp Apr 20, 2023
844365e
Merge branch 'main' of github.com:itsankitkp/RustPython into add-fork…
itsankitkp Apr 22, 2023
d69426b
Merge branch 'main' of github.com:itsankitkp/RustPython into add-fork…
itsankitkp Apr 23, 2023
20f5abf
skip cases which needs _at_fork_reinit support
itsankitkp Apr 23, 2023
3bfe8c8
add type error if arg is not callable
itsankitkp Apr 23, 2023
e2e6844
reverse function calls for before forkers as per as cpython implement…
itsankitkp Apr 23, 2023
e5d9bb5
handle non callable args
itsankitkp Apr 23, 2023
0fd1a9f
skip test cases which has dependencies on Lock._at_fork_reinit
itsankitkp Apr 23, 2023
47f13b2
skip test case
itsankitkp Apr 23, 2023
35f23be
remove un-needed expected failure decorator as test is passing
itsankitkp Apr 23, 2023
e19c8fd
debug test failure
itsankitkp Apr 23, 2023
c79e2fd
increase verbosity
itsankitkp Apr 23, 2023
f237e5e
print more lines for debug
itsankitkp Apr 23, 2023
870ef53
disable assertion failure to check error
itsankitkp Apr 23, 2023
fdaefd0
more debugging
itsankitkp Apr 23, 2023
eb01160
minor change
itsankitkp Apr 23, 2023
669ff59
add logs
itsankitkp Apr 23, 2023
76b89ce
more experimentations to get error
itsankitkp Apr 23, 2023
68a78e4
last shot to find issue
itsankitkp Apr 23, 2023
a854bb6
remove debug logs
itsankitkp Apr 23, 2023
110fc33
skip test_input_no_stdout_fileno for macos
youknowone Apr 23, 2023
0a2c6fd
Add todo item
itsankitkp Apr 23, 2023
af28311
Add todo item
itsankitkp Apr 23, 2023
dc087c3
not to remove it when update to next CPython version
itsankitkp Apr 23, 2023
511a448
add todo item
itsankitkp Apr 23, 2023
e4aacf9
handle optional args for register at fork
itsankitkp Apr 23, 2023
72b53a9
add ignore kwargs
itsankitkp Apr 23, 2023
f8b966d
remove clone
itsankitkp Apr 23, 2023
9963280
chore! add ignorable kwargs
itsankitkp Apr 23, 2023
6cd534b
add cfg to unix only structs
itsankitkp Apr 23, 2023
0b8f237
Update Lib/test/test_pty.py
itsankitkp Apr 24, 2023
a85ac15
Update vm/src/stdlib/os.rs
itsankitkp Apr 24, 2023
68fc959
use cleaner way of param validation and remove take on state
itsankitkp Apr 24, 2023
90b472b
add skip cfg
itsankitkp Apr 24, 2023
7b52282
Update Lib/test/test_os.py
itsankitkp Apr 24, 2023
898e084
Update Lib/test/test_fcntl.py
youknowone Apr 24, 2023
5ab1706
relocate fork to posix
youknowone Apr 24, 2023
0b61077
simplify fork functoins
youknowone Apr 24, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Lib/concurrent/futures/thread.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ def _python_exit():
threading._register_atexit(_python_exit)

# At fork, reinitialize the `_global_shutdown_lock` lock in the child process
if hasattr(os, 'register_at_fork'):
# TODO RUSTPYTHON - _at_fork_reinit is not implemented yet
if hasattr(os, 'register_at_fork') and hasattr(_global_shutdown_lock, '_at_fork_reinit'):
os.register_at_fork(before=_global_shutdown_lock.acquire,
after_in_child=_global_shutdown_lock._at_fork_reinit,
after_in_parent=_global_shutdown_lock.release)
Expand Down
28 changes: 28 additions & 0 deletions Lib/test/test_os.py
Original file line number Diff line number Diff line change
Expand Up @@ -4611,6 +4611,34 @@ def test_fork(self):
assert_python_ok("-c", code)
assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug")

@unittest.skipUnless(sys.platform in ("linux", "darwin"),
"Only Linux and macOS detect this today.")
def test_fork_warns_when_non_python_thread_exists(self):
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this originated from CPython source code?

Copy link
Copy Markdown
Contributor Author

@itsankitkp itsankitkp Apr 24, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes

    @unittest.skipUnless(sys.platform in ("linux", "darwin"),
                         "Only Linux and macOS detect this today.")
    def test_fork_warns_when_non_python_thread_exists(self):
       ...

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, got it. it came from later then 3.11.

code = """if 1:
import os, threading, warnings
from _testcapi import _spawn_pthread_waiter, _end_spawned_pthread
_spawn_pthread_waiter()
try:
with warnings.catch_warnings(record=True) as ws:
warnings.filterwarnings(
"always", category=DeprecationWarning)
if os.fork() == 0:
assert not ws, f"unexpected warnings in child: {ws}"
os._exit(0) # child
else:
assert ws[0].category == DeprecationWarning, ws[0]
assert 'fork' in str(ws[0].message), ws[0]
# Waiting allows an error in the child to hit stderr.
exitcode = os.wait()[1]
assert exitcode == 0, f"child exited {exitcode}"
assert threading.active_count() == 1, threading.enumerate()
finally:
_end_spawned_pthread()
"""
_, out, err = assert_python_ok("-c", code, PYTHONOPTIMIZE='0')
self.assertEqual(err.decode("utf-8"), "")
self.assertEqual(out.decode("utf-8"), "")


# Only test if the C version is provided, otherwise TestPEP519 already tested
# the pure Python implementation.
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_socketserver.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class ForkingUnixDatagramServer(socketserver.ForkingMixIn,
socketserver.UnixDatagramServer):
pass


@test.support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems
Comment thread
youknowone marked this conversation as resolved.
@contextlib.contextmanager
def simple_subprocess(testcase):
"""Tests that a custom child process is not waited on (Issue 1540386)"""
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_uuid.py
Original file line number Diff line number Diff line change
Expand Up @@ -641,7 +641,7 @@ def test_uuid5(self):

# TODO: RUSTPYTHON
@unittest.expectedFailure
@unittest.skipUnless(os.name == 'posix', 'requires Posix')
@support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems
Comment thread
fanninpm marked this conversation as resolved.
Outdated
def testIssue8621(self):
Comment thread
fanninpm marked this conversation as resolved.
# On at least some versions of OSX self.uuid.uuid4 generates
# the same sequence of UUIDs in the parent and any
Expand Down
81 changes: 81 additions & 0 deletions vm/src/stdlib/os.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1127,6 +1127,87 @@ pub(super) mod _os {
OutputMode::String.process_path(curdir_inner(vm)?, vm)
}

#[cfg(unix)]
#[pyfunction]
fn register_at_fork(kwargs: crate::function::KwArgs, vm: &VirtualMachine) -> PyResult<()> {
let mut match_found = false; // better way to handle this?
for (key, value) in kwargs.into_iter() {
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no error when key matches none of 3 keys?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For this reason I was using let mut match_found = false hack but it is not needed if I use #[derive(FromArgs)]

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If no errors raised for wrong keys, adding a useless KwArgs will be helpful

e.g.

fn register_at_fork(args: RegisterAtForkArgs, _ignored: Kwargs, vm: ..) -> ...

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used this but I am getting

TypeError: Unexpected keyword argument

if key == "before" {
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rust conventions prefer match key {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this be rewritten using a struct with #[derive(FromArgs)]?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rewriting using #[derive(FromArgs)]

match_found = true;
vm.state.before_forkers.lock().push(value.clone());
}
if key == "after_in_parent" {
match_found = true;
vm.state.after_forkers_parent.lock().push(value.clone());
}
if key == "after_in_child" {
match_found = true;
vm.state.after_forkers_child.lock().push(value.clone());
}
}

if !match_found {
return Err(vm.new_value_error("At least one argument is required.".to_owned()));
}

Ok(())
}

#[cfg(unix)]
fn run_at_forkers(funcs: Vec<PyObjectRef>, vm: &VirtualMachine) {
if !funcs.is_empty() {
for func in funcs.into_iter().rev() {
if let Err(e) = func.call((), vm) {
let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit);
vm.run_unraisable(e, Some("Exception ignored in".to_owned()), func);
if exit {
// Do nothing!
}
}
}
}
}

#[cfg(unix)]
fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> {
let before_forkers: Vec<PyObjectRef> = std::mem::take(&mut *vm.state.before_forkers.lock());
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this not clone but take? So that second run of fork doesn't be affect by it?

Copy link
Copy Markdown
Contributor Author

@itsankitkp itsankitkp Apr 24, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually you are right.

import os
def a():
   print('a')
def b():
   print('b')
def c():
   os.register_at_fork(before=a)
   os.fork()
   os.register_at_fork(before=b)
   os.fork()
c()

yields

a
b
a
b
a

in cpython. I have remove take operation and now rustpython as same results.
This should be a test case in cpython.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cpython PR: python/cpython#103759

run_at_forkers(before_forkers, vm);

Ok(())
}

#[cfg(unix)]
fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> {
let after_forkers_child: Vec<PyObjectRef> =
std::mem::take(&mut *vm.state.after_forkers_child.lock());
run_at_forkers(after_forkers_child, vm);
Ok(())
}

#[cfg(unix)]
fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> {
let after_forkers_parent: Vec<PyObjectRef> =
std::mem::take(&mut *vm.state.after_forkers_parent.lock());
run_at_forkers(after_forkers_parent, vm);
Ok(())
}

#[cfg(unix)]
#[pyfunction]
fn fork(vm: &VirtualMachine) -> PyResult {
Comment thread
itsankitkp marked this conversation as resolved.
Outdated
let pid: i32;
py_os_before_fork(vm)?;
unsafe {
pid = libc::fork();
}
if pid == 0 {
py_os_after_fork_child(vm)?;
} else {
py_os_after_fork_parent(vm)?;
}
Ok(vm.ctx.new_int(pid).into())
}

#[pyfunction]
fn getcwdb(vm: &VirtualMachine) -> PyResult {
OutputMode::Bytes.process_path(curdir_inner(vm)?, vm)
Expand Down
6 changes: 6 additions & 0 deletions vm/src/vm/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ pub struct PyGlobalState {
pub finalizing: AtomicBool,
pub warnings: WarningsState,
pub override_frozen_modules: AtomicCell<isize>,
pub before_forkers: PyMutex<Vec<PyObjectRef>>,
pub after_forkers_child: PyMutex<Vec<PyObjectRef>>,
pub after_forkers_parent: PyMutex<Vec<PyObjectRef>>,
}

pub fn process_hash_secret_seed() -> u32 {
Expand Down Expand Up @@ -175,6 +178,9 @@ impl VirtualMachine {
finalizing: AtomicBool::new(false),
warnings,
override_frozen_modules: AtomicCell::new(0),
before_forkers: PyMutex::default(),
after_forkers_child: PyMutex::default(),
after_forkers_parent: PyMutex::default(),
}),
initialized: false,
recursion_depth: Cell::new(0),
Expand Down