diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3d3ac2dbc72..f79415891a0 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -6820,6 +6820,7 @@ class IranTest(ZoneInfoTest): zonename = 'Asia/Tehran' +@unittest.skip("TODO: RUSTPYTHON; setUp needs _testcapi.test_datetime_capi") @unittest.skipIf(_testcapi is None, 'need _testcapi module') class CapiTest(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 19582e75716..a05361d8e73 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4593,6 +4593,7 @@ def test_call_builtins(self): with self.assertRaises(OverflowError): ba.__buffer__(sys.maxsize + 1) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.testBuf @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_c_buffer(self): buf = _testcapi.testBuf() @@ -4610,12 +4611,14 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.testBuf @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_c_buffer_invalid_flags(self): buf = _testcapi.testBuf() self.assertRaises(SystemError, buf.__buffer__, PyBUF_READ) self.assertRaises(SystemError, buf.__buffer__, PyBUF_WRITE) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.buffer_fill_info @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_c_fill_buffer_invalid_flags(self): # PyBuffer_FillInfo @@ -4625,6 +4628,7 @@ def test_c_fill_buffer_invalid_flags(self): self.assertRaises(SystemError, _testcapi.buffer_fill_info, source, 0, PyBUF_WRITE) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.buffer_fill_info @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_c_fill_buffer_readonly_and_writable(self): source = b"abc" diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index ffc59dab9e8..8545a7bbcb7 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1171,6 +1171,7 @@ def __bytes__(self): BytesSubclass(b'abc')) # Test PyBytes_FromFormat() + @unittest.expectedFailure # TODO: RUSTPYTHON; needs ctypes.pythonapi def test_from_format(self): ctypes = import_helper.import_module('ctypes') _testcapi = import_helper.import_module('_testcapi') diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index b18f16a6efc..e396989edbc 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -306,16 +306,19 @@ def test_o(self): def test_o_ext(self): self.assertEqual(self.obj.meth_o(*[1]), (self.expected_self, 1)) + @unittest.expectedFailure # TODO: RUSTPYTHON; built-in argument count error message differs def test_o_error_no_arg(self): msg = r"meth_o\(\) takes exactly one argument \(0 given\)" self.assertRaisesRegex(TypeError, msg, self.obj.meth_o) + @unittest.expectedFailure # TODO: RUSTPYTHON; built-in argument count error message differs def test_o_error_two_args(self): msg = r"meth_o\(\) takes exactly one argument \(2 given\)" self.assertRaisesRegex( TypeError, msg, lambda: self.obj.meth_o(1, 2), ) + @unittest.expectedFailure # TODO: RUSTPYTHON; built-in argument count error message differs def test_o_error_ext(self): msg = r"meth_o\(\) takes exactly one argument \(3 given\)" self.assertRaisesRegex( @@ -340,18 +343,21 @@ def test_noargs(self): def test_noargs_ext(self): self.assertEqual(self.obj.meth_noargs(*[]), self.expected_self) + @unittest.expectedFailure # TODO: RUSTPYTHON; built-in argument count error message differs def test_noargs_error_arg(self): msg = r"meth_noargs\(\) takes no arguments \(1 given\)" self.assertRaisesRegex( TypeError, msg, lambda: self.obj.meth_noargs(1), ) + @unittest.expectedFailure # TODO: RUSTPYTHON; built-in argument count error message differs def test_noargs_error_arg2(self): msg = r"meth_noargs\(\) takes no arguments \(2 given\)" self.assertRaisesRegex( TypeError, msg, lambda: self.obj.meth_noargs(1, 2), ) + @unittest.expectedFailure # TODO: RUSTPYTHON; built-in argument count error message differs def test_noargs_error_ext(self): msg = r"meth_noargs\(\) takes no arguments \(3 given\)" self.assertRaisesRegex( @@ -541,6 +547,7 @@ def check_result(self, result, expected): expected = (*expected[:-1], result[-1]) self.assertEqual(result, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.pyobject_fastcalldict @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall_dict(self): # Test PyObject_VectorcallDict() @@ -561,6 +568,7 @@ def test_vectorcall_dict(self): result = _testcapi.pyobject_fastcalldict(func, args, kwargs) self.check_result(result, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.pyobject_vectorcall @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall(self): # Test PyObject_Vectorcall() @@ -626,6 +634,7 @@ def testfunction_kw(self, *, kw): @unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; wrapper_descriptor lacks Py_TPFLAGS_METHOD_DESCRIPTOR def test_method_descriptor_flag(self): import functools cached = functools.lru_cache(1)(testfunction) @@ -645,6 +654,7 @@ class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.MethodDescriptorBase def test_vectorcall_flag(self): self.assertTrue(_testcapi.MethodDescriptorBase.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertTrue(_testcapi.MethodDescriptorDerived.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) @@ -666,6 +676,7 @@ def __call__(self): pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.MethodDescriptorNopGet def test_vectorcall_override(self): # Check that tp_call can correctly override vectorcall. # MethodDescriptorNopGet implements tp_call but it inherits from @@ -676,6 +687,7 @@ def test_vectorcall_override(self): f = _testcapi.MethodDescriptorNopGet() self.assertIs(f(*args), args) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.make_vectorcall_class def test_vectorcall_override_on_mutable_class(self): """Setting __call__ should disable vectorcall""" TestType = _testcapi.make_vectorcall_class() @@ -686,6 +698,7 @@ def test_vectorcall_override_on_mutable_class(self): TestType.__call__ = lambda self: "custom" self.assertEqual(instance(), "custom") + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.make_vectorcall_class def test_vectorcall_override_with_subclass(self): """Setting __call__ on a superclass should disable vectorcall""" SuperType = _testcapi.make_vectorcall_class() @@ -728,6 +741,7 @@ class DerivedType(SuperType): self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.MethodDescriptorBase def test_vectorcall(self): # Test a bunch of different ways to call objects: # 1. vectorcall using PyVectorcall_Call() @@ -804,6 +818,7 @@ def __call__(self, *args): self.assertEqual(expected, meth(*args1, **kwargs)) self.assertEqual(expected, wrapped(*args, **kwargs)) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.function_setvectorcall def test_setvectorcall(self): from _testcapi import function_setvectorcall _testinternalcapi = import_helper.import_module("_testinternalcapi") @@ -815,6 +830,7 @@ def f(num): return num + 1 for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("overridden", f(num)) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.function_setvectorcall def test_setvectorcall_load_attr_specialization_skip(self): from _testcapi import function_setvectorcall _testinternalcapi = import_helper.import_module("_testinternalcapi") @@ -832,6 +848,7 @@ def __getattribute__(self, attr): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("overridden", x.a) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.function_setvectorcall def test_setvectorcall_load_attr_specialization_deopt(self): from _testcapi import function_setvectorcall _testinternalcapi = import_helper.import_module("_testinternalcapi") @@ -1117,6 +1134,7 @@ def test_function_with_many_args(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') class TestCAPI(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.pycfunction_call def test_cfunction_call(self): def func(*args, **kwargs): return (args, kwargs) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 3923412b9a0..bee4444f797 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -815,6 +815,7 @@ class X(C, int()): class X(int(), C): pass + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.HeapCTypeMetaclassNullNew @unittest.skipIf(_testcapi is None, 'need the _testcapi module') def test_type_with_null_new_metaclass(self): metaclass = _testcapi.HeapCTypeMetaclassNullNew @@ -5099,6 +5100,7 @@ def __call__(self, arg): self.assertRegex(repr(method), r">") + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.bad_get @unittest.skipIf(_testcapi is None, 'need the _testcapi module') def test_bpo25750(self): # bpo-25750: calling a descriptor (implemented as built-in diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index a103601a23a..c327fcf8758 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1100,6 +1100,7 @@ def test_dis_with_some_positions(self): ]) self.do_disassembly_test(f, expect, show_positions=True) + @unittest.expectedFailure # TODO: RUSTPYTHON; AST nodes without columns compile to column 0 @requires_debug_ranges() def test_dis_with_linenos_but_no_columns(self): code = "def f():\n\tx = 1" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c7c9a2930ad..d8350f87e42 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3387,6 +3387,7 @@ def test_signature_on_builtins_no_signature(self): self.assertEqual(inspect.signature(builtin), inspect.signature(template)) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.DocStringUnrepresentableSignatureTest @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_parsing_with_defaults(self): diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 64e4156f6e3..acda8e42afc 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -721,6 +721,10 @@ def test_int_from_other_bases(self): with self.subTest(base=base): self._other_base_helper(base) + @unittest.expectedFailureIf( + sys.platform == "wasi", + "TODO: RUSTPYTHON; subinterpreters need the threading feature", + ) def test_int_max_str_digits_is_per_interpreter(self): # Changing the limit in one interpreter does not change others. code = """if 1: diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 144acb7afd7..8cb187fafe7 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -1007,6 +1007,7 @@ class ExternalEntityParserCreateErrorTest(unittest.TestCase): def setUpClass(cls): cls.testcapi = import_helper.import_module('_testcapi') + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.set_nomemory @unittest.skipIf(support.Py_TRACE_REFS, 'Py_TRACE_REFS conflicts with testcapi.set_nomemory') def test_error_path_no_crash(self): @@ -1294,6 +1295,10 @@ def test_set_activation_threshold__fail_for_subparser(self): def test_set_activation_threshold__invalid_threshold_type(self): return super().test_set_activation_threshold__invalid_threshold_type() + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionActivationThreshold' + def test_set_activation_threshold__invalid_threshold_range(self): + return super().test_set_activation_threshold__invalid_threshold_range() + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetBillionLaughsAttackProtectionMaximumAmplification' def test_set_maximum_amplification__fail_for_subparser(self): return super().test_set_maximum_amplification__fail_for_subparser() diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index a4af92d52eb..7561dced635 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -353,6 +353,7 @@ def check_signum(signals): assert_python_ok('-c', code) + @unittest.expectedFailure # TODO: RUSTPYTHON; wakeup fd write errors are not reported @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_wakeup_write_error(self): @@ -534,6 +535,7 @@ def handler(signum, frame): assert_python_ok('-c', code) + @unittest.expectedFailureIf(os.name != 'nt', "TODO: RUSTPYTHON; wakeup fd write errors are not reported") @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_send_error(self): # Use a subprocess to have only one thread. @@ -577,6 +579,7 @@ def handler(signum, frame): """.format(action=action) assert_python_ok('-c', code) + @unittest.expectedFailure # TODO: RUSTPYTHON; wakeup fd write errors are not reported @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_warn_on_full_buffer(self): # Use a subprocess to have only one thread. diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 0e20723b6f3..76fb146d1d9 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1758,6 +1758,7 @@ def test_getaddrinfo_localized_error(self): flags=socket.AI_NUMERICSERV) str(cm.exception) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError for an out-of-range int port @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_getaddrinfo_int_port_overflow(self): # gh-74895: Test that getaddrinfo does not raise OverflowError on port. diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index e5b946bbaf2..7e487395fa6 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -177,6 +177,7 @@ def handler(*a, **kw): self.cx.set_authorizer(handler) self.assert_not_authorized(self.cx.execute, "select * from test") + @unittest.expectedFailure # TODO: RUSTPYTHON; unraisable exception handling not implemented @with_tracebacks(OverflowError) def test_authorizer_concurrent_mutation_with_overflown_value(self): _testcapi = import_helper.import_module("_testcapi") diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 605dffa5214..3cd9a994dd6 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1630,7 +1630,6 @@ def pipe(self): os.set_blocking(r, False) return (r, w) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_threads_join(self): # Non-daemon threads should be joined at subinterpreter shutdown # (issue #18808) @@ -1659,7 +1658,6 @@ def f(): # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_threads_join_2(self): # Same as above, but a delay gets introduced after the thread's # Python code returned but before the thread state is deleted. diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index 3a58afd8194..c211c5f75eb 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -197,6 +197,7 @@ class X: self.assertIsNone(wr()) + @unittest.expectedFailure # TODO: RUSTPYTHON; needs _testcapi.call_in_temporary_c_thread def test_threading_local_clear_race(self): # See https://github.com/python/cpython/issues/100892 diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index d448e3df5d7..6e3cb586d2b 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -843,7 +843,6 @@ def doTraceback(self, module): else: raise AssertionError("This ought to be impossible") - @unittest.expectedFailure # TODO: RUSTPYTHON; empty caret lines from equal col/end_col def testTraceback(self): files = {TESTMOD + ".py": raise_src} self.doTest(None, files, TESTMOD, call=self.doTraceback) diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index 8132c561318..ce08e28b765 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -3,7 +3,10 @@ use crate::{ marshal::MarshalError, - varint::{read_varint, read_varint_with_start, write_varint_be, write_varint_with_start}, + varint::{ + read_varint, read_varint_with_start, write_signed_varint, write_varint_be, + write_varint_with_start, + }, {OneIndexed, SourceLocation}, }; use alloc::{borrow::ToOwned, boxed::Box, collections::BTreeSet, fmt, string::String, vec::Vec}; @@ -143,6 +146,66 @@ pub fn decode_exception_table(table: &[u8]) -> Vec { entries } +/// scan_varint +fn scan_varint(data: &[u8]) -> u32 { + let mut iter = data.iter(); + let mut read = iter.next().copied().unwrap_or(0); + let mut val = u32::from(read & 63); + let mut shift = 0; + while read & 64 != 0 { + read = iter.next().copied().unwrap_or(0); + shift += 6; + val |= u32::from(read & 63) << shift; + } + val +} + +/// scan_signed_varint +fn scan_signed_varint(data: &[u8]) -> i32 { + let uval = scan_varint(data); + if uval & 1 != 0 { + -((uval >> 1) as i32) + } else { + (uval >> 1) as i32 + } +} + +/// get_line_delta +fn get_line_delta(entry: &[u8]) -> i32 { + match PyCodeLocationInfoKind::from_code((entry[0] >> 3) & 15) { + Some(PyCodeLocationInfoKind::NoColumns | PyCodeLocationInfoKind::Long) => { + scan_signed_varint(&entry[1..]) + } + Some(PyCodeLocationInfoKind::OneLine1) => 1, + Some(PyCodeLocationInfoKind::OneLine2) => 2, + // None, OneLine0 and the short forms stay on the same line. + _ => 0, + } +} + +/// remove_column_info: rewrite every entry of a linetable that has a +/// location as a NO_COLUMNS entry with the same length and line delta. +#[must_use] +pub fn remove_column_info(locations: &[u8]) -> Vec { + let mut output = Vec::with_capacity(locations.len()); + let mut offset = 0; + while offset < locations.len() { + let header = locations[offset]; + if (header >> 3) & 15 == PyCodeLocationInfoKind::None as u8 { + output.push(header); + } else { + let blength = (header & 7) + 1; + output.push(0x80 | ((PyCodeLocationInfoKind::NoColumns as u8) << 3) | (blength - 1)); + write_signed_varint(&mut output, get_line_delta(&locations[offset..])); + } + offset += 1; + while offset < locations.len() && locations[offset] & 128 == 0 { + offset += 1; + } + } + output +} + /// Parse linetable to build a boolean mask indicating which code units /// have NO_LOCATION (line == -1). Returns a Vec of length `num_units`. #[must_use] diff --git a/crates/stdlib/src/syslog.rs b/crates/stdlib/src/syslog.rs index 5966dced299..2ea4bbc1633 100644 --- a/crates/stdlib/src/syslog.rs +++ b/crates/stdlib/src/syslog.rs @@ -76,6 +76,11 @@ mod syslog { #[pyfunction] fn openlog(args: OpenLogArgs, vm: &VirtualMachine) -> PyResult<()> { + // Since the sys.openlog changes the process level state of syslog library, + // this operation is only allowed for the main interpreter. + if !vm.state.is_main_interpreter() { + return Err(vm.new_runtime_error("subinterpreter can't use syslog.openlog()")); + } let logoption = args.logoption; let facility = args.facility; let ident = match args.ident.clone() { @@ -115,7 +120,13 @@ mod syslog { vm.audit("syslog.syslog", || (priority, msg.clone()))?; + // if log is not opened, open it now if !host_syslog::is_open() { + if !vm.state.is_main_interpreter() { + return Err(vm.new_runtime_error( + "subinterpreter can't use syslog.syslog() until the syslog is opened by the main interpreter", + )); + } openlog( OpenLogArgs { facility: LOG_USER, @@ -132,6 +143,11 @@ mod syslog { #[pyfunction] fn closelog(vm: &VirtualMachine) -> PyResult<()> { + // Since the sys.closelog changes the process level state of syslog library, + // this operation is only allowed for the main interpreter. + if !vm.state.is_main_interpreter() { + return Err(vm.new_runtime_error("subinterpreter can't use syslog.closelog()")); + } vm.audit("syslog.closelog", || ())?; host_syslog::closelog(); diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index 89f760ad26f..fcfedece2c7 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -642,6 +642,24 @@ impl PyCode { } } + /// Compact the linetable if we are opted out of debug ranges, as + /// _PyCode_New does. + pub(crate) fn remove_column_info(mut self, vm: &VirtualMachine) -> Self { + if vm.state.config.settings.code_debug_ranges { + return self; + } + let linetable = bytecode::remove_column_info(&self.code.linetable); + let first_line = self.code.first_line_number.map_or(0, |n| n.get() as i32); + self.code.locations = rustpython_compiler_core::marshal::linetable_to_locations( + &linetable, + first_line, + self.code.instructions.len(), + ); + self.co_linetable = vm.ctx.new_bytes(linetable.clone()); + self.code.linetable = linetable.into_boxed_slice(); + self + } + /// The code object `r_object` creates from a TYPE_CODE record. It keeps /// the objects read for the object-valued fields. pub(crate) fn from_marshal( @@ -1173,7 +1191,7 @@ impl Constructor for PyCode { code.set_names(args.names); code.co_linetable = args.linetable; code.co_exceptiontable = args.exceptiontable; - Ok(code) + Ok(code.remove_column_info(vm)) } } @@ -1744,7 +1762,7 @@ impl Py { code.set_names(names); code.co_linetable = linetable; code.co_exceptiontable = exceptiontable; - Ok(code) + Ok(code.remove_column_info(vm)) } #[pymethod] diff --git a/crates/vm/src/stdlib/_testcapi.rs b/crates/vm/src/stdlib/_testcapi.rs new file mode 100644 index 00000000000..6dfdfe0432d --- /dev/null +++ b/crates/vm/src/stdlib/_testcapi.rs @@ -0,0 +1,585 @@ +pub(crate) use _testcapi::module_def; + +#[pymodule] +mod _testcapi { + use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{ + PyBaseException, PyCode, PyInt, PyModule, PyTupleRef, PyType, PyTypeRef, PyUtf8StrRef, + }, + common::str::wchar_t, + function::{ArgBytesLike, ItemDoc, KwArgs, PosArgs, PyMethodDef, PyMethodFlags}, + types::{Constructor, DefaultConstructor}, + }; + + #[pyattr] + const CHAR_MAX: core::ffi::c_char = core::ffi::c_char::MAX; + #[pyattr] + const CHAR_MIN: core::ffi::c_char = core::ffi::c_char::MIN; + #[pyattr] + const UCHAR_MAX: u8 = u8::MAX; + #[pyattr] + const SHRT_MAX: i16 = i16::MAX; + #[pyattr] + const SHRT_MIN: i16 = i16::MIN; + #[pyattr] + const USHRT_MAX: u16 = u16::MAX; + #[pyattr] + const INT_MAX: core::ffi::c_int = core::ffi::c_int::MAX; + #[pyattr] + const INT_MIN: core::ffi::c_int = core::ffi::c_int::MIN; + #[pyattr] + const UINT_MAX: core::ffi::c_uint = core::ffi::c_uint::MAX; + #[pyattr] + const LONG_MAX: core::ffi::c_long = core::ffi::c_long::MAX; + #[pyattr] + const LONG_MIN: core::ffi::c_long = core::ffi::c_long::MIN; + #[pyattr] + const ULONG_MAX: core::ffi::c_ulong = core::ffi::c_ulong::MAX; + #[pyattr] + const FLT_MAX: f64 = f32::MAX as f64; + #[pyattr] + const FLT_MIN: f64 = f32::MIN_POSITIVE as f64; + #[pyattr] + const DBL_MAX: f64 = f64::MAX; + #[pyattr] + const DBL_MIN: f64 = f64::MIN_POSITIVE; + #[pyattr] + const LLONG_MAX: i64 = i64::MAX; + #[pyattr] + const LLONG_MIN: i64 = i64::MIN; + #[pyattr] + const ULLONG_MAX: u64 = u64::MAX; + #[pyattr] + const PY_SSIZE_T_MAX: isize = isize::MAX; + #[pyattr] + const PY_SSIZE_T_MIN: isize = isize::MIN; + #[pyattr] + const SIZE_MAX: usize = usize::MAX; + #[pyattr] + const SIZEOF_WCHAR_T: usize = core::mem::size_of::(); + #[pyattr] + const SIZEOF_VOID_P: usize = core::mem::size_of::<*const ()>(); + #[cfg(any(unix, windows))] + #[pyattr] + const SIZEOF_TIME_T: usize = core::mem::size_of::(); + #[cfg(unix)] + #[pyattr] + const SIZEOF_PID_T: usize = core::mem::size_of::(); + #[cfg(windows)] + #[pyattr] + const SIZEOF_PID_T: usize = core::mem::size_of::(); + #[pyattr(name = "Py_Version")] + const PY_VERSION: usize = crate::version::VERSION_HEX; + #[pyattr(name = "the_number_three")] + const THE_NUMBER_THREE: i32 = 3; + #[pyattr] + const INT32_MIN: i32 = i32::MIN; + #[pyattr] + const INT32_MAX: i32 = i32::MAX; + #[pyattr] + const UINT32_MAX: u32 = u32::MAX; + #[pyattr] + const INT64_MIN: i64 = i64::MIN; + #[pyattr] + const INT64_MAX: i64 = i64::MAX; + #[pyattr] + const UINT64_MAX: u64 = u64::MAX; + + // MIPS without NaN2008 and PA-RISC use the most significant mantissa + // bit as the signaling bit. + #[pyattr(name = "nan_msb_is_signaling")] + const NAN_MSB_IS_SIGNALING: bool = cfg!(any(target_arch = "mips", target_arch = "mips64")); + + /// PyConfig_Get + fn config_get_value(name: &str, vm: &VirtualMachine) -> PyResult { + let settings = &vm.state.config.settings; + let ctx = &vm.ctx; + let sys_attr = match name { + "argv" => Some("argv"), + "base_exec_prefix" => Some("base_exec_prefix"), + "base_executable" => Some("_base_executable"), + "base_prefix" => Some("base_prefix"), + "exec_prefix" => Some("exec_prefix"), + "executable" => Some("executable"), + "module_search_paths" => Some("path"), + "platlibdir" => Some("platlibdir"), + "prefix" => Some("prefix"), + "pycache_prefix" => Some("pycache_prefix"), + "stdlib_dir" => Some("_stdlib_dir"), + "warnoptions" => Some("warnoptions"), + "xoptions" => Some("_xoptions"), + "orig_argv" => Some("orig_argv"), + _ => None, + }; + if let Some(attr) = sys_attr { + return vm.sys_module.get_attr(attr, vm); + } + let value: PyObjectRef = match name { + "bytes_warning" => ctx.new_int(settings.bytes_warning).into(), + "cpu_count" => ctx + .new_int(settings.cpu_count.map_or(-1, |n| n.get())) + .into(), + "inspect" => ctx.new_bool(settings.inspect).into(), + "int_max_str_digits" => ctx.new_int(vm.state.int_max_str_digits.load()).into(), + "interactive" => ctx.new_bool(settings.interactive).into(), + "optimization_level" => ctx.new_int(settings.optimize).into(), + "parser_debug" => ctx.new_bool(settings.debug != 0).into(), + "quiet" => ctx.new_bool(settings.quiet).into(), + "use_environment" => ctx.new_bool(!settings.ignore_environment).into(), + "verbose" => ctx.new_int(settings.verbose).into(), + "write_bytecode" => { + let dont_write_bytecode = vm.sys_module.get_attr("dont_write_bytecode", vm)?; + ctx.new_bool(!dont_write_bytecode.try_to_bool(vm)?).into() + } + "buffered_stdio" => ctx.new_bool(settings.buffered_stdio).into(), + "check_hash_pycs_mode" => ctx + .new_str(settings.check_hash_pycs_mode.to_string()) + .into(), + "code_debug_ranges" => ctx.new_bool(settings.code_debug_ranges).into(), + "dev_mode" => ctx.new_bool(settings.dev_mode).into(), + "faulthandler" => ctx.new_bool(settings.faulthandler).into(), + "hash_seed" => ctx.new_int(settings.hash_seed.unwrap_or(0)).into(), + "thread_inherit_context" => ctx.new_int(settings.thread_inherit_context as i32).into(), + "context_aware_warnings" => ctx.new_int(settings.context_aware_warnings as i32).into(), + "install_signal_handlers" => ctx.new_bool(settings.install_signal_handlers).into(), + "isolated" => ctx.new_bool(settings.isolated).into(), + "safe_path" => ctx.new_bool(settings.safe_path).into(), + "site_import" => ctx.new_bool(settings.import_site).into(), + "use_hash_seed" => ctx.new_bool(settings.hash_seed.is_some()).into(), + "user_site_directory" => ctx.new_bool(settings.user_site_directory).into(), + "warn_default_encoding" => ctx.new_bool(settings.warn_default_encoding).into(), + "utf8_mode" => ctx.new_bool(settings.utf8_mode != 0).into(), + _ => { + return Err(vm.new_value_error(format!("unknown config option name: {name}"))); + } + }; + Ok(value) + } + + #[pyfunction] + fn config_get(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + config_get_value(name.as_str(), vm) + } + + #[pyfunction] + fn config_getint(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + let name = name.as_str(); + let obj = config_get_value(name, vm)?; + let Some(int) = obj.downcast_ref::() else { + return Err(vm.new_type_error(format!("config option {name} is not an int"))); + }; + int.try_to_primitive::(vm).map_err(|_| { + vm.new_overflow_error(format!( + "config option {name} value does not fit into a C int" + )) + }) + } + + #[pyfunction] + fn code_offset_to_line(code: PyObjectRef, offset: i32, vm: &VirtualMachine) -> PyResult { + let Some(code) = code.downcast_ref::() else { + return Err(vm.new_type_error("first arg must be a code object")); + }; + Ok(code.addr2line(offset)) + } + + #[pyfunction] + fn run_in_subinterp(code: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + #[cfg(feature = "threading")] + { + crate::stdlib::_testinternalcapi::run_string_in_new_subinterp( + code.as_str(), + crate::vm::InterpreterConfig::LEGACY, + vm, + ) + } + #[cfg(not(feature = "threading"))] + { + let _ = code; + Err(vm.new_runtime_error("sub-interpreter creation failed")) + } + } + + #[cfg(all(not(windows), feature = "threading"))] + static WAIT_DONE: std::sync::Mutex>> = + std::sync::Mutex::new(None); + + // These can be used to test things that care about the existence of another + // thread that the threading module doesn't know about. + #[cfg(all(not(windows), feature = "threading"))] + #[pyfunction] + fn _spawn_pthread_waiter(vm: &VirtualMachine) -> PyResult<()> { + let mut wait_done = WAIT_DONE.lock().unwrap(); + if wait_done.is_some() { + return Err(vm.new_runtime_error("thread already running")); + } + let (sender, receiver) = std::sync::mpsc::channel(); + *wait_done = Some(sender); + std::thread::spawn(move || { + let _ = receiver.recv(); + *WAIT_DONE.lock().unwrap() = None; + }); + Ok(()) + } + + #[cfg(all(not(windows), feature = "threading"))] + #[pyfunction] + fn _end_spawned_pthread(vm: &VirtualMachine) -> PyResult<()> { + let wait_done = WAIT_DONE.lock().unwrap(); + let Some(sender) = wait_done.as_ref() else { + return Err(vm.new_runtime_error("call _spawn_pthread_waiter 1st")); + }; + let _ = sender.send(()); + Ok(()) + } + + /// PyUnicode_FromFormat for the object conversions. + fn unicode_from_format( + format: &str, + objs: &[PyObjectRef], + vm: &VirtualMachine, + ) -> PyResult { + let mut out = String::with_capacity(format.len()); + let mut objs = objs.iter(); + let mut chars = format.chars(); + while let Some(c) = chars.next() { + if c != '%' { + out.push(c); + continue; + } + let spec = chars.next(); + if spec == Some('%') { + out.push('%'); + continue; + } + let obj = objs + .next() + .ok_or_else(|| vm.new_system_error("not enough arguments for format string"))?; + let text = match spec { + Some('R') => obj.repr(vm)?, + Some('S' | 'U') => obj.str(vm)?, + Some('A') => obj.ascii(vm)?, + _ => { + return Err(vm.new_system_error(format!( + "PyUnicode_FromFormatV() expects a valid format character, got {format:?}" + ))); + } + }; + out.push_str(&text.to_string_lossy()); + } + Ok(out) + } + + #[pyfunction] + fn err_formatunraisable( + exc: PyObjectRef, + fmt: Option, + objs: PosArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if vm.is_none(&exc) { + return Ok(()); + } + let exc = exc + .downcast::() + .map_err(|_| vm.new_type_error("exceptions must derive from BaseException"))?; + let msg = fmt + .map(|fmt| unicode_from_format(fmt.as_str(), &objs.into_vec(), vm)) + .transpose()?; + vm.run_unraisable(exc, msg, vm.ctx.none()); + Ok(()) + } + + #[pyfunction(name = "Py_CompileStringExFlags")] + fn py_compile_string_ex_flags( + the_string: ArgBytesLike, + filename: PyUtf8StrRef, + start: i32, + flags: i32, + vm: &VirtualMachine, + ) -> PyResult { + const PY_CF_IGNORE_COOKIE: i32 = 0x0800; + let mode = match start { + 256 => "single", + 257 => "exec", + 258 => "eval", + 345 => "func_type", + _ => return Err(vm.new_system_error("invalid start symbol")), + }; + let bytes = the_string.borrow_buf().to_vec(); + if bytes.contains(&0) { + return Err(vm.new_value_error("embedded null byte")); + } + let source: PyObjectRef = vm.ctx.new_bytes(bytes).into(); + let source = if flags & PY_CF_IGNORE_COOKIE != 0 { + vm.call_method(&source, "decode", ("utf-8",))? + } else { + source + }; + let compile = vm.builtins.get_attr("compile", vm)?; + compile.call( + (source, filename, mode, flags & !PY_CF_IGNORE_COOKIE, true), + vm, + ) + } + + #[pyfunction(name = "PyTime_AsSecondsDouble")] + fn test_pytime_as_seconds_double(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + use crate::stdlib::_testinternalcapi::pytime; + let ts = pytime::pylong_as_int64(&obj, vm)?; + Ok(pytime::pytime_as_seconds_double(ts)) + } + + fn type_arg<'a>(arg: &'a PyObjectRef, vm: &VirtualMachine) -> PyResult<&'a Py> { + arg.downcast_ref::() + .ok_or_else(|| vm.new_type_error("argument must be a type")) + } + + /// type->tp_version_tag + #[pyfunction] + fn type_get_version(ty: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(type_arg(&ty, vm)? + .tp_version_tag + .load(core::sync::atomic::Ordering::Acquire)) + } + + /// PyType_Modified + #[pyfunction] + fn type_modified(ty: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + type_arg(&ty, vm)?.modified(); + Ok(()) + } + + /// PyUnstable_Type_AssignVersionTag + #[pyfunction] + fn type_assign_version(ty: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok((type_arg(&ty, vm)?.version_for_specialization(vm) != 0) as i32) + } + + fn meth_varargs(zelf: PyObjectRef, args: PosArgs, vm: &VirtualMachine) -> PyTupleRef { + vm.new_tuple((zelf, vm.ctx.new_tuple(args.into_vec()))) + } + + fn meth_varargs_keywords( + zelf: PyObjectRef, + args: PosArgs, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let kwargs: PyObjectRef = if kwargs.is_empty() { + vm.ctx.none() + } else { + let dict = vm.ctx.new_dict(); + for (key, value) in kwargs { + dict.set_item(vm.ctx.new_str(key).as_object(), value, vm)?; + } + dict.into() + }; + Ok(vm.new_tuple((zelf, vm.ctx.new_tuple(args.into_vec()), kwargs))) + } + + fn meth_o(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyTupleRef { + vm.new_tuple((zelf, obj)) + } + + fn meth_noargs(zelf: PyObjectRef) -> PyObjectRef { + zelf + } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + const METHODS: &[PyMethodDef] = &[ + PyMethodDef::new_const( + "meth_varargs", + meth_varargs, + PyMethodFlags::empty(), + ItemDoc::NONE, + ), + PyMethodDef::new_const( + "meth_varargs_keywords", + meth_varargs_keywords, + PyMethodFlags::empty(), + ItemDoc::NONE, + ), + PyMethodDef::new_const("meth_o", meth_o, PyMethodFlags::empty(), ItemDoc::NONE), + PyMethodDef::new_const( + "meth_noargs", + meth_noargs, + PyMethodFlags::empty(), + ItemDoc::NONE, + ), + PyMethodDef::new_const( + "meth_fastcall", + meth_varargs, + PyMethodFlags::empty(), + ItemDoc::NONE, + ), + PyMethodDef::new_const( + "meth_fastcall_keywords", + meth_varargs_keywords, + PyMethodFlags::empty(), + ItemDoc::NONE, + ), + ]; + for definition in METHODS { + let function = definition.build_bound_function(&vm.ctx, module.to_owned().into()); + function + .as_object() + .set_attr("__module__", vm.ctx.new_str("_testcapi"), vm)?; + module + .dict() + .set_item(definition.name, function.into(), vm)?; + } + Ok(()) + } + + #[pyattr] + #[pyclass(name = "MethInstance", module = "_testcapi")] + #[derive(Debug, Default, PyPayload)] + struct MethInstance; + + impl DefaultConstructor for MethInstance {} + + #[pyclass(with(Constructor))] + impl MethInstance { + #[pymethod] + fn meth_varargs(zelf: PyObjectRef, args: PosArgs, vm: &VirtualMachine) -> PyTupleRef { + meth_varargs(zelf, args, vm) + } + + #[pymethod] + fn meth_varargs_keywords( + zelf: PyObjectRef, + args: PosArgs, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + meth_varargs_keywords(zelf, args, kwargs, vm) + } + + #[pymethod] + fn meth_o(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyTupleRef { + meth_o(zelf, obj, vm) + } + + #[pymethod] + fn meth_noargs(zelf: PyObjectRef) -> PyObjectRef { + meth_noargs(zelf) + } + + #[pymethod] + fn meth_fastcall(zelf: PyObjectRef, args: PosArgs, vm: &VirtualMachine) -> PyTupleRef { + meth_varargs(zelf, args, vm) + } + + #[pymethod] + fn meth_fastcall_keywords( + zelf: PyObjectRef, + args: PosArgs, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + meth_varargs_keywords(zelf, args, kwargs, vm) + } + } + + #[pyattr] + #[pyclass(name = "MethClass", module = "_testcapi")] + #[derive(Debug, Default, PyPayload)] + struct MethClass; + + impl DefaultConstructor for MethClass {} + + #[pyclass(with(Constructor))] + impl MethClass { + #[pyclassmethod] + fn meth_varargs(cls: PyTypeRef, args: PosArgs, vm: &VirtualMachine) -> PyTupleRef { + meth_varargs(cls.into(), args, vm) + } + + #[pyclassmethod] + fn meth_varargs_keywords( + cls: PyTypeRef, + args: PosArgs, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + meth_varargs_keywords(cls.into(), args, kwargs, vm) + } + + #[pyclassmethod] + fn meth_o(cls: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyTupleRef { + meth_o(cls.into(), obj, vm) + } + + #[pyclassmethod] + fn meth_noargs(cls: PyTypeRef) -> PyObjectRef { + meth_noargs(cls.into()) + } + + #[pyclassmethod] + fn meth_fastcall(cls: PyTypeRef, args: PosArgs, vm: &VirtualMachine) -> PyTupleRef { + meth_varargs(cls.into(), args, vm) + } + + #[pyclassmethod] + fn meth_fastcall_keywords( + cls: PyTypeRef, + args: PosArgs, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + meth_varargs_keywords(cls.into(), args, kwargs, vm) + } + } + + #[pyattr] + #[pyclass(name = "MethStatic", module = "_testcapi")] + #[derive(Debug, Default, PyPayload)] + struct MethStatic; + + impl DefaultConstructor for MethStatic {} + + // A static method has no self; _null_to_none turns it into None. + #[pyclass(with(Constructor))] + impl MethStatic { + #[pystaticmethod] + fn meth_varargs(args: PosArgs, vm: &VirtualMachine) -> PyTupleRef { + meth_varargs(vm.ctx.none(), args, vm) + } + + #[pystaticmethod] + fn meth_varargs_keywords( + args: PosArgs, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + meth_varargs_keywords(vm.ctx.none(), args, kwargs, vm) + } + + #[pystaticmethod] + fn meth_o(obj: PyObjectRef, vm: &VirtualMachine) -> PyTupleRef { + meth_o(vm.ctx.none(), obj, vm) + } + + #[pystaticmethod] + fn meth_noargs(vm: &VirtualMachine) -> PyObjectRef { + meth_noargs(vm.ctx.none()) + } + + #[pystaticmethod] + fn meth_fastcall(args: PosArgs, vm: &VirtualMachine) -> PyTupleRef { + meth_varargs(vm.ctx.none(), args, vm) + } + + #[pystaticmethod] + fn meth_fastcall_keywords( + args: PosArgs, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + meth_varargs_keywords(vm.ctx.none(), args, kwargs, vm) + } + } +} diff --git a/crates/vm/src/stdlib/_testinternalcapi.rs b/crates/vm/src/stdlib/_testinternalcapi.rs index f378eed11c4..32d400f2163 100644 --- a/crates/vm/src/stdlib/_testinternalcapi.rs +++ b/crates/vm/src/stdlib/_testinternalcapi.rs @@ -24,7 +24,9 @@ pub(crate) fn note_func_modification() { RARE_FUNC_MODIFICATION.fetch_add(1, Ordering::Relaxed); } -#[pymodule] +pub(crate) mod pytime; + +#[pymodule(with(pytime::pytime))] mod _testinternalcapi { use super::*; use crate::{ @@ -486,26 +488,6 @@ mod _testinternalcapi { } } - #[pyfunction(name = "_PyTime_AsTimespec")] - fn pytime_as_timespec(ns: i64) -> (i64, i64) { - let sec = ns.div_euclid(1_000_000_000); - let nsec = ns.rem_euclid(1_000_000_000); - (sec, nsec) - } - - #[pyfunction(name = "_PyTime_AsTimespec_clamp")] - fn pytime_as_timespec_clamp(ns: i64) -> (i64, i64) { - pytime_as_timespec(ns) - } - - #[pyfunction(name = "_PyTime_AsTimeval_clamp")] - fn pytime_as_timeval_clamp(ns: i64, _rnd: OptionalArg) -> (i64, i64) { - let us = ns.div_euclid(1000); - let sec = us.div_euclid(1_000_000); - let usec = us.rem_euclid(1_000_000); - (sec, usec) - } - #[pyfunction] fn test_edit_cost() {} @@ -1634,7 +1616,7 @@ fn sep_or_end(path: &[char], idx: usize, size: usize) -> bool { } #[cfg(feature = "threading")] -fn run_string_in_new_subinterp( +pub(crate) fn run_string_in_new_subinterp( source: &str, config: crate::vm::InterpreterConfig, vm: &VirtualMachine, diff --git a/crates/vm/src/stdlib/_testinternalcapi/pytime.rs b/crates/vm/src/stdlib/_testinternalcapi/pytime.rs new file mode 100644 index 00000000000..78b8aa96c64 --- /dev/null +++ b/crates/vm/src/stdlib/_testinternalcapi/pytime.rs @@ -0,0 +1,424 @@ +//! Python/pytime.c conversions and the Modules/_testinternalcapi/pytime.c +//! functions that test them. + +use crate::{ + Py, PyObject, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyInt}, +}; +use num_traits::ToPrimitive; + +pub(crate) type PyTime = i64; + +#[cfg(any(unix, windows))] +type TimeT = libc::time_t; +#[cfg(not(any(unix, windows)))] +type TimeT = i64; + +// On Windows, timeval.tv_sec type is long. +#[cfg(windows)] +type TimevalSec = core::ffi::c_long; +#[cfg(not(windows))] +type TimevalSec = TimeT; + +const SEC_TO_MS: PyTime = 1000; +const MS_TO_US: PyTime = 1000; +const SEC_TO_US: PyTime = SEC_TO_MS * MS_TO_US; +const US_TO_NS: PyTime = 1000; +const MS_TO_NS: PyTime = MS_TO_US * US_TO_NS; +const SEC_TO_NS: PyTime = SEC_TO_MS * MS_TO_NS; +const NS_TO_MS: PyTime = 1000 * 1000; +const NS_TO_US: PyTime = 1000; + +/// _PyTime_round_t +#[derive(Clone, Copy, PartialEq, Eq)] +enum Round { + Floor, + Ceiling, + HalfEven, + Up, +} + +/// check_time_rounding +fn check_time_rounding(round: i32, vm: &VirtualMachine) -> PyResult { + match round { + 0 => Ok(Round::Floor), + 1 => Ok(Round::Ceiling), + 2 => Ok(Round::HalfEven), + 3 => Ok(Round::Up), + _ => Err(vm.new_value_error("invalid rounding")), + } +} + +fn pytime_time_t_overflow(vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_overflow_error("timestamp out of range for platform time_t") +} + +fn pytime_overflow(vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_overflow_error("timestamp out of range for C PyTime_t") +} + +/// Compute t * k. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +fn pytime_mul(t: &mut PyTime, k: PyTime) -> bool { + debug_assert!(k >= 0); + match t.checked_mul(k) { + Some(value) => { + *t = value; + true + } + None => { + *t = if *t >= 0 { PyTime::MAX } else { PyTime::MIN }; + false + } + } +} + +/// _PyLong_AsTime_t +fn pylong_as_time_t(int: &Py, vm: &VirtualMachine) -> PyResult { + int.as_bigint() + .to_i64() + .and_then(|value| TimeT::try_from(value).ok()) + .ok_or_else(|| pytime_time_t_overflow(vm)) +} + +/// _PyTime_AsTime_t: clamp the value on overflow. +fn pytime_as_time_t + num_traits::Bounded>(t: PyTime) -> (T, bool) { + match T::try_from(t) { + Ok(value) => (value, true), + Err(_) if t > 0 => (T::max_value(), false), + Err(_) => (T::min_value(), false), + } +} + +/// Round to nearest with ties going to nearest even integer +/// (_PyTime_ROUND_HALF_EVEN) +fn pytime_round_half_even(x: f64) -> f64 { + let rounded = x.round(); + if (x - rounded).abs() == 0.5 { + // halfway case: round to even + 2.0 * (x / 2.0).round() + } else { + rounded + } +} + +fn pytime_round(x: f64, round: Round) -> f64 { + match round { + Round::HalfEven => pytime_round_half_even(x), + Round::Ceiling => x.ceil(), + Round::Floor => x.floor(), + Round::Up if x >= 0.0 => x.ceil(), + Round::Up => x.floor(), + } +} + +/// modf +fn modf(x: f64) -> (f64, f64) { + let intpart = x.trunc(); + let floatpart = if x.is_infinite() { + 0.0_f64.copysign(x) + } else { + x - intpart + }; + (floatpart, intpart) +} + +/// Whether `intpart` is in the time_t range. The condition is true for NaN. +fn time_t_out_of_range(intpart: f64) -> bool { + !((TimeT::MIN as f64) <= intpart && intpart < -(TimeT::MIN as f64)) +} + +fn pytime_double_to_denominator( + d: f64, + idenominator: i64, + round: Round, + vm: &VirtualMachine, +) -> PyResult<(TimeT, i64)> { + let denominator = idenominator as f64; + let (floatpart, mut intpart) = modf(d); + let mut floatpart = pytime_round(floatpart * denominator, round); + if floatpart >= denominator { + floatpart -= denominator; + intpart += 1.0; + } else if floatpart < 0.0 { + floatpart += denominator; + intpart -= 1.0; + } + debug_assert!((0.0..denominator).contains(&floatpart)); + + if time_t_out_of_range(intpart) { + return Err(pytime_time_t_overflow(vm)); + } + Ok((intpart as TimeT, floatpart as i64)) +} + +/// PyFloat_AsDouble, rejecting NaN. +fn pyfloat_as_double_not_nan(obj: &PyObject, vm: &VirtualMachine) -> PyResult { + let d = obj.try_float(vm)?.to_f64(); + if d.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)")); + } + Ok(d) +} + +fn pytime_object_to_denominator( + obj: &PyObject, + denominator: i64, + round: Round, + vm: &VirtualMachine, +) -> PyResult<(TimeT, i64)> { + debug_assert!(denominator >= 1); + if let Some(int) = obj.try_index_opt(vm) { + let int = int?; + return Ok((pylong_as_time_t(&int, vm)?, 0)); + } + let d = pyfloat_as_double_not_nan(obj, vm)?; + pytime_double_to_denominator(d, denominator, round, vm) +} + +/// _PyTime_ObjectToTime_t +fn pytime_object_to_time_t(obj: &PyObject, round: Round, vm: &VirtualMachine) -> PyResult { + if let Some(int) = obj.try_index_opt(vm) { + let int = int?; + return pylong_as_time_t(&int, vm); + } + let d = pytime_round(pyfloat_as_double_not_nan(obj, vm)?, round); + let (_, intpart) = modf(d); + if time_t_out_of_range(intpart) { + return Err(pytime_time_t_overflow(vm)); + } + Ok(intpart as TimeT) +} + +fn pytime_from_double( + value: f64, + round: Round, + unit_to_ns: PyTime, + vm: &VirtualMachine, +) -> PyResult { + // convert to a number of nanoseconds + let d = pytime_round(value * unit_to_ns as f64, round); + if !((PyTime::MIN as f64) <= d && d < -(PyTime::MIN as f64)) { + return Err(pytime_overflow(vm)); + } + Ok(d as PyTime) +} + +fn pytime_from_object( + obj: &PyObject, + round: Round, + unit_to_ns: PyTime, + vm: &VirtualMachine, +) -> PyResult { + if let Some(int) = obj.try_index_opt(vm) { + let mut ns = int? + .as_bigint() + .to_i64() + .ok_or_else(|| pytime_overflow(vm))?; + if !pytime_mul(&mut ns, unit_to_ns) { + return Err(pytime_overflow(vm)); + } + return Ok(ns); + } + let d = pyfloat_as_double_not_nan(obj, vm)?; + pytime_from_double(d, round, unit_to_ns, vm) +} + +/// PyTime_AsSecondsDouble +pub(crate) fn pytime_as_seconds_double(ns: PyTime) -> f64 { + if ns % SEC_TO_NS == 0 { + // Divide using integers to avoid rounding issues on the integer part. + // 1e-9 cannot be stored exactly in IEEE 64-bit. + (ns / SEC_TO_NS) as f64 + } else { + ns as f64 / 1e9 + } +} + +fn pytime_divide_round_up(t: PyTime, k: PyTime) -> PyTime { + debug_assert!(k > 1); + let q = t / k; + match (t % k != 0, t >= 0) { + (false, _) => q, + (true, true) => q + 1, + (true, false) => q - 1, + } +} + +fn pytime_divide(t: PyTime, k: PyTime, round: Round) -> PyTime { + debug_assert!(k > 1); + match round { + Round::HalfEven => { + let x = t / k; + let abs_r = (t % k).abs(); + if abs_r > k / 2 || (abs_r == k / 2 && x.abs() & 1 != 0) { + if t >= 0 { x + 1 } else { x - 1 } + } else { + x + } + } + Round::Ceiling if t >= 0 => pytime_divide_round_up(t, k), + Round::Ceiling => t / k, + Round::Floor if t >= 0 => t / k, + Round::Floor | Round::Up => pytime_divide_round_up(t, k), + } +} + +/// Compute (t / k, t % k), making sure that 0 <= r < k. On underflow, +/// return (PyTime_MIN, 0) and false. +fn pytime_divmod(t: PyTime, k: PyTime) -> (PyTime, PyTime, bool) { + debug_assert!(k > 1); + let q = t / k; + let r = t % k; + if r >= 0 { + (q, r, true) + } else if q == PyTime::MIN { + (PyTime::MIN, 0, false) + } else { + (q - 1, r + k, true) + } +} + +fn pytime_as_timeval_struct( + t: PyTime, + round: Round, + raise_exc: bool, + vm: &VirtualMachine, +) -> PyResult<(TimevalSec, PyTime)> { + let us = pytime_divide(t, US_TO_NS, round); + let (tv_sec, tv_usec, res) = pytime_divmod(us, SEC_TO_US); + let (tv_sec, res2) = pytime_as_time_t::(tv_sec); + let tv_usec = if res2 { tv_usec } else { 0 }; + if raise_exc && !(res && res2) { + return Err(pytime_time_t_overflow(vm)); + } + Ok((tv_sec, tv_usec)) +} + +fn pytime_as_timespec( + ns: PyTime, + raise_exc: bool, + vm: &VirtualMachine, +) -> PyResult<(TimeT, PyTime)> { + let (tv_sec, tv_nsec, res) = pytime_divmod(ns, SEC_TO_NS); + let (tv_sec, res2) = pytime_as_time_t::(tv_sec); + let tv_nsec = if res2 { tv_nsec } else { 0 }; + if raise_exc && !(res && res2) { + return Err(pytime_time_t_overflow(vm)); + } + Ok((tv_sec, tv_nsec)) +} + +/// PyLong_AsInt64 +pub(crate) fn pylong_as_int64(obj: &PyObject, vm: &VirtualMachine) -> PyResult { + obj.try_index(vm)?.try_to_primitive::(vm) +} + +#[pymodule(sub, name = "_testinternalcapi")] +pub(super) mod pytime { + use super::*; + use crate::PyObjectRef; + + #[pyfunction(name = "_PyTime_FromSeconds")] + fn test_pytime_fromseconds(seconds: i32) -> PyTime { + PyTime::from(seconds) * SEC_TO_NS + } + + #[pyfunction(name = "_PyTime_FromSecondsObject")] + fn test_pytime_fromsecondsobject( + obj: PyObjectRef, + round: i32, + vm: &VirtualMachine, + ) -> PyResult { + let round = check_time_rounding(round, vm)?; + pytime_from_object(&obj, round, SEC_TO_NS, vm) + } + + #[pyfunction(name = "_PyTime_AsTimeval")] + fn test_pytime_as_timeval( + obj: PyObjectRef, + round: i32, + vm: &VirtualMachine, + ) -> PyResult<(TimevalSec, PyTime)> { + let round = check_time_rounding(round, vm)?; + let t = pylong_as_int64(&obj, vm)?; + pytime_as_timeval_struct(t, round, true, vm) + } + + #[pyfunction(name = "_PyTime_AsTimeval_clamp")] + fn test_pytime_as_timeval_clamp( + obj: PyObjectRef, + round: i32, + vm: &VirtualMachine, + ) -> PyResult<(TimevalSec, PyTime)> { + let round = check_time_rounding(round, vm)?; + let t = pylong_as_int64(&obj, vm)?; + pytime_as_timeval_struct(t, round, false, vm) + } + + #[pyfunction(name = "_PyTime_AsTimespec")] + fn test_pytime_as_timespec(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<(TimeT, PyTime)> { + let t = pylong_as_int64(&obj, vm)?; + pytime_as_timespec(t, true, vm) + } + + #[pyfunction(name = "_PyTime_AsTimespec_clamp")] + fn test_pytime_as_timespec_clamp( + obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<(TimeT, PyTime)> { + let t = pylong_as_int64(&obj, vm)?; + pytime_as_timespec(t, false, vm) + } + + #[pyfunction(name = "_PyTime_AsMilliseconds")] + fn test_pytime_as_milliseconds( + obj: PyObjectRef, + round: i32, + vm: &VirtualMachine, + ) -> PyResult { + let t = pylong_as_int64(&obj, vm)?; + let round = check_time_rounding(round, vm)?; + Ok(pytime_divide(t, NS_TO_MS, round)) + } + + #[pyfunction(name = "_PyTime_AsMicroseconds")] + fn test_pytime_as_microseconds( + obj: PyObjectRef, + round: i32, + vm: &VirtualMachine, + ) -> PyResult { + let t = pylong_as_int64(&obj, vm)?; + let round = check_time_rounding(round, vm)?; + Ok(pytime_divide(t, NS_TO_US, round)) + } + + #[pyfunction(name = "_PyTime_ObjectToTime_t")] + fn test_pytime_object_to_time_t( + obj: PyObjectRef, + round: i32, + vm: &VirtualMachine, + ) -> PyResult { + let round = check_time_rounding(round, vm)?; + pytime_object_to_time_t(&obj, round, vm) + } + + #[pyfunction(name = "_PyTime_ObjectToTimeval")] + fn test_pytime_object_to_timeval( + obj: PyObjectRef, + round: i32, + vm: &VirtualMachine, + ) -> PyResult<(TimeT, i64)> { + let round = check_time_rounding(round, vm)?; + pytime_object_to_denominator(&obj, SEC_TO_US, round, vm) + } + + #[pyfunction(name = "_PyTime_ObjectToTimespec")] + fn test_pytime_object_to_timespec( + obj: PyObjectRef, + round: i32, + vm: &VirtualMachine, + ) -> PyResult<(TimeT, i64)> { + let round = check_time_rounding(round, vm)?; + pytime_object_to_denominator(&obj, SEC_TO_NS, round, vm) + } +} diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index 7e6d1700007..c5a1f527600 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -652,7 +652,8 @@ mod decl { .new_value_error("unmarshalling code objects is disallowed"), )); } - let code = PyCode::from_marshal(code, fields, &self.vm.ctx)?; + let code = + PyCode::from_marshal(code, fields, &self.vm.ctx)?.remove_column_info(self.vm); Ok(code.into_ref(&self.vm.ctx).into()) } fn make_stop_iter(&self) -> Result { diff --git a/crates/vm/src/stdlib/mod.rs b/crates/vm/src/stdlib/mod.rs index d5dde116ec1..435017a88ca 100644 --- a/crates/vm/src/stdlib/mod.rs +++ b/crates/vm/src/stdlib/mod.rs @@ -15,6 +15,7 @@ mod _string; mod _symtable; mod _sysconfig; mod _sysconfigdata; +mod _testcapi; pub(crate) mod _testinternalcapi; mod _types; pub mod _typing; @@ -135,6 +136,7 @@ pub fn builtin_module_defs(ctx: &Context) -> Vec<&'static PyModuleDef> { _sre::module_def(ctx), _stat::module_def(ctx), _string::module_def(ctx), + _testcapi::module_def(ctx), _testinternalcapi::module_def(ctx), #[cfg(feature = "compiler")] _symtable::module_def(ctx), diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 7ae959a3b75..9dc8c9cb629 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1708,7 +1708,18 @@ impl VirtualMachine { err_msg: self.new_pyobj(msg), object, }; - if let Err(e) = unraisablehook.call((args,), self) { + let hook_args = args.to_pyobject(self); + if let Err(e) = self.audit("sys.unraisablehook", || { + (unraisablehook.clone(), hook_args.clone()) + }) { + self.write_unraisable_to_stderr( + &e, + Some("Exception ignored in audit hook"), + &self.ctx.none(), + ); + return; + } + if let Err(e) = unraisablehook.call((hook_args,), self) { println!("{}", e.as_object().repr(self).unwrap()); } }