From 6a4f9983666b456669d75a35231a86e2186469f4 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 17 May 2026 14:33:48 +0300 Subject: [PATCH 1/4] Update `test_xxlimited.py` & `test_xxtestfuzz.py` to 3.14.5 --- Lib/test/test_xxlimited.py | 90 +++++++++++++++++++++++++++++++++++++ Lib/test/test_xxtestfuzz.py | 25 +++++++++++ 2 files changed, 115 insertions(+) create mode 100644 Lib/test/test_xxlimited.py create mode 100644 Lib/test/test_xxtestfuzz.py diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py new file mode 100644 index 00000000000..b52e78bc4fb --- /dev/null +++ b/Lib/test/test_xxlimited.py @@ -0,0 +1,90 @@ +import unittest +from test.support import import_helper +import types + +xxlimited = import_helper.import_module('xxlimited') +xxlimited_35 = import_helper.import_module('xxlimited_35') + + +class CommonTests: + module: types.ModuleType + + def test_xxo_new(self): + xxo = self.module.Xxo() + + def test_xxo_attributes(self): + xxo = self.module.Xxo() + with self.assertRaises(AttributeError): + xxo.foo + with self.assertRaises(AttributeError): + del xxo.foo + + xxo.foo = 1234 + self.assertEqual(xxo.foo, 1234) + + del xxo.foo + with self.assertRaises(AttributeError): + xxo.foo + + def test_foo(self): + # the foo function adds 2 numbers + self.assertEqual(self.module.foo(1, 2), 3) + + def test_str(self): + self.assertIsSubclass(self.module.Str, str) + self.assertIsNot(self.module.Str, str) + + custom_string = self.module.Str("abcd") + self.assertEqual(custom_string, "abcd") + self.assertEqual(custom_string.upper(), "ABCD") + + def test_new(self): + xxo = self.module.new() + self.assertEqual(xxo.demo("abc"), "abc") + + +class TestXXLimited(CommonTests, unittest.TestCase): + module = xxlimited + + def test_xxo_demo(self): + xxo = self.module.Xxo() + other = self.module.Xxo() + self.assertEqual(xxo.demo("abc"), "abc") + self.assertEqual(xxo.demo(xxo), xxo) + self.assertEqual(xxo.demo(other), other) + self.assertEqual(xxo.demo(0), None) + + def test_error(self): + with self.assertRaises(self.module.Error): + raise self.module.Error + + def test_buffer(self): + xxo = self.module.Xxo() + self.assertEqual(xxo.x_exports, 0) + b1 = memoryview(xxo) + self.assertEqual(xxo.x_exports, 1) + b2 = memoryview(xxo) + self.assertEqual(xxo.x_exports, 2) + b1[0] = 1 + self.assertEqual(b1[0], 1) + self.assertEqual(b2[0], 1) + + +class TestXXLimited35(CommonTests, unittest.TestCase): + module = xxlimited_35 + + def test_xxo_demo(self): + xxo = self.module.Xxo() + other = self.module.Xxo() + self.assertEqual(xxo.demo("abc"), "abc") + self.assertEqual(xxo.demo(0), None) + + def test_roj(self): + # the roj function always fails + with self.assertRaises(SystemError): + self.module.roj(0) + + def test_null(self): + null1 = self.module.Null() + null2 = self.module.Null() + self.assertNotEqual(null1, null2) diff --git a/Lib/test/test_xxtestfuzz.py b/Lib/test/test_xxtestfuzz.py new file mode 100644 index 00000000000..3304c6e703a --- /dev/null +++ b/Lib/test/test_xxtestfuzz.py @@ -0,0 +1,25 @@ +import faulthandler +from test.support import import_helper +import unittest + +_xxtestfuzz = import_helper.import_module('_xxtestfuzz') + + +class TestFuzzer(unittest.TestCase): + """To keep our https://github.com/google/oss-fuzz API working.""" + + def test_sample_input_smoke_test(self): + """This is only a regression test: Check that it doesn't crash.""" + _xxtestfuzz.run(b"") + _xxtestfuzz.run(b"\0") + _xxtestfuzz.run(b"{") + _xxtestfuzz.run(b" ") + _xxtestfuzz.run(b"x") + _xxtestfuzz.run(b"1") + _xxtestfuzz.run(b"AAAAAAA") + _xxtestfuzz.run(b"AAAAAA\0") + + +if __name__ == "__main__": + faulthandler.enable() + unittest.main() From f484c1ab2d1d2a971ddd77518302f16bb0d96585 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 17 May 2026 14:44:16 +0300 Subject: [PATCH 2/4] Add more NOP tests --- Lib/test/test_asdl_parser.py | 131 + Lib/test/test_capi/__init__.py | 12 + Lib/test/test_capi/__main__.py | 3 + Lib/test/test_capi/check_config.py | 77 + Lib/test/test_capi/test_abstract.py | 1082 ++++ Lib/test/test_capi/test_bytearray.py | 174 + Lib/test/test_capi/test_bytes.py | 303 ++ Lib/test/test_capi/test_codecs.py | 879 ++++ Lib/test/test_capi/test_complex.py | 299 ++ Lib/test/test_capi/test_config.py | 404 ++ Lib/test/test_capi/test_dict.py | 550 ++ Lib/test/test_capi/test_emscripten.py | 25 + Lib/test/test_capi/test_eval.py | 103 + Lib/test/test_capi/test_eval_code_ex.py | 126 + Lib/test/test_capi/test_exceptions.py | 680 +++ Lib/test/test_capi/test_file.py | 325 ++ Lib/test/test_capi/test_float.py | 259 + Lib/test/test_capi/test_frame.py | 56 + Lib/test/test_capi/test_function.py | 340 ++ Lib/test/test_capi/test_getargs.py | 1421 +++++ Lib/test/test_capi/test_hash.py | 93 + Lib/test/test_capi/test_immortal.py | 42 + Lib/test/test_capi/test_import.py | 381 ++ Lib/test/test_capi/test_list.py | 359 ++ Lib/test/test_capi/test_long.py | 818 +++ Lib/test/test_capi/test_mem.py | 183 + Lib/test/test_capi/test_misc.py | 2922 +++++++++++ Lib/test/test_capi/test_number.py | 352 ++ Lib/test/test_capi/test_object.py | 258 + Lib/test/test_capi/test_opt.py | 2021 ++++++++ Lib/test/test_capi/test_pyatomic.py | 15 + Lib/test/test_capi/test_run.py | 118 + Lib/test/test_capi/test_set.py | 271 + Lib/test/test_capi/test_structmembers.py | 186 + Lib/test/test_capi/test_sys.py | 154 + Lib/test/test_capi/test_time.py | 78 + Lib/test/test_capi/test_tuple.py | 301 ++ Lib/test/test_capi/test_type.py | 276 + Lib/test/test_capi/test_unicode.py | 2039 ++++++++ Lib/test/test_capi/test_watchers.py | 678 +++ Lib/test/test_cext/__init__.py | 127 + Lib/test/test_cext/extension.c | 142 + Lib/test/test_cext/setup.py | 135 + Lib/test/test_clinic.py | 4594 +++++++++++++++++ Lib/test/test_generated_cases.py | 2074 ++++++++ Lib/test/test_peg_generator/__init__.py | 12 + Lib/test/test_peg_generator/__main__.py | 4 + Lib/test/test_peg_generator/test_c_parser.py | 523 ++ .../test_peg_generator/test_first_sets.py | 286 + .../test_grammar_validator.py | 77 + Lib/test/test_peg_generator/test_pegen.py | 1132 ++++ .../test_tools/i18n_data/ascii-escapes.pot | 8 +- Lib/test/test_tools/i18n_data/comments.pot | 110 + Lib/test/test_tools/i18n_data/comments.py | 78 + .../test_tools/i18n_data/custom_keywords.pot | 51 + .../test_tools/i18n_data/custom_keywords.py | 34 + Lib/test/test_tools/i18n_data/docstrings.pot | 28 +- Lib/test/test_tools/i18n_data/docstrings.py | 12 +- Lib/test/test_tools/i18n_data/escapes.pot | 6 + Lib/test/test_tools/i18n_data/messages.pot | 56 +- Lib/test/test_tools/i18n_data/messages.py | 67 +- .../i18n_data/multiple_keywords.pot | 38 + .../test_tools/i18n_data/multiple_keywords.py | 11 + Lib/test/test_tools/test_compute_changes.py | 144 + Lib/test/test_tools/test_i18n.py | 215 +- Lib/test/test_tools/test_makefile.py | 17 +- Lib/test/test_tools/test_msgfmt.py | 159 +- Lib/test/test_type_cache.py | 265 + 68 files changed, 29141 insertions(+), 58 deletions(-) create mode 100644 Lib/test/test_asdl_parser.py create mode 100644 Lib/test/test_capi/__init__.py create mode 100644 Lib/test/test_capi/__main__.py create mode 100644 Lib/test/test_capi/check_config.py create mode 100644 Lib/test/test_capi/test_abstract.py create mode 100644 Lib/test/test_capi/test_bytearray.py create mode 100644 Lib/test/test_capi/test_bytes.py create mode 100644 Lib/test/test_capi/test_codecs.py create mode 100644 Lib/test/test_capi/test_complex.py create mode 100644 Lib/test/test_capi/test_config.py create mode 100644 Lib/test/test_capi/test_dict.py create mode 100644 Lib/test/test_capi/test_emscripten.py create mode 100644 Lib/test/test_capi/test_eval.py create mode 100644 Lib/test/test_capi/test_eval_code_ex.py create mode 100644 Lib/test/test_capi/test_exceptions.py create mode 100644 Lib/test/test_capi/test_file.py create mode 100644 Lib/test/test_capi/test_float.py create mode 100644 Lib/test/test_capi/test_frame.py create mode 100644 Lib/test/test_capi/test_function.py create mode 100644 Lib/test/test_capi/test_getargs.py create mode 100644 Lib/test/test_capi/test_hash.py create mode 100644 Lib/test/test_capi/test_immortal.py create mode 100644 Lib/test/test_capi/test_import.py create mode 100644 Lib/test/test_capi/test_list.py create mode 100644 Lib/test/test_capi/test_long.py create mode 100644 Lib/test/test_capi/test_mem.py create mode 100644 Lib/test/test_capi/test_misc.py create mode 100644 Lib/test/test_capi/test_number.py create mode 100644 Lib/test/test_capi/test_object.py create mode 100644 Lib/test/test_capi/test_opt.py create mode 100644 Lib/test/test_capi/test_pyatomic.py create mode 100644 Lib/test/test_capi/test_run.py create mode 100644 Lib/test/test_capi/test_set.py create mode 100644 Lib/test/test_capi/test_structmembers.py create mode 100644 Lib/test/test_capi/test_sys.py create mode 100644 Lib/test/test_capi/test_time.py create mode 100644 Lib/test/test_capi/test_tuple.py create mode 100644 Lib/test/test_capi/test_type.py create mode 100644 Lib/test/test_capi/test_unicode.py create mode 100644 Lib/test/test_capi/test_watchers.py create mode 100644 Lib/test/test_cext/__init__.py create mode 100644 Lib/test/test_cext/extension.c create mode 100644 Lib/test/test_cext/setup.py create mode 100644 Lib/test/test_clinic.py create mode 100644 Lib/test/test_generated_cases.py create mode 100644 Lib/test/test_peg_generator/__init__.py create mode 100644 Lib/test/test_peg_generator/__main__.py create mode 100644 Lib/test/test_peg_generator/test_c_parser.py create mode 100644 Lib/test/test_peg_generator/test_first_sets.py create mode 100644 Lib/test/test_peg_generator/test_grammar_validator.py create mode 100644 Lib/test/test_peg_generator/test_pegen.py create mode 100644 Lib/test/test_tools/i18n_data/comments.pot create mode 100644 Lib/test/test_tools/i18n_data/comments.py create mode 100644 Lib/test/test_tools/i18n_data/custom_keywords.pot create mode 100644 Lib/test/test_tools/i18n_data/custom_keywords.py create mode 100644 Lib/test/test_tools/i18n_data/multiple_keywords.pot create mode 100644 Lib/test/test_tools/i18n_data/multiple_keywords.py create mode 100644 Lib/test/test_tools/test_compute_changes.py create mode 100644 Lib/test/test_type_cache.py diff --git a/Lib/test/test_asdl_parser.py b/Lib/test/test_asdl_parser.py new file mode 100644 index 00000000000..b9df6568123 --- /dev/null +++ b/Lib/test/test_asdl_parser.py @@ -0,0 +1,131 @@ +"""Tests for the asdl parser in Parser/asdl.py""" + +import importlib.machinery +import importlib.util +import os +from os.path import dirname +import sys +import sysconfig +import unittest + + +# This test is only relevant for from-source builds of Python. +if not sysconfig.is_python_build(): + raise unittest.SkipTest('test irrelevant for an installed Python') + +src_base = dirname(dirname(dirname(__file__))) +parser_dir = os.path.join(src_base, 'Parser') + + +class TestAsdlParser(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Loads the asdl module dynamically, since it's not in a real importable + # package. + # Parses Python.asdl into an ast.Module and run the check on it. + # There's no need to do this for each test method, hence setUpClass. + sys.path.insert(0, parser_dir) + loader = importlib.machinery.SourceFileLoader( + 'asdl', os.path.join(parser_dir, 'asdl.py')) + spec = importlib.util.spec_from_loader('asdl', loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + cls.asdl = module + cls.mod = cls.asdl.parse(os.path.join(parser_dir, 'Python.asdl')) + cls.assertTrue(cls.asdl.check(cls.mod), 'Module validation failed') + + @classmethod + def tearDownClass(cls): + del sys.path[0] + + def setUp(self): + # alias stuff from the class, for convenience + self.asdl = TestAsdlParser.asdl + self.mod = TestAsdlParser.mod + self.types = self.mod.types + + def test_module(self): + self.assertEqual(self.mod.name, 'Python') + self.assertIn('stmt', self.types) + self.assertIn('expr', self.types) + self.assertIn('mod', self.types) + + def test_definitions(self): + defs = self.mod.dfns + self.assertIsInstance(defs[0], self.asdl.Type) + self.assertIsInstance(defs[0].value, self.asdl.Sum) + + self.assertIsInstance(self.types['withitem'], self.asdl.Product) + self.assertIsInstance(self.types['alias'], self.asdl.Product) + + def test_product(self): + alias = self.types['alias'] + self.assertEqual( + str(alias), + 'Product([Field(identifier, name), Field(identifier, asname, quantifiers=[OPTIONAL])], ' + '[Field(int, lineno), Field(int, col_offset), ' + 'Field(int, end_lineno, quantifiers=[OPTIONAL]), Field(int, end_col_offset, quantifiers=[OPTIONAL])])') + + def test_attributes(self): + stmt = self.types['stmt'] + self.assertEqual(len(stmt.attributes), 4) + self.assertEqual(repr(stmt.attributes[0]), 'Field(int, lineno)') + self.assertEqual(repr(stmt.attributes[1]), 'Field(int, col_offset)') + self.assertEqual(repr(stmt.attributes[2]), 'Field(int, end_lineno, quantifiers=[OPTIONAL])') + self.assertEqual(repr(stmt.attributes[3]), 'Field(int, end_col_offset, quantifiers=[OPTIONAL])') + + def test_constructor_fields(self): + ehandler = self.types['excepthandler'] + self.assertEqual(len(ehandler.types), 1) + self.assertEqual(len(ehandler.attributes), 4) + + cons = ehandler.types[0] + self.assertIsInstance(cons, self.asdl.Constructor) + self.assertEqual(len(cons.fields), 3) + + f0 = cons.fields[0] + self.assertEqual(f0.type, 'expr') + self.assertEqual(f0.name, 'type') + self.assertTrue(f0.opt) + + f1 = cons.fields[1] + self.assertEqual(f1.type, 'identifier') + self.assertEqual(f1.name, 'name') + self.assertTrue(f1.opt) + + f2 = cons.fields[2] + self.assertEqual(f2.type, 'stmt') + self.assertEqual(f2.name, 'body') + self.assertFalse(f2.opt) + self.assertTrue(f2.seq) + + def test_visitor(self): + class CustomVisitor(self.asdl.VisitorBase): + def __init__(self): + super().__init__() + self.names_with_seq = [] + + def visitModule(self, mod): + for dfn in mod.dfns: + self.visit(dfn) + + def visitType(self, type): + self.visit(type.value) + + def visitSum(self, sum): + for t in sum.types: + self.visit(t) + + def visitConstructor(self, cons): + for f in cons.fields: + if f.seq: + self.names_with_seq.append(cons.name) + + v = CustomVisitor() + v.visit(self.types['mod']) + self.assertEqual(v.names_with_seq, + ['Module', 'Module', 'Interactive', 'FunctionType']) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_capi/__init__.py b/Lib/test/test_capi/__init__.py new file mode 100644 index 00000000000..5a8ba684539 --- /dev/null +++ b/Lib/test/test_capi/__init__.py @@ -0,0 +1,12 @@ +import os +import unittest +from test.support import load_package_tests +from test.support import TEST_MODULES_ENABLED + + +if not TEST_MODULES_ENABLED: + raise unittest.SkipTest("requires test modules") + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_capi/__main__.py b/Lib/test/test_capi/__main__.py new file mode 100644 index 00000000000..05d01775ddf --- /dev/null +++ b/Lib/test/test_capi/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_capi') diff --git a/Lib/test/test_capi/check_config.py b/Lib/test/test_capi/check_config.py new file mode 100644 index 00000000000..bc4f3a0beb9 --- /dev/null +++ b/Lib/test/test_capi/check_config.py @@ -0,0 +1,77 @@ +# This script is used by test_misc. + +import _imp +import _testinternalcapi +import json +import os +import sys + + +def import_singlephase(): + assert '_testsinglephase' not in sys.modules + try: + import _testsinglephase # noqa: F401 + except ImportError: + sys.modules.pop('_testsinglephase', None) + return False + else: + del sys.modules['_testsinglephase'] + return True + + +def check_singlephase(override): + # Check using the default setting. + settings_initial = _testinternalcapi.get_interp_settings() + allowed_initial = import_singlephase() + assert(_testinternalcapi.get_interp_settings() == settings_initial) + + # Apply the override and check. + override_initial = _imp._override_multi_interp_extensions_check(override) + settings_after = _testinternalcapi.get_interp_settings() + allowed_after = import_singlephase() + + # Apply the override again and check. + noop = {} + override_after = _imp._override_multi_interp_extensions_check(override) + settings_noop = _testinternalcapi.get_interp_settings() + if settings_noop != settings_after: + noop['settings_noop'] = settings_noop + allowed_noop = import_singlephase() + if allowed_noop != allowed_after: + noop['allowed_noop'] = allowed_noop + + # Restore the original setting and check. + override_noop = _imp._override_multi_interp_extensions_check(override_initial) + if override_noop != override_after: + noop['override_noop'] = override_noop + settings_restored = _testinternalcapi.get_interp_settings() + allowed_restored = import_singlephase() + + # Restore the original setting again. + override_restored = _imp._override_multi_interp_extensions_check(override_initial) + assert(_testinternalcapi.get_interp_settings() == settings_restored) + + return dict({ + 'requested': override, + 'override__initial': override_initial, + 'override_after': override_after, + 'override_restored': override_restored, + 'settings__initial': settings_initial, + 'settings_after': settings_after, + 'settings_restored': settings_restored, + 'allowed__initial': allowed_initial, + 'allowed_after': allowed_after, + 'allowed_restored': allowed_restored, + }, **noop) + + +def run_singlephase_check(override, outfd): + with os.fdopen(outfd, 'w') as outfile: + sys.stdout = outfile + sys.stderr = outfile + try: + results = check_singlephase(override) + json.dump(results, outfile) + finally: + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py new file mode 100644 index 00000000000..7d548ae87c0 --- /dev/null +++ b/Lib/test/test_capi/test_abstract.py @@ -0,0 +1,1082 @@ +import unittest +from collections import OrderedDict +from test import support +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + +NULL = None + +class StrSubclass(str): + pass + +class BytesSubclass(bytes): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + +class TestObject: + @property + def evil(self): + raise RuntimeError('do not get evil') + @evil.setter + def evil(self, value): + raise RuntimeError('do not set evil') + @evil.deleter + def evil(self): + raise RuntimeError('do not del evil') + +class ProxyGetItem: + def __init__(self, obj): + self.obj = obj + def __getitem__(self, key): + return self.obj[key] + +class ProxySetItem: + def __init__(self, obj): + self.obj = obj + def __setitem__(self, key, value): + self.obj[key] = value + +class ProxyDelItem: + def __init__(self, obj): + self.obj = obj + def __delitem__(self, key): + del self.obj[key] + +def gen(): + yield 'a' + yield 'b' + yield 'c' + + +class CAPITest(unittest.TestCase): + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + + def test_object_str(self): + # Test PyObject_Str() + object_str = _testlimitedcapi.object_str + self.assertTypedEqual(object_str(''), '') + self.assertTypedEqual(object_str('abc'), 'abc') + self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') + self.assertTypedEqual(object_str(StrSubclass('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(object_str(WithRepr('')), '') + self.assertTypedEqual(object_str(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_str(NULL), '') + + def test_object_repr(self): + # Test PyObject_Repr() + object_repr = _testlimitedcapi.object_repr + self.assertTypedEqual(object_repr(''), "''") + self.assertTypedEqual(object_repr('abc'), "'abc'") + self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(object_repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_repr(WithRepr('')), '') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertTypedEqual(object_repr(NULL), '') + + def test_object_ascii(self): + # Test PyObject_ASCII() + object_ascii = _testlimitedcapi.object_ascii + self.assertTypedEqual(object_ascii(''), "''") + self.assertTypedEqual(object_ascii('abc'), "'abc'") + self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(object_ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_ascii(WithRepr('')), '') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass(''))), StrSubclass('')) + self.assertTypedEqual(object_ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(NULL), '') + + def test_object_bytes(self): + # Test PyObject_Bytes() + object_bytes = _testlimitedcapi.object_bytes + self.assertTypedEqual(object_bytes(b''), b'') + self.assertTypedEqual(object_bytes(b'abc'), b'abc') + self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(object_bytes(bytearray(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(memoryview(b'abc')), b'abc') + self.assertTypedEqual(object_bytes([97, 98, 99]), b'abc') + self.assertTypedEqual(object_bytes((97, 98, 99)), b'abc') + self.assertTypedEqual(object_bytes(iter([97, 98, 99])), b'abc') + self.assertRaises(TypeError, object_bytes, WithBytes(bytearray(b'abc'))) + self.assertRaises(TypeError, object_bytes, WithBytes([97, 98, 99])) + self.assertRaises(TypeError, object_bytes, 3) + self.assertRaises(TypeError, object_bytes, 'abc') + self.assertRaises(TypeError, object_bytes, object()) + self.assertTypedEqual(object_bytes(NULL), b'') + + def test_object_getattr(self): + xgetattr = _testlimitedcapi.object_getattr + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(xgetattr(obj, 'a'), 11) + self.assertRaises(AttributeError, xgetattr, obj, 'b') + self.assertEqual(xgetattr(obj, '\U0001f40d'), 22) + + self.assertRaises(RuntimeError, xgetattr, obj, 'evil') + self.assertRaises(TypeError, xgetattr, obj, 1) + # CRASHES xgetattr(obj, NULL) + # CRASHES xgetattr(NULL, 'a') + + def test_object_getattrstring(self): + getattrstring = _testlimitedcapi.object_getattrstring + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getattrstring(obj, b'a'), 11) + self.assertRaises(AttributeError, getattrstring, obj, b'b') + self.assertEqual(getattrstring(obj, '\U0001f40d'.encode()), 22) + + self.assertRaises(RuntimeError, getattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, getattrstring, obj, b'\xff') + # CRASHES getattrstring(obj, NULL) + # CRASHES getattrstring(NULL, b'a') + + def test_object_getoptionalattr(self): + getoptionalattr = _testcapi.object_getoptionalattr + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getoptionalattr(obj, 'a'), 11) + self.assertIs(getoptionalattr(obj, 'b'), AttributeError) + self.assertEqual(getoptionalattr(obj, '\U0001f40d'), 22) + + self.assertRaises(RuntimeError, getoptionalattr, obj, 'evil') + self.assertRaises(TypeError, getoptionalattr, obj, 1) + # CRASHES getoptionalattr(obj, NULL) + # CRASHES getoptionalattr(NULL, 'a') + + def test_object_getoptionalattrstring(self): + getoptionalattrstring = _testcapi.object_getoptionalattrstring + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getoptionalattrstring(obj, b'a'), 11) + self.assertIs(getoptionalattrstring(obj, b'b'), AttributeError) + self.assertEqual(getoptionalattrstring(obj, '\U0001f40d'.encode()), 22) + + self.assertRaises(RuntimeError, getoptionalattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, getoptionalattrstring, obj, b'\xff') + # CRASHES getoptionalattrstring(obj, NULL) + # CRASHES getoptionalattrstring(NULL, b'a') + + def test_object_hasattr(self): + xhasattr = _testlimitedcapi.object_hasattr + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(xhasattr(obj, 'a')) + self.assertFalse(xhasattr(obj, 'b')) + self.assertTrue(xhasattr(obj, '\U0001f40d')) + + with support.catch_unraisable_exception() as cm: + self.assertFalse(xhasattr(obj, 'evil')) + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + self.assertEqual(str(cm.unraisable.exc_value), + 'do not get evil') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(xhasattr(obj, 1)) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "attribute name must be string, not 'int'") + + # CRASHES xhasattr(obj, NULL) + # CRASHES xhasattr(NULL, 'a') + + def test_object_hasattrstring(self): + hasattrstring = _testlimitedcapi.object_hasattrstring + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(hasattrstring(obj, b'a')) + self.assertFalse(hasattrstring(obj, b'b')) + self.assertTrue(hasattrstring(obj, '\U0001f40d'.encode())) + + with support.catch_unraisable_exception() as cm: + self.assertFalse(hasattrstring(obj, b'evil')) + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + self.assertEqual(str(cm.unraisable.exc_value), + 'do not get evil') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(hasattrstring(obj, b'\xff')) + self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) + self.assertRegex(str(cm.unraisable.exc_value), + "'utf-8' codec can't decode") + + # CRASHES hasattrstring(obj, NULL) + # CRASHES hasattrstring(NULL, b'a') + + def test_object_hasattrwitherror(self): + xhasattr = _testcapi.object_hasattrwitherror + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(xhasattr(obj, 'a')) + self.assertFalse(xhasattr(obj, 'b')) + self.assertTrue(xhasattr(obj, '\U0001f40d')) + + self.assertRaises(RuntimeError, xhasattr, obj, 'evil') + self.assertRaises(TypeError, xhasattr, obj, 1) + # CRASHES xhasattr(obj, NULL) + # CRASHES xhasattr(NULL, 'a') + + def test_object_hasattrstringwitherror(self): + hasattrstring = _testcapi.object_hasattrstringwitherror + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(hasattrstring(obj, b'a')) + self.assertFalse(hasattrstring(obj, b'b')) + self.assertTrue(hasattrstring(obj, '\U0001f40d'.encode())) + + self.assertRaises(RuntimeError, hasattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, hasattrstring, obj, b'\xff') + # CRASHES hasattrstring(obj, NULL) + # CRASHES hasattrstring(NULL, b'a') + + def test_object_setattr(self): + xsetattr = _testlimitedcapi.object_setattr + obj = TestObject() + xsetattr(obj, 'a', 5) + self.assertEqual(obj.a, 5) + xsetattr(obj, '\U0001f40d', 8) + self.assertEqual(getattr(obj, '\U0001f40d'), 8) + + # PyObject_SetAttr(obj, attr_name, NULL) removes the attribute + xsetattr(obj, 'a', NULL) + self.assertNotHasAttr(obj, 'a') + self.assertRaises(AttributeError, xsetattr, obj, 'b', NULL) + self.assertRaises(RuntimeError, xsetattr, obj, 'evil', NULL) + + self.assertRaises(RuntimeError, xsetattr, obj, 'evil', 'good') + self.assertRaises(AttributeError, xsetattr, 42, 'a', 5) + self.assertRaises(TypeError, xsetattr, obj, 1, 5) + # CRASHES xsetattr(obj, NULL, 5) + # CRASHES xsetattr(NULL, 'a', 5) + + def test_object_setattrstring(self): + setattrstring = _testlimitedcapi.object_setattrstring + obj = TestObject() + setattrstring(obj, b'a', 5) + self.assertEqual(obj.a, 5) + setattrstring(obj, '\U0001f40d'.encode(), 8) + self.assertEqual(getattr(obj, '\U0001f40d'), 8) + + # PyObject_SetAttrString(obj, attr_name, NULL) removes the attribute + setattrstring(obj, b'a', NULL) + self.assertNotHasAttr(obj, 'a') + self.assertRaises(AttributeError, setattrstring, obj, b'b', NULL) + self.assertRaises(RuntimeError, setattrstring, obj, b'evil', NULL) + + self.assertRaises(RuntimeError, setattrstring, obj, b'evil', 'good') + self.assertRaises(AttributeError, setattrstring, 42, b'a', 5) + self.assertRaises(TypeError, setattrstring, obj, 1, 5) + self.assertRaises(UnicodeDecodeError, setattrstring, obj, b'\xff', 5) + # CRASHES setattrstring(obj, NULL, 5) + # CRASHES setattrstring(NULL, b'a', 5) + + def test_object_delattr(self): + xdelattr = _testlimitedcapi.object_delattr + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + xdelattr(obj, 'a') + self.assertNotHasAttr(obj, 'a') + self.assertRaises(AttributeError, xdelattr, obj, 'b') + xdelattr(obj, '\U0001f40d') + self.assertNotHasAttr(obj, '\U0001f40d') + + self.assertRaises(AttributeError, xdelattr, 42, 'numerator') + self.assertRaises(RuntimeError, xdelattr, obj, 'evil') + self.assertRaises(TypeError, xdelattr, obj, 1) + # CRASHES xdelattr(obj, NULL) + # CRASHES xdelattr(NULL, 'a') + + def test_object_delattrstring(self): + delattrstring = _testlimitedcapi.object_delattrstring + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + delattrstring(obj, b'a') + self.assertNotHasAttr(obj, 'a') + self.assertRaises(AttributeError, delattrstring, obj, b'b') + delattrstring(obj, '\U0001f40d'.encode()) + self.assertNotHasAttr(obj, '\U0001f40d') + + self.assertRaises(AttributeError, delattrstring, 42, b'numerator') + self.assertRaises(RuntimeError, delattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, delattrstring, obj, b'\xff') + # CRASHES delattrstring(obj, NULL) + # CRASHES delattrstring(NULL, b'a') + + + def test_mapping_check(self): + check = _testlimitedcapi.mapping_check + self.assertTrue(check({1: 2})) + self.assertTrue(check([1, 2])) + self.assertTrue(check((1, 2))) + self.assertTrue(check('abc')) + self.assertTrue(check(b'abc')) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + self.assertFalse(check(NULL)) + + def test_mapping_size(self): + for size in _testlimitedcapi.mapping_size, _testlimitedcapi.mapping_length: + self.assertEqual(size({1: 2}), 1) + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size('abc'), 3) + self.assertEqual(size(b'abc'), 3) + + self.assertRaises(TypeError, size, 42) + self.assertRaises(TypeError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_object_getitem(self): + getitem = _testlimitedcapi.object_getitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertRaises(KeyError, getitem, dct, 'b') + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertRaises(KeyError, getitem, dct2, 'b') + + self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') + + self.assertRaises(TypeError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, {}, NULL) + self.assertRaises(IndexError, getitem, [], 1) + self.assertRaises(TypeError, getitem, [], 'a') + self.assertRaises(SystemError, getitem, NULL, 'a') + + def test_mapping_getitemstring(self): + getitemstring = _testlimitedcapi.mapping_getitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertRaises(KeyError, getitemstring, dct, b'b') + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertRaises(KeyError, getitemstring, dct2, b'b') + + self.assertRaises(TypeError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(SystemError, getitemstring, {}, NULL) + self.assertRaises(TypeError, getitemstring, [], b'a') + self.assertRaises(SystemError, getitemstring, NULL, b'a') + + def test_mapping_getoptionalitem(self): + getitem = _testcapi.mapping_getoptionalitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertEqual(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertEqual(getitem(dct2, 'b'), KeyError) + + self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') + + self.assertRaises(TypeError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(IndexError, getitem, [], 1) + self.assertRaises(TypeError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_mapping_getoptionalitemstring(self): + getitemstring = _testcapi.mapping_getoptionalitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertEqual(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertEqual(getitemstring(dct2, b'b'), KeyError) + + self.assertRaises(TypeError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(SystemError, getitemstring, {}, NULL) + self.assertRaises(TypeError, getitemstring, [], b'a') + # CRASHES getitemstring(NULL, b'a') + + def test_mapping_haskey(self): + haskey = _testlimitedcapi.mapping_haskey + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskey(dct, 'a')) + self.assertFalse(haskey(dct, 'b')) + self.assertTrue(haskey(dct, '\U0001f40d')) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskey(dct2, 'a')) + self.assertFalse(haskey(dct2, 'b')) + + self.assertTrue(haskey(['a', 'b', 'c'], 1)) + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey(42, 'a')) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "'int' object is not subscriptable") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey({}, [])) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "cannot use 'list' as a dict key " + "(unhashable type: 'list')") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey([], 1)) + self.assertEqual(cm.unraisable.exc_type, IndexError) + self.assertEqual(str(cm.unraisable.exc_value), + 'list index out of range') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey([], 'a')) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + 'list indices must be integers or slices, not str') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey({}, NULL)) + self.assertEqual(cm.unraisable.exc_type, SystemError) + self.assertEqual(str(cm.unraisable.exc_value), + 'null argument to internal routine') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskey(NULL, 'a')) + self.assertEqual(cm.unraisable.exc_type, SystemError) + self.assertEqual(str(cm.unraisable.exc_value), + 'null argument to internal routine') + + def test_mapping_haskeystring(self): + haskeystring = _testlimitedcapi.mapping_haskeystring + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskeystring(dct, b'a')) + self.assertFalse(haskeystring(dct, b'b')) + self.assertTrue(haskeystring(dct, '\U0001f40d'.encode())) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskeystring(dct2, b'a')) + self.assertFalse(haskeystring(dct2, b'b')) + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring(42, b'a')) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "'int' object is not subscriptable") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring({}, b'\xff')) + self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) + self.assertRegex(str(cm.unraisable.exc_value), + "'utf-8' codec can't decode") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring({}, NULL)) + self.assertEqual(cm.unraisable.exc_type, SystemError) + self.assertEqual(str(cm.unraisable.exc_value), + "null argument to internal routine") + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring([], b'a')) + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + 'list indices must be integers or slices, not str') + + with support.catch_unraisable_exception() as cm: + self.assertFalse(haskeystring(NULL, b'a')) + self.assertEqual(cm.unraisable.exc_type, SystemError) + self.assertEqual(str(cm.unraisable.exc_value), + "null argument to internal routine") + + def test_mapping_haskeywitherror(self): + haskey = _testlimitedcapi.mapping_haskeywitherror + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskey(dct, 'a')) + self.assertFalse(haskey(dct, 'b')) + self.assertTrue(haskey(dct, '\U0001f40d')) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskey(dct2, 'a')) + self.assertFalse(haskey(dct2, 'b')) + + self.assertTrue(haskey(['a', 'b', 'c'], 1)) + + self.assertRaises(TypeError, haskey, 42, 'a') + self.assertRaises(TypeError, haskey, {}, []) # unhashable + self.assertRaises(IndexError, haskey, [], 1) + self.assertRaises(TypeError, haskey, [], 'a') + + # CRASHES haskey({}, NULL)) + # CRASHES haskey(NULL, 'a')) + + def test_mapping_haskeystringwitherror(self): + haskeystring = _testlimitedcapi.mapping_haskeystringwitherror + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskeystring(dct, b'a')) + self.assertFalse(haskeystring(dct, b'b')) + self.assertTrue(haskeystring(dct, '\U0001f40d'.encode())) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskeystring(dct2, b'a')) + self.assertFalse(haskeystring(dct2, b'b')) + + self.assertRaises(TypeError, haskeystring, 42, b'a') + self.assertRaises(UnicodeDecodeError, haskeystring, {}, b'\xff') + self.assertRaises(SystemError, haskeystring, {}, NULL) + self.assertRaises(TypeError, haskeystring, [], b'a') + # CRASHES haskeystring(NULL, b'a') + + def test_object_setitem(self): + setitem = _testlimitedcapi.object_setitem + dct = {} + setitem(dct, 'a', 5) + self.assertEqual(dct, {'a': 5}) + setitem(dct, '\U0001f40d', 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct = {} + dct2 = ProxySetItem(dct) + setitem(dct2, 'a', 5) + self.assertEqual(dct, {'a': 5}) + + lst = ['a', 'b', 'c'] + setitem(lst, 1, 'x') + self.assertEqual(lst, ['a', 'x', 'c']) + + self.assertRaises(TypeError, setitem, 42, 'a', 5) + self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable + self.assertRaises(SystemError, setitem, {}, NULL, 5) + self.assertRaises(SystemError, setitem, {}, 'a', NULL) + self.assertRaises(IndexError, setitem, [], 1, 5) + self.assertRaises(TypeError, setitem, [], 'a', 5) + self.assertRaises(TypeError, setitem, (), 1, 5) + self.assertRaises(SystemError, setitem, NULL, 'a', 5) + + def test_mapping_setitemstring(self): + setitemstring = _testlimitedcapi.mapping_setitemstring + dct = {} + setitemstring(dct, b'a', 5) + self.assertEqual(dct, {'a': 5}) + setitemstring(dct, '\U0001f40d'.encode(), 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct = {} + dct2 = ProxySetItem(dct) + setitemstring(dct2, b'a', 5) + self.assertEqual(dct, {'a': 5}) + + self.assertRaises(TypeError, setitemstring, 42, b'a', 5) + self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) + self.assertRaises(SystemError, setitemstring, {}, NULL, 5) + self.assertRaises(SystemError, setitemstring, {}, b'a', NULL) + self.assertRaises(TypeError, setitemstring, [], b'a', 5) + self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) + + def test_object_delitem(self): + for delitem in _testlimitedcapi.object_delitem, _testlimitedcapi.mapping_delitem: + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitem(dct, 'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitem, dct, 'b') + delitem(dct, '\U0001f40d') + self.assertEqual(dct, {'c': 2}) + + dct = {'a': 1, 'c': 2} + dct2 = ProxyDelItem(dct) + delitem(dct2, 'a') + self.assertEqual(dct, {'c': 2}) + self.assertRaises(KeyError, delitem, dct2, 'b') + + lst = ['a', 'b', 'c'] + delitem(lst, 1) + self.assertEqual(lst, ['a', 'c']) + + self.assertRaises(TypeError, delitem, 42, 'a') + self.assertRaises(TypeError, delitem, {}, []) # unhashable + self.assertRaises(SystemError, delitem, {}, NULL) + self.assertRaises(IndexError, delitem, [], 1) + self.assertRaises(TypeError, delitem, [], 'a') + self.assertRaises(SystemError, delitem, NULL, 'a') + + def test_mapping_delitemstring(self): + delitemstring = _testlimitedcapi.mapping_delitemstring + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitemstring(dct, b'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitemstring, dct, b'b') + delitemstring(dct, '\U0001f40d'.encode()) + self.assertEqual(dct, {'c': 2}) + + dct = {'a': 1, 'c': 2} + dct2 = ProxyDelItem(dct) + delitemstring(dct2, b'a') + self.assertEqual(dct, {'c': 2}) + self.assertRaises(KeyError, delitemstring, dct2, b'b') + + self.assertRaises(TypeError, delitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') + self.assertRaises(SystemError, delitemstring, {}, NULL) + self.assertRaises(TypeError, delitemstring, [], b'a') + self.assertRaises(SystemError, delitemstring, NULL, b'a') + + def test_mapping_keys_valuesitems(self): + class Mapping1(dict): + def keys(self): + return list(super().keys()) + def values(self): + return list(super().values()) + def items(self): + return list(super().items()) + class Mapping2(dict): + def keys(self): + return tuple(super().keys()) + def values(self): + return tuple(super().values()) + def items(self): + return tuple(super().items()) + dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} + + for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), + dict_obj, OrderedDict(dict_obj), + Mapping1(dict_obj), Mapping2(dict_obj)]: + self.assertListEqual(_testlimitedcapi.mapping_keys(mapping), + list(mapping.keys())) + self.assertListEqual(_testlimitedcapi.mapping_values(mapping), + list(mapping.values())) + self.assertListEqual(_testlimitedcapi.mapping_items(mapping), + list(mapping.items())) + + def test_mapping_keys_valuesitems_bad_arg(self): + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, []) + self.assertRaises(SystemError, _testlimitedcapi.mapping_keys, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_values, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_items, NULL) + + class BadMapping: + def keys(self): + return None + def values(self): + return None + def items(self): + return None + bad_mapping = BadMapping() + self.assertRaises(TypeError, _testlimitedcapi.mapping_keys, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_values, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_items, bad_mapping) + + def test_sequence_check(self): + check = _testlimitedcapi.sequence_check + self.assertFalse(check({1: 2})) + self.assertTrue(check([1, 2])) + self.assertTrue(check((1, 2))) + self.assertTrue(check('abc')) + self.assertTrue(check(b'abc')) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + # CRASHES check(NULL) + + def test_sequence_size(self): + for size in _testlimitedcapi.sequence_size, _testlimitedcapi.sequence_length: + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size('abc'), 3) + self.assertEqual(size(b'abc'), 3) + + self.assertRaises(TypeError, size, {}) + self.assertRaises(TypeError, size, 42) + self.assertRaises(TypeError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_sequence_getitem(self): + getitem = _testlimitedcapi.sequence_getitem + lst = ['a', 'b', 'c'] + self.assertEqual(getitem(lst, 1), 'b') + self.assertEqual(getitem(lst, -1), 'c') + self.assertRaises(IndexError, getitem, lst, 3) + self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) + self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) + + self.assertRaises(TypeError, getitem, 42, 1) + self.assertRaises(TypeError, getitem, {}, 1) + self.assertRaises(SystemError, getitem, NULL, 1) + + def test_sequence_concat(self): + concat = _testlimitedcapi.sequence_concat + self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) + self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) + + self.assertRaises(TypeError, concat, [], ()) + self.assertRaises(TypeError, concat, (), []) + self.assertRaises(TypeError, concat, [], 42) + self.assertRaises(TypeError, concat, 42, []) + self.assertRaises(TypeError, concat, 42, 43) + self.assertRaises(SystemError, concat, [], NULL) + self.assertRaises(SystemError, concat, NULL, []) + + def test_sequence_repeat(self): + repeat = _testlimitedcapi.sequence_repeat + self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) + self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) + self.assertEqual(repeat(['a', 'b'], 0), []) + self.assertEqual(repeat(['a', 'b'], -1), []) + self.assertEqual(repeat(['a', 'b'], PY_SSIZE_T_MIN), []) + self.assertEqual(repeat([], PY_SSIZE_T_MAX), []) + self.assertRaises(MemoryError, repeat, ['a', 'b'], PY_SSIZE_T_MAX) + + self.assertRaises(TypeError, repeat, set(), 2) + self.assertRaises(TypeError, repeat, 42, 2) + self.assertRaises(SystemError, repeat, NULL, 2) + + def test_sequence_inplaceconcat(self): + inplaceconcat = _testlimitedcapi.sequence_inplaceconcat + lst = ['a', 'b'] + res = inplaceconcat(lst, [1, 2]) + self.assertEqual(res, ['a', 'b', 1, 2]) + self.assertIs(res, lst) + lst = ['a', 'b'] + res = inplaceconcat(lst, (1, 2)) + self.assertEqual(res, ['a', 'b', 1, 2]) + self.assertIs(res, lst) + self.assertEqual(inplaceconcat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) + + self.assertRaises(TypeError, inplaceconcat, (), []) + self.assertRaises(TypeError, inplaceconcat, [], 42) + self.assertRaises(TypeError, inplaceconcat, 42, []) + self.assertRaises(TypeError, inplaceconcat, 42, 43) + self.assertRaises(SystemError, inplaceconcat, [], NULL) + self.assertRaises(SystemError, inplaceconcat, NULL, []) + + def test_sequence_inplacerepeat(self): + inplacerepeat = _testlimitedcapi.sequence_inplacerepeat + lst = ['a', 'b'] + res = inplacerepeat(lst, 2) + self.assertEqual(res, ['a', 'b', 'a', 'b']) + self.assertIs(res, lst) + self.assertEqual(inplacerepeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) + self.assertEqual(inplacerepeat(['a', 'b'], 0), []) + self.assertEqual(inplacerepeat(['a', 'b'], -1), []) + self.assertEqual(inplacerepeat(['a', 'b'], PY_SSIZE_T_MIN), []) + self.assertEqual(inplacerepeat([], PY_SSIZE_T_MAX), []) + self.assertRaises(MemoryError, inplacerepeat, ['a', 'b'], PY_SSIZE_T_MAX) + + self.assertRaises(TypeError, inplacerepeat, set(), 2) + self.assertRaises(TypeError, inplacerepeat, 42, 2) + self.assertRaises(SystemError, inplacerepeat, NULL, 2) + + def test_sequence_setitem(self): + setitem = _testlimitedcapi.sequence_setitem + lst = ['a', 'b', 'c'] + setitem(lst, 1, 'x') + self.assertEqual(lst, ['a', 'x', 'c']) + setitem(lst, -1, 'y') + self.assertEqual(lst, ['a', 'x', 'y']) + + setitem(lst, 0, NULL) + self.assertEqual(lst, ['x', 'y']) + self.assertRaises(IndexError, setitem, lst, 3, 'x') + self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MAX, 'x') + self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MIN, 'x') + + self.assertRaises(TypeError, setitem, 42, 1, 'x') + self.assertRaises(TypeError, setitem, {}, 1, 'x') + self.assertRaises(SystemError, setitem, NULL, 1, 'x') + + def test_sequence_delitem(self): + delitem = _testlimitedcapi.sequence_delitem + lst = ['a', 'b', 'c'] + delitem(lst, 1) + self.assertEqual(lst, ['a', 'c']) + delitem(lst, -1) + self.assertEqual(lst, ['a']) + self.assertRaises(IndexError, delitem, lst, 3) + self.assertRaises(IndexError, delitem, lst, PY_SSIZE_T_MAX) + self.assertRaises(IndexError, delitem, lst, PY_SSIZE_T_MIN) + + self.assertRaises(TypeError, delitem, 42, 1) + self.assertRaises(TypeError, delitem, {}, 1) + self.assertRaises(SystemError, delitem, NULL, 1) + + def test_sequence_setslice(self): + setslice = _testlimitedcapi.sequence_setslice + + # Correct case: + for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + for stop in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + data = [1, 2, 3, 4, 5] + data_copy = [1, 2, 3, 4, 5] + setslice(data, start, stop, [8, 9]) + data_copy[start:stop] = [8, 9] + self.assertEqual(data, data_copy) + + data = [1, 2, 3, 4, 5] + data_copy = [1, 2, 3, 4, 5] + setslice(data, start, stop, NULL) + del data_copy[start:stop] + self.assertEqual(data, data_copy) + + # Custom class: + class Custom: + def __setitem__(self, index, value): + self.index = index + self.value = value + + c = Custom() + setslice(c, 0, 5, 'abc') + self.assertEqual(c.index, slice(0, 5)) + self.assertEqual(c.value, 'abc') + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + self.assertRaises(TypeError, setslice, bad_seq1, 1, 3, (8, 9)) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + self.assertRaises(TypeError, setslice, bad_seq2, 1, 3, 'xy') + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + self.assertRaises(TypeError, setslice, object(), 1, 3, 'xy') + self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') + + def test_sequence_delslice(self): + delslice = _testlimitedcapi.sequence_delslice + + # Correct case: + for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + for stop in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + data = [1, 2, 3, 4, 5] + data_copy = [1, 2, 3, 4, 5] + delslice(data, start, stop) + del data_copy[start:stop] + self.assertEqual(data, data_copy) + + # Custom class: + class Custom: + def __delitem__(self, index): + self.index = index + + c = Custom() + delslice(c, 0, 5) + self.assertEqual(c.index, slice(0, 5)) + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + self.assertRaises(TypeError, delslice, bad_seq1, 1, 3) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + self.assertRaises(TypeError, delslice, bad_seq2, 1, 3) + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + self.assertRaises(TypeError, delslice, object(), 1, 3) + self.assertRaises(SystemError, delslice, NULL, 1, 3) + + mapping = {1: 'a', 2: 'b', 3: 'c'} + self.assertRaises(KeyError, delslice, mapping, 1, 3) + self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) + + def test_sequence_count(self): + count = _testlimitedcapi.sequence_count + + lst = ['a', 'b', 'a'] + self.assertEqual(count(lst, 'a'), 2) + self.assertEqual(count(lst, 'c'), 0) + self.assertEqual(count(iter(lst), 'a'), 2) + self.assertEqual(count(iter(lst), 'c'), 0) + self.assertEqual(count({'a': 2}, 'a'), 1) + + self.assertRaises(TypeError, count, 42, 'a') + self.assertRaises(SystemError, count, [], NULL) + self.assertRaises(SystemError, count, [1], NULL) + self.assertRaises(SystemError, count, NULL, 'a') + + def test_sequence_contains(self): + contains = _testlimitedcapi.sequence_contains + + lst = ['a', 'b', 'a'] + self.assertEqual(contains(lst, 'a'), 1) + self.assertEqual(contains(lst, 'c'), 0) + self.assertEqual(contains(iter(lst), 'a'), 1) + self.assertEqual(contains(iter(lst), 'c'), 0) + self.assertEqual(contains({'a': 2}, 'a'), 1) + + # XXX Only for empty sequences. Should be SystemError? + self.assertEqual(contains([], NULL), 0) + + self.assertRaises(TypeError, contains, 42, 'a') + self.assertRaises(SystemError, contains, [1], NULL) + # CRASHES contains({}, NULL) + # CRASHES contains(set(), NULL) + # CRASHES contains(NULL, 'a') + + def test_sequence_index(self): + index = _testlimitedcapi.sequence_index + + lst = ['a', 'b', 'a'] + self.assertEqual(index(lst, 'a'), 0) + self.assertEqual(index(lst, 'b'), 1) + self.assertRaises(ValueError, index, lst, 'c') + self.assertEqual(index(iter(lst), 'a'), 0) + self.assertEqual(index(iter(lst), 'b'), 1) + self.assertRaises(ValueError, index, iter(lst), 'c') + dct = {'a': 2, 'b': 3} + self.assertEqual(index(dct, 'a'), 0) + self.assertEqual(index(dct, 'b'), 1) + self.assertRaises(ValueError, index, dct, 'c') + + self.assertRaises(TypeError, index, 42, 'a') + self.assertRaises(SystemError, index, [], NULL) + self.assertRaises(SystemError, index, [1], NULL) + self.assertRaises(SystemError, index, NULL, 'a') + + def test_sequence_list(self): + xlist = _testlimitedcapi.sequence_list + self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) + self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) + self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) + self.assertEqual(xlist(gen()), ['a', 'b', 'c']) + + self.assertRaises(TypeError, xlist, 42) + self.assertRaises(SystemError, xlist, NULL) + + def test_sequence_tuple(self): + xtuple = _testlimitedcapi.sequence_tuple + self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) + self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) + self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) + self.assertEqual(xtuple(gen()), ('a', 'b', 'c')) + + self.assertRaises(TypeError, xtuple, 42) + self.assertRaises(SystemError, xtuple, NULL) + + def test_sequence_fast(self): + # Test PySequence_Fast() + sequence_fast = _testlimitedcapi.sequence_fast + sequence_fast_get_size = _testcapi.sequence_fast_get_size + sequence_fast_get_item = _testcapi.sequence_fast_get_item + + tpl = ('a', 'b', 'c') + fast = sequence_fast(tpl, "err_msg") + self.assertIs(fast, tpl) + self.assertEqual(sequence_fast_get_size(fast), 3) + self.assertEqual(sequence_fast_get_item(fast, 2), 'c') + + lst = ['a', 'b', 'c'] + fast = sequence_fast(lst, "err_msg") + self.assertIs(fast, lst) + self.assertEqual(sequence_fast_get_size(fast), 3) + self.assertEqual(sequence_fast_get_item(fast, 2), 'c') + + it = iter(['A', 'B']) + fast = sequence_fast(it, "err_msg") + self.assertEqual(fast, ['A', 'B']) + self.assertEqual(sequence_fast_get_size(fast), 2) + self.assertEqual(sequence_fast_get_item(fast, 1), 'B') + + text = 'fast' + fast = sequence_fast(text, "err_msg") + self.assertEqual(fast, ['f', 'a', 's', 't']) + self.assertEqual(sequence_fast_get_size(fast), 4) + self.assertEqual(sequence_fast_get_item(fast, 0), 'f') + + self.assertRaises(TypeError, sequence_fast, 42, "err_msg") + self.assertRaises(SystemError, sequence_fast, NULL, "err_msg") + + # CRASHES sequence_fast_get_size(NULL) + # CRASHES sequence_fast_get_item(NULL, 0) + + def test_object_generichash(self): + # Test PyObject_GenericHash() + generichash = _testcapi.object_generichash + for obj in object(), 1, 'string', []: + self.assertEqual(generichash(obj), object.__hash__(obj)) + + def run_iter_api_test(self, next_func): + for data in (), [], (1, 2, 3), [1 , 2, 3], "123": + with self.subTest(data=data): + items = [] + it = iter(data) + while (item := next_func(it)) is not None: + items.append(item) + self.assertEqual(items, list(data)) + + class Broken: + def __init__(self): + self.count = 0 + + def __next__(self): + if self.count < 3: + self.count += 1 + return self.count + else: + raise TypeError('bad type') + + it = Broken() + self.assertEqual(next_func(it), 1) + self.assertEqual(next_func(it), 2) + self.assertEqual(next_func(it), 3) + with self.assertRaisesRegex(TypeError, 'bad type'): + next_func(it) + + def test_iter_next(self): + from _testcapi import PyIter_Next + self.run_iter_api_test(PyIter_Next) + # CRASHES PyIter_Next(10) + + def test_iter_nextitem(self): + from _testcapi import PyIter_NextItem + self.run_iter_api_test(PyIter_NextItem) + + regex = "expected.*iterator.*got.*'int'" + with self.assertRaisesRegex(TypeError, regex): + PyIter_NextItem(10) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py new file mode 100644 index 00000000000..52565ea34c6 --- /dev/null +++ b/Lib/test/test_capi/test_bytearray.py @@ -0,0 +1,174 @@ +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + +NULL = None + +class ByteArraySubclass(bytearray): + pass + +class BytesLike: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyByteArray_Check() + check = _testlimitedcapi.bytearray_check + self.assertTrue(check(bytearray(b''))) + self.assertTrue(check(bytearray(b'abc'))) + self.assertFalse(check(b'abc')) + self.assertTrue(check(ByteArraySubclass(b'abc'))) + self.assertFalse(check(BytesLike(b'abc'))) + self.assertFalse(check(3)) + self.assertFalse(check([])) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_checkexact(self): + # Test PyByteArray_CheckExact() + check = _testlimitedcapi.bytearray_checkexact + self.assertTrue(check(bytearray(b''))) + self.assertTrue(check(bytearray(b'abc'))) + self.assertFalse(check(b'abc')) + self.assertFalse(check(ByteArraySubclass(b'abc'))) + self.assertFalse(check(BytesLike(b'abc'))) + self.assertFalse(check(3)) + self.assertFalse(check([])) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_fromstringandsize(self): + # Test PyByteArray_FromStringAndSize() + fromstringandsize = _testlimitedcapi.bytearray_fromstringandsize + + self.assertEqual(fromstringandsize(b'abc'), bytearray(b'abc')) + self.assertEqual(fromstringandsize(b'abc', 2), bytearray(b'ab')) + self.assertEqual(fromstringandsize(b'abc\0def'), bytearray(b'abc\0def')) + self.assertEqual(fromstringandsize(b'', 0), bytearray()) + self.assertEqual(fromstringandsize(NULL, 0), bytearray()) + self.assertEqual(len(fromstringandsize(NULL, 3)), 3) + self.assertRaises(MemoryError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + + self.assertRaises(SystemError, fromstringandsize, b'abc', -1) + self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) + self.assertRaises(SystemError, fromstringandsize, NULL, -1) + self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) + + def test_fromobject(self): + # Test PyByteArray_FromObject() + fromobject = _testlimitedcapi.bytearray_fromobject + + self.assertEqual(fromobject(b''), bytearray(b'')) + self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) + self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) + self.assertEqual(fromobject(ByteArraySubclass(b'abc')), bytearray(b'abc')) + self.assertEqual(fromobject([97, 98, 99]), bytearray(b'abc')) + self.assertEqual(fromobject(3), bytearray(b'\0\0\0')) + self.assertRaises(TypeError, fromobject, BytesLike(b'abc')) + self.assertRaises(TypeError, fromobject, 'abc') + self.assertRaises(TypeError, fromobject, object()) + + # CRASHES fromobject(NULL) + + def test_size(self): + # Test PyByteArray_Size() + size = _testlimitedcapi.bytearray_size + self.assertEqual(size(bytearray(b'')), 0) + self.assertEqual(size(bytearray(b'abc')), 3) + self.assertEqual(size(ByteArraySubclass(b'abc')), 3) + + # CRASHES size(b'abc') + # CRASHES size(object()) + # CRASHES size(NULL) + + def test_asstring(self): + """Test PyByteArray_AsString()""" + asstring = _testlimitedcapi.bytearray_asstring + self.assertEqual(asstring(bytearray(b''), 1), b'\0') + self.assertEqual(asstring(bytearray(b'abc'), 4), b'abc\0') + self.assertEqual(asstring(ByteArraySubclass(b'abc'), 4), b'abc\0') + self.assertEqual(asstring(bytearray(b'abc\0def'), 8), b'abc\0def\0') + + # CRASHES asstring(b'abc', 0) + # CRASHES asstring(object()', 0) + # CRASHES asstring(NULL, 0) + + def test_concat(self): + """Test PyByteArray_Concat()""" + concat = _testlimitedcapi.bytearray_concat + + ba = bytearray(b'abc') + self.assertEqual(concat(ba, b'def'), bytearray(b'abcdef')) + self.assertEqual(ba, b'abc') + self.assertEqual(concat(ba, ba), bytearray(b'abcabc')) + + self.assertEqual(concat(b'abc', b'def'), bytearray(b'abcdef')) + self.assertEqual(concat(b'a\0b', b'c\0d'), bytearray(b'a\0bc\0d')) + self.assertEqual(concat(bytearray(b'abc'), b'def'), bytearray(b'abcdef')) + self.assertEqual(concat(b'abc', bytearray(b'def')), bytearray(b'abcdef')) + self.assertEqual(concat(bytearray(b'abc'), b''), bytearray(b'abc')) + self.assertEqual(concat(b'', bytearray(b'def')), bytearray(b'def')) + self.assertEqual(concat(bytearray(b''), bytearray(b'')), bytearray(b'')) + self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), + bytearray(b'abcdef')) + self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), + bytearray(b'abcdef')) + + self.assertRaises(TypeError, concat, memoryview(b'axbycz')[::2], b'def') + self.assertRaises(TypeError, concat, b'abc', memoryview(b'dxeyfz')[::2]) + self.assertRaises(TypeError, concat, b'abc', 'def') + self.assertRaises(TypeError, concat, 'abc', b'def') + self.assertRaises(TypeError, concat, 'abc', 'def') + self.assertRaises(TypeError, concat, [], b'def') + self.assertRaises(TypeError, concat, b'abc', []) + self.assertRaises(TypeError, concat, [], []) + + # CRASHES concat(NULL, bytearray(b'def')) + # CRASHES concat(bytearray(b'abc'), NULL) + # CRASHES concat(NULL, object()) + # CRASHES concat(object(), NULL) + + def test_resize(self): + """Test PyByteArray_Resize()""" + resize = _testlimitedcapi.bytearray_resize + + ba = bytearray(b'abcdef') + self.assertEqual(resize(ba, 3), 0) + self.assertEqual(ba, bytearray(b'abc')) + self.assertEqual(resize(ba, 10), 0) + self.assertEqual(len(ba), 10) + self.assertEqual(ba[:3], bytearray(b'abc')) + self.assertEqual(resize(ba, 2**20), 0) + self.assertEqual(len(ba), 2**20) + self.assertEqual(ba[:3], bytearray(b'abc')) + self.assertEqual(resize(ba, 0), 0) + self.assertEqual(ba, bytearray()) + + ba = bytearray(b'') + self.assertEqual(resize(ba, 0), 0) + self.assertEqual(ba, bytearray()) + + ba = ByteArraySubclass(b'abcdef') + self.assertEqual(resize(ba, 3), 0) + self.assertEqual(ba, bytearray(b'abc')) + + self.assertRaises(ValueError, resize, bytearray(), -1) + self.assertRaises(ValueError, resize, bytearray(), -200) + self.assertRaises(MemoryError, resize, bytearray(), PY_SSIZE_T_MAX) + self.assertRaises(MemoryError, resize, bytearray(1000), PY_SSIZE_T_MAX) + + # CRASHES resize(b'abc', 0) + # CRASHES resize(object(), 0) + # CRASHES resize(NULL, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py new file mode 100644 index 00000000000..bc820bd68d9 --- /dev/null +++ b/Lib/test/test_capi/test_bytes.py @@ -0,0 +1,303 @@ +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testcapi = import_helper.import_module('_testcapi') +from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + +NULL = None + +class BytesSubclass(bytes): + pass + +class BytesLike: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyBytes_Check() + check = _testlimitedcapi.bytes_check + self.assertTrue(check(b'abc')) + self.assertTrue(check(b'')) + self.assertFalse(check('abc')) + self.assertFalse(check(bytearray(b'abc'))) + self.assertTrue(check(BytesSubclass(b'abc'))) + self.assertFalse(check(BytesLike(b'abc'))) + self.assertFalse(check(3)) + self.assertFalse(check([])) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_checkexact(self): + # Test PyBytes_CheckExact() + check = _testlimitedcapi.bytes_checkexact + self.assertTrue(check(b'abc')) + self.assertTrue(check(b'')) + self.assertFalse(check('abc')) + self.assertFalse(check(bytearray(b'abc'))) + self.assertFalse(check(BytesSubclass(b'abc'))) + self.assertFalse(check(BytesLike(b'abc'))) + self.assertFalse(check(3)) + self.assertFalse(check([])) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_fromstringandsize(self): + # Test PyBytes_FromStringAndSize() + fromstringandsize = _testlimitedcapi.bytes_fromstringandsize + + self.assertEqual(fromstringandsize(b'abc'), b'abc') + self.assertEqual(fromstringandsize(b'abc', 2), b'ab') + self.assertEqual(fromstringandsize(b'abc\0def'), b'abc\0def') + self.assertEqual(fromstringandsize(b'a'), b'a') + self.assertEqual(fromstringandsize(b'a', 1), b'a') + self.assertEqual(fromstringandsize(b'', 0), b'') + self.assertEqual(fromstringandsize(NULL, 0), b'') + self.assertEqual(len(fromstringandsize(NULL, 3)), 3) + self.assertRaises((MemoryError, OverflowError), + fromstringandsize, NULL, PY_SSIZE_T_MAX) + + self.assertRaises(SystemError, fromstringandsize, b'abc', -1) + self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) + self.assertRaises(SystemError, fromstringandsize, NULL, -1) + self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) + + def test_fromstring(self): + # Test PyBytes_FromString() + fromstring = _testlimitedcapi.bytes_fromstring + + self.assertEqual(fromstring(b'abc\0def'), b'abc') + self.assertEqual(fromstring(b''), b'') + + # CRASHES fromstring(NULL) + + def test_fromobject(self): + # Test PyBytes_FromObject() + fromobject = _testlimitedcapi.bytes_fromobject + + self.assertEqual(fromobject(b''), b'') + self.assertEqual(fromobject(b'abc'), b'abc') + self.assertEqual(fromobject(bytearray(b'abc')), b'abc') + self.assertEqual(fromobject(BytesSubclass(b'abc')), b'abc') + self.assertEqual(fromobject([97, 98, 99]), b'abc') + self.assertRaises(TypeError, fromobject, 3) + self.assertRaises(TypeError, fromobject, BytesLike(b'abc')) + self.assertRaises(TypeError, fromobject, 'abc') + self.assertRaises(TypeError, fromobject, object()) + self.assertRaises(SystemError, fromobject, NULL) + + def test_size(self): + # Test PyBytes_Size() + size = _testlimitedcapi.bytes_size + + self.assertEqual(size(b''), 0) + self.assertEqual(size(b'abc'), 3) + self.assertEqual(size(BytesSubclass(b'abc')), 3) + self.assertRaises(TypeError, size, bytearray(b'abc')) + self.assertRaises(TypeError, size, 'abc') + self.assertRaises(TypeError, size, object()) + + # CRASHES size(NULL) + + def test_asstring(self): + """Test PyBytes_AsString()""" + asstring = _testlimitedcapi.bytes_asstring + + self.assertEqual(asstring(b'abc', 4), b'abc\0') + self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') + self.assertEqual(asstring(b'', 1), b'\0') + self.assertRaises(TypeError, asstring, 'abc', 0) + self.assertRaises(TypeError, asstring, object(), 0) + + # CRASHES asstring(NULL, 0) + + def test_asstringandsize(self): + """Test PyBytes_AsStringAndSize()""" + asstringandsize = _testlimitedcapi.bytes_asstringandsize + asstringandsize_null = _testlimitedcapi.bytes_asstringandsize_null + + self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) + self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) + self.assertEqual(asstringandsize(b'', 1), (b'\0', 0)) + self.assertEqual(asstringandsize_null(b'abc', 4), b'abc\0') + self.assertRaises(ValueError, asstringandsize_null, b'abc\0def', 8) + self.assertRaises(TypeError, asstringandsize, 'abc', 0) + self.assertRaises(TypeError, asstringandsize_null, 'abc', 0) + self.assertRaises(TypeError, asstringandsize, object(), 0) + self.assertRaises(TypeError, asstringandsize_null, object(), 0) + + # CRASHES asstringandsize(NULL, 0) + # CRASHES asstringandsize_null(NULL, 0) + + def test_repr(self): + # Test PyBytes_Repr() + bytes_repr = _testlimitedcapi.bytes_repr + + self.assertEqual(bytes_repr(b'', 0), r"""b''""") + self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") + self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") + self.assertEqual(bytes_repr(b'''a'b"c"d''', 0), r"""b'a\'b"c"d'""") + self.assertEqual(bytes_repr(b'''a'b"c"d''', 1), r"""b'a\'b"c"d'""") + self.assertEqual(bytes_repr(b'''a'b"c''', 0), r"""b'a\'b"c'""") + self.assertEqual(bytes_repr(b'''a'b"c''', 1), r"""b'a\'b"c'""") + self.assertEqual(bytes_repr(b'''a'b'c"d''', 0), r"""b'a\'b\'c"d'""") + self.assertEqual(bytes_repr(b'''a'b'c"d''', 1), r"""b'a\'b\'c"d'""") + self.assertEqual(bytes_repr(b'''a'b'c'd''', 0), r"""b'a\'b\'c\'d'""") + self.assertEqual(bytes_repr(b'''a'b'c'd''', 1), r'''b"a'b'c'd"''') + + self.assertEqual(bytes_repr(BytesSubclass(b'abc'), 0), r"""b'abc'""") + + # UDEFINED bytes_repr(object(), 0) + # CRASHES bytes_repr(NULL, 0) + + def test_concat(self, concat=None): + """Test PyBytes_Concat()""" + if concat is None: + concat = _testlimitedcapi.bytes_concat + + self.assertEqual(concat(b'abc', b'def'), b'abcdef') + self.assertEqual(concat(b'a\0b', b'c\0d'), b'a\0bc\0d') + self.assertEqual(concat(bytearray(b'abc'), b'def'), b'abcdef') + self.assertEqual(concat(b'abc', bytearray(b'def')), b'abcdef') + self.assertEqual(concat(bytearray(b'abc'), b''), b'abc') + self.assertEqual(concat(b'', bytearray(b'def')), b'def') + self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), b'abcdef') + self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), b'abcdef') + self.assertEqual(concat(b'', b''), b'') + + self.assertEqual(concat(b'abc', b'def', True), b'abcdef') + self.assertEqual(concat(b'abc', bytearray(b'def'), True), b'abcdef') + # Check that it does not change the singleton + self.assertEqual(concat(bytes(), b'def', True), b'def') + self.assertEqual(len(bytes()), 0) + + self.assertRaises(TypeError, concat, memoryview(b'axbycz')[::2], b'def') + self.assertRaises(TypeError, concat, b'abc', memoryview(b'dxeyfz')[::2]) + self.assertRaises(TypeError, concat, b'abc', 'def') + self.assertRaises(TypeError, concat, 'abc', b'def') + self.assertRaises(TypeError, concat, 'abc', 'def') + self.assertRaises(TypeError, concat, [], b'def') + self.assertRaises(TypeError, concat, b'abc', []) + self.assertRaises(TypeError, concat, [], []) + + self.assertEqual(concat(NULL, b'def'), NULL) + self.assertEqual(concat(b'abc', NULL), NULL) + self.assertEqual(concat(NULL, object()), NULL) + self.assertEqual(concat(object(), NULL), NULL) + + def test_concatanddel(self): + """Test PyBytes_ConcatAndDel()""" + self.test_concat(_testlimitedcapi.bytes_concatanddel) + + def test_decodeescape(self): + """Test PyBytes_DecodeEscape()""" + decodeescape = _testlimitedcapi.bytes_decodeescape + + self.assertEqual(decodeescape(b''), b'') + self.assertEqual(decodeescape(b'abc'), b'abc') + self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), + b'''\t\n\r\v\f\0\\'"''') + self.assertEqual(decodeescape(b'\t\n\r\x0b\x0c\x00'), b'\t\n\r\v\f\0') + self.assertEqual(decodeescape(br'\xa1\xa2'), b'\xa1\xa2') + self.assertEqual(decodeescape(br'\2\24\241'), b'\x02\x14\xa1') + self.assertEqual(decodeescape(b'\xa1\xa2'), b'\xa1\xa2') + with self.assertWarns(DeprecationWarning): + self.assertEqual(decodeescape(br'\u4f60'), br'\u4f60') + with self.assertWarns(DeprecationWarning): + self.assertEqual(decodeescape(br'\z'), br'\z') + with self.assertWarns(DeprecationWarning): + self.assertEqual(decodeescape(br'\541'), b'a') + + for b in b'\\', br'\x', br'\xa', br'\xz', br'\xaz': + self.assertRaises(ValueError, decodeescape, b) + self.assertRaises(ValueError, decodeescape, b, 'strict') + self.assertEqual(decodeescape(br'x\xa', 'replace'), b'x?') + self.assertEqual(decodeescape(br'x\xay', 'replace'), b'x?y') + self.assertEqual(decodeescape(br'x\xa\xy', 'replace'), b'x??y') + self.assertEqual(decodeescape(br'x\xa\xy', 'ignore'), b'xy') + self.assertRaises(ValueError, decodeescape, b'\\', 'spam') + self.assertEqual(decodeescape(NULL), b'') + self.assertRaises(OverflowError, decodeescape, b'abc', NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, decodeescape, NULL, NULL, PY_SSIZE_T_MAX) + + # CRASHES decodeescape(b'abc', NULL, -1) + # CRASHES decodeescape(NULL, NULL, 1) + + def test_resize(self): + """Test _PyBytes_Resize()""" + resize = _testcapi.bytes_resize + + for new in True, False: + self.assertEqual(resize(b'abc', 0, new), b'') + self.assertEqual(resize(b'abc', 1, new), b'a') + self.assertEqual(resize(b'abc', 2, new), b'ab') + self.assertEqual(resize(b'abc', 3, new), b'abc') + b = resize(b'abc', 4, new) + self.assertEqual(len(b), 4) + self.assertEqual(b[:3], b'abc') + + self.assertEqual(resize(b'a', 0, new), b'') + self.assertEqual(resize(b'a', 1, new), b'a') + b = resize(b'a', 2, new) + self.assertEqual(len(b), 2) + self.assertEqual(b[:1], b'a') + + self.assertEqual(resize(b'', 0, new), b'') + self.assertEqual(len(resize(b'', 1, new)), 1) + self.assertEqual(len(resize(b'', 2, new)), 2) + + self.assertRaises(SystemError, resize, b'abc', -1, False) + self.assertRaises(SystemError, resize, bytearray(b'abc'), 3, False) + + # CRASHES resize(NULL, 0, False) + # CRASHES resize(NULL, 3, False) + + def test_join(self): + """Test PyBytes_Join()""" + bytes_join = _testcapi.bytes_join + + self.assertEqual(bytes_join(b'', []), b'') + self.assertEqual(bytes_join(b'sep', []), b'') + + self.assertEqual(bytes_join(b'', [b'a', b'b', b'c']), b'abc') + self.assertEqual(bytes_join(b'-', [b'a', b'b', b'c']), b'a-b-c') + self.assertEqual(bytes_join(b' - ', [b'a', b'b', b'c']), b'a - b - c') + self.assertEqual(bytes_join(b'-', [bytearray(b'abc'), + memoryview(b'def')]), + b'abc-def') + + self.assertEqual(bytes_join(b'-', iter([b'a', b'b', b'c'])), b'a-b-c') + + # invalid 'sep' argument + with self.assertRaises(TypeError): + bytes_join(bytearray(b'sep'), []) + with self.assertRaises(TypeError): + bytes_join(memoryview(b'sep'), []) + with self.assertRaises(TypeError): + bytes_join('', []) # empty Unicode string + with self.assertRaises(TypeError): + bytes_join('unicode', []) + with self.assertRaises(TypeError): + bytes_join(123, []) + with self.assertRaises(SystemError): + self.assertEqual(bytes_join(NULL, [b'a', b'b', b'c']), b'abc') + + # invalid 'iterable' argument + with self.assertRaises(TypeError): + bytes_join(b'', [b'bytes', 'unicode']) + with self.assertRaises(TypeError): + bytes_join(b'', [b'bytes', 123]) + with self.assertRaises(TypeError): + bytes_join(b'', 123) + with self.assertRaises(SystemError): + bytes_join(b'', NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py new file mode 100644 index 00000000000..a0355c7a388 --- /dev/null +++ b/Lib/test/test_capi/test_codecs.py @@ -0,0 +1,879 @@ +import codecs +import contextlib +import io +import re +import sys +import unittest +import unittest.mock as mock +import _testcapi +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None +BAD_ARGUMENT = re.escape('bad argument type for built-in operation') + + +class CAPIUnicodeTest(unittest.TestCase): + # TODO: Test the following functions: + # + # PyUnicode_BuildEncodingMap + # PyUnicode_FSConverter + # PyUnicode_FSDecoder + # PyUnicode_DecodeMBCS + # PyUnicode_DecodeMBCSStateful + # PyUnicode_DecodeCodePageStateful + # PyUnicode_AsMBCSString + # PyUnicode_EncodeCodePage + # PyUnicode_DecodeLocaleAndSize + # PyUnicode_DecodeLocale + # PyUnicode_EncodeLocale + # PyUnicode_DecodeFSDefault + # PyUnicode_DecodeFSDefaultAndSize + # PyUnicode_EncodeFSDefault + + def test_fromencodedobject(self): + """Test PyUnicode_FromEncodedObject()""" + fromencodedobject = _testlimitedcapi.unicode_fromencodedobject + + self.assertEqual(fromencodedobject(b'abc', NULL), 'abc') + self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc') + b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80' + s = 'a\xa1\u4f60\U0001f600' + self.assertEqual(fromencodedobject(b, NULL), s) + self.assertEqual(fromencodedobject(b, 'utf-8'), s) + self.assertEqual(fromencodedobject(b, 'latin1'), b.decode('latin1')) + self.assertRaises(UnicodeDecodeError, fromencodedobject, b, 'ascii') + self.assertEqual(fromencodedobject(b, 'ascii', 'replace'), + 'a' + '\ufffd'*9) + self.assertEqual(fromencodedobject(bytearray(b), NULL), s) + self.assertEqual(fromencodedobject(bytearray(b), 'utf-8'), s) + self.assertRaises(LookupError, fromencodedobject, b'abc', 'foo') + self.assertRaises(LookupError, fromencodedobject, b, 'ascii', 'foo') + self.assertRaises(TypeError, fromencodedobject, 'abc', NULL) + self.assertRaises(TypeError, fromencodedobject, 'abc', 'ascii') + self.assertRaises(TypeError, fromencodedobject, [], NULL) + self.assertRaises(TypeError, fromencodedobject, [], 'ascii') + self.assertRaises(SystemError, fromencodedobject, NULL, NULL) + self.assertRaises(SystemError, fromencodedobject, NULL, 'ascii') + + def test_decode(self): + """Test PyUnicode_Decode()""" + decode = _testlimitedcapi.unicode_decode + + self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]') + self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]') + self.assertEqual(decode(b'[\xa4]', 'iso8859-15', 'strict'), '[\u20ac]') + self.assertRaises(UnicodeDecodeError, decode, b'[\xa4]', 'utf-8') + self.assertEqual(decode(b'[\xa4]', 'utf-8', 'replace'), '[\ufffd]') + + self.assertEqual(decode(b'[\xe2\x82\xac]', NULL), '[\u20ac]') + self.assertEqual(decode(b'[\xa4]', NULL, 'replace'), '[\ufffd]') + + self.assertRaises(LookupError, decode, b'\xa4', 'foo') + self.assertRaises(LookupError, decode, b'\xa4', 'utf-8', 'foo') + # TODO: Test PyUnicode_Decode() with NULL as data and + # negative size. + + def test_asencodedstring(self): + """Test PyUnicode_AsEncodedString()""" + asencodedstring = _testlimitedcapi.unicode_asencodedstring + + self.assertEqual(asencodedstring('abc', NULL), b'abc') + self.assertEqual(asencodedstring('abc', 'ascii'), b'abc') + s = 'a\xa1\u4f60\U0001f600' + b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80' + self.assertEqual(asencodedstring(s, NULL), b) + self.assertEqual(asencodedstring(s, 'utf-8'), b) + self.assertEqual(asencodedstring('\xa1\xa2', 'latin1'), b'\xa1\xa2') + self.assertRaises(UnicodeEncodeError, asencodedstring, '\xa1\xa2', 'ascii') + self.assertEqual(asencodedstring(s, 'ascii', 'replace'), b'a???') + + self.assertRaises(LookupError, asencodedstring, 'abc', 'foo') + self.assertRaises(LookupError, asencodedstring, s, 'ascii', 'foo') + self.assertRaises(TypeError, asencodedstring, b'abc', NULL) + self.assertRaises(TypeError, asencodedstring, b'abc', 'ascii') + self.assertRaises(TypeError, asencodedstring, [], NULL) + self.assertRaises(TypeError, asencodedstring, [], 'ascii') + # CRASHES asencodedstring(NULL, NULL) + # CRASHES asencodedstring(NULL, 'ascii') + + def test_decodeutf8(self): + """Test PyUnicode_DecodeUTF8()""" + decodeutf8 = _testlimitedcapi.unicode_decodeutf8 + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8(b), s) + self.assertEqual(decodeutf8(b, 'strict'), s) + + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') + self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') + self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and + # negative size. + + def test_decodeutf8stateful(self): + """Test PyUnicode_DecodeUTF8Stateful()""" + decodeutf8stateful = _testlimitedcapi.unicode_decodeutf8stateful + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8stateful(b), (s, len(b))) + self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) + + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) + + self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of + # "consumed". + + def test_asutf8string(self): + """Test PyUnicode_AsUTF8String()""" + asutf8string = _testlimitedcapi.unicode_asutf8string + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + self.assertEqual(asutf8string(s), s.encode('utf-8')) + + self.assertRaises(UnicodeEncodeError, asutf8string, '\ud8ff') + self.assertRaises(TypeError, asutf8string, b'abc') + self.assertRaises(TypeError, asutf8string, []) + # CRASHES asutf8string(NULL) + + def test_decodeutf16(self): + """Test PyUnicode_DecodeUTF16()""" + decodeutf16 = _testlimitedcapi.unicode_decodeutf16 + + naturalbyteorder = -1 if sys.byteorder == 'little' else 1 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-16') + self.assertEqual(decodeutf16(0, b), (naturalbyteorder, s)) + b = s.encode('utf-16le') + self.assertEqual(decodeutf16(-1, b), (-1, s)) + self.assertEqual(decodeutf16(0, b'\xff\xfe'+b), (-1, s)) + b = s.encode('utf-16be') + self.assertEqual(decodeutf16(1, b), (1, s)) + self.assertEqual(decodeutf16(0, b'\xfe\xff'+b), (1, s)) + + self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'a') + self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'a') + self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xff\xfea') + self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xfe\xffa') + + self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x00\xde') + self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xde\x00') + self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xde\xde') + self.assertEqual(decodeutf16(-1, b'\x00\xde', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf16(1, b'\xde\x00', 'replace'), (1, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xde\xde', 'replace'), (0, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd')) + + self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x3d\xd8') + self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xd8\x3d') + self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xd8\xd8') + self.assertEqual(decodeutf16(-1, b'\x3d\xd8', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf16(1, b'\xd8\x3d', 'replace'), (1, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xd8\xd8', 'replace'), (0, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xff\xfe\x3d\xd8', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf16(0, b'\xfe\xff\xd8\x3d', 'replace'), (1, '\ufffd')) + + self.assertRaises(LookupError, decodeutf16, -1, b'\x00\xde', 'foo') + self.assertRaises(LookupError, decodeutf16, 1, b'\xde\x00', 'foo') + self.assertRaises(LookupError, decodeutf16, 0, b'\xde\xde', 'foo') + # TODO: Test PyUnicode_DecodeUTF16() with NULL as data and + # negative size. + + def test_decodeutf16stateful(self): + """Test PyUnicode_DecodeUTF16Stateful()""" + decodeutf16stateful = _testlimitedcapi.unicode_decodeutf16stateful + + naturalbyteorder = -1 if sys.byteorder == 'little' else 1 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-16') + self.assertEqual(decodeutf16stateful(0, b), (naturalbyteorder, s, len(b))) + b = s.encode('utf-16le') + self.assertEqual(decodeutf16stateful(-1, b), (-1, s, len(b))) + self.assertEqual(decodeutf16stateful(0, b'\xff\xfe'+b), (-1, s, len(b)+2)) + b = s.encode('utf-16be') + self.assertEqual(decodeutf16stateful(1, b), (1, s, len(b))) + self.assertEqual(decodeutf16stateful(0, b'\xfe\xff'+b), (1, s, len(b)+2)) + + self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d'), (-1, 'a', 2)) + self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8'), (-1, 'a', 2)) + self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8\x00'), (-1, 'a', 2)) + self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8'), (1, 'a', 2)) + self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d'), (1, 'a', 2)) + self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d\xde'), (1, 'a', 2)) + self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x61\x00\x3d\xd8\x00'), (-1, 'a', 4)) + self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\x00\x61\xd8\x3d\xde'), (1, 'a', 4)) + + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x00\xde') + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xde\x00') + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 0, b'\xde\xde') + self.assertEqual(decodeutf16stateful(-1, b'\x00\xde', 'replace'), (-1, '\ufffd', 2)) + self.assertEqual(decodeutf16stateful(1, b'\xde\x00', 'replace'), (1, '\ufffd', 2)) + self.assertEqual(decodeutf16stateful(0, b'\xde\xde', 'replace'), (0, '\ufffd', 2)) + self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd', 4)) + self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd', 4)) + + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x3d\xd8\x61\x00') + self.assertEqual(decodeutf16stateful(-1, b'\x3d\xd8\x61\x00', 'replace'), (-1, '\ufffda', 4)) + self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xd8\x3d\x00\x61') + self.assertEqual(decodeutf16stateful(1, b'\xd8\x3d\x00\x61', 'replace'), (1, '\ufffda', 4)) + + self.assertRaises(LookupError, decodeutf16stateful, -1, b'\x00\xde', 'foo') + self.assertRaises(LookupError, decodeutf16stateful, 1, b'\xde\x00', 'foo') + self.assertRaises(LookupError, decodeutf16stateful, 0, b'\xde\xde', 'foo') + # TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as the address of + # "consumed". + + def test_asutf16string(self): + """Test PyUnicode_AsUTF16String()""" + asutf16string = _testlimitedcapi.unicode_asutf16string + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + self.assertEqual(asutf16string(s), s.encode('utf-16')) + + self.assertRaises(UnicodeEncodeError, asutf16string, '\ud8ff') + self.assertRaises(TypeError, asutf16string, b'abc') + self.assertRaises(TypeError, asutf16string, []) + # CRASHES asutf16string(NULL) + + def test_decodeutf32(self): + """Test PyUnicode_DecodeUTF8()""" + decodeutf32 = _testlimitedcapi.unicode_decodeutf32 + + naturalbyteorder = -1 if sys.byteorder == 'little' else 1 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-32') + self.assertEqual(decodeutf32(0, b), (naturalbyteorder, s)) + b = s.encode('utf-32le') + self.assertEqual(decodeutf32(-1, b), (-1, s)) + self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00'+b), (-1, s)) + b = s.encode('utf-32be') + self.assertEqual(decodeutf32(1, b), (1, s)) + self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff'+b), (1, s)) + + self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x61\x00\x00\x00\x00') + self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\x00\x61\x00') + self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00') + self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00') + + self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\xff\xff\xff\xff') + self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\xff\xff\xff\xff') + self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xff\xff\xff') + self.assertEqual(decodeutf32(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf32(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd')) + self.assertEqual(decodeutf32(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd')) + self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd')) + self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd')) + + self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x3d\xd8\x00\x00') + self.assertEqual(decodeutf32(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd')) + self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\xd8\x3d') + self.assertEqual(decodeutf32(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd')) + + self.assertRaises(LookupError, decodeutf32, -1, b'\xff\xff\xff\xff', 'foo') + self.assertRaises(LookupError, decodeutf32, 1, b'\xff\xff\xff\xff', 'foo') + self.assertRaises(LookupError, decodeutf32, 0, b'\xff\xff\xff\xff', 'foo') + # TODO: Test PyUnicode_DecodeUTF32() with NULL as data and + # negative size. + + def test_decodeutf32stateful(self): + """Test PyUnicode_DecodeUTF32Stateful()""" + decodeutf32stateful = _testlimitedcapi.unicode_decodeutf32stateful + + naturalbyteorder = -1 if sys.byteorder == 'little' else 1 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-32') + self.assertEqual(decodeutf32stateful(0, b), (naturalbyteorder, s, len(b))) + b = s.encode('utf-32le') + self.assertEqual(decodeutf32stateful(-1, b), (-1, s, len(b))) + self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, s, len(b)+4)) + b = s.encode('utf-32be') + self.assertEqual(decodeutf32stateful(1, b), (1, s, len(b))) + self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, s, len(b)+4)) + + self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00'), (-1, 'a', 4)) + self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6'), (-1, 'a', 4)) + self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 4)) + self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00'), (1, 'a', 4)) + self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01'), (1, 'a', 4)) + self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 4)) + self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 8)) + self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 8)) + + for b in b'\xff', b'\xff\xff', b'\xff\xff\xff': + self.assertEqual(decodeutf32stateful(-1, b), (-1, '', 0)) + self.assertEqual(decodeutf32stateful(1, b), (1, '', 0)) + self.assertEqual(decodeutf32stateful(0, b), (0, '', 0)) + self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, '', 4)) + self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, '', 4)) + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\xff\xff\xff\xff') + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\xff\xff\xff\xff') + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 0, b'\xff\xff\xff\xff') + self.assertEqual(decodeutf32stateful(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 4)) + self.assertEqual(decodeutf32stateful(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 4)) + self.assertEqual(decodeutf32stateful(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd', 4)) + self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 8)) + self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 8)) + + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\x3d\xd8\x00\x00') + self.assertEqual(decodeutf32stateful(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd', 4)) + self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\x00\x00\xd8\x3d') + self.assertEqual(decodeutf32stateful(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd', 4)) + + self.assertRaises(LookupError, decodeutf32stateful, -1, b'\xff\xff\xff\xff', 'foo') + self.assertRaises(LookupError, decodeutf32stateful, 1, b'\xff\xff\xff\xff', 'foo') + self.assertRaises(LookupError, decodeutf32stateful, 0, b'\xff\xff\xff\xff', 'foo') + # TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as the address of + # "consumed". + + def test_asutf32string(self): + """Test PyUnicode_AsUTF32String()""" + asutf32string = _testlimitedcapi.unicode_asutf32string + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + self.assertEqual(asutf32string(s), s.encode('utf-32')) + + self.assertRaises(UnicodeEncodeError, asutf32string, '\ud8ff') + self.assertRaises(TypeError, asutf32string, b'abc') + self.assertRaises(TypeError, asutf32string, []) + # CRASHES asutf32string(NULL) + + def test_decodelatin1(self): + """Test PyUnicode_DecodeLatin1()""" + decodelatin1 = _testlimitedcapi.unicode_decodelatin1 + + self.assertEqual(decodelatin1(b'abc'), 'abc') + self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc') + self.assertEqual(decodelatin1(b'\xa1\xa2'), '\xa1\xa2') + self.assertEqual(decodelatin1(b'\xa1\xa2', 'strict'), '\xa1\xa2') + # TODO: Test PyUnicode_DecodeLatin1() with NULL as data and + # negative size. + + def test_aslatin1string(self): + """Test PyUnicode_AsLatin1String()""" + aslatin1string = _testlimitedcapi.unicode_aslatin1string + + self.assertEqual(aslatin1string('abc'), b'abc') + self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2') + + self.assertRaises(UnicodeEncodeError, aslatin1string, '\u4f60') + self.assertRaises(TypeError, aslatin1string, b'abc') + self.assertRaises(TypeError, aslatin1string, []) + # CRASHES aslatin1string(NULL) + + def test_decodeascii(self): + """Test PyUnicode_DecodeASCII()""" + decodeascii = _testlimitedcapi.unicode_decodeascii + + self.assertEqual(decodeascii(b'abc'), 'abc') + self.assertEqual(decodeascii(b'abc', 'strict'), 'abc') + + self.assertRaises(UnicodeDecodeError, decodeascii, b'\xff') + self.assertEqual(decodeascii(b'a\xff', 'replace'), 'a\ufffd') + self.assertEqual(decodeascii(b'a\xffb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeascii, b'a\xff', 'foo') + # TODO: Test PyUnicode_DecodeASCII() with NULL as data and + # negative size. + + def test_asasciistring(self): + """Test PyUnicode_AsASCIIString()""" + asasciistring = _testlimitedcapi.unicode_asasciistring + + self.assertEqual(asasciistring('abc'), b'abc') + + self.assertRaises(UnicodeEncodeError, asasciistring, '\x80') + self.assertRaises(TypeError, asasciistring, b'abc') + self.assertRaises(TypeError, asasciistring, []) + # CRASHES asasciistring(NULL) + + def test_decodecharmap(self): + """Test PyUnicode_DecodeCharmap()""" + decodecharmap = _testlimitedcapi.unicode_decodecharmap + + self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac') + self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac') + self.assertEqual(decodecharmap(b'\1\0\2', 'abc'), 'bac') + self.assertEqual(decodecharmap(b'\1\0\2', ['\xa1', '\xa2', '\xa3']), '\xa2\xa1\xa3') + self.assertEqual(decodecharmap(b'\1\0\2', ['\u4f60', '\u597d', '\u4e16']), '\u597d\u4f60\u4e16') + self.assertEqual(decodecharmap(b'\1\0\2', ['\U0001f600', '\U0001f601', '\U0001f602']), '\U0001f601\U0001f600\U0001f602') + + self.assertEqual(decodecharmap(b'\1\0\2', [97, 98, 99]), 'bac') + self.assertEqual(decodecharmap(b'\1\0\2', ['', 'b', 'cd']), 'bcd') + + self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {}) + self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {0: None}) + self.assertEqual(decodecharmap(b'\1\0\2', [None, 'b', 'c'], 'replace'), 'b\ufffdc') + self.assertEqual(decodecharmap(b'\1\0\2\xff', NULL), '\1\0\2\xff') + self.assertRaises(TypeError, decodecharmap, b'\0', 42) + + # TODO: Test PyUnicode_DecodeCharmap() with NULL as data and + # negative size. + + def test_ascharmapstring(self): + """Test PyUnicode_AsCharmapString()""" + ascharmapstring = _testlimitedcapi.unicode_ascharmapstring + + self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7') + self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7') + self.assertEqual(ascharmapstring('\u4f60\u597d\u4e16', {0x4f60: 3, 0x597d: 0, 0x4e16: 7}), b'\3\0\7') + self.assertEqual(ascharmapstring('\U0001f600\U0001f601\U0001f602', {0x1f600: 3, 0x1f601: 0, 0x1f602: 7}), b'\3\0\7') + self.assertEqual(ascharmapstring('abc', {97: 3, 98: b'', 99: b'spam'}), b'\3spam') + + self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {}) + self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {97: None}) + self.assertRaises(TypeError, ascharmapstring, b'a', {}) + self.assertRaises(TypeError, ascharmapstring, [], {}) + self.assertRaises(TypeError, ascharmapstring, 'a', NULL) + # CRASHES ascharmapstring(NULL, {}) + + def test_decodeunicodeescape(self): + """Test PyUnicode_DecodeUnicodeEscape()""" + decodeunicodeescape = _testlimitedcapi.unicode_decodeunicodeescape + + self.assertEqual(decodeunicodeescape(b'abc'), 'abc') + self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\') + self.assertEqual(decodeunicodeescape(b'\t\n\r\x0b\x0c\x00'), '\t\n\r\v\f\0') + self.assertEqual(decodeunicodeescape(br'\xa1\xa2'), '\xa1\xa2') + self.assertEqual(decodeunicodeescape(b'\xa1\xa2'), '\xa1\xa2') + self.assertEqual(decodeunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d') + self.assertEqual(decodeunicodeescape(br'\U0001f600'), '\U0001f600') + with self.assertWarns(DeprecationWarning): + self.assertEqual(decodeunicodeescape(br'\z'), r'\z') + + for b in b'\\', br'\xa', br'\u4f6', br'\U0001f60': + self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b) + self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b, 'strict') + self.assertEqual(decodeunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd') + self.assertEqual(decodeunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy') + + self.assertRaises(LookupError, decodeunicodeescape, b'\\', 'foo') + # TODO: Test PyUnicode_DecodeUnicodeEscape() with NULL as data and + # negative size. + + def test_asunicodeescapestring(self): + """Test PyUnicode_AsUnicodeEscapeString()""" + asunicodeescapestring = _testlimitedcapi.unicode_asunicodeescapestring + + self.assertEqual(asunicodeescapestring('abc'), b'abc') + self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\') + self.assertEqual(asunicodeescapestring('\xa1\xa2'), br'\xa1\xa2') + self.assertEqual(asunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d') + self.assertEqual(asunicodeescapestring('\U0001f600'), br'\U0001f600') + + self.assertRaises(TypeError, asunicodeescapestring, b'abc') + self.assertRaises(TypeError, asunicodeescapestring, []) + # CRASHES asunicodeescapestring(NULL) + + def test_decoderawunicodeescape(self): + """Test PyUnicode_DecodeRawUnicodeEscape()""" + decoderawunicodeescape = _testlimitedcapi.unicode_decoderawunicodeescape + + self.assertEqual(decoderawunicodeescape(b'abc'), 'abc') + self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\') + self.assertEqual(decoderawunicodeescape(b'\xa1\xa2'), '\xa1\xa2') + self.assertEqual(decoderawunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d') + self.assertEqual(decoderawunicodeescape(br'\U0001f600'), '\U0001f600') + self.assertEqual(decoderawunicodeescape(br'\xa1\xa2'), r'\xa1\xa2') + self.assertEqual(decoderawunicodeescape(br'\z'), r'\z') + + for b in br'\u4f6', br'\U0001f60': + self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b) + self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b, 'strict') + self.assertEqual(decoderawunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd') + self.assertEqual(decoderawunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy') + + self.assertRaises(LookupError, decoderawunicodeescape, br'\U0001f60', 'foo') + # TODO: Test PyUnicode_DecodeRawUnicodeEscape() with NULL as data and + # negative size. + + def test_asrawunicodeescapestring(self): + """Test PyUnicode_AsRawUnicodeEscapeString()""" + asrawunicodeescapestring = _testlimitedcapi.unicode_asrawunicodeescapestring + + self.assertEqual(asrawunicodeescapestring('abc'), b'abc') + self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\') + self.assertEqual(asrawunicodeescapestring('\xa1\xa2'), b'\xa1\xa2') + self.assertEqual(asrawunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d') + self.assertEqual(asrawunicodeescapestring('\U0001f600'), br'\U0001f600') + + self.assertRaises(TypeError, asrawunicodeescapestring, b'abc') + self.assertRaises(TypeError, asrawunicodeescapestring, []) + # CRASHES asrawunicodeescapestring(NULL) + + +class CAPICodecs(unittest.TestCase): + + def setUp(self): + # Encoding names are normalized internally by converting them + # to lowercase and their hyphens are replaced by underscores. + self.encoding_name = 'test.test_capi.test_codecs.codec_reversed' + # Make sure that our custom codec is not already registered (that + # way we know whether we correctly unregistered the custom codec + # after a test or not). + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + # create the search function without registering yet + self._create_custom_codec() + + def _create_custom_codec(self): + def codec_encoder(m, errors='strict'): + return (type(m)().join(reversed(m)), len(m)) + + def codec_decoder(c, errors='strict'): + return (type(c)().join(reversed(c)), len(c)) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codec_encoder(input) + + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return codec_decoder(input) + + class StreamReader(codecs.StreamReader): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + info = codecs.CodecInfo( + encode=codec_encoder, + decode=codec_decoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + name=self.encoding_name + ) + + def search_function(encoding): + if encoding == self.encoding_name: + return info + return None + + self.codec_info = info + self.search_function = search_function + + @contextlib.contextmanager + def use_custom_encoder(self): + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + codecs.register(self.search_function) + yield + codecs.unregister(self.search_function) + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + + def test_codec_register(self): + search_function, encoding = self.search_function, self.encoding_name + # register the search function using the C API + self.assertIsNone(_testcapi.codec_register(search_function)) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIs(codecs.lookup(encoding), search_function(encoding)) + self.assertEqual(codecs.encode('123', encoding=encoding), '321') + # unregister the search function using the regular API + codecs.unregister(search_function) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_unregister(self): + search_function, encoding = self.search_function, self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding) + # register the search function using the regular API + codecs.register(search_function) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIsNotNone(codecs.lookup(encoding)) + # unregister the search function using the C API + self.assertIsNone(_testcapi.codec_unregister(search_function)) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_known_encoding(self): + self.assertRaises(LookupError, codecs.lookup, 'unknown-codec') + self.assertFalse(_testcapi.codec_known_encoding('unknown-codec')) + self.assertFalse(_testcapi.codec_known_encoding('unknown_codec')) + self.assertFalse(_testcapi.codec_known_encoding('UNKNOWN-codec')) + + encoding_name = self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding_name) + + codecs.register(self.search_function) + self.addCleanup(codecs.unregister, self.search_function) + + for name in [ + encoding_name, + encoding_name.upper(), + encoding_name.replace('_', '-'), + ]: + with self.subTest(name): + self.assertTrue(_testcapi.codec_known_encoding(name)) + + def test_codec_encode(self): + encode = _testcapi.codec_encode + self.assertEqual(encode('a', 'utf-8', NULL), b'a') + self.assertEqual(encode('a', 'utf-8', 'strict'), b'a') + self.assertEqual(encode('[é]', 'ascii', 'ignore'), b'[]') + + self.assertRaises(TypeError, encode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + encode('a', NULL, 'strict') + + def test_codec_decode(self): + decode = _testcapi.codec_decode + + s = 'a\xa1\u4f60\U0001f600' + b = s.encode() + + self.assertEqual(decode(b, 'utf-8', 'strict'), s) + self.assertEqual(decode(b, 'utf-8', NULL), s) + self.assertEqual(decode(b, 'latin1', 'strict'), b.decode('latin1')) + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', 'strict') + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', NULL) + self.assertEqual(decode(b, 'ascii', 'replace'), 'a' + '\ufffd'*9) + + # _codecs.decode() only reports an unknown error handling name when + # the corresponding error handling function is used; this difers + # from PyUnicode_Decode() which checks that both the encoding and + # the error handling name are recognized before even attempting to + # call the decoder. + self.assertEqual(decode(b'', 'utf-8', 'unknown-error-handler'), '') + self.assertEqual(decode(b'a', 'utf-8', 'unknown-error-handler'), 'a') + + self.assertRaises(TypeError, decode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + decode(b, NULL, 'strict') + + def test_codec_encoder(self): + codec_encoder = _testcapi.codec_encoder + + with self.use_custom_encoder(): + encoder = codec_encoder(self.encoding_name) + self.assertIs(encoder, self.codec_info.encode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_encoder(NULL) + + def test_codec_decoder(self): + codec_decoder = _testcapi.codec_decoder + + with self.use_custom_encoder(): + decoder = codec_decoder(self.encoding_name) + self.assertIs(decoder, self.codec_info.decode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_decoder(NULL) + + def test_codec_incremental_encoder(self): + codec_incremental_encoder = _testcapi.codec_incremental_encoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + encoder = codec_incremental_encoder(encoding, errors) + self.assertIsInstance(encoder, self.codec_info.incrementalencoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_encoder(NULL, 'strict') + + def test_codec_incremental_decoder(self): + codec_incremental_decoder = _testcapi.codec_incremental_decoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + decoder = codec_incremental_decoder(encoding, errors) + self.assertIsInstance(decoder, self.codec_info.incrementaldecoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_decoder(NULL, 'strict') + + def test_codec_stream_reader(self): + codec_stream_reader = _testcapi.codec_stream_reader + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_reader(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamreader) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_reader(NULL, stream, 'strict') + + def test_codec_stream_writer(self): + codec_stream_writer = _testcapi.codec_stream_writer + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_writer(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamwriter) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_writer(NULL, stream, 'strict') + + +class CAPICodecErrors(unittest.TestCase): + + @classmethod + def _generate_exception_args(cls): + for objlen in range(5): + maxind = 2 * max(2, objlen) + for start in range(-maxind, maxind + 1): + for end in range(-maxind, maxind + 1): + yield objlen, start, end + + @classmethod + def generate_encode_errors(cls): + return tuple( + UnicodeEncodeError('utf-8', '0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_decode_errors(cls): + return tuple( + UnicodeDecodeError('utf-8', b'0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_translate_errors(cls): + return tuple( + UnicodeTranslateError('0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def setUpClass(cls): + cls.unicode_encode_errors = cls.generate_encode_errors() + cls.unicode_decode_errors = cls.generate_decode_errors() + cls.unicode_translate_errors = cls.generate_translate_errors() + cls.all_unicode_errors = ( + cls.unicode_encode_errors + + cls.unicode_decode_errors + + cls.unicode_translate_errors + ) + cls.bad_unicode_errors = ( + ValueError(), + ) + + def test_codec_register_error(self): + # for cleaning up between tests + from _codecs import _unregister_error as _codecs_unregister_error + + self.assertRaises(LookupError, _testcapi.codec_lookup_error, 'custom') + + def custom_error_handler(exc): + raise exc + + error_handler = mock.Mock(wraps=custom_error_handler) + _testcapi.codec_register_error('custom', error_handler) + self.addCleanup(_codecs_unregister_error, 'custom') + + self.assertRaises(UnicodeEncodeError, codecs.encode, + '\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + error_handler.reset_mock() + + self.assertRaises(UnicodeDecodeError, codecs.decode, + b'\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + + # _codecs._unregister_error directly delegates to the internal C + # function so a Python-level function test is sufficient (it is + # tested in test_codeccallbacks). + + def test_codec_lookup_error(self): + codec_lookup_error = _testcapi.codec_lookup_error + self.assertIs(codec_lookup_error(NULL), codecs.strict_errors) + self.assertIs(codec_lookup_error('strict'), codecs.strict_errors) + self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) + self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) + self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) + self.assertIs(codec_lookup_error('backslashreplace'), codecs.backslashreplace_errors) + self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) + self.assertRaises(LookupError, codec_lookup_error, 'unknown') + + def test_codec_strict_errors_handler(self): + handler = _testcapi.codec_strict_errors + for exc in self.all_unicode_errors + self.bad_unicode_errors: + with self.subTest(handler=handler, exc=exc): + self.assertRaises(type(exc), handler, exc) + + def test_codec_ignore_errors_handler(self): + handler = _testcapi.codec_ignore_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_replace_errors_handler(self): + handler = _testcapi.codec_replace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_xmlcharrefreplace_errors_handler(self): + handler = _testcapi.codec_xmlcharrefreplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def test_codec_backslashreplace_errors_handler(self): + handler = _testcapi.codec_backslashreplace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_namereplace_errors_handler(self): + handler = _testlimitedcapi.codec_namereplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def do_test_codec_errors_handler(self, handler, exceptions): + self.assertNotEqual(len(exceptions), 0) + for exc in exceptions: + with self.subTest(handler=handler, exc=exc): + # test that the handler does not crash + res = handler(exc) + self.assertIsInstance(res, tuple) + self.assertEqual(len(res), 2) + replacement, continue_from = res + self.assertIsInstance(replacement, str) + self.assertIsInstance(continue_from, int) + self.assertGreaterEqual(continue_from, 0) + self.assertLessEqual(continue_from, len(exc.object)) + + for bad_exc in ( + self.bad_unicode_errors + + tuple(e for e in self.all_unicode_errors if e not in exceptions) + ): + with self.subTest('bad type', handler=handler, exc=bad_exc): + self.assertRaises(TypeError, handler, bad_exc) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py new file mode 100644 index 00000000000..c3189a67cc7 --- /dev/null +++ b/Lib/test/test_capi/test_complex.py @@ -0,0 +1,299 @@ +from math import isnan +import errno +import unittest +import warnings + +from test.test_capi.test_getargs import (BadComplex, BadComplex2, Complex, + FloatSubclass, Float, BadFloat, + BadFloat2, ComplexSubclass) +from test.support import import_helper +from test.support.testcase import ComplexesAreIdenticalMixin + + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') + +NULL = None +INF = float("inf") +NAN = float("nan") +DBL_MAX = _testcapi.DBL_MAX + + +class BadComplex3: + def __complex__(self): + raise RuntimeError + + +class CAPIComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase): + def test_check(self): + # Test PyComplex_Check() + check = _testlimitedcapi.complex_check + + self.assertTrue(check(1+2j)) + self.assertTrue(check(ComplexSubclass(1+2j))) + self.assertFalse(check(Complex())) + self.assertFalse(check(3)) + self.assertFalse(check(3.0)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_checkexact(self): + # PyComplex_CheckExact() + checkexact = _testlimitedcapi.complex_checkexact + + self.assertTrue(checkexact(1+2j)) + self.assertFalse(checkexact(ComplexSubclass(1+2j))) + self.assertFalse(checkexact(Complex())) + self.assertFalse(checkexact(3)) + self.assertFalse(checkexact(3.0)) + self.assertFalse(checkexact(object())) + + # CRASHES checkexact(NULL) + + def test_fromccomplex(self): + # Test PyComplex_FromCComplex() + fromccomplex = _testcapi.complex_fromccomplex + + self.assertEqual(fromccomplex(1+2j), 1.0+2.0j) + + def test_fromdoubles(self): + # Test PyComplex_FromDoubles() + fromdoubles = _testlimitedcapi.complex_fromdoubles + + self.assertEqual(fromdoubles(1.0, 2.0), 1.0+2.0j) + + def test_realasdouble(self): + # Test PyComplex_RealAsDouble() + realasdouble = _testlimitedcapi.complex_realasdouble + + self.assertEqual(realasdouble(1+2j), 1.0) + self.assertEqual(realasdouble(-1+0j), -1.0) + self.assertEqual(realasdouble(4.25), 4.25) + self.assertEqual(realasdouble(-1.0), -1.0) + self.assertEqual(realasdouble(42), 42.) + self.assertEqual(realasdouble(-1), -1.0) + + # Test subclasses of complex/float + self.assertEqual(realasdouble(ComplexSubclass(1+2j)), 1.0) + self.assertEqual(realasdouble(FloatSubclass(4.25)), 4.25) + + # Test types with __complex__ dunder method + self.assertEqual(realasdouble(Complex()), 4.25) + self.assertRaises(TypeError, realasdouble, BadComplex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(realasdouble(BadComplex2()), 4.25) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, realasdouble, BadComplex2()) + self.assertRaises(RuntimeError, realasdouble, BadComplex3()) + + # Test types with __float__ dunder method + self.assertEqual(realasdouble(Float()), 4.25) + self.assertRaises(TypeError, realasdouble, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(realasdouble(BadFloat2()), 4.25) + + self.assertRaises(TypeError, realasdouble, object()) + + # CRASHES realasdouble(NULL) + + def test_imagasdouble(self): + # Test PyComplex_ImagAsDouble() + imagasdouble = _testlimitedcapi.complex_imagasdouble + + self.assertEqual(imagasdouble(1+2j), 2.0) + self.assertEqual(imagasdouble(1-1j), -1.0) + self.assertEqual(imagasdouble(4.25), 0.0) + self.assertEqual(imagasdouble(42), 0.0) + + # Test subclasses of complex/float + self.assertEqual(imagasdouble(ComplexSubclass(1+2j)), 2.0) + self.assertEqual(imagasdouble(FloatSubclass(4.25)), 0.0) + + # Test types with __complex__ dunder method + self.assertEqual(imagasdouble(Complex()), 0.5) + self.assertRaises(TypeError, imagasdouble, BadComplex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(imagasdouble(BadComplex2()), 0.5) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, imagasdouble, BadComplex2()) + self.assertRaises(RuntimeError, imagasdouble, BadComplex3()) + + # Test types with __float__ dunder method + self.assertEqual(imagasdouble(Float()), 0.0) + self.assertRaises(TypeError, imagasdouble, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(imagasdouble(BadFloat2()), 0.0) + + self.assertRaises(TypeError, imagasdouble, object()) + + # CRASHES imagasdouble(NULL) + + def test_asccomplex(self): + # Test PyComplex_AsCComplex() + asccomplex = _testcapi.complex_asccomplex + + self.assertEqual(asccomplex(1+2j), 1.0+2.0j) + self.assertEqual(asccomplex(-1+2j), -1.0+2.0j) + self.assertEqual(asccomplex(4.25), 4.25+0.0j) + self.assertEqual(asccomplex(-1.0), -1.0+0.0j) + self.assertEqual(asccomplex(42), 42+0j) + self.assertEqual(asccomplex(-1), -1.0+0.0j) + + # Test subclasses of complex/float + self.assertEqual(asccomplex(ComplexSubclass(1+2j)), 1.0+2.0j) + self.assertEqual(asccomplex(FloatSubclass(4.25)), 4.25+0.0j) + + # Test types with __complex__ dunder method + self.assertEqual(asccomplex(Complex()), 4.25+0.5j) + self.assertRaises(TypeError, asccomplex, BadComplex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(asccomplex(BadComplex2()), 4.25+0.5j) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, asccomplex, BadComplex2()) + self.assertRaises(RuntimeError, asccomplex, BadComplex3()) + + # Test types with __float__ dunder method + self.assertEqual(asccomplex(Float()), 4.25+0.0j) + self.assertRaises(TypeError, asccomplex, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(asccomplex(BadFloat2()), 4.25+0.0j) + + self.assertRaises(TypeError, asccomplex, object()) + + # CRASHES asccomplex(NULL) + + def test_py_c_sum(self): + # Test _Py_c_sum() + _py_c_sum = _testcapi._py_c_sum + + self.assertEqual(_py_c_sum(1, 1j), (1+1j, 0)) + + def test_py_cr_sum(self): + # Test _Py_cr_sum() + _py_cr_sum = _testinternalcapi._py_cr_sum + + self.assertComplexesAreIdentical(_py_cr_sum(-0j, -0.0)[0], + complex(-0.0, -0.0)) + + def test_py_c_diff(self): + # Test _Py_c_diff() + _py_c_diff = _testcapi._py_c_diff + + self.assertEqual(_py_c_diff(1, 1j), (1-1j, 0)) + + def test_py_cr_diff(self): + # Test _Py_cr_diff() + _py_cr_diff = _testinternalcapi._py_cr_diff + + self.assertComplexesAreIdentical(_py_cr_diff(-0j, 0.0)[0], + complex(-0.0, -0.0)) + + def test_py_rc_diff(self): + # Test _Py_rc_diff() + _py_rc_diff = _testinternalcapi._py_rc_diff + + self.assertComplexesAreIdentical(_py_rc_diff(-0.0, 0j)[0], + complex(-0.0, -0.0)) + + def test_py_c_neg(self): + # Test _Py_c_neg() + _py_c_neg = _testcapi._py_c_neg + + self.assertEqual(_py_c_neg(1+1j), -1-1j) + + def test_py_c_prod(self): + # Test _Py_c_prod() + _py_c_prod = _testcapi._py_c_prod + + self.assertEqual(_py_c_prod(2, 1j), (2j, 0)) + + def test_py_cr_prod(self): + # Test _Py_cr_prod() + _py_cr_prod = _testinternalcapi._py_cr_prod + + self.assertComplexesAreIdentical(_py_cr_prod(complex('inf+1j'), INF)[0], + complex('inf+infj')) + + def test_py_c_quot(self): + # Test _Py_c_quot() + _py_c_quot = _testcapi._py_c_quot + + self.assertEqual(_py_c_quot(1, 1j), (-1j, 0)) + self.assertEqual(_py_c_quot(1, -1j), (1j, 0)) + self.assertEqual(_py_c_quot(1j, 2), (0.5j, 0)) + self.assertEqual(_py_c_quot(1j, -2), (-0.5j, 0)) + self.assertEqual(_py_c_quot(1, 2j), (-0.5j, 0)) + + z, e = _py_c_quot(NAN, 1j) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + self.assertEqual(e, 0) + + z, e = _py_c_quot(1j, NAN) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + self.assertEqual(e, 0) + + self.assertEqual(_py_c_quot(1, 0j)[1], errno.EDOM) + + def test_py_cr_quot(self): + # Test _Py_cr_quot() + _py_cr_quot = _testinternalcapi._py_cr_quot + + self.assertComplexesAreIdentical(_py_cr_quot(complex('inf+1j'), 2**1000)[0], + INF + 2**-1000*1j) + + def test_py_rc_quot(self): + # Test _Py_rc_quot() + _py_rc_quot = _testinternalcapi._py_rc_quot + + self.assertComplexesAreIdentical(_py_rc_quot(1.0, complex('nan-infj'))[0], + 0j) + + def test_py_c_pow(self): + # Test _Py_c_pow() + _py_c_pow = _testcapi._py_c_pow + + self.assertEqual(_py_c_pow(1j, 0j), (1+0j, 0)) + self.assertEqual(_py_c_pow(1, 1j), (1+0j, 0)) + self.assertEqual(_py_c_pow(0j, 1), (0j, 0)) + self.assertAlmostEqual(_py_c_pow(1j, 2)[0], -1.0+0j) + + r, e = _py_c_pow(1+1j, -1) + self.assertAlmostEqual(r, 0.5-0.5j) + self.assertEqual(e, 0) + + self.assertEqual(_py_c_pow(0j, -1)[1], errno.EDOM) + self.assertEqual(_py_c_pow(0j, 1j)[1], errno.EDOM) + max_num = DBL_MAX+1j + self.assertEqual(_py_c_pow(max_num, max_num), + (complex(INF, INF), errno.ERANGE)) + self.assertEqual(_py_c_pow(max_num, 2), + (complex(INF, INF), errno.ERANGE)) + + + def test_py_c_abs(self): + # Test _Py_c_abs() + _py_c_abs = _testcapi._py_c_abs + + self.assertEqual(_py_c_abs(-1), (1.0, 0)) + self.assertEqual(_py_c_abs(1j), (1.0, 0)) + + self.assertEqual(_py_c_abs(complex('+inf+1j')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('-inf+1j')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('1.25+infj')), (INF, 0)) + self.assertEqual(_py_c_abs(complex('1.25-infj')), (INF, 0)) + + self.assertTrue(isnan(_py_c_abs(complex('1.25+nanj'))[0])) + self.assertTrue(isnan(_py_c_abs(complex('nan-1j'))[0])) + + self.assertEqual(_py_c_abs(complex(*[DBL_MAX]*2))[1], errno.ERANGE) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py new file mode 100644 index 00000000000..a2d70dd3af4 --- /dev/null +++ b/Lib/test/test_capi/test_config.py @@ -0,0 +1,404 @@ +""" +Tests PyConfig_Get() and PyConfig_Set() C API (PEP 741). +""" +import os +import sys +import sysconfig +import types +import unittest +from test import support +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +# Is the Py_STATS macro defined? +Py_STATS = hasattr(sys, '_stats_on') + + +class CAPITests(unittest.TestCase): + def test_config_get(self): + # Test PyConfig_Get() + config_get = _testcapi.config_get + config_names = _testcapi.config_names + + TEST_VALUE = { + str: "TEST_MARKER_STR", + str | None: "TEST_MARKER_OPT_STR", + list[str]: ("TEST_MARKER_STR_TUPLE",), + dict[str, str | bool]: {"x": "value", "y": True}, + } + + # read config options and check their type + options = [ + ("allocator", int, None), + ("argv", list[str], "argv"), + ("base_exec_prefix", str | None, "base_exec_prefix"), + ("base_executable", str | None, "_base_executable"), + ("base_prefix", str | None, "base_prefix"), + ("buffered_stdio", bool, None), + ("bytes_warning", int, None), + ("check_hash_pycs_mode", str, None), + ("code_debug_ranges", bool, None), + ("configure_c_stdio", bool, None), + ("coerce_c_locale", bool, None), + ("coerce_c_locale_warn", bool, None), + ("configure_locale", bool, None), + ("cpu_count", int, None), + ("dev_mode", bool, None), + ("dump_refs", bool, None), + ("dump_refs_file", str | None, None), + ("exec_prefix", str | None, "exec_prefix"), + ("executable", str | None, "executable"), + ("faulthandler", bool, None), + ("filesystem_encoding", str, None), + ("filesystem_errors", str, None), + ("hash_seed", int, None), + ("home", str | None, None), + ("thread_inherit_context", int, None), + ("context_aware_warnings", int, None), + ("import_time", int, None), + ("inspect", bool, None), + ("install_signal_handlers", bool, None), + ("int_max_str_digits", int, None), + ("interactive", bool, None), + ("isolated", bool, None), + ("malloc_stats", bool, None), + ("module_search_paths", list[str], "path"), + ("optimization_level", int, None), + ("orig_argv", list[str], "orig_argv"), + ("parser_debug", bool, None), + ("parse_argv", bool, None), + ("pathconfig_warnings", bool, None), + ("perf_profiling", int, None), + ("platlibdir", str, "platlibdir"), + ("prefix", str | None, "prefix"), + ("program_name", str, None), + ("pycache_prefix", str | None, "pycache_prefix"), + ("quiet", bool, None), + ("remote_debug", int, None), + ("run_command", str | None, None), + ("run_filename", str | None, None), + ("run_module", str | None, None), + ("safe_path", bool, None), + ("show_ref_count", bool, None), + ("site_import", bool, None), + ("skip_source_first_line", bool, None), + ("stdio_encoding", str, None), + ("stdio_errors", str, None), + ("stdlib_dir", str | None, "_stdlib_dir"), + ("tracemalloc", int, None), + ("use_environment", bool, None), + ("use_frozen_modules", bool, None), + ("use_hash_seed", bool, None), + ("user_site_directory", bool, None), + ("utf8_mode", bool, None), + ("verbose", int, None), + ("warn_default_encoding", bool, None), + ("warnoptions", list[str], "warnoptions"), + ("write_bytecode", bool, None), + ("xoptions", dict[str, str | bool], "_xoptions"), + ] + if support.Py_DEBUG: + options.append(("run_presite", str | None, None)) + if support.Py_GIL_DISABLED: + options.append(("enable_gil", int, None)) + options.append(("tlbc_enabled", int, None)) + if support.MS_WINDOWS: + options.extend(( + ("legacy_windows_stdio", bool, None), + ("legacy_windows_fs_encoding", bool, None), + )) + if Py_STATS: + options.extend(( + ("_pystats", bool, None), + )) + if support.is_apple: + options.extend(( + ("use_system_logger", bool, None), + )) + + for name, option_type, sys_attr in options: + with self.subTest(name=name, option_type=option_type, + sys_attr=sys_attr): + value = config_get(name) + if isinstance(option_type, types.GenericAlias): + self.assertIsInstance(value, option_type.__origin__) + if option_type.__origin__ == dict: + key_type = option_type.__args__[0] + value_type = option_type.__args__[1] + for item in value.items(): + self.assertIsInstance(item[0], key_type) + self.assertIsInstance(item[1], value_type) + else: + item_type = option_type.__args__[0] + for item in value: + self.assertIsInstance(item, item_type) + else: + self.assertIsInstance(value, option_type) + + if sys_attr is not None: + expected = getattr(sys, sys_attr) + self.assertEqual(expected, value) + + override = TEST_VALUE[option_type] + with support.swap_attr(sys, sys_attr, override): + self.assertEqual(config_get(name), override) + + # check that the test checks all options + self.assertEqual(sorted(name for name, option_type, sys_attr in options), + sorted(config_names())) + + def test_config_get_sys_flags(self): + # Test PyConfig_Get() + config_get = _testcapi.config_get + + # compare config options with sys.flags + for flag, name, negate in ( + ("debug", "parser_debug", False), + ("inspect", "inspect", False), + ("interactive", "interactive", False), + ("optimize", "optimization_level", False), + ("dont_write_bytecode", "write_bytecode", True), + ("no_user_site", "user_site_directory", True), + ("no_site", "site_import", True), + ("ignore_environment", "use_environment", True), + ("verbose", "verbose", False), + ("bytes_warning", "bytes_warning", False), + ("quiet", "quiet", False), + # "hash_randomization" is tested below + ("isolated", "isolated", False), + ("dev_mode", "dev_mode", False), + ("utf8_mode", "utf8_mode", False), + ("warn_default_encoding", "warn_default_encoding", False), + ("safe_path", "safe_path", False), + ("int_max_str_digits", "int_max_str_digits", False), + # "gil", "thread_inherit_context" and "context_aware_warnings" are tested below + ): + with self.subTest(flag=flag, name=name, negate=negate): + value = config_get(name) + if negate: + value = not value + self.assertEqual(getattr(sys.flags, flag), value) + + self.assertEqual(sys.flags.hash_randomization, + config_get('use_hash_seed') == 0 + or config_get('hash_seed') != 0) + + if support.Py_GIL_DISABLED: + value = config_get('enable_gil') + expected = (value if value != -1 else None) + self.assertEqual(sys.flags.gil, expected) + + expected_inherit_context = 1 if support.Py_GIL_DISABLED else 0 + self.assertEqual(sys.flags.thread_inherit_context, expected_inherit_context) + + expected_safe_warnings = 1 if support.Py_GIL_DISABLED else 0 + self.assertEqual(sys.flags.context_aware_warnings, expected_safe_warnings) + + def test_config_get_non_existent(self): + # Test PyConfig_Get() on non-existent option name + config_get = _testcapi.config_get + nonexistent_key = 'NONEXISTENT_KEY' + err_msg = f'unknown config option name: {nonexistent_key}' + with self.assertRaisesRegex(ValueError, err_msg): + config_get(nonexistent_key) + + def test_config_get_write_bytecode(self): + # PyConfig_Get("write_bytecode") gets sys.dont_write_bytecode + # as an integer + config_get = _testcapi.config_get + with support.swap_attr(sys, "dont_write_bytecode", 0): + self.assertEqual(config_get('write_bytecode'), 1) + with support.swap_attr(sys, "dont_write_bytecode", "yes"): + self.assertEqual(config_get('write_bytecode'), 0) + with support.swap_attr(sys, "dont_write_bytecode", []): + self.assertEqual(config_get('write_bytecode'), 1) + + def test_config_getint(self): + # Test PyConfig_GetInt() + config_getint = _testcapi.config_getint + + # PyConfig_MEMBER_INT type + self.assertEqual(config_getint('verbose'), sys.flags.verbose) + + # PyConfig_MEMBER_UINT type + self.assertEqual(config_getint('isolated'), sys.flags.isolated) + + # PyConfig_MEMBER_ULONG type + self.assertIsInstance(config_getint('hash_seed'), int) + + # PyPreConfig member + self.assertIsInstance(config_getint('allocator'), int) + + # platlibdir type is str + with self.assertRaises(TypeError): + config_getint('platlibdir') + + def test_get_config_names(self): + names = _testcapi.config_names() + self.assertIsInstance(names, frozenset) + for name in names: + self.assertIsInstance(name, str) + + def test_config_set_sys_attr(self): + # Test PyConfig_Set() with sys attributes + config_get = _testcapi.config_get + config_set = _testcapi.config_set + + # mutable configuration option mapped to sys attributes + for name, sys_attr, option_type in ( + ('argv', 'argv', list[str]), + ('base_exec_prefix', 'base_exec_prefix', str | None), + ('base_executable', '_base_executable', str | None), + ('base_prefix', 'base_prefix', str | None), + ('exec_prefix', 'exec_prefix', str | None), + ('executable', 'executable', str | None), + ('module_search_paths', 'path', list[str]), + ('platlibdir', 'platlibdir', str), + ('prefix', 'prefix', str | None), + ('pycache_prefix', 'pycache_prefix', str | None), + ('stdlib_dir', '_stdlib_dir', str | None), + ('warnoptions', 'warnoptions', list[str]), + ('xoptions', '_xoptions', dict[str, str | bool]), + ): + with self.subTest(name=name): + if option_type == str: + test_values = ('TEST_REPLACE',) + invalid_types = (1, None) + elif option_type == str | None: + test_values = ('TEST_REPLACE', None) + invalid_types = (123,) + elif option_type == list[str]: + test_values = (['TEST_REPLACE'], []) + invalid_types = ('text', 123, [123]) + else: # option_type == dict[str, str | bool]: + test_values = ({"x": "value", "y": True},) + invalid_types = ('text', 123, ['option'], + {123: 'value'}, + {'key': b'bytes'}) + + old_opt_value = config_get(name) + old_sys_value = getattr(sys, sys_attr) + try: + for value in test_values: + config_set(name, value) + self.assertEqual(config_get(name), value) + self.assertEqual(getattr(sys, sys_attr), value) + + for value in invalid_types: + with self.assertRaises(TypeError): + config_set(name, value) + finally: + setattr(sys, sys_attr, old_sys_value) + config_set(name, old_opt_value) + + def test_config_set_sys_flag(self): + # Test PyConfig_Set() with sys.flags + config_get = _testcapi.config_get + config_set = _testcapi.config_set + + # mutable configuration option mapped to sys.flags + class unsigned_int(int): + pass + + def expect_int(value): + value = int(value) + return (value, value) + + def expect_bool(value): + value = int(bool(value)) + return (value, value) + + def expect_bool_not(value): + value = bool(value) + return (int(value), int(not value)) + + for name, sys_flag, option_type, expect_func in ( + # (some flags cannot be set, see comments below.) + ('parser_debug', 'debug', bool, expect_bool), + ('inspect', 'inspect', bool, expect_bool), + ('interactive', 'interactive', bool, expect_bool), + ('optimization_level', 'optimize', unsigned_int, expect_int), + ('write_bytecode', 'dont_write_bytecode', bool, expect_bool_not), + # user_site_directory + # site_import + ('use_environment', 'ignore_environment', bool, expect_bool_not), + ('verbose', 'verbose', unsigned_int, expect_int), + ('bytes_warning', 'bytes_warning', unsigned_int, expect_int), + ('quiet', 'quiet', bool, expect_bool), + # hash_randomization + # isolated + # dev_mode + # utf8_mode + # warn_default_encoding + # safe_path + ('int_max_str_digits', 'int_max_str_digits', unsigned_int, expect_int), + # gil + ): + if name == "int_max_str_digits": + new_values = (0, 5_000, 999_999) + invalid_values = (-1, 40) # value must 0 or >= 4300 + invalid_types = (1.0, "abc") + elif option_type == int: + new_values = (False, True, 0, 1, 5, -5) + invalid_values = () + invalid_types = (1.0, "abc") + else: + new_values = (False, True, 0, 1, 5) + invalid_values = (-5,) + invalid_types = (1.0, "abc") + + with self.subTest(name=name): + old_value = config_get(name) + try: + for value in new_values: + expected, expect_flag = expect_func(value) + + config_set(name, value) + self.assertEqual(config_get(name), expected) + self.assertEqual(getattr(sys.flags, sys_flag), expect_flag) + if name == "write_bytecode": + self.assertEqual(getattr(sys, "dont_write_bytecode"), + expect_flag) + if name == "int_max_str_digits": + self.assertEqual(sys.get_int_max_str_digits(), + expect_flag) + + for value in invalid_values: + with self.assertRaises(ValueError): + config_set(name, value) + + for value in invalid_types: + with self.assertRaises(TypeError): + config_set(name, value) + finally: + config_set(name, old_value) + + def test_config_set_cpu_count(self): + config_get = _testcapi.config_get + config_set = _testcapi.config_set + + old_value = config_get('cpu_count') + try: + config_set('cpu_count', 123) + self.assertEqual(os.cpu_count(), 123) + finally: + config_set('cpu_count', old_value) + + def test_config_set_read_only(self): + # Test PyConfig_Set() on read-only options + config_set = _testcapi.config_set + for name, value in ( + ("allocator", 0), # PyPreConfig member + ("perf_profiling", 8), + ("dev_mode", True), + ("filesystem_encoding", "utf-8"), + ): + with self.subTest(name=name, value=value): + with self.assertRaisesRegex(ValueError, r"read-only"): + config_set(name, value) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py new file mode 100644 index 00000000000..e726e3d813d --- /dev/null +++ b/Lib/test/test_capi/test_dict.py @@ -0,0 +1,550 @@ +import unittest +from collections import OrderedDict, UserDict +from types import MappingProxyType +from test import support +from test.support import import_helper + + +_testcapi = import_helper.import_module("_testcapi") +_testlimitedcapi = import_helper.import_module("_testlimitedcapi") + + +NULL = None +INVALID_UTF8 = b'\xff' + +class DictSubclass(dict): + def __getitem__(self, key): + raise RuntimeError('do not get evil') + def __setitem__(self, key, value): + raise RuntimeError('do not set evil') + def __delitem__(self, key): + raise RuntimeError('do not del evil') + +def gen(): + yield 'a' + yield 'b' + yield 'c' + + +class CAPITest(unittest.TestCase): + + def test_dict_check(self): + check = _testlimitedcapi.dict_check + self.assertTrue(check({1: 2})) + self.assertTrue(check(OrderedDict({1: 2}))) + self.assertFalse(check(UserDict({1: 2}))) + self.assertFalse(check([1, 2])) + self.assertFalse(check(object())) + # CRASHES check(NULL) + + def test_dict_checkexact(self): + check = _testlimitedcapi.dict_checkexact + self.assertTrue(check({1: 2})) + self.assertFalse(check(OrderedDict({1: 2}))) + self.assertFalse(check(UserDict({1: 2}))) + self.assertFalse(check([1, 2])) + self.assertFalse(check(object())) + # CRASHES check(NULL) + + def test_dict_new(self): + dict_new = _testlimitedcapi.dict_new + dct = dict_new() + self.assertEqual(dct, {}) + self.assertIs(type(dct), dict) + dct2 = dict_new() + self.assertIsNot(dct2, dct) + + def test_dictproxy_new(self): + dictproxy_new = _testlimitedcapi.dictproxy_new + for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): + proxy = dictproxy_new(dct) + self.assertIs(type(proxy), MappingProxyType) + self.assertEqual(proxy, dct) + with self.assertRaises(TypeError): + proxy[1] = 3 + self.assertEqual(proxy[1], 2) + dct[1] = 4 + self.assertEqual(proxy[1], 4) + + self.assertRaises(TypeError, dictproxy_new, []) + self.assertRaises(TypeError, dictproxy_new, 42) + # CRASHES dictproxy_new(NULL) + + def test_dict_copy(self): + copy = _testlimitedcapi.dict_copy + for dct in {1: 2}, OrderedDict({1: 2}): + dct_copy = copy(dct) + self.assertIs(type(dct_copy), dict) + self.assertEqual(dct_copy, dct) + + self.assertRaises(SystemError, copy, UserDict()) + self.assertRaises(SystemError, copy, []) + self.assertRaises(SystemError, copy, 42) + self.assertRaises(SystemError, copy, NULL) + + def test_dict_clear(self): + clear = _testlimitedcapi.dict_clear + dct = {1: 2} + clear(dct) + self.assertEqual(dct, {}) + + # NOTE: It is not safe to call it with OrderedDict. + + # Has no effect for non-dicts. + dct = UserDict({1: 2}) + clear(dct) + self.assertEqual(dct, {1: 2}) + lst = [1, 2] + clear(lst) + self.assertEqual(lst, [1, 2]) + clear(object()) + + # CRASHES? clear(NULL) + + def test_dict_size(self): + size = _testlimitedcapi.dict_size + self.assertEqual(size({1: 2}), 1) + self.assertEqual(size(OrderedDict({1: 2})), 1) + + self.assertRaises(SystemError, size, UserDict()) + self.assertRaises(SystemError, size, []) + self.assertRaises(SystemError, size, 42) + self.assertRaises(SystemError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_dict_getitem(self): + getitem = _testlimitedcapi.dict_getitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + with support.catch_unraisable_exception() as cm: + self.assertIs(getitem({}, []), KeyError) # unhashable + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "unhashable type: 'list'") + + self.assertIs(getitem(42, 'a'), KeyError) + self.assertIs(getitem([1], 0), KeyError) + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_getitemstring(self): + getitemstring = _testlimitedcapi.dict_getitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertIs(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertIs(getitemstring(dct2, b'b'), KeyError) + + with support.catch_unraisable_exception() as cm: + self.assertIs(getitemstring({}, INVALID_UTF8), KeyError) + self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) + self.assertRegex(str(cm.unraisable.exc_value), + "'utf-8' codec can't decode") + + self.assertIs(getitemstring(42, b'a'), KeyError) + self.assertIs(getitemstring([], b'a'), KeyError) + # CRASHES getitemstring({}, NULL) + # CRASHES getitemstring(NULL, b'a') + + def test_dict_getitemref(self): + getitem = _testcapi.dict_getitemref + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertRaises(SystemError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, [], 1) + self.assertRaises(SystemError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_getitemstringref(self): + getitemstring = _testcapi.dict_getitemstringref + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertIs(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertIs(getitemstring(dct2, b'b'), KeyError) + + self.assertRaises(SystemError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, INVALID_UTF8) + self.assertRaises(SystemError, getitemstring, [], b'a') + # CRASHES getitemstring({}, NULL) + # CRASHES getitemstring(NULL, b'a') + + def test_dict_getitemwitherror(self): + getitem = _testlimitedcapi.dict_getitemwitherror + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertRaises(SystemError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, [], 1) + self.assertRaises(SystemError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_contains(self): + contains = _testlimitedcapi.dict_contains + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(contains(dct, 'a')) + self.assertFalse(contains(dct, 'b')) + self.assertTrue(contains(dct, '\U0001f40d')) + + dct2 = DictSubclass(dct) + self.assertTrue(contains(dct2, 'a')) + self.assertFalse(contains(dct2, 'b')) + + self.assertRaises(TypeError, contains, {}, []) # unhashable + # CRASHES contains({}, NULL) + # CRASHES contains(UserDict(), 'a') + # CRASHES contains(42, 'a') + # CRASHES contains(NULL, 'a') + + def test_dict_contains_string(self): + contains_string = _testcapi.dict_containsstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(contains_string(dct, b'a')) + self.assertFalse(contains_string(dct, b'b')) + self.assertTrue(contains_string(dct, '\U0001f40d'.encode())) + self.assertRaises(UnicodeDecodeError, contains_string, dct, INVALID_UTF8) + + dct2 = DictSubclass(dct) + self.assertTrue(contains_string(dct2, b'a')) + self.assertFalse(contains_string(dct2, b'b')) + + # CRASHES contains({}, NULL) + # CRASHES contains(NULL, b'a') + + def test_dict_setitem(self): + setitem = _testlimitedcapi.dict_setitem + dct = {} + setitem(dct, 'a', 5) + self.assertEqual(dct, {'a': 5}) + setitem(dct, '\U0001f40d', 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct2 = DictSubclass() + setitem(dct2, 'a', 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable + self.assertRaises(SystemError, setitem, UserDict(), 'a', 5) + self.assertRaises(SystemError, setitem, [1], 0, 5) + self.assertRaises(SystemError, setitem, 42, 'a', 5) + # CRASHES setitem({}, NULL, 5) + # CRASHES setitem({}, 'a', NULL) + # CRASHES setitem(NULL, 'a', 5) + + def test_dict_setitemstring(self): + setitemstring = _testlimitedcapi.dict_setitemstring + dct = {} + setitemstring(dct, b'a', 5) + self.assertEqual(dct, {'a': 5}) + setitemstring(dct, '\U0001f40d'.encode(), 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct2 = DictSubclass() + setitemstring(dct2, b'a', 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(UnicodeDecodeError, setitemstring, {}, INVALID_UTF8, 5) + self.assertRaises(SystemError, setitemstring, UserDict(), b'a', 5) + self.assertRaises(SystemError, setitemstring, 42, b'a', 5) + # CRASHES setitemstring({}, NULL, 5) + # CRASHES setitemstring({}, b'a', NULL) + # CRASHES setitemstring(NULL, b'a', 5) + + def test_dict_delitem(self): + delitem = _testlimitedcapi.dict_delitem + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitem(dct, 'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitem, dct, 'b') + delitem(dct, '\U0001f40d') + self.assertEqual(dct, {'c': 2}) + + dct2 = DictSubclass({'a': 1, 'c': 2}) + delitem(dct2, 'a') + self.assertEqual(dct2, {'c': 2}) + self.assertRaises(KeyError, delitem, dct2, 'b') + + self.assertRaises(TypeError, delitem, {}, []) # unhashable + self.assertRaises(SystemError, delitem, UserDict({'a': 1}), 'a') + self.assertRaises(SystemError, delitem, [1], 0) + self.assertRaises(SystemError, delitem, 42, 'a') + # CRASHES delitem({}, NULL) + # CRASHES delitem(NULL, 'a') + + def test_dict_delitemstring(self): + delitemstring = _testlimitedcapi.dict_delitemstring + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitemstring(dct, b'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitemstring, dct, b'b') + delitemstring(dct, '\U0001f40d'.encode()) + self.assertEqual(dct, {'c': 2}) + + dct2 = DictSubclass({'a': 1, 'c': 2}) + delitemstring(dct2, b'a') + self.assertEqual(dct2, {'c': 2}) + self.assertRaises(KeyError, delitemstring, dct2, b'b') + + self.assertRaises(UnicodeDecodeError, delitemstring, {}, INVALID_UTF8) + self.assertRaises(SystemError, delitemstring, UserDict({'a': 1}), b'a') + self.assertRaises(SystemError, delitemstring, 42, b'a') + # CRASHES delitemstring({}, NULL) + # CRASHES delitemstring(NULL, b'a') + + def test_dict_setdefault(self): + setdefault = _testcapi.dict_setdefault + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + + def test_dict_setdefaultref(self): + setdefault = _testcapi.dict_setdefaultref + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + + def test_mapping_keys_valuesitems(self): + class BadMapping(dict): + def keys(self): + return None + def values(self): + return None + def items(self): + return None + dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} + for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: + self.assertListEqual(_testlimitedcapi.dict_keys(mapping), + list(dict_obj.keys())) + self.assertListEqual(_testlimitedcapi.dict_values(mapping), + list(dict_obj.values())) + self.assertListEqual(_testlimitedcapi.dict_items(mapping), + list(dict_obj.items())) + + def test_dict_keys_valuesitems_bad_arg(self): + for mapping in UserDict(), [], object(): + self.assertRaises(SystemError, _testlimitedcapi.dict_keys, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_values, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_items, mapping) + + def test_dict_next(self): + dict_next = _testlimitedcapi.dict_next + self.assertIsNone(dict_next({}, 0)) + dct = {'a': 1, 'b': 2, 'c': 3} + pos = 0 + pairs = [] + while True: + res = dict_next(dct, pos) + if res is None: + break + rc, pos, key, value = res + self.assertEqual(rc, 1) + pairs.append((key, value)) + self.assertEqual(pairs, list(dct.items())) + + # CRASHES dict_next(NULL, 0) + + def test_dict_update(self): + update = _testlimitedcapi.dict_update + for cls1 in dict, DictSubclass: + for cls2 in dict, DictSubclass, UserDict: + dct = cls1({'a': 1, 'b': 2}) + update(dct, cls2({'b': 3, 'c': 4})) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(AttributeError, update, {}, []) + self.assertRaises(AttributeError, update, {}, 42) + self.assertRaises(SystemError, update, UserDict(), {}) + self.assertRaises(SystemError, update, 42, {}) + self.assertRaises(SystemError, update, {}, NULL) + self.assertRaises(SystemError, update, NULL, {}) + + def test_dict_merge(self): + merge = _testlimitedcapi.dict_merge + for cls1 in dict, DictSubclass: + for cls2 in dict, DictSubclass, UserDict: + dct = cls1({'a': 1, 'b': 2}) + merge(dct, cls2({'b': 3, 'c': 4}), 0) + self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) + dct = cls1({'a': 1, 'b': 2}) + merge(dct, cls2({'b': 3, 'c': 4}), 1) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(AttributeError, merge, {}, [], 0) + self.assertRaises(AttributeError, merge, {}, 42, 0) + self.assertRaises(SystemError, merge, UserDict(), {}, 0) + self.assertRaises(SystemError, merge, 42, {}, 0) + self.assertRaises(SystemError, merge, {}, NULL, 0) + self.assertRaises(SystemError, merge, NULL, {}, 0) + + def test_dict_mergefromseq2(self): + mergefromseq2 = _testlimitedcapi.dict_mergefromseq2 + for cls1 in dict, DictSubclass: + for cls2 in list, iter: + dct = cls1({'a': 1, 'b': 2}) + mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 0) + self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) + dct = cls1({'a': 1, 'b': 2}) + mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 1) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(ValueError, mergefromseq2, {}, [(1,)], 0) + self.assertRaises(ValueError, mergefromseq2, {}, [(1, 2, 3)], 0) + self.assertRaises(TypeError, mergefromseq2, {}, [1], 0) + self.assertRaises(TypeError, mergefromseq2, {}, 42, 0) + # CRASHES mergefromseq2(UserDict(), [], 0) + # CRASHES mergefromseq2(42, [], 0) + # CRASHES mergefromseq2({}, NULL, 0) + # CRASHES mergefromseq2(NULL, {}, 0) + + def test_dict_pop(self): + # Test PyDict_Pop() + dict_pop = _testcapi.dict_pop + dict_pop_null = _testcapi.dict_pop_null + + # key present, get removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_pop(mydict, "key"), (1, "value")) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_pop(mydict, "key2"), (1, "value2")) + self.assertEqual(mydict, {}) + + # key present, ignore removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_pop_null(mydict, "key"), 1) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_pop_null(mydict, "key2"), 1) + self.assertEqual(mydict, {}) + + # key missing, expect removed value; empty dict has a fast path + self.assertEqual(dict_pop({}, "key"), (0, NULL)) + self.assertEqual(dict_pop({"a": 1}, "key"), (0, NULL)) + + # key missing, ignored removed value; empty dict has a fast path + self.assertEqual(dict_pop_null({}, "key"), 0) + self.assertEqual(dict_pop_null({"a": 1}, "key"), 0) + + # dict error + not_dict = UserDict({1: 2}) + self.assertRaises(SystemError, dict_pop, not_dict, "key") + self.assertRaises(SystemError, dict_pop_null, not_dict, "key") + + # key error; don't hash key if dict is empty + not_hashable_key = ["list"] + self.assertEqual(dict_pop({}, not_hashable_key), (0, NULL)) + with self.assertRaises(TypeError): + dict_pop({'key': 1}, not_hashable_key) + dict_pop({}, NULL) # key is not checked if dict is empty + + # CRASHES dict_pop(NULL, "key") + # CRASHES dict_pop({"a": 1}, NULL) + + def test_dict_popstring(self): + # Test PyDict_PopString() + dict_popstring = _testcapi.dict_popstring + dict_popstring_null = _testcapi.dict_popstring_null + + # key present, get removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_popstring(mydict, "key"), (1, "value")) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_popstring(mydict, "key2"), (1, "value2")) + self.assertEqual(mydict, {}) + + # key present, ignore removed value + mydict = {"key": "value", "key2": "value2"} + self.assertEqual(dict_popstring_null(mydict, "key"), 1) + self.assertEqual(mydict, {"key2": "value2"}) + self.assertEqual(dict_popstring_null(mydict, "key2"), 1) + self.assertEqual(mydict, {}) + + # key missing; empty dict has a fast path + self.assertEqual(dict_popstring({}, "key"), (0, NULL)) + self.assertEqual(dict_popstring_null({}, "key"), 0) + self.assertEqual(dict_popstring({"a": 1}, "key"), (0, NULL)) + self.assertEqual(dict_popstring_null({"a": 1}, "key"), 0) + + # non-ASCII key + non_ascii = '\U0001f40d' + dct = {'\U0001f40d': 123} + self.assertEqual(dict_popstring(dct, '\U0001f40d'.encode()), (1, 123)) + dct = {'\U0001f40d': 123} + self.assertEqual(dict_popstring_null(dct, '\U0001f40d'.encode()), 1) + + # dict error + not_dict = UserDict({1: 2}) + self.assertRaises(SystemError, dict_popstring, not_dict, "key") + self.assertRaises(SystemError, dict_popstring_null, not_dict, "key") + + # key error + self.assertRaises(UnicodeDecodeError, dict_popstring, {1: 2}, INVALID_UTF8) + self.assertRaises(UnicodeDecodeError, dict_popstring_null, {1: 2}, INVALID_UTF8) + + # CRASHES dict_popstring(NULL, "key") + # CRASHES dict_popstring({}, NULL) + # CRASHES dict_popstring({"a": 1}, NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_emscripten.py b/Lib/test/test_capi/test_emscripten.py new file mode 100644 index 00000000000..272d9a10ceb --- /dev/null +++ b/Lib/test/test_capi/test_emscripten.py @@ -0,0 +1,25 @@ +import unittest +from test.support import is_emscripten + +if not is_emscripten: + raise unittest.SkipTest("Emscripten-only test") + +from _testinternalcapi import emscripten_set_up_async_input_device +from pathlib import Path + + +class EmscriptenAsyncInputDeviceTest(unittest.TestCase): + def test_emscripten_async_input_device(self): + jspi_supported = emscripten_set_up_async_input_device() + p = Path("/dev/blah") + self.addCleanup(p.unlink) + if not jspi_supported: + with open(p, "r") as f: + self.assertRaises(OSError, f.readline) + return + + with open(p, "r") as f: + for _ in range(10): + self.assertEqual(f.readline().strip(), "ab") + self.assertEqual(f.readline().strip(), "fi") + self.assertEqual(f.readline().strip(), "xy") diff --git a/Lib/test/test_capi/test_eval.py b/Lib/test/test_capi/test_eval.py new file mode 100644 index 00000000000..20ef2695ef3 --- /dev/null +++ b/Lib/test/test_capi/test_eval.py @@ -0,0 +1,103 @@ +import sys +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +class Tests(unittest.TestCase): + def test_eval_get_func_name(self): + eval_get_func_name = _testlimitedcapi.eval_get_func_name + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_name(function_example), + "function_example") + self.assertEqual(eval_get_func_name(A.method_example), + "method_example") + self.assertEqual(eval_get_func_name(A().method_example), + "method_example") + self.assertEqual(eval_get_func_name(sum), "sum") # c function + self.assertEqual(eval_get_func_name(A), "type") + + def test_eval_get_func_desc(self): + eval_get_func_desc = _testlimitedcapi.eval_get_func_desc + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_desc(function_example), + "()") + self.assertEqual(eval_get_func_desc(A.method_example), + "()") + self.assertEqual(eval_get_func_desc(A().method_example), + "()") + self.assertEqual(eval_get_func_desc(sum), "()") # c function + self.assertEqual(eval_get_func_desc(A), " object") + + def test_eval_getlocals(self): + # Test PyEval_GetLocals() + x = 1 + self.assertEqual(_testlimitedcapi.eval_getlocals(), + {'self': self, + 'x': 1}) + + y = 2 + self.assertEqual(_testlimitedcapi.eval_getlocals(), + {'self': self, + 'x': 1, + 'y': 2}) + + def test_eval_getglobals(self): + # Test PyEval_GetGlobals() + self.assertEqual(_testlimitedcapi.eval_getglobals(), + globals()) + + def test_eval_getbuiltins(self): + # Test PyEval_GetBuiltins() + self.assertEqual(_testlimitedcapi.eval_getbuiltins(), + globals()['__builtins__']) + + def test_eval_getframe(self): + # Test PyEval_GetFrame() + self.assertEqual(_testlimitedcapi.eval_getframe(), + sys._getframe()) + + def test_eval_getframe_builtins(self): + # Test PyEval_GetFrameBuiltins() + self.assertEqual(_testlimitedcapi.eval_getframe_builtins(), + sys._getframe().f_builtins) + + def test_eval_getframe_globals(self): + # Test PyEval_GetFrameGlobals() + self.assertEqual(_testlimitedcapi.eval_getframe_globals(), + sys._getframe().f_globals) + + def test_eval_getframe_locals(self): + # Test PyEval_GetFrameLocals() + self.assertEqual(_testlimitedcapi.eval_getframe_locals(), + sys._getframe().f_locals) + + def test_eval_get_recursion_limit(self): + # Test Py_GetRecursionLimit() + self.assertEqual(_testlimitedcapi.eval_get_recursion_limit(), + sys.getrecursionlimit()) + + def test_eval_set_recursion_limit(self): + # Test Py_SetRecursionLimit() + old_limit = sys.getrecursionlimit() + try: + limit = old_limit + 123 + _testlimitedcapi.eval_set_recursion_limit(limit) + self.assertEqual(sys.getrecursionlimit(), limit) + finally: + sys.setrecursionlimit(old_limit) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_eval_code_ex.py b/Lib/test/test_capi/test_eval_code_ex.py new file mode 100644 index 00000000000..b298e5007e5 --- /dev/null +++ b/Lib/test/test_capi/test_eval_code_ex.py @@ -0,0 +1,126 @@ +import unittest +import builtins +from collections import UserDict + +from test.support import import_helper +from test.support import swap_attr + + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + +NULL = None + + +class PyEval_EvalCodeExTests(unittest.TestCase): + + def test_simple(self): + def f(): + return a + + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, dict(a=1)), 1) + + self.assertRaises(NameError, eval_code_ex, code, {}) + self.assertRaises(SystemError, eval_code_ex, code, UserDict(a=1)) + self.assertRaises(SystemError, eval_code_ex, code, []) + self.assertRaises(SystemError, eval_code_ex, code, 1) + # CRASHES eval_code_ex(code, NULL) + # CRASHES eval_code_ex(1, {}) + # CRASHES eval_code_ex(NULL, {}) + + def test_custom_locals(self): + # Monkey-patch __build_class__ to get a class code object. + code = None + def build_class(func, name, /, *bases, **kwds): + nonlocal code + code = func.__code__ + + with swap_attr(builtins, '__build_class__', build_class): + class A: + # Uses LOAD_NAME for a + r[:] = [a] + + eval_code_ex = _testcapi.eval_code_ex + results = [] + g = dict(a=1, r=results) + self.assertIsNone(eval_code_ex(code, g)) + self.assertEqual(results, [1]) + self.assertIsNone(eval_code_ex(code, g, dict(a=2))) + self.assertEqual(results, [2]) + self.assertIsNone(eval_code_ex(code, g, UserDict(a=3))) + self.assertEqual(results, [3]) + self.assertIsNone(eval_code_ex(code, g, {})) + self.assertEqual(results, [1]) + self.assertIsNone(eval_code_ex(code, g, NULL)) + self.assertEqual(results, [1]) + + self.assertRaises(TypeError, eval_code_ex, code, g, []) + self.assertRaises(TypeError, eval_code_ex, code, g, 1) + self.assertRaises(NameError, eval_code_ex, code, dict(r=results), {}) + self.assertRaises(NameError, eval_code_ex, code, dict(r=results), NULL) + self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), []) + self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), 1) + + def test_with_args(self): + def f(a, b, c): + return a + + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (1, 2, 3)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2, 3, 4)) + + def test_with_kwargs(self): + def f(a, b, c): + return a + + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), dict(a=1, b=2, c=3)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2, c=3, d=4)) + + def test_with_default(self): + def f(a): + return a + + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (1,)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, ()) + + def test_with_kwarg_default(self): + def f(*, a): + return a + + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), dict(a=1)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), {}) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), NULL) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), UserDict(a=1)) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), []) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), 1) + + def test_with_closure(self): + a = 1 + b = 2 + def f(): + b + return a + + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__), 1) + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__[::-1]), 2) + + # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, ()), 1) + # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, NULL), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py new file mode 100644 index 00000000000..6a32201cce5 --- /dev/null +++ b/Lib/test/test_capi/test_exceptions.py @@ -0,0 +1,680 @@ +import errno +import os +import re +import sys +import unittest +import textwrap + +from test import support +from test.support import import_helper +from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE +from test.support.script_helper import assert_python_failure, assert_python_ok +from test.support.testcase import ExceptionIsLikeMixin + +from .test_misc import decode_stderr + +# Skip this test if the _testcapi or _testinternalcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') + +NULL = None + +class CustomError(Exception): + pass + + +class Test_Exceptions(unittest.TestCase): + + def test_exception(self): + raised_exception = ValueError("5") + new_exc = TypeError("TEST") + try: + raise raised_exception + except ValueError as e: + orig_sys_exception = sys.exception() + orig_exception = _testcapi.set_exception(new_exc) + new_sys_exception = sys.exception() + new_exception = _testcapi.set_exception(orig_exception) + reset_sys_exception = sys.exception() + + self.assertEqual(orig_exception, e) + + self.assertEqual(orig_exception, raised_exception) + self.assertEqual(orig_sys_exception, orig_exception) + self.assertEqual(reset_sys_exception, orig_exception) + self.assertEqual(new_exception, new_exc) + self.assertEqual(new_sys_exception, new_exception) + else: + self.fail("Exception not raised") + + def test_exc_info(self): + raised_exception = ValueError("5") + new_exc = TypeError("TEST") + try: + raise raised_exception + except ValueError as e: + tb = e.__traceback__ + orig_sys_exc_info = sys.exc_info() + orig_exc_info = _testcapi.set_exc_info(new_exc.__class__, new_exc, None) + new_sys_exc_info = sys.exc_info() + new_exc_info = _testcapi.set_exc_info(*orig_exc_info) + reset_sys_exc_info = sys.exc_info() + + self.assertEqual(orig_exc_info[1], e) + + self.assertSequenceEqual(orig_exc_info, (raised_exception.__class__, raised_exception, tb)) + self.assertSequenceEqual(orig_sys_exc_info, orig_exc_info) + self.assertSequenceEqual(reset_sys_exc_info, orig_exc_info) + self.assertSequenceEqual(new_exc_info, (new_exc.__class__, new_exc, None)) + self.assertSequenceEqual(new_sys_exc_info, new_exc_info) + else: + self.assertTrue(False) + + def test_warn_with_stacklevel(self): + code = textwrap.dedent('''\ + import _testcapi + + def foo(): + _testcapi.function_set_warning() + + foo() # line 6 + + + foo() # line 9 + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + self.assertEqual(warnings, [ + b':6: RuntimeWarning: Testing PyErr_WarnEx', + b':9: RuntimeWarning: Testing PyErr_WarnEx', + ]) + + def test_warn_during_finalization(self): + code = textwrap.dedent('''\ + import _testcapi + + class Foo: + def foo(self): + _testcapi.function_set_warning() + def __del__(self): + self.foo() + + ref = Foo() + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + # Due to the finalization of the interpreter, the source will be omitted + # because the ``warnings`` module cannot be imported at this time + self.assertEqual(warnings, [ + b':7: RuntimeWarning: Testing PyErr_WarnEx', + ]) + + def test__pyerr_setkeyerror(self): + # Test _PyErr_SetKeyError() + _pyerr_setkeyerror = _testinternalcapi._pyerr_setkeyerror + for arg in ( + "key", + # check that a tuple argument is not unpacked + (1, 2, 3), + # PyErr_SetObject(exc_type, exc_value) uses exc_value if it's + # already an exception, but _PyErr_SetKeyError() always creates a + # new KeyError. + KeyError('arg'), + ): + with self.subTest(arg=arg): + with self.assertRaises(KeyError) as cm: + # Test calling _PyErr_SetKeyError() with an exception set + # to check that the function overrides the current + # exception. + _pyerr_setkeyerror(arg) + self.assertEqual(cm.exception.args, (arg,)) + + +class Test_FatalError(unittest.TestCase): + + def check_fatal_error(self, code, expected, not_expected=()): + with support.SuppressCrashReport(): + rc, out, err = assert_python_failure('-sSI', '-c', code) + + err = decode_stderr(err) + self.assertIn('Fatal Python error: _testcapi_fatal_error_impl: MESSAGE\n', + err) + + match = re.search(r'^Extension modules:(.*) \(total: ([0-9]+)\)$', + err, re.MULTILINE) + if not match: + self.fail(f"Cannot find 'Extension modules:' in {err!r}") + modules = set(match.group(1).strip().split(', ')) + total = int(match.group(2)) + + for name in expected: + self.assertIn(name, modules) + for name in not_expected: + self.assertNotIn(name, modules) + self.assertEqual(len(modules), total) + + @support.requires_subprocess() + def test_fatal_error(self): + # By default, stdlib extension modules are ignored, + # but not test modules. + expected = ('_testcapi',) + not_expected = ('sys',) + code = 'import _testcapi, sys; _testcapi.fatal_error(b"MESSAGE")' + self.check_fatal_error(code, expected, not_expected) + + # Mark _testcapi as stdlib module, but not sys + expected = ('sys',) + not_expected = ('_testcapi',) + code = """if True: + import _testcapi, sys + sys.stdlib_module_names = frozenset({"_testcapi"}) + _testcapi.fatal_error(b"MESSAGE") + """ + self.check_fatal_error(code, expected) + + +class Test_ErrSetAndRestore(unittest.TestCase): + + def test_err_set_raised(self): + with self.assertRaises(ValueError): + _testcapi.err_set_raised(ValueError()) + v = ValueError() + try: + _testcapi.err_set_raised(v) + except ValueError as ex: + self.assertIs(v, ex) + + def test_err_restore(self): + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError) + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError, 1) + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError, 1, None) + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError, ValueError()) + try: + _testcapi.err_restore(KeyError, "hi") + except KeyError as k: + self.assertEqual("hi", k.args[0]) + try: + 1/0 + except Exception as e: + tb = e.__traceback__ + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError, 1, tb) + with self.assertRaises(TypeError): + _testcapi.err_restore(ValueError, 1, 0) + try: + _testcapi.err_restore(ValueError, 1, tb) + except ValueError as v: + self.assertEqual(1, v.args[0]) + self.assertIs(tb, v.__traceback__.tb_next) + + def test_set_object(self): + + # new exception as obj is not an exception + with self.assertRaises(ValueError) as e: + _testcapi.exc_set_object(ValueError, 42) + self.assertEqual(e.exception.args, (42,)) + + # wraps the exception because unrelated types + with self.assertRaises(ValueError) as e: + _testcapi.exc_set_object(ValueError, TypeError(1,2,3)) + wrapped = e.exception.args[0] + self.assertIsInstance(wrapped, TypeError) + self.assertEqual(wrapped.args, (1, 2, 3)) + + # is superclass, so does not wrap + with self.assertRaises(PermissionError) as e: + _testcapi.exc_set_object(OSError, PermissionError(24)) + self.assertEqual(e.exception.args, (24,)) + + class Meta(type): + def __subclasscheck__(cls, sub): + 1/0 + + class Broken(Exception, metaclass=Meta): + pass + + with self.assertRaises(ZeroDivisionError) as e: + _testcapi.exc_set_object(Broken, Broken()) + + def test_set_object_and_fetch(self): + class Broken(Exception): + def __init__(self, *arg): + raise ValueError("Broken __init__") + + exc = _testcapi.exc_set_object_fetch(Broken, 'abcd') + self.assertIsInstance(exc, ValueError) + self.assertEqual(exc.__notes__[0], + "Normalization failed: type=Broken args='abcd'") + + class BadArg: + def __repr__(self): + raise TypeError('Broken arg type') + + exc = _testcapi.exc_set_object_fetch(Broken, BadArg()) + self.assertIsInstance(exc, ValueError) + self.assertEqual(exc.__notes__[0], + 'Normalization failed: type=Broken args=') + + def test_set_string(self): + """Test PyErr_SetString()""" + setstring = _testcapi.err_setstring + with self.assertRaises(ZeroDivisionError) as e: + setstring(ZeroDivisionError, b'error') + self.assertEqual(e.exception.args, ('error',)) + with self.assertRaises(ZeroDivisionError) as e: + setstring(ZeroDivisionError, 'помилка'.encode()) + self.assertEqual(e.exception.args, ('помилка',)) + + with self.assertRaises(UnicodeDecodeError): + setstring(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, setstring, list, b'error') + # CRASHES setstring(ZeroDivisionError, NULL) + # CRASHES setstring(NULL, b'error') + + def test_format(self): + """Test PyErr_Format()""" + import_helper.import_module('ctypes') + from ctypes import pythonapi, py_object, c_char_p, c_int + name = "PyErr_Format" + PyErr_Format = getattr(pythonapi, name) + PyErr_Format.argtypes = (py_object, c_char_p,) + PyErr_Format.restype = py_object + with self.assertRaises(ZeroDivisionError) as e: + PyErr_Format(ZeroDivisionError, b'%s %d', b'error', c_int(42)) + self.assertEqual(e.exception.args, ('error 42',)) + with self.assertRaises(ZeroDivisionError) as e: + PyErr_Format(ZeroDivisionError, b'%s', 'помилка'.encode()) + self.assertEqual(e.exception.args, ('помилка',)) + + with self.assertRaisesRegex(OverflowError, 'not in range'): + PyErr_Format(ZeroDivisionError, b'%c', c_int(-1)) + with self.assertRaisesRegex(ValueError, 'format string'): + PyErr_Format(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, PyErr_Format, list, b'error') + # CRASHES PyErr_Format(ZeroDivisionError, NULL) + # CRASHES PyErr_Format(py_object(), b'error') + + def test_setfromerrnowithfilename(self): + """Test PyErr_SetFromErrnoWithFilename()""" + setfromerrnowithfilename = _testcapi.err_setfromerrnowithfilename + ENOENT = errno.ENOENT + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, b'file') + self.assertEqual(e.exception.args, + (ENOENT, 'No such file or directory')) + self.assertEqual(e.exception.errno, ENOENT) + self.assertEqual(e.exception.filename, 'file') + + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, os.fsencode(TESTFN)) + self.assertEqual(e.exception.filename, TESTFN) + + if TESTFN_UNDECODABLE: + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, TESTFN_UNDECODABLE) + self.assertEqual(e.exception.filename, + os.fsdecode(TESTFN_UNDECODABLE)) + + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, NULL) + self.assertIsNone(e.exception.filename) + + with self.assertRaises(OSError) as e: + setfromerrnowithfilename(0, OSError, b'file') + self.assertEqual(e.exception.args, (0, 'Error')) + self.assertEqual(e.exception.errno, 0) + self.assertEqual(e.exception.filename, 'file') + + with self.assertRaises(ZeroDivisionError) as e: + setfromerrnowithfilename(ENOENT, ZeroDivisionError, b'file') + self.assertEqual(e.exception.args, + (ENOENT, 'No such file or directory', 'file')) + # CRASHES setfromerrnowithfilename(ENOENT, NULL, b'error') + + def test_err_writeunraisable(self): + # Test PyErr_WriteUnraisable() + writeunraisable = _testcapi.err_writeunraisable + firstline = self.test_err_writeunraisable.__code__.co_firstlineno + + with support.catch_unraisable_exception() as cm: + writeunraisable(CustomError('oops!'), hex) + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 6) + self.assertIsNone(cm.unraisable.err_msg) + self.assertEqual(cm.unraisable.object, hex) + + with support.catch_unraisable_exception() as cm: + writeunraisable(CustomError('oops!'), NULL) + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 15) + self.assertIsNone(cm.unraisable.err_msg) + self.assertIsNone(cm.unraisable.object) + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + writeunraisable(CustomError('oops!'), hex) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Exception ignored in: {hex!r}') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + writeunraisable(CustomError('oops!'), NULL) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + # CRASHES writeunraisable(NULL, hex) + # CRASHES writeunraisable(NULL, NULL) + + def test_err_formatunraisable(self): + # Test PyErr_FormatUnraisable() + formatunraisable = _testcapi.err_formatunraisable + firstline = self.test_err_formatunraisable.__code__.co_firstlineno + + with support.catch_unraisable_exception() as cm: + formatunraisable(CustomError('oops!'), b'Error in %R', []) + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 6) + self.assertEqual(cm.unraisable.err_msg, 'Error in []') + self.assertIsNone(cm.unraisable.object) + + with support.catch_unraisable_exception() as cm: + formatunraisable(CustomError('oops!'), b'undecodable \xff') + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 15) + self.assertIsNone(cm.unraisable.err_msg) + self.assertIsNone(cm.unraisable.object) + + with support.catch_unraisable_exception() as cm: + formatunraisable(CustomError('oops!'), NULL) + self.assertEqual(cm.unraisable.exc_type, CustomError) + self.assertEqual(str(cm.unraisable.exc_value), 'oops!') + self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, + firstline + 24) + self.assertIsNone(cm.unraisable.err_msg) + self.assertIsNone(cm.unraisable.object) + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + formatunraisable(CustomError('oops!'), b'Error in %R', []) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Error in []:') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + formatunraisable(CustomError('oops!'), b'undecodable \xff') + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + with (support.swap_attr(sys, 'unraisablehook', None), + support.captured_stderr() as stderr): + formatunraisable(CustomError('oops!'), NULL) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') + + # CRASHES formatunraisable(NULL, b'Error in %R', []) + # CRASHES formatunraisable(NULL, NULL) + + +class TestUnicodeTranslateError(UnicodeTranslateError): + # UnicodeTranslateError takes 4 arguments instead of 5, + # so we just make a UnicodeTranslateError class that is + # compatible with the UnicodeError.__init__. + def __init__(self, encoding, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class TestUnicodeError(unittest.TestCase): + + def _check_no_crash(self, exc): + # ensure that the __str__() method does not crash + _ = str(exc) + + def test_unicode_encode_error_get_start(self): + get_start = _testcapi.unicode_encode_get_start + self._test_unicode_error_get_start('x', UnicodeEncodeError, get_start) + + def test_unicode_decode_error_get_start(self): + get_start = _testcapi.unicode_decode_get_start + self._test_unicode_error_get_start(b'x', UnicodeDecodeError, get_start) + + def test_unicode_translate_error_get_start(self): + get_start = _testcapi.unicode_translate_get_start + self._test_unicode_error_get_start('x', TestUnicodeTranslateError, get_start) + + def _test_unicode_error_get_start(self, literal, exc_type, get_start): + for obj_len, start, c_start in [ + # normal cases + (5, 0, 0), + (5, 1, 1), + (5, 2, 2), + # out of range start is clamped to max(0, obj_len - 1) + (0, 0, 0), + (0, 1, 0), + (0, 10, 0), + (5, 5, 4), + (5, 10, 4), + # negative values are allowed but clipped in the getter + (0, -1, 0), + (1, -1, 0), + (2, -1, 0), + (2, -2, 0), + ]: + obj = literal * obj_len + with self.subTest(obj, exc_type=exc_type, start=start): + exc = exc_type('utf-8', obj, start, obj_len, 'reason') + self.assertEqual(get_start(exc), c_start) + self._check_no_crash(exc) + + def test_unicode_encode_error_set_start(self): + set_start = _testcapi.unicode_encode_set_start + self._test_unicode_error_set_start('x', UnicodeEncodeError, set_start) + + def test_unicode_decode_error_set_start(self): + set_start = _testcapi.unicode_decode_set_start + self._test_unicode_error_set_start(b'x', UnicodeDecodeError, set_start) + + def test_unicode_translate_error_set_start(self): + set_start = _testcapi.unicode_translate_set_start + self._test_unicode_error_set_start('x', TestUnicodeTranslateError, set_start) + + def _test_unicode_error_set_start(self, literal, exc_type, set_start): + obj_len = 5 + obj = literal * obj_len + for new_start in range(-2 * obj_len, 2 * obj_len): + with self.subTest('C-API', obj=obj, exc_type=exc_type, new_start=new_start): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the C API setter + set_start(exc, new_start) + self.assertEqual(exc.start, new_start) + self._check_no_crash(exc) + + with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_start=new_start): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the attribute setter + exc.start = new_start + self.assertEqual(exc.start, new_start) + self._check_no_crash(exc) + + def test_unicode_encode_error_get_end(self): + get_end = _testcapi.unicode_encode_get_end + self._test_unicode_error_get_end('x', UnicodeEncodeError, get_end) + + def test_unicode_decode_error_get_end(self): + get_end = _testcapi.unicode_decode_get_end + self._test_unicode_error_get_end(b'x', UnicodeDecodeError, get_end) + + def test_unicode_translate_error_get_end(self): + get_end = _testcapi.unicode_translate_get_end + self._test_unicode_error_get_end('x', TestUnicodeTranslateError, get_end) + + def _test_unicode_error_get_end(self, literal, exc_type, get_end): + for obj_len, end, c_end in [ + # normal cases + (5, 0, 1), + (5, 1, 1), + (5, 2, 2), + # out-of-range clipped in [MIN(1, OBJLEN), MAX(MIN(1, OBJLEN), OBJLEN)] + (0, 0, 0), + (0, 1, 0), + (0, 10, 0), + (1, 1, 1), + (1, 2, 1), + (5, 5, 5), + (5, 5, 5), + (5, 10, 5), + # negative values are allowed but clipped in the getter + (0, -1, 0), + (1, -1, 1), + (2, -1, 1), + (2, -2, 1), + ]: + obj = literal * obj_len + with self.subTest(obj, exc_type=exc_type, end=end): + exc = exc_type('utf-8', obj, 0, end, 'reason') + self.assertEqual(get_end(exc), c_end) + self._check_no_crash(exc) + + def test_unicode_encode_error_set_end(self): + set_end = _testcapi.unicode_encode_set_end + self._test_unicode_error_set_end('x', UnicodeEncodeError, set_end) + + def test_unicode_decode_error_set_end(self): + set_end = _testcapi.unicode_decode_set_end + self._test_unicode_error_set_end(b'x', UnicodeDecodeError, set_end) + + def test_unicode_translate_error_set_end(self): + set_end = _testcapi.unicode_translate_set_end + self._test_unicode_error_set_end('x', TestUnicodeTranslateError, set_end) + + def _test_unicode_error_set_end(self, literal, exc_type, set_end): + obj_len = 5 + obj = literal * obj_len + for new_end in range(-2 * obj_len, 2 * obj_len): + with self.subTest('C-API', obj=obj, exc_type=exc_type, new_end=new_end): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the C API setter + set_end(exc, new_end) + self.assertEqual(exc.end, new_end) + self._check_no_crash(exc) + + with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_end=new_end): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the attribute setter + exc.end = new_end + self.assertEqual(exc.end, new_end) + self._check_no_crash(exc) + + +class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): + + def setUp(self): + super().setUp() + try: + raise ExceptionGroup("eg", [TypeError('bad type'), ValueError(42)]) + except ExceptionGroup as e: + self.orig = e + + def test_invalid_args(self): + with self.assertRaisesRegex(TypeError, "orig must be an exception"): + _testcapi.unstable_exc_prep_reraise_star(42, [None]) + + with self.assertRaisesRegex(TypeError, "excs must be a list"): + _testcapi.unstable_exc_prep_reraise_star(self.orig, 42) + + with self.assertRaisesRegex(TypeError, "not an exception"): + _testcapi.unstable_exc_prep_reraise_star(self.orig, [TypeError(42), 42]) + + with self.assertRaisesRegex(ValueError, "orig must be a raised exception"): + _testcapi.unstable_exc_prep_reraise_star(ValueError(42), [TypeError(42)]) + + with self.assertRaisesRegex(ValueError, "orig must be a raised exception"): + _testcapi.unstable_exc_prep_reraise_star(ExceptionGroup("eg", [ValueError(42)]), + [TypeError(42)]) + + + def test_nothing_to_reraise(self): + self.assertEqual( + _testcapi.unstable_exc_prep_reraise_star(self.orig, [None]), None) + + try: + raise ValueError(42) + except ValueError as e: + orig = e + self.assertEqual( + _testcapi.unstable_exc_prep_reraise_star(orig, [None]), None) + + def test_reraise_orig(self): + orig = self.orig + res = _testcapi.unstable_exc_prep_reraise_star(orig, [orig]) + self.assertExceptionIsLike(res, orig) + + def test_raise_orig_parts(self): + orig = self.orig + match, rest = orig.split(TypeError) + + test_cases = [ + ([match, rest], orig), + ([rest, match], orig), + ([match], match), + ([rest], rest), + ([], None), + ] + + for input, expected in test_cases: + with self.subTest(input=input): + res = _testcapi.unstable_exc_prep_reraise_star(orig, input) + self.assertExceptionIsLike(res, expected) + + + def test_raise_with_new_exceptions(self): + orig = self.orig + + match, rest = orig.split(TypeError) + new1 = OSError('bad file') + new2 = RuntimeError('bad runtime') + + test_cases = [ + ([new1, match, rest], ExceptionGroup("", [new1, orig])), + ([match, new1, rest], ExceptionGroup("", [new1, orig])), + ([match, rest, new1], ExceptionGroup("", [new1, orig])), + + ([new1, new2, match, rest], ExceptionGroup("", [new1, new2, orig])), + ([new1, match, new2, rest], ExceptionGroup("", [new1, new2, orig])), + ([new2, rest, match, new1], ExceptionGroup("", [new2, new1, orig])), + ([rest, new2, match, new1], ExceptionGroup("", [new2, new1, orig])), + + + ([new1, new2, rest], ExceptionGroup("", [new1, new2, rest])), + ([new1, match, new2], ExceptionGroup("", [new1, new2, match])), + ([rest, new2, new1], ExceptionGroup("", [new2, new1, rest])), + ([new1, new2], ExceptionGroup("", [new1, new2])), + ([new2, new1], ExceptionGroup("", [new2, new1])), + ] + + for (input, expected) in test_cases: + with self.subTest(input=input): + res = _testcapi.unstable_exc_prep_reraise_star(orig, input) + self.assertExceptionIsLike(res, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_file.py b/Lib/test/test_capi/test_file.py new file mode 100644 index 00000000000..7c1e0026188 --- /dev/null +++ b/Lib/test/test_capi/test_file.py @@ -0,0 +1,325 @@ +import io +import os +import unittest +from test import support +from test.support import import_helper, os_helper, warnings_helper + + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_io = import_helper.import_module('_io') +NULL = None +STDOUT_FD = 1 + +with open(__file__, 'rb') as fp: + FIRST_LINE = next(fp).decode() +FIRST_LINE_NORM = FIRST_LINE.rstrip() + '\n' + + +class CAPIFileTest(unittest.TestCase): + def test_pyfile_fromfd(self): + # Test PyFile_FromFd() which is a thin wrapper to _io.open() + pyfile_fromfd = _testlimitedcapi.pyfile_fromfd + filename = __file__ + with open(filename, "rb") as fp: + fd = fp.fileno() + + # FileIO + fp.seek(0) + obj = pyfile_fromfd(fd, filename, "rb", 0, NULL, NULL, NULL, 0) + try: + self.assertIsInstance(obj, _io.FileIO) + self.assertEqual(obj.readline(), FIRST_LINE.encode()) + finally: + obj.close() + + # BufferedReader + fp.seek(0) + obj = pyfile_fromfd(fd, filename, "rb", 1024, NULL, NULL, NULL, 0) + try: + self.assertIsInstance(obj, _io.BufferedReader) + self.assertEqual(obj.readline(), FIRST_LINE.encode()) + finally: + obj.close() + + # TextIOWrapper + fp.seek(0) + obj = pyfile_fromfd(fd, filename, "r", 1, + "utf-8", "replace", NULL, 0) + try: + self.assertIsInstance(obj, _io.TextIOWrapper) + self.assertEqual(obj.encoding, "utf-8") + self.assertEqual(obj.errors, "replace") + self.assertEqual(obj.readline(), FIRST_LINE_NORM) + finally: + obj.close() + + def test_pyfile_getline(self): + # Test PyFile_GetLine(file, n): call file.readline() + # and strip "\n" suffix if n < 0. + pyfile_getline = _testlimitedcapi.pyfile_getline + + # Test Unicode + with open(__file__, "r") as fp: + fp.seek(0) + self.assertEqual(pyfile_getline(fp, -1), + FIRST_LINE_NORM.rstrip('\n')) + fp.seek(0) + self.assertEqual(pyfile_getline(fp, 0), + FIRST_LINE_NORM) + fp.seek(0) + self.assertEqual(pyfile_getline(fp, 6), + FIRST_LINE_NORM[:6]) + + # Test bytes + with open(__file__, "rb") as fp: + fp.seek(0) + self.assertEqual(pyfile_getline(fp, -1), + FIRST_LINE.rstrip('\n').encode()) + fp.seek(0) + self.assertEqual(pyfile_getline(fp, 0), + FIRST_LINE.encode()) + fp.seek(0) + self.assertEqual(pyfile_getline(fp, 6), + FIRST_LINE.encode()[:6]) + + def test_pyfile_writestring(self): + # Test PyFile_WriteString(str, file): call file.write(str) + writestr = _testlimitedcapi.pyfile_writestring + + with io.StringIO() as fp: + self.assertEqual(writestr("a\xe9\u20ac\U0010FFFF".encode(), fp), 0) + with self.assertRaises(UnicodeDecodeError): + writestr(b"\xff", fp) + with self.assertRaises(UnicodeDecodeError): + writestr("\udc80".encode("utf-8", "surrogatepass"), fp) + + text = fp.getvalue() + self.assertEqual(text, "a\xe9\u20ac\U0010FFFF") + + with self.assertRaises(SystemError): + writestr(b"abc", NULL) + + def test_pyfile_writeobject(self): + # Test PyFile_WriteObject(obj, file, flags): + # - Call file.write(str(obj)) if flags equals Py_PRINT_RAW. + # - Call file.write(repr(obj)) otherwise. + writeobject = _testlimitedcapi.pyfile_writeobject + Py_PRINT_RAW = 1 + + with io.StringIO() as fp: + # Test flags=Py_PRINT_RAW + self.assertEqual(writeobject("raw", fp, Py_PRINT_RAW), 0) + writeobject(NULL, fp, Py_PRINT_RAW) + + # Test flags=0 + self.assertEqual(writeobject("repr", fp, 0), 0) + writeobject(NULL, fp, 0) + + text = fp.getvalue() + self.assertEqual(text, "raw'repr'") + + # invalid file type + for invalid_file in (123, "abc", object()): + with self.subTest(file=invalid_file): + with self.assertRaises(AttributeError): + writeobject("abc", invalid_file, Py_PRINT_RAW) + + with self.assertRaises(TypeError): + writeobject("abc", NULL, 0) + + def test_pyobject_asfiledescriptor(self): + # Test PyObject_AsFileDescriptor(obj): + # - Return obj if obj is an integer. + # - Return obj.fileno() otherwise. + # File descriptor must be >= 0. + asfd = _testlimitedcapi.pyobject_asfiledescriptor + + self.assertEqual(asfd(123), 123) + self.assertEqual(asfd(0), 0) + + with open(__file__, "rb") as fp: + self.assertEqual(asfd(fp), fp.fileno()) + + # bool emits RuntimeWarning + msg = r"bool is used as a file descriptor" + with warnings_helper.check_warnings((msg, RuntimeWarning)): + self.assertEqual(asfd(True), 1) + + class FakeFile: + def __init__(self, fd): + self.fd = fd + def fileno(self): + return self.fd + + # file descriptor must be positive + with self.assertRaises(ValueError): + asfd(-1) + with self.assertRaises(ValueError): + asfd(FakeFile(-1)) + + # fileno() result must be an integer + with self.assertRaises(TypeError): + asfd(FakeFile("text")) + + # unsupported types + for obj in ("string", ["list"], object()): + with self.subTest(obj=obj): + with self.assertRaises(TypeError): + asfd(obj) + + # CRASHES asfd(NULL) + + def test_pyfile_newstdprinter(self): + # Test PyFile_NewStdPrinter() + pyfile_newstdprinter = _testcapi.pyfile_newstdprinter + + file = pyfile_newstdprinter(STDOUT_FD) + self.assertEqual(file.closed, False) + self.assertIsNone(file.encoding) + self.assertEqual(file.mode, "w") + + self.assertEqual(file.fileno(), STDOUT_FD) + self.assertEqual(file.isatty(), os.isatty(STDOUT_FD)) + + # flush() is a no-op + self.assertIsNone(file.flush()) + + # close() is a no-op + self.assertIsNone(file.close()) + self.assertEqual(file.closed, False) + + support.check_disallow_instantiation(self, type(file)) + + def test_pyfile_newstdprinter_write(self): + # Test the write() method of PyFile_NewStdPrinter() + pyfile_newstdprinter = _testcapi.pyfile_newstdprinter + + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + try: + old_stdout = os.dup(STDOUT_FD) + except OSError as exc: + # os.dup(STDOUT_FD) is not supported on WASI + self.skipTest(f"os.dup() failed with {exc!r}") + + try: + with open(filename, "wb") as fp: + # PyFile_NewStdPrinter() only accepts fileno(stdout) + # or fileno(stderr) file descriptor. + fd = fp.fileno() + os.dup2(fd, STDOUT_FD) + + file = pyfile_newstdprinter(STDOUT_FD) + self.assertEqual(file.write("text"), 4) + # The surrogate character is encoded with + # the "surrogateescape" error handler + self.assertEqual(file.write("[\udc80]"), 8) + finally: + os.dup2(old_stdout, STDOUT_FD) + os.close(old_stdout) + + with open(filename, "r") as fp: + self.assertEqual(fp.read(), "text[\\udc80]") + + def test_py_fopen(self): + # Test Py_fopen() and Py_fclose() + py_fopen = _testcapi.py_fopen + + with open(__file__, "rb") as fp: + source = fp.read() + + for filename in (__file__, os.fsencode(__file__)): + with self.subTest(filename=filename): + data = py_fopen(filename, "rb") + self.assertEqual(data, source[:256]) + + data = py_fopen(os_helper.FakePath(filename), "rb") + self.assertEqual(data, source[:256]) + + filenames = [ + os_helper.TESTFN, + os.fsencode(os_helper.TESTFN), + ] + if os_helper.TESTFN_UNDECODABLE is not None: + filenames.append(os_helper.TESTFN_UNDECODABLE) + filenames.append(os.fsdecode(os_helper.TESTFN_UNDECODABLE)) + if os_helper.TESTFN_UNENCODABLE is not None: + filenames.append(os_helper.TESTFN_UNENCODABLE) + for filename in filenames: + with self.subTest(filename=filename): + try: + with open(filename, "wb") as fp: + fp.write(source) + except OSError: + # TESTFN_UNDECODABLE cannot be used to create a file + # on macOS/WASI. + filename = None + continue + try: + data = py_fopen(filename, "rb") + self.assertEqual(data, source[:256]) + finally: + os_helper.unlink(filename) + + # embedded null character/byte in the filename + with self.assertRaises(ValueError): + py_fopen("a\x00b", "rb") + with self.assertRaises(ValueError): + py_fopen(b"a\x00b", "rb") + + # non-ASCII mode failing with "Invalid argument" + with self.assertRaises(OSError): + py_fopen(__file__, b"\xc2\x80") + with self.assertRaises(OSError): + # \x98 is invalid in cp1250, cp1251, cp1257 + # \x9d is invalid in cp1252-cp1255, cp1258 + py_fopen(__file__, b"\xc2\x98\xc2\x9d") + # UnicodeDecodeError can come from the audit hook code + with self.assertRaises((UnicodeDecodeError, OSError)): + py_fopen(__file__, b"\x98\x9d") + + # invalid filename type + for invalid_type in (123, object()): + with self.subTest(filename=invalid_type): + with self.assertRaises(TypeError): + py_fopen(invalid_type, "rb") + + if support.MS_WINDOWS: + with self.assertRaises(OSError): + # On Windows, the file mode is limited to 10 characters + py_fopen(__file__, "rt+, ccs=UTF-8") + + # CRASHES py_fopen(NULL, 'rb') + # CRASHES py_fopen(__file__, NULL) + + def test_py_universalnewlinefgets(self): + py_universalnewlinefgets = _testcapi.py_universalnewlinefgets + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + with open(filename, "wb") as fp: + fp.write(b"line1\nline2") + + line = py_universalnewlinefgets(filename, 1000) + self.assertEqual(line, b"line1\n") + + with open(filename, "wb") as fp: + fp.write(b"line2\r\nline3") + + line = py_universalnewlinefgets(filename, 1000) + self.assertEqual(line, b"line2\n") + + with open(filename, "wb") as fp: + fp.write(b"line3\rline4") + + line = py_universalnewlinefgets(filename, 1000) + self.assertEqual(line, b"line3\n") + + # PyFile_SetOpenCodeHook() and PyFile_OpenCode() are tested by + # test_embed.test_open_code_hook() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py new file mode 100644 index 00000000000..8b25607b6d5 --- /dev/null +++ b/Lib/test/test_capi/test_float.py @@ -0,0 +1,259 @@ +import math +import random +import sys +import unittest +import warnings + +from test.test_capi.test_getargs import (Float, FloatSubclass, FloatSubclass2, + BadIndex2, BadFloat2, Index, BadIndex, + BadFloat) +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None + +# For PyFloat_Pack/Unpack* +BIG_ENDIAN = 0 +LITTLE_ENDIAN = 1 +EPSILON = { + 2: 2.0 ** -11, # binary16 + 4: 2.0 ** -24, # binary32 + 8: 2.0 ** -53, # binary64 +} + +HAVE_IEEE_754 = float.__getformat__("double").startswith("IEEE") +INF = float("inf") +NAN = float("nan") + + +def make_nan(size, sign, quiet, payload=None): + if size == 8: + payload_mask = 0x7ffffffffffff + i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + elif size == 4: + payload_mask = 0x3fffff + i = (sign << 31) + (0xff << 23) + (quiet << 22) + elif size == 2: + payload_mask = 0x1ff + i = (sign << 15) + (0x1f << 10) + (quiet << 9) + else: + raise ValueError("size must be either 2, 4, or 8") + if payload is None: + payload = random.randint(not quiet, payload_mask) + return i + payload + + +class CAPIFloatTest(unittest.TestCase): + def test_check(self): + # Test PyFloat_Check() + check = _testlimitedcapi.float_check + + self.assertTrue(check(4.25)) + self.assertTrue(check(FloatSubclass(4.25))) + self.assertFalse(check(Float())) + self.assertFalse(check(3)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_checkexact(self): + # Test PyFloat_CheckExact() + checkexact = _testlimitedcapi.float_checkexact + + self.assertTrue(checkexact(4.25)) + self.assertFalse(checkexact(FloatSubclass(4.25))) + self.assertFalse(checkexact(Float())) + self.assertFalse(checkexact(3)) + self.assertFalse(checkexact(object())) + + # CRASHES checkexact(NULL) + + def test_fromstring(self): + # Test PyFloat_FromString() + fromstring = _testlimitedcapi.float_fromstring + + self.assertEqual(fromstring("4.25"), 4.25) + self.assertEqual(fromstring(b"4.25"), 4.25) + self.assertRaises(ValueError, fromstring, "4.25\0") + self.assertRaises(ValueError, fromstring, b"4.25\0") + + self.assertEqual(fromstring(bytearray(b"4.25")), 4.25) + + self.assertEqual(fromstring(memoryview(b"4.25")), 4.25) + self.assertEqual(fromstring(memoryview(b"4.255")[:-1]), 4.25) + self.assertRaises(TypeError, fromstring, memoryview(b"4.25")[::2]) + + self.assertRaises(TypeError, fromstring, 4.25) + + # CRASHES fromstring(NULL) + + def test_fromdouble(self): + # Test PyFloat_FromDouble() + fromdouble = _testlimitedcapi.float_fromdouble + + self.assertEqual(fromdouble(4.25), 4.25) + + def test_asdouble(self): + # Test PyFloat_AsDouble() + asdouble = _testlimitedcapi.float_asdouble + + class BadFloat3: + def __float__(self): + raise RuntimeError + + self.assertEqual(asdouble(4.25), 4.25) + self.assertEqual(asdouble(-1.0), -1.0) + self.assertEqual(asdouble(42), 42.0) + self.assertEqual(asdouble(-1), -1.0) + self.assertEqual(asdouble(2**1000), float(2**1000)) + + self.assertEqual(asdouble(FloatSubclass(4.25)), 4.25) + self.assertEqual(asdouble(FloatSubclass2(4.25)), 4.25) + self.assertEqual(asdouble(Index()), 99.) + + self.assertRaises(TypeError, asdouble, BadIndex()) + self.assertRaises(TypeError, asdouble, BadFloat()) + self.assertRaises(RuntimeError, asdouble, BadFloat3()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(asdouble(BadIndex2()), 1.) + with self.assertWarns(DeprecationWarning): + self.assertEqual(asdouble(BadFloat2()), 4.25) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, asdouble, BadFloat2()) + self.assertRaises(TypeError, asdouble, object()) + self.assertRaises(TypeError, asdouble, NULL) + + def test_getinfo(self): + # Test PyFloat_GetInfo() + getinfo = _testlimitedcapi.float_getinfo + + self.assertEqual(getinfo(), sys.float_info) + + def test_getmax(self): + # Test PyFloat_GetMax() + getmax = _testlimitedcapi.float_getmax + + self.assertEqual(getmax(), sys.float_info.max) + + def test_getmin(self): + # Test PyFloat_GetMax() + getmin = _testlimitedcapi.float_getmin + + self.assertEqual(getmin(), sys.float_info.min) + + def test_pack(self): + # Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8() + pack = _testcapi.float_pack + + self.assertEqual(pack(2, 1.5, BIG_ENDIAN), b'>\x00') + self.assertEqual(pack(4, 1.5, BIG_ENDIAN), b'?\xc0\x00\x00') + self.assertEqual(pack(8, 1.5, BIG_ENDIAN), + b'?\xf8\x00\x00\x00\x00\x00\x00') + self.assertEqual(pack(2, 1.5, LITTLE_ENDIAN), b'\x00>') + self.assertEqual(pack(4, 1.5, LITTLE_ENDIAN), b'\x00\x00\xc0?') + self.assertEqual(pack(8, 1.5, LITTLE_ENDIAN), + b'\x00\x00\x00\x00\x00\x00\xf8?') + + def test_unpack(self): + # Test PyFloat_Unpack2(), PyFloat_Unpack4() and PyFloat_Unpack8() + unpack = _testcapi.float_unpack + + self.assertEqual(unpack(b'>\x00', BIG_ENDIAN), 1.5) + self.assertEqual(unpack(b'?\xc0\x00\x00', BIG_ENDIAN), 1.5) + self.assertEqual(unpack(b'?\xf8\x00\x00\x00\x00\x00\x00', BIG_ENDIAN), + 1.5) + self.assertEqual(unpack(b'\x00>', LITTLE_ENDIAN), 1.5) + self.assertEqual(unpack(b'\x00\x00\xc0?', LITTLE_ENDIAN), 1.5) + self.assertEqual(unpack(b'\x00\x00\x00\x00\x00\x00\xf8?', LITTLE_ENDIAN), + 1.5) + + def test_pack_unpack_roundtrip(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + large = 2.0 ** 100 + values = [1.0, 1.5, large, 1.0/7, math.pi] + if HAVE_IEEE_754: + values.extend((INF, NAN)) + for value in values: + for size in (2, 4, 8,): + if size == 2 and value == large: + # too large for 16-bit float + continue + rel_tol = EPSILON[size] + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(value=value, size=size, endian=endian): + data = pack(size, value, endian) + value2 = unpack(data, endian) + if math.isnan(value): + self.assertTrue(math.isnan(value2), (value, value2)) + elif size < 8: + self.assertTrue(math.isclose(value2, value, rel_tol=rel_tol), + (value, value2)) + else: + self.assertEqual(value2, value) + + @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") + def test_pack_unpack_roundtrip_for_nans(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + for _ in range(10): + for size in (2, 4, 8): + sign = random.randint(0, 1) + if sys.maxsize != 2147483647: # not it 32-bit mode + signaling = random.randint(0, 1) + else: + # Skip sNaN's on x86 (32-bit). The problem is that sNaN + # doubles become qNaN doubles just by the C calling + # convention, there is no way to preserve sNaN doubles + # between C function calls with the current + # PyFloat_Pack/Unpack*() API. See also gh-130317 and + # e.g. https://developercommunity.visualstudio.com/t/155064 + signaling = 0 + if _testcapi.nan_msb_is_signaling: + # HP PA RISC and some MIPS CPUs use 0 for quiet, see: + # https://en.wikipedia.org/wiki/NaN#Encoding + signaling = 1 + i = make_nan(size, sign, not signaling) + data = bytes.fromhex(f'{i:x}') + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(data=data, size=size, endian=endian): + data1 = data if endian == BIG_ENDIAN else data[::-1] + value = unpack(data1, endian) + data2 = pack(size, value, endian) + self.assertTrue(math.isnan(value)) + self.assertEqual(data1, data2) + + @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") + @unittest.skipUnless(sys.maxsize != 2147483647, "requires 64-bit mode") + def test_pack_unpack_nans_for_different_formats(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(endian=endian): + byteorder = "big" if endian == BIG_ENDIAN else "little" + + # Convert sNaN to qNaN, if payload got truncated + data = make_nan(8, 0, False, 0x80001).to_bytes(8, byteorder) + snan_low = unpack(data, endian) + qnan4 = make_nan(4, 0, True, 0).to_bytes(4, byteorder) + qnan2 = make_nan(2, 0, True, 0).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_low, endian), qnan4) + self.assertEqual(pack(2, snan_low, endian), qnan2) + + # Preserve NaN type, if payload not truncated + data = make_nan(8, 0, False, 0x80000000001).to_bytes(8, byteorder) + snan_high = unpack(data, endian) + snan4 = make_nan(4, 0, False, 16384).to_bytes(4, byteorder) + snan2 = make_nan(2, 0, False, 2).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_high, endian), snan4) + self.assertEqual(pack(2, snan_high, endian), snan2) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_frame.py b/Lib/test/test_capi/test_frame.py new file mode 100644 index 00000000000..23cb8e3dada --- /dev/null +++ b/Lib/test/test_capi/test_frame.py @@ -0,0 +1,56 @@ +import sys +import unittest +from test.support import import_helper + + +_testcapi = import_helper.import_module('_testcapi') + + +class FrameTest(unittest.TestCase): + def getframe(self): + return sys._getframe() + + def test_frame_getters(self): + frame = self.getframe() + self.assertEqual(frame.f_locals, _testcapi.frame_getlocals(frame)) + self.assertIs(frame.f_globals, _testcapi.frame_getglobals(frame)) + self.assertIs(frame.f_builtins, _testcapi.frame_getbuiltins(frame)) + self.assertEqual(frame.f_lasti, _testcapi.frame_getlasti(frame)) + + def test_getvar(self): + current_frame = sys._getframe() + x = 1 + self.assertEqual(_testcapi.frame_getvar(current_frame, "x"), 1) + self.assertEqual(_testcapi.frame_getvarstring(current_frame, b"x"), 1) + with self.assertRaises(NameError): + _testcapi.frame_getvar(current_frame, "y") + with self.assertRaises(NameError): + _testcapi.frame_getvarstring(current_frame, b"y") + + # wrong name type + with self.assertRaises(TypeError): + _testcapi.frame_getvar(current_frame, b'x') + with self.assertRaises(TypeError): + _testcapi.frame_getvar(current_frame, 123) + + def getgenframe(self): + yield sys._getframe() + + def test_frame_get_generator(self): + gen = self.getgenframe() + frame = next(gen) + self.assertIs(gen, _testcapi.frame_getgenerator(frame)) + + def test_frame_fback_api(self): + """Test that accessing `f_back` does not cause a segmentation fault on + a frame created with `PyFrame_New` (GH-99110).""" + def dummy(): + pass + + frame = _testcapi.frame_new(dummy.__code__, globals(), locals()) + # The following line should not cause a segmentation fault. + self.assertIsNone(frame.f_back) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_function.py b/Lib/test/test_capi/test_function.py new file mode 100644 index 00000000000..c1a278e5d4d --- /dev/null +++ b/Lib/test/test_capi/test_function.py @@ -0,0 +1,340 @@ +import unittest +from test.support import import_helper + + +_testcapi = import_helper.import_module('_testcapi') + + +class FunctionTest(unittest.TestCase): + def test_function_get_code(self): + # Test PyFunction_GetCode() + import types + + def some(): + pass + + code = _testcapi.function_get_code(some) + self.assertIsInstance(code, types.CodeType) + self.assertEqual(code, some.__code__) + + with self.assertRaises(SystemError): + _testcapi.function_get_code(None) # not a function + + def test_function_get_globals(self): + # Test PyFunction_GetGlobals() + def some(): + pass + + globals_ = _testcapi.function_get_globals(some) + self.assertIsInstance(globals_, dict) + self.assertEqual(globals_, some.__globals__) + + with self.assertRaises(SystemError): + _testcapi.function_get_globals(None) # not a function + + def test_function_get_module(self): + # Test PyFunction_GetModule() + def some(): + pass + + module = _testcapi.function_get_module(some) + self.assertIsInstance(module, str) + self.assertEqual(module, some.__module__) + + with self.assertRaises(SystemError): + _testcapi.function_get_module(None) # not a function + + def test_function_get_defaults(self): + # Test PyFunction_GetDefaults() + def some( + pos_only1, pos_only2='p', + /, + zero=0, optional=None, + *, + kw1, + kw2=True, + ): + pass + + defaults = _testcapi.function_get_defaults(some) + self.assertEqual(defaults, ('p', 0, None)) + self.assertEqual(defaults, some.__defaults__) + + with self.assertRaises(SystemError): + _testcapi.function_get_defaults(None) # not a function + + def test_function_set_defaults(self): + # Test PyFunction_SetDefaults() + def some( + pos_only1, pos_only2='p', + /, + zero=0, optional=None, + *, + kw1, + kw2=True, + ): + pass + + old_defaults = ('p', 0, None) + self.assertEqual(_testcapi.function_get_defaults(some), old_defaults) + self.assertEqual(some.__defaults__, old_defaults) + + with self.assertRaises(SystemError): + _testcapi.function_set_defaults(some, 1) # not tuple or None + self.assertEqual(_testcapi.function_get_defaults(some), old_defaults) + self.assertEqual(some.__defaults__, old_defaults) + + with self.assertRaises(SystemError): + _testcapi.function_set_defaults(1, ()) # not a function + self.assertEqual(_testcapi.function_get_defaults(some), old_defaults) + self.assertEqual(some.__defaults__, old_defaults) + + new_defaults = ('q', 1, None) + _testcapi.function_set_defaults(some, new_defaults) + self.assertEqual(_testcapi.function_get_defaults(some), new_defaults) + self.assertEqual(some.__defaults__, new_defaults) + + # Empty tuple is fine: + new_defaults = () + _testcapi.function_set_defaults(some, new_defaults) + self.assertEqual(_testcapi.function_get_defaults(some), new_defaults) + self.assertEqual(some.__defaults__, new_defaults) + + class tuplesub(tuple): ... # tuple subclasses must work + + new_defaults = tuplesub(((1, 2), ['a', 'b'], None)) + _testcapi.function_set_defaults(some, new_defaults) + self.assertEqual(_testcapi.function_get_defaults(some), new_defaults) + self.assertEqual(some.__defaults__, new_defaults) + + # `None` is special, it sets `defaults` to `NULL`, + # it needs special handling in `_testcapi`: + _testcapi.function_set_defaults(some, None) + self.assertEqual(_testcapi.function_get_defaults(some), None) + self.assertEqual(some.__defaults__, None) + + def test_function_get_kw_defaults(self): + # Test PyFunction_GetKwDefaults() + def some( + pos_only1, pos_only2='p', + /, + zero=0, optional=None, + *, + kw1, + kw2=True, + ): + pass + + defaults = _testcapi.function_get_kw_defaults(some) + self.assertEqual(defaults, {'kw2': True}) + self.assertEqual(defaults, some.__kwdefaults__) + + with self.assertRaises(SystemError): + _testcapi.function_get_kw_defaults(None) # not a function + + def test_function_set_kw_defaults(self): + # Test PyFunction_SetKwDefaults() + def some( + pos_only1, pos_only2='p', + /, + zero=0, optional=None, + *, + kw1, + kw2=True, + ): + pass + + old_defaults = {'kw2': True} + self.assertEqual(_testcapi.function_get_kw_defaults(some), old_defaults) + self.assertEqual(some.__kwdefaults__, old_defaults) + + with self.assertRaises(SystemError): + _testcapi.function_set_kw_defaults(some, 1) # not dict or None + self.assertEqual(_testcapi.function_get_kw_defaults(some), old_defaults) + self.assertEqual(some.__kwdefaults__, old_defaults) + + with self.assertRaises(SystemError): + _testcapi.function_set_kw_defaults(1, {}) # not a function + self.assertEqual(_testcapi.function_get_kw_defaults(some), old_defaults) + self.assertEqual(some.__kwdefaults__, old_defaults) + + new_defaults = {'kw2': (1, 2, 3)} + _testcapi.function_set_kw_defaults(some, new_defaults) + self.assertEqual(_testcapi.function_get_kw_defaults(some), new_defaults) + self.assertEqual(some.__kwdefaults__, new_defaults) + + # Empty dict is fine: + new_defaults = {} + _testcapi.function_set_kw_defaults(some, new_defaults) + self.assertEqual(_testcapi.function_get_kw_defaults(some), new_defaults) + self.assertEqual(some.__kwdefaults__, new_defaults) + + class dictsub(dict): ... # dict subclasses must work + + new_defaults = dictsub({'kw2': None}) + _testcapi.function_set_kw_defaults(some, new_defaults) + self.assertEqual(_testcapi.function_get_kw_defaults(some), new_defaults) + self.assertEqual(some.__kwdefaults__, new_defaults) + + # `None` is special, it sets `kwdefaults` to `NULL`, + # it needs special handling in `_testcapi`: + _testcapi.function_set_kw_defaults(some, None) + self.assertEqual(_testcapi.function_get_kw_defaults(some), None) + self.assertEqual(some.__kwdefaults__, None) + + def test_function_get_closure(self): + # Test PyFunction_GetClosure() + from types import CellType + + def regular_function(): ... + def unused_one_level(arg1): + def inner(arg2, arg3): ... + return inner + def unused_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): ... + return inner + return decorator + def with_one_level(arg1): + def inner(arg2, arg3): + return arg1 + arg2 + arg3 + return inner + def with_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): + return arg1 + arg2 + arg3 + arg4 + arg5 + arg6 + return inner + return decorator + + # Functions without closures: + self.assertIsNone(_testcapi.function_get_closure(regular_function)) + self.assertIsNone(regular_function.__closure__) + + func = unused_one_level(1) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + func = unused_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + # Functions with closures: + func = with_one_level(5) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 1) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + for cell in closure: + self.assertIsInstance(cell, CellType) + self.assertTrue(closure[0].cell_contents, 5) + + func = with_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 4) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + for cell in closure: + self.assertIsInstance(cell, CellType) + self.assertEqual([cell.cell_contents for cell in closure], + [1, 2, 3, 4]) + + def test_function_get_closure_error(self): + # Test PyFunction_GetClosure() + with self.assertRaises(SystemError): + _testcapi.function_get_closure(1) + with self.assertRaises(SystemError): + _testcapi.function_get_closure(None) + + def test_function_set_closure(self): + # Test PyFunction_SetClosure() + from types import CellType + + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + func = function_without_closure + _testcapi.function_set_closure(func, (CellType(1), CellType(1))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 1]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 1]) + + func = function_with_closure(1) + _testcapi.function_set_closure(func, + (CellType(1), CellType(2), CellType(3))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 2, 3]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 2, 3]) + + def test_function_set_closure_none(self): + # Test PyFunction_SetClosure() + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + _testcapi.function_set_closure(function_without_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_without_closure)) + self.assertIsNone(function_without_closure.__closure__) + + _testcapi.function_set_closure(function_with_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_with_closure)) + self.assertIsNone(function_with_closure.__closure__) + + def test_function_set_closure_errors(self): + # Test PyFunction_SetClosure() + def function_without_closure(): ... + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(None, ()) # not a function + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(function_without_closure, 1) + self.assertIsNone(function_without_closure.__closure__) # no change + + # NOTE: this works, but goes against the docs: + _testcapi.function_set_closure(function_without_closure, (1, 2)) + self.assertEqual( + _testcapi.function_get_closure(function_without_closure), (1, 2)) + self.assertEqual(function_without_closure.__closure__, (1, 2)) + + def test_function_get_annotations(self): + # Test PyFunction_GetAnnotations() + def normal(): + pass + + def annofn(arg: int) -> str: + return f'arg = {arg}' + + annotations = _testcapi.function_get_annotations(normal) + self.assertIsNone(annotations) + + annotations = _testcapi.function_get_annotations(annofn) + self.assertIsInstance(annotations, dict) + self.assertEqual(annotations, annofn.__annotations__) + + with self.assertRaises(SystemError): + _testcapi.function_get_annotations(None) + + # TODO: test PyFunction_New() + # TODO: test PyFunction_NewWithQualName() + # TODO: test PyFunction_SetVectorcall() + # TODO: test PyFunction_SetAnnotations() + # TODO: test PyClassMethod_New() + # TODO: test PyStaticMethod_New() + # + # PyFunction_AddWatcher() and PyFunction_ClearWatcher() are tested by + # test_capi.test_watchers. + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py new file mode 100644 index 00000000000..2f047200daf --- /dev/null +++ b/Lib/test/test_capi/test_getargs.py @@ -0,0 +1,1421 @@ +import string +import sys +import unittest +from test import support +from test.support import import_helper +from test.support import script_helper +from test.support import warnings_helper +from test.support.testcase import FloatsAreIdenticalMixin +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') +from _testcapi import getargs_keywords, getargs_keyword_only + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + +# > How about the following counterproposal. This also changes some of +# > the other format codes to be a little more regular. +# > +# > Code C type Range check +# > +# > b unsigned char 0..UCHAR_MAX +# > h signed short SHRT_MIN..SHRT_MAX +# > B unsigned char none ** +# > H unsigned short none ** +# > k * unsigned long none +# > I * unsigned int 0..UINT_MAX +# +# +# > i int INT_MIN..INT_MAX +# > l long LONG_MIN..LONG_MAX +# +# > K * unsigned long long none +# > L long long LLONG_MIN..LLONG_MAX +# +# > Notes: +# > +# > * New format codes. +# > +# > ** Changed from previous "range-and-a-half" to "none"; the +# > range-and-a-half checking wasn't particularly useful. +# +# Plus a C API or two, e.g. PyLong_AsUnsignedLongMask() -> +# unsigned long and PyLong_AsUnsignedLongLongMask() -> unsigned +# long long (if that exists). + +LARGE = 0x7FFFFFFF +VERY_LARGE = 0xFF0000121212121212121242 + +from _testcapi import UCHAR_MAX, USHRT_MAX, UINT_MAX, ULONG_MAX, INT_MAX, \ + INT_MIN, LONG_MIN, LONG_MAX, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, \ + SHRT_MIN, SHRT_MAX, FLT_MIN, FLT_MAX, DBL_MIN, DBL_MAX + +DBL_MAX_EXP = sys.float_info.max_exp +INF = float('inf') +NAN = float('nan') + +# fake, they are not defined in Python's header files +LLONG_MAX = 2**63-1 +LLONG_MIN = -2**63 +ULLONG_MAX = 2**64-1 + +NULL = None + +class CustomError(Exception): + pass + +class Index: + def __index__(self): + return 99 + +class IndexIntSubclass(int): + def __index__(self): + return 99 + +class BadIndex: + def __index__(self): + return 1.0 + +class BadIndex2: + def __index__(self): + return True + +class BadIndex3(int): + def __index__(self): + return True + + +class Int: + def __int__(self): + return 99 + +class IntSubclass(int): + def __int__(self): + return 99 + +class BadInt: + def __int__(self): + return 1.0 + +class BadInt2: + def __int__(self): + return True + +class BadInt3(int): + def __int__(self): + return True + + +class Float: + def __float__(self): + return 4.25 + +class FloatSubclass(float): + pass + +class FloatSubclass2(float): + def __float__(self): + return 4.25 + +class BadFloat: + def __float__(self): + return 687 + +class BadFloat2: + def __float__(self): + return FloatSubclass(4.25) + +class BadFloat3(float): + def __float__(self): + return FloatSubclass(4.25) + + +class Complex: + def __complex__(self): + return 4.25+0.5j + +class ComplexSubclass(complex): + pass + +class ComplexSubclass2(complex): + def __complex__(self): + return 4.25+0.5j + +class BadComplex: + def __complex__(self): + return 1.25 + +class BadComplex2: + def __complex__(self): + return ComplexSubclass(4.25+0.5j) + +class BadComplex3(complex): + def __complex__(self): + return ComplexSubclass(4.25+0.5j) + + +class TupleSubclass(tuple): + pass + +class DictSubclass(dict): + pass + +NONCONTIG_WRITABLE = memoryview(bytearray(b'noncontig'))[::-2] +NONCONTIG_READONLY = memoryview(b'noncontig')[::-2] + +class Unsigned_TestCase(unittest.TestCase): + def test_b(self): + from _testcapi import getargs_b + # b returns 'unsigned char', and does range checking (0 ... UCHAR_MAX) + self.assertRaises(TypeError, getargs_b, 3.14) + self.assertEqual(99, getargs_b(Index())) + self.assertEqual(0, getargs_b(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_b, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_b(BadIndex2())) + self.assertEqual(0, getargs_b(BadIndex3())) + self.assertRaises(TypeError, getargs_b, Int()) + self.assertEqual(0, getargs_b(IntSubclass())) + self.assertRaises(TypeError, getargs_b, BadInt()) + self.assertRaises(TypeError, getargs_b, BadInt2()) + self.assertEqual(0, getargs_b(BadInt3())) + + self.assertRaises(OverflowError, getargs_b, -1) + self.assertEqual(0, getargs_b(0)) + self.assertEqual(UCHAR_MAX, getargs_b(UCHAR_MAX)) + self.assertRaises(OverflowError, getargs_b, UCHAR_MAX + 1) + + self.assertEqual(42, getargs_b(42)) + self.assertRaises(OverflowError, getargs_b, VERY_LARGE) + + def test_B(self): + from _testcapi import getargs_B + # B returns 'unsigned char', no range checking + self.assertRaises(TypeError, getargs_B, 3.14) + self.assertEqual(99, getargs_B(Index())) + self.assertEqual(0, getargs_B(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_B, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_B(BadIndex2())) + self.assertEqual(0, getargs_B(BadIndex3())) + self.assertRaises(TypeError, getargs_B, Int()) + self.assertEqual(0, getargs_B(IntSubclass())) + self.assertRaises(TypeError, getargs_B, BadInt()) + self.assertRaises(TypeError, getargs_B, BadInt2()) + self.assertEqual(0, getargs_B(BadInt3())) + + self.assertEqual(UCHAR_MAX, getargs_B(-1)) + self.assertEqual(0, getargs_B(0)) + self.assertEqual(UCHAR_MAX, getargs_B(UCHAR_MAX)) + self.assertEqual(0, getargs_B(UCHAR_MAX+1)) + + self.assertEqual(42, getargs_B(42)) + self.assertEqual(UCHAR_MAX & VERY_LARGE, getargs_B(VERY_LARGE)) + + def test_H(self): + from _testcapi import getargs_H + # H returns 'unsigned short', no range checking + self.assertRaises(TypeError, getargs_H, 3.14) + self.assertEqual(99, getargs_H(Index())) + self.assertEqual(0, getargs_H(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_H, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_H(BadIndex2())) + self.assertEqual(0, getargs_H(BadIndex3())) + self.assertRaises(TypeError, getargs_H, Int()) + self.assertEqual(0, getargs_H(IntSubclass())) + self.assertRaises(TypeError, getargs_H, BadInt()) + self.assertRaises(TypeError, getargs_H, BadInt2()) + self.assertEqual(0, getargs_H(BadInt3())) + + self.assertEqual(USHRT_MAX, getargs_H(-1)) + self.assertEqual(0, getargs_H(0)) + self.assertEqual(USHRT_MAX, getargs_H(USHRT_MAX)) + self.assertEqual(0, getargs_H(USHRT_MAX+1)) + + self.assertEqual(42, getargs_H(42)) + + self.assertEqual(VERY_LARGE & USHRT_MAX, getargs_H(VERY_LARGE)) + + def test_I(self): + from _testcapi import getargs_I + # I returns 'unsigned int', no range checking + self.assertRaises(TypeError, getargs_I, 3.14) + self.assertEqual(99, getargs_I(Index())) + self.assertEqual(0, getargs_I(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_I, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_I(BadIndex2())) + self.assertEqual(0, getargs_I(BadIndex3())) + self.assertRaises(TypeError, getargs_I, Int()) + self.assertEqual(0, getargs_I(IntSubclass())) + self.assertRaises(TypeError, getargs_I, BadInt()) + self.assertRaises(TypeError, getargs_I, BadInt2()) + self.assertEqual(0, getargs_I(BadInt3())) + + self.assertEqual(UINT_MAX, getargs_I(-1)) + self.assertEqual(0, getargs_I(0)) + self.assertEqual(UINT_MAX, getargs_I(UINT_MAX)) + self.assertEqual(0, getargs_I(UINT_MAX+1)) + + self.assertEqual(42, getargs_I(42)) + + self.assertEqual(VERY_LARGE & UINT_MAX, getargs_I(VERY_LARGE)) + + def test_k(self): + from _testcapi import getargs_k + # k returns 'unsigned long', no range checking + self.assertRaises(TypeError, getargs_k, 3.14) + self.assertEqual(99, getargs_k(Index())) + self.assertEqual(0, getargs_k(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_k, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_k(BadIndex2())) + self.assertEqual(0, getargs_k(BadIndex3())) + self.assertRaises(TypeError, getargs_k, Int()) + self.assertEqual(0, getargs_k(IntSubclass())) + self.assertRaises(TypeError, getargs_k, BadInt()) + self.assertRaises(TypeError, getargs_k, BadInt2()) + self.assertEqual(0, getargs_k(BadInt3())) + + self.assertEqual(ULONG_MAX, getargs_k(-1)) + self.assertEqual(0, getargs_k(0)) + self.assertEqual(ULONG_MAX, getargs_k(ULONG_MAX)) + self.assertEqual(0, getargs_k(ULONG_MAX+1)) + + self.assertEqual(42, getargs_k(42)) + + self.assertEqual(VERY_LARGE & ULONG_MAX, getargs_k(VERY_LARGE)) + +class Signed_TestCase(unittest.TestCase): + def test_h(self): + from _testcapi import getargs_h + # h returns 'short', and does range checking (SHRT_MIN ... SHRT_MAX) + self.assertRaises(TypeError, getargs_h, 3.14) + self.assertEqual(99, getargs_h(Index())) + self.assertEqual(0, getargs_h(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_h, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_h(BadIndex2())) + self.assertEqual(0, getargs_h(BadIndex3())) + self.assertRaises(TypeError, getargs_h, Int()) + self.assertEqual(0, getargs_h(IntSubclass())) + self.assertRaises(TypeError, getargs_h, BadInt()) + self.assertRaises(TypeError, getargs_h, BadInt2()) + self.assertEqual(0, getargs_h(BadInt3())) + + self.assertRaises(OverflowError, getargs_h, SHRT_MIN-1) + self.assertEqual(SHRT_MIN, getargs_h(SHRT_MIN)) + self.assertEqual(SHRT_MAX, getargs_h(SHRT_MAX)) + self.assertRaises(OverflowError, getargs_h, SHRT_MAX+1) + + self.assertEqual(42, getargs_h(42)) + self.assertRaises(OverflowError, getargs_h, VERY_LARGE) + + def test_i(self): + from _testcapi import getargs_i + # i returns 'int', and does range checking (INT_MIN ... INT_MAX) + self.assertRaises(TypeError, getargs_i, 3.14) + self.assertEqual(99, getargs_i(Index())) + self.assertEqual(0, getargs_i(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_i, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_i(BadIndex2())) + self.assertEqual(0, getargs_i(BadIndex3())) + self.assertRaises(TypeError, getargs_i, Int()) + self.assertEqual(0, getargs_i(IntSubclass())) + self.assertRaises(TypeError, getargs_i, BadInt()) + self.assertRaises(TypeError, getargs_i, BadInt2()) + self.assertEqual(0, getargs_i(BadInt3())) + + self.assertRaises(OverflowError, getargs_i, INT_MIN-1) + self.assertEqual(INT_MIN, getargs_i(INT_MIN)) + self.assertEqual(INT_MAX, getargs_i(INT_MAX)) + self.assertRaises(OverflowError, getargs_i, INT_MAX+1) + + self.assertEqual(42, getargs_i(42)) + self.assertRaises(OverflowError, getargs_i, VERY_LARGE) + + def test_l(self): + from _testcapi import getargs_l + # l returns 'long', and does range checking (LONG_MIN ... LONG_MAX) + self.assertRaises(TypeError, getargs_l, 3.14) + self.assertEqual(99, getargs_l(Index())) + self.assertEqual(0, getargs_l(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_l, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_l(BadIndex2())) + self.assertEqual(0, getargs_l(BadIndex3())) + self.assertRaises(TypeError, getargs_l, Int()) + self.assertEqual(0, getargs_l(IntSubclass())) + self.assertRaises(TypeError, getargs_l, BadInt()) + self.assertRaises(TypeError, getargs_l, BadInt2()) + self.assertEqual(0, getargs_l(BadInt3())) + + self.assertRaises(OverflowError, getargs_l, LONG_MIN-1) + self.assertEqual(LONG_MIN, getargs_l(LONG_MIN)) + self.assertEqual(LONG_MAX, getargs_l(LONG_MAX)) + self.assertRaises(OverflowError, getargs_l, LONG_MAX+1) + + self.assertEqual(42, getargs_l(42)) + self.assertRaises(OverflowError, getargs_l, VERY_LARGE) + + def test_n(self): + from _testcapi import getargs_n + # n returns 'Py_ssize_t', and does range checking + # (PY_SSIZE_T_MIN ... PY_SSIZE_T_MAX) + self.assertRaises(TypeError, getargs_n, 3.14) + self.assertEqual(99, getargs_n(Index())) + self.assertEqual(0, getargs_n(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_n, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_n(BadIndex2())) + self.assertEqual(0, getargs_n(BadIndex3())) + self.assertRaises(TypeError, getargs_n, Int()) + self.assertEqual(0, getargs_n(IntSubclass())) + self.assertRaises(TypeError, getargs_n, BadInt()) + self.assertRaises(TypeError, getargs_n, BadInt2()) + self.assertEqual(0, getargs_n(BadInt3())) + + self.assertRaises(OverflowError, getargs_n, PY_SSIZE_T_MIN-1) + self.assertEqual(PY_SSIZE_T_MIN, getargs_n(PY_SSIZE_T_MIN)) + self.assertEqual(PY_SSIZE_T_MAX, getargs_n(PY_SSIZE_T_MAX)) + self.assertRaises(OverflowError, getargs_n, PY_SSIZE_T_MAX+1) + + self.assertEqual(42, getargs_n(42)) + self.assertRaises(OverflowError, getargs_n, VERY_LARGE) + + +class LongLong_TestCase(unittest.TestCase): + def test_L(self): + from _testcapi import getargs_L + # L returns 'long long', and does range checking (LLONG_MIN + # ... LLONG_MAX) + self.assertRaises(TypeError, getargs_L, 3.14) + self.assertRaises(TypeError, getargs_L, "Hello") + self.assertEqual(99, getargs_L(Index())) + self.assertEqual(0, getargs_L(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_L, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_L(BadIndex2())) + self.assertEqual(0, getargs_L(BadIndex3())) + self.assertRaises(TypeError, getargs_L, Int()) + self.assertEqual(0, getargs_L(IntSubclass())) + self.assertRaises(TypeError, getargs_L, BadInt()) + self.assertRaises(TypeError, getargs_L, BadInt2()) + self.assertEqual(0, getargs_L(BadInt3())) + + self.assertRaises(OverflowError, getargs_L, LLONG_MIN-1) + self.assertEqual(LLONG_MIN, getargs_L(LLONG_MIN)) + self.assertEqual(LLONG_MAX, getargs_L(LLONG_MAX)) + self.assertRaises(OverflowError, getargs_L, LLONG_MAX+1) + + self.assertEqual(42, getargs_L(42)) + self.assertRaises(OverflowError, getargs_L, VERY_LARGE) + + def test_K(self): + from _testcapi import getargs_K + # K return 'unsigned long long', no range checking + self.assertRaises(TypeError, getargs_K, 3.14) + self.assertEqual(99, getargs_K(Index())) + self.assertEqual(0, getargs_K(IndexIntSubclass())) + self.assertRaises(TypeError, getargs_K, BadIndex()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_K(BadIndex2())) + self.assertEqual(0, getargs_K(BadIndex3())) + self.assertRaises(TypeError, getargs_K, Int()) + self.assertEqual(0, getargs_K(IntSubclass())) + self.assertRaises(TypeError, getargs_K, BadInt()) + self.assertRaises(TypeError, getargs_K, BadInt2()) + self.assertEqual(0, getargs_K(BadInt3())) + + self.assertEqual(ULLONG_MAX, getargs_K(ULLONG_MAX)) + self.assertEqual(0, getargs_K(0)) + self.assertEqual(ULLONG_MAX, getargs_K(ULLONG_MAX)) + self.assertEqual(0, getargs_K(ULLONG_MAX+1)) + + self.assertEqual(42, getargs_K(42)) + + self.assertEqual(VERY_LARGE & ULLONG_MAX, getargs_K(VERY_LARGE)) + + +class Float_TestCase(unittest.TestCase, FloatsAreIdenticalMixin): + def test_f(self): + from _testcapi import getargs_f + self.assertEqual(getargs_f(4.25), 4.25) + self.assertEqual(getargs_f(4), 4.0) + self.assertRaises(TypeError, getargs_f, 4.25+0j) + self.assertEqual(getargs_f(Float()), 4.25) + self.assertEqual(getargs_f(FloatSubclass(7.5)), 7.5) + self.assertEqual(getargs_f(FloatSubclass2(7.5)), 7.5) + self.assertRaises(TypeError, getargs_f, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(getargs_f(BadFloat2()), 4.25) + self.assertEqual(getargs_f(BadFloat3(7.5)), 7.5) + self.assertEqual(getargs_f(Index()), 99.0) + self.assertRaises(TypeError, getargs_f, Int()) + + for x in (FLT_MIN, -FLT_MIN, FLT_MAX, -FLT_MAX, INF, -INF): + self.assertEqual(getargs_f(x), x) + if FLT_MAX < DBL_MAX: + self.assertEqual(getargs_f(DBL_MAX), INF) + self.assertEqual(getargs_f(-DBL_MAX), -INF) + if FLT_MIN > DBL_MIN: + self.assertFloatsAreIdentical(getargs_f(DBL_MIN), 0.0) + self.assertFloatsAreIdentical(getargs_f(-DBL_MIN), -0.0) + self.assertFloatsAreIdentical(getargs_f(0.0), 0.0) + self.assertFloatsAreIdentical(getargs_f(-0.0), -0.0) + r = getargs_f(NAN) + self.assertNotEqual(r, r) + + @support.requires_IEEE_754 + def test_f_rounding(self): + from _testcapi import getargs_f + self.assertEqual(getargs_f(3.40282356e38), FLT_MAX) + self.assertEqual(getargs_f(-3.40282356e38), -FLT_MAX) + + def test_d(self): + from _testcapi import getargs_d + self.assertEqual(getargs_d(4.25), 4.25) + self.assertEqual(getargs_d(4), 4.0) + self.assertRaises(TypeError, getargs_d, 4.25+0j) + self.assertEqual(getargs_d(Float()), 4.25) + self.assertEqual(getargs_d(FloatSubclass(7.5)), 7.5) + self.assertEqual(getargs_d(FloatSubclass2(7.5)), 7.5) + self.assertRaises(TypeError, getargs_d, BadFloat()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(getargs_d(BadFloat2()), 4.25) + self.assertEqual(getargs_d(BadFloat3(7.5)), 7.5) + self.assertEqual(getargs_d(Index()), 99.0) + self.assertRaises(TypeError, getargs_d, Int()) + + for x in (DBL_MIN, -DBL_MIN, DBL_MAX, -DBL_MAX, INF, -INF): + self.assertEqual(getargs_d(x), x) + self.assertRaises(OverflowError, getargs_d, 1< 1 + self.assertEqual(getargs_c(b'a'), 97) + self.assertEqual(getargs_c(bytearray(b'a')), 97) + self.assertRaises(TypeError, getargs_c, memoryview(b'a')) + self.assertRaises(TypeError, getargs_c, 's') + self.assertRaises(TypeError, getargs_c, 97) + self.assertRaises(TypeError, getargs_c, None) + + def test_y(self): + from _testcapi import getargs_y + self.assertRaises(TypeError, getargs_y, 'abc\xe9') + self.assertEqual(getargs_y(b'bytes'), b'bytes') + self.assertRaises(ValueError, getargs_y, b'nul:\0') + self.assertRaises(TypeError, getargs_y, bytearray(b'bytearray')) + self.assertRaises(TypeError, getargs_y, memoryview(b'memoryview')) + self.assertRaises(TypeError, getargs_y, None) + + def test_y_star(self): + from _testcapi import getargs_y_star + self.assertRaises(TypeError, getargs_y_star, 'abc\xe9') + self.assertEqual(getargs_y_star(b'bytes'), b'bytes') + self.assertEqual(getargs_y_star(b'nul:\0'), b'nul:\0') + self.assertEqual(getargs_y_star(bytearray(b'bytearray')), b'bytearray') + self.assertEqual(getargs_y_star(memoryview(b'memoryview')), b'memoryview') + self.assertRaises(TypeError, getargs_y_star, None) + self.assertRaises(BufferError, getargs_y_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_y_star, NONCONTIG_READONLY) + + def test_y_hash(self): + from _testcapi import getargs_y_hash + self.assertRaises(TypeError, getargs_y_hash, 'abc\xe9') + self.assertEqual(getargs_y_hash(b'bytes'), b'bytes') + self.assertEqual(getargs_y_hash(b'nul:\0'), b'nul:\0') + self.assertRaises(TypeError, getargs_y_hash, bytearray(b'bytearray')) + self.assertRaises(TypeError, getargs_y_hash, memoryview(b'memoryview')) + self.assertRaises(TypeError, getargs_y_hash, None) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_READONLY) + + def test_w_star(self): + # getargs_w_star() modifies first and last byte + # getargs_w_star_opt() takes additional optional args: with one + # argument it should behave the same as getargs_w_star + from _testcapi import getargs_w_star, getargs_w_star_opt + for func in (getargs_w_star, getargs_w_star_opt): + with self.subTest(func=func): + self.assertRaises(TypeError, func, 'abc\xe9') + self.assertRaises(TypeError, func, b'bytes') + self.assertRaises(TypeError, func, b'nul:\0') + self.assertRaises(TypeError, func, memoryview(b'bytes')) + buf = bytearray(b'bytearray') + self.assertEqual(func(buf), b'[ytearra]') + self.assertEqual(buf, bytearray(b'[ytearra]')) + buf = bytearray(b'memoryview') + self.assertEqual(func(memoryview(buf)), b'[emoryvie]') + self.assertEqual(buf, bytearray(b'[emoryvie]')) + self.assertRaises(TypeError, func, None) + self.assertRaises(TypeError, func, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, func, NONCONTIG_READONLY) + + def test_getargs_empty(self): + from _testcapi import getargs_empty + self.assertTrue(getargs_empty()) + self.assertRaises(TypeError, getargs_empty, 1) + self.assertRaises(TypeError, getargs_empty, 1, 2, 3) + self.assertRaises(TypeError, getargs_empty, a=1) + self.assertRaises(TypeError, getargs_empty, a=1, b=2) + self.assertRaises(TypeError, getargs_empty, 'x', 'y', 'z', a=1, b=2) + + +class String_TestCase(unittest.TestCase): + def test_C(self): + from _testcapi import getargs_C + self.assertRaises(TypeError, getargs_C, 'abc') # len > 1 + self.assertEqual(getargs_C('a'), 97) + self.assertEqual(getargs_C('\u20ac'), 0x20ac) + self.assertEqual(getargs_C('\U0001f40d'), 0x1f40d) + self.assertRaises(TypeError, getargs_C, b'a') + self.assertRaises(TypeError, getargs_C, bytearray(b'a')) + self.assertRaises(TypeError, getargs_C, memoryview(b'a')) + self.assertRaises(TypeError, getargs_C, 97) + self.assertRaises(TypeError, getargs_C, None) + + def test_s(self): + from _testcapi import getargs_s + self.assertEqual(getargs_s('abc\xe9'), b'abc\xc3\xa9') + self.assertRaises(ValueError, getargs_s, 'nul:\0') + self.assertRaises(TypeError, getargs_s, b'bytes') + self.assertRaises(TypeError, getargs_s, bytearray(b'bytearray')) + self.assertRaises(TypeError, getargs_s, memoryview(b'memoryview')) + self.assertRaises(TypeError, getargs_s, None) + + def test_s_star(self): + from _testcapi import getargs_s_star + self.assertEqual(getargs_s_star('abc\xe9'), b'abc\xc3\xa9') + self.assertEqual(getargs_s_star('nul:\0'), b'nul:\0') + self.assertEqual(getargs_s_star(b'bytes'), b'bytes') + self.assertEqual(getargs_s_star(bytearray(b'bytearray')), b'bytearray') + self.assertEqual(getargs_s_star(memoryview(b'memoryview')), b'memoryview') + self.assertRaises(TypeError, getargs_s_star, None) + self.assertRaises(BufferError, getargs_s_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_s_star, NONCONTIG_READONLY) + + def test_s_hash(self): + from _testcapi import getargs_s_hash + self.assertEqual(getargs_s_hash('abc\xe9'), b'abc\xc3\xa9') + self.assertEqual(getargs_s_hash('nul:\0'), b'nul:\0') + self.assertEqual(getargs_s_hash(b'bytes'), b'bytes') + self.assertRaises(TypeError, getargs_s_hash, bytearray(b'bytearray')) + self.assertRaises(TypeError, getargs_s_hash, memoryview(b'memoryview')) + self.assertRaises(TypeError, getargs_s_hash, None) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_READONLY) + + def test_z(self): + from _testcapi import getargs_z + self.assertEqual(getargs_z('abc\xe9'), b'abc\xc3\xa9') + self.assertRaises(ValueError, getargs_z, 'nul:\0') + self.assertRaises(TypeError, getargs_z, b'bytes') + self.assertRaises(TypeError, getargs_z, bytearray(b'bytearray')) + self.assertRaises(TypeError, getargs_z, memoryview(b'memoryview')) + self.assertIsNone(getargs_z(None)) + + def test_z_star(self): + from _testcapi import getargs_z_star + self.assertEqual(getargs_z_star('abc\xe9'), b'abc\xc3\xa9') + self.assertEqual(getargs_z_star('nul:\0'), b'nul:\0') + self.assertEqual(getargs_z_star(b'bytes'), b'bytes') + self.assertEqual(getargs_z_star(bytearray(b'bytearray')), b'bytearray') + self.assertEqual(getargs_z_star(memoryview(b'memoryview')), b'memoryview') + self.assertIsNone(getargs_z_star(None)) + self.assertRaises(BufferError, getargs_z_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_z_star, NONCONTIG_READONLY) + + def test_z_hash(self): + from _testcapi import getargs_z_hash + self.assertEqual(getargs_z_hash('abc\xe9'), b'abc\xc3\xa9') + self.assertEqual(getargs_z_hash('nul:\0'), b'nul:\0') + self.assertEqual(getargs_z_hash(b'bytes'), b'bytes') + self.assertRaises(TypeError, getargs_z_hash, bytearray(b'bytearray')) + self.assertRaises(TypeError, getargs_z_hash, memoryview(b'memoryview')) + self.assertIsNone(getargs_z_hash(None)) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_READONLY) + + def test_es(self): + from _testcapi import getargs_es + self.assertEqual(getargs_es('abc\xe9'), b'abc\xc3\xa9') + self.assertEqual(getargs_es('abc\xe9', 'latin1'), b'abc\xe9') + self.assertRaises(UnicodeEncodeError, getargs_es, 'abc\xe9', 'ascii') + self.assertRaises(LookupError, getargs_es, 'abc\xe9', 'spam') + self.assertRaises(TypeError, getargs_es, b'bytes', 'latin1') + self.assertRaises(TypeError, getargs_es, bytearray(b'bytearray'), 'latin1') + self.assertRaises(TypeError, getargs_es, memoryview(b'memoryview'), 'latin1') + self.assertRaises(TypeError, getargs_es, None, 'latin1') + self.assertRaises(TypeError, getargs_es, 'nul:\0', 'latin1') + + def test_et(self): + from _testcapi import getargs_et + self.assertEqual(getargs_et('abc\xe9'), b'abc\xc3\xa9') + self.assertEqual(getargs_et('abc\xe9', 'latin1'), b'abc\xe9') + self.assertRaises(UnicodeEncodeError, getargs_et, 'abc\xe9', 'ascii') + self.assertRaises(LookupError, getargs_et, 'abc\xe9', 'spam') + self.assertEqual(getargs_et(b'bytes', 'latin1'), b'bytes') + self.assertEqual(getargs_et(bytearray(b'bytearray'), 'latin1'), b'bytearray') + self.assertRaises(TypeError, getargs_et, memoryview(b'memoryview'), 'latin1') + self.assertRaises(TypeError, getargs_et, None, 'latin1') + self.assertRaises(TypeError, getargs_et, 'nul:\0', 'latin1') + self.assertRaises(TypeError, getargs_et, b'nul:\0', 'latin1') + self.assertRaises(TypeError, getargs_et, bytearray(b'nul:\0'), 'latin1') + + def test_es_hash(self): + from _testcapi import getargs_es_hash + self.assertEqual(getargs_es_hash('abc\xe9'), b'abc\xc3\xa9') + self.assertEqual(getargs_es_hash('abc\xe9', 'latin1'), b'abc\xe9') + self.assertRaises(UnicodeEncodeError, getargs_es_hash, 'abc\xe9', 'ascii') + self.assertRaises(LookupError, getargs_es_hash, 'abc\xe9', 'spam') + self.assertRaises(TypeError, getargs_es_hash, b'bytes', 'latin1') + self.assertRaises(TypeError, getargs_es_hash, bytearray(b'bytearray'), 'latin1') + self.assertRaises(TypeError, getargs_es_hash, memoryview(b'memoryview'), 'latin1') + self.assertRaises(TypeError, getargs_es_hash, None, 'latin1') + self.assertEqual(getargs_es_hash('nul:\0', 'latin1'), b'nul:\0') + + buf = bytearray(b'x'*8) + self.assertEqual(getargs_es_hash('abc\xe9', 'latin1', buf), b'abc\xe9') + self.assertEqual(buf, bytearray(b'abc\xe9\x00xxx')) + buf = bytearray(b'x'*5) + self.assertEqual(getargs_es_hash('abc\xe9', 'latin1', buf), b'abc\xe9') + self.assertEqual(buf, bytearray(b'abc\xe9\x00')) + buf = bytearray(b'x'*4) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertEqual(buf, bytearray(b'x'*4)) + buf = bytearray() + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + + def test_et_hash(self): + from _testcapi import getargs_et_hash + self.assertEqual(getargs_et_hash('abc\xe9'), b'abc\xc3\xa9') + self.assertEqual(getargs_et_hash('abc\xe9', 'latin1'), b'abc\xe9') + self.assertRaises(UnicodeEncodeError, getargs_et_hash, 'abc\xe9', 'ascii') + self.assertRaises(LookupError, getargs_et_hash, 'abc\xe9', 'spam') + self.assertEqual(getargs_et_hash(b'bytes', 'latin1'), b'bytes') + self.assertEqual(getargs_et_hash(bytearray(b'bytearray'), 'latin1'), b'bytearray') + self.assertRaises(TypeError, getargs_et_hash, memoryview(b'memoryview'), 'latin1') + self.assertRaises(TypeError, getargs_et_hash, None, 'latin1') + self.assertEqual(getargs_et_hash('nul:\0', 'latin1'), b'nul:\0') + self.assertEqual(getargs_et_hash(b'nul:\0', 'latin1'), b'nul:\0') + self.assertEqual(getargs_et_hash(bytearray(b'nul:\0'), 'latin1'), b'nul:\0') + + buf = bytearray(b'x'*8) + self.assertEqual(getargs_et_hash('abc\xe9', 'latin1', buf), b'abc\xe9') + self.assertEqual(buf, bytearray(b'abc\xe9\x00xxx')) + buf = bytearray(b'x'*5) + self.assertEqual(getargs_et_hash('abc\xe9', 'latin1', buf), b'abc\xe9') + self.assertEqual(buf, bytearray(b'abc\xe9\x00')) + buf = bytearray(b'x'*4) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertEqual(buf, bytearray(b'x'*4)) + buf = bytearray() + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + + def test_gh_99240_clear_args(self): + from _testcapi import gh_99240_clear_args + self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') + + +class Object_TestCase(unittest.TestCase): + def test_S(self): + from _testcapi import getargs_S + obj = b'bytes' + self.assertIs(getargs_S(obj), obj) + self.assertRaises(TypeError, getargs_S, bytearray(b'bytearray')) + self.assertRaises(TypeError, getargs_S, 'str') + self.assertRaises(TypeError, getargs_S, None) + self.assertRaises(TypeError, getargs_S, memoryview(obj)) + + def test_Y(self): + from _testcapi import getargs_Y + obj = bytearray(b'bytearray') + self.assertIs(getargs_Y(obj), obj) + self.assertRaises(TypeError, getargs_Y, b'bytes') + self.assertRaises(TypeError, getargs_Y, 'str') + self.assertRaises(TypeError, getargs_Y, None) + self.assertRaises(TypeError, getargs_Y, memoryview(obj)) + + def test_U(self): + from _testcapi import getargs_U + obj = 'str' + self.assertIs(getargs_U(obj), obj) + self.assertRaises(TypeError, getargs_U, b'bytes') + self.assertRaises(TypeError, getargs_U, bytearray(b'bytearray')) + self.assertRaises(TypeError, getargs_U, None) + + +# Bug #6012 +class Test6012(unittest.TestCase): + def test(self): + self.assertEqual(_testcapi.argparsing("Hello", "World"), 1) + + +class SkipitemTest(unittest.TestCase): + + # u, and Z raises DeprecationWarning + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_skipitem(self): + """ + If this test failed, you probably added a new "format unit" + in Python/getargs.c, but neglected to update our poor friend + skipitem() in the same file. (If so, shame on you!) + + With a few exceptions**, this function brute-force tests all + printable ASCII*** characters (32 to 126 inclusive) as format units, + checking to see that PyArg_ParseTupleAndKeywords() return consistent + errors both when the unit is attempted to be used and when it is + skipped. If the format unit doesn't exist, we'll get one of two + specific error messages (one for used, one for skipped); if it does + exist we *won't* get that error--we'll get either no error or some + other error. If we get the specific "does not exist" error for one + test and not for the other, there's a mismatch, and the test fails. + + ** Some format units have special funny semantics and it would + be difficult to accommodate them here. Since these are all + well-established and properly skipped in skipitem() we can + get away with not testing them--this test is really intended + to catch *new* format units. + + *** Python C source files must be ASCII. Therefore it's impossible + to have non-ASCII format units. + + """ + empty_tuple = () + tuple_1 = (0,) + dict_b = {'b':1} + keywords = ["a", "b"] + + for i in range(32, 127): + c = chr(i) + + # skip parentheses, the error reporting is inconsistent about them + # skip 'e' and 'w', they're always two-character codes + # skip '|' and '$', they don't represent arguments anyway + if c in '()ew|$': + continue + + # test the format unit when not skipped + format = c + "i" + try: + _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, + format, keywords) + when_not_skipped = False + except SystemError as e: + s = "argument 1 (impossible)" + when_not_skipped = (str(e) == s) + except TypeError: + when_not_skipped = False + + # test the format unit when skipped + optional_format = "|" + format + try: + _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, + optional_format, keywords) + when_skipped = False + except SystemError as e: + s = "impossible: '{}'".format(format) + when_skipped = (str(e) == s) + + message = ("test_skipitem_parity: " + "detected mismatch between convertsimple and skipitem " + "for format unit '{}' ({}), not skipped {}, skipped {}".format( + c, i, when_skipped, when_not_skipped)) + self.assertIs(when_skipped, when_not_skipped, message) + + def test_skipitem_with_suffix(self): + parse = _testcapi.parse_tuple_and_keywords + empty_tuple = () + tuple_1 = (0,) + dict_b = {'b':1} + keywords = ["a", "b"] + + supported = ('s#', 's*', 'z#', 'z*', 'y#', 'y*', 'w*') + for c in string.ascii_letters: + for c2 in '#*': + f = c + c2 + with self.subTest(format=f): + optional_format = "|" + f + "i" + if f in supported: + parse(empty_tuple, dict_b, optional_format, keywords) + else: + with self.assertRaisesRegex(SystemError, + 'impossible'): + parse(empty_tuple, dict_b, optional_format, keywords) + + for c in map(chr, range(32, 128)): + f = 'e' + c + optional_format = "|" + f + "i" + with self.subTest(format=f): + if c in 'st': + parse(empty_tuple, dict_b, optional_format, keywords) + else: + with self.assertRaisesRegex(SystemError, + 'impossible'): + parse(empty_tuple, dict_b, optional_format, keywords) + + +class ParseTupleAndKeywords_Test(unittest.TestCase): + + def test_parse_tuple_and_keywords(self): + # Test handling errors in the parse_tuple_and_keywords helper itself + self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords, + (), {}, 42, []) + self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, + (), {}, '', 42) + self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, + (), {}, '', [''] * 42) + self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, + (), {}, '', [42]) + + def test_basic(self): + parse = _testcapi.parse_tuple_and_keywords + + self.assertEqual(parse((), {'a': 1}, 'O', ['a']), (1,)) + self.assertEqual(parse((), {}, '|O', ['a']), (NULL,)) + self.assertEqual(parse((1, 2), {}, 'OO', ['a', 'b']), (1, 2)) + self.assertEqual(parse((1,), {'b': 2}, 'OO', ['a', 'b']), (1, 2)) + self.assertEqual(parse((), {'a': 1, 'b': 2}, 'OO', ['a', 'b']), (1, 2)) + self.assertEqual(parse((), {'b': 2}, '|OO', ['a', 'b']), (NULL, 2)) + + with self.assertRaisesRegex(TypeError, + "function missing required argument 'a'"): + parse((), {}, 'O', ['a']) + with self.assertRaisesRegex(TypeError, + "this function got an unexpected keyword argument 'b'"): + parse((), {'b': 1}, '|O', ['a']) + with self.assertRaisesRegex(TypeError, + fr"argument for function given by name \('a'\) " + fr"and position \(1\)"): + parse((1,), {'a': 2}, 'O|O', ['a', 'b']) + + def test_bad_use(self): + # Test handling invalid format and keywords in + # PyArg_ParseTupleAndKeywords() + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '||O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1, 2), {}, '|O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$$O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O$O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$|O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|OO', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|$O', ['']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|OO', ['a', '']) + + def test_positional_only(self): + parse = _testcapi.parse_tuple_and_keywords + + self.assertEqual(parse((1, 2, 3), {}, 'OOO', ['', '', 'a']), (1, 2, 3)) + self.assertEqual(parse((1, 2), {'a': 3}, 'OOO', ['', '', 'a']), (1, 2, 3)) + with self.assertRaisesRegex(TypeError, + r'function takes at least 2 positional arguments \(1 given\)'): + parse((1,), {'a': 3}, 'OOO', ['', '', 'a']) + self.assertEqual(parse((1,), {}, 'O|OO', ['', '', 'a']), + (1, NULL, NULL)) + with self.assertRaisesRegex(TypeError, + r'function takes at least 1 positional argument \(0 given\)'): + parse((), {}, 'O|OO', ['', '', 'a']) + self.assertEqual(parse((1, 2), {'a': 3}, 'OO$O', ['', '', 'a']), + (1, 2, 3)) + with self.assertRaisesRegex(TypeError, + r'function takes exactly 2 positional arguments \(1 given\)'): + parse((1,), {'a': 3}, 'OO$O', ['', '', 'a']) + self.assertEqual(parse((1,), {}, 'O|O$O', ['', '', 'a']), + (1, NULL, NULL)) + with self.assertRaisesRegex(TypeError, + r'function takes at least 1 positional argument \(0 given\)'): + parse((), {}, 'O|O$O', ['', '', 'a']) + with self.assertRaisesRegex(SystemError, r'Empty parameter name after \$'): + parse((1,), {}, 'O|$OO', ['', '', 'a']) + with self.assertRaisesRegex(SystemError, 'Empty keyword'): + parse((1,), {}, 'O|OO', ['', 'a', '']) + + def test_nonascii_keywords(self): + parse = _testcapi.parse_tuple_and_keywords + + for name in ('a', 'ä', 'ŷ', '㷷', '𐀀'): + with self.subTest(name=name): + self.assertEqual(parse((), {name: 1}, 'O', [name]), (1,)) + self.assertEqual(parse((), {}, '|O', [name]), (NULL,)) + with self.assertRaisesRegex(TypeError, + f"function missing required argument '{name}'"): + parse((), {}, 'O', [name]) + with self.assertRaisesRegex(TypeError, + fr"argument for function given by name \('{name}'\) " + fr"and position \(1\)"): + parse((1,), {name: 2}, 'O|O', [name, 'b']) + with self.assertRaisesRegex(TypeError, + f"this function got an unexpected keyword argument '{name}'"): + parse((), {name: 1}, '|O', ['b']) + with self.assertRaisesRegex(TypeError, + "this function got an unexpected keyword argument 'b'"): + parse((), {'b': 1}, '|O', [name]) + + invalid = name.encode() + (name.encode()[:-1] or b'\x80') + self.assertEqual(parse((), {}, '|O', [invalid]), (NULL,)) + self.assertEqual(parse((1,), {'b': 2}, 'O|O', [invalid, 'b']), + (1, 2)) + with self.assertRaisesRegex(TypeError, + f"function missing required argument '{name}\ufffd'"): + parse((), {}, 'O', [invalid]) + with self.assertRaisesRegex(UnicodeDecodeError, + f"'utf-8' codec can't decode bytes? "): + parse((), {'b': 1}, '|OO', [invalid, 'b']) + with self.assertRaisesRegex(UnicodeDecodeError, + f"'utf-8' codec can't decode bytes? "): + parse((), {'b': 1}, '|O', [invalid]) + + for name2 in ('b', 'ë', 'ĉ', 'Ɐ', '𐀁'): + with self.subTest(name2=name2): + with self.assertRaisesRegex(TypeError, + f"this function got an unexpected keyword argument '{name2}'"): + parse((), {name2: 1}, '|O', [name]) + + name2 = name.encode().decode('latin1') + if name2 != name: + with self.assertRaisesRegex(TypeError, + f"this function got an unexpected keyword argument '{name2}'"): + parse((), {name2: 1}, '|O', [name]) + name3 = name + '3' + with self.assertRaisesRegex(TypeError, + f"this function got an unexpected keyword argument '{name2}'"): + parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) + + def test_nested_sequence(self): + parse = _testcapi.parse_tuple_and_keywords + + self.assertEqual(parse(((1, 2, 3),), {}, '(OOO)', ['a']), (1, 2, 3)) + self.assertEqual(parse((1, (2, 3), 4), {}, 'O(OO)O', ['a', 'b', 'c']), + (1, 2, 3, 4)) + parse(((1, 2, 3),), {}, '(iii)', ['a']) + parse(([1, 2, 3],), {}, '(iii)', ['a']) + + with self.assertRaisesRegex(TypeError, + "argument 1 must be tuple of length 2, not 3"): + parse(((1, 2, 3),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be tuple of length 2, not 1"): + parse(((1,),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 3"): + parse(([1, 2, 3],), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 2, not 1"): + parse(([1,],), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not int"): + parse((1,), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not None$"): + parse((None,), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not str"): + parse(('ab',), {}, '(CC)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not bytes"): + parse((b'ab',), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not bytearray"): + parse((bytearray(b'ab'),), {}, '(ii)', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be 2-item tuple, not dict"): + parse(({},), {}, '(ii)', ['a']) + + with self.assertWarnsRegex(DeprecationWarning, + "argument must be 3-item tuple, not list"): + self.assertEqual(parse(([1, 2, 3],), {}, '(OOO)', ['a']), (1, 2, 3)) + with self.assertWarnsRegex(DeprecationWarning, + "argument must be 2-item tuple, not list"): + with self.assertRaisesRegex(TypeError, + "argument 1 must be tuple of length 2, not 3"): + parse(([1, 2, 3],), {}, '(OO)', ['a']) + with self.assertWarnsRegex(DeprecationWarning, + "argument must be 2-item tuple, not list"): + with self.assertRaisesRegex(TypeError, + "argument 1 must be tuple of length 2, not 1"): + parse(([1,],), {}, '(OO)', ['a']) + + for f in 'es', 'et', 'es#', 'et#': + with self.assertRaises(LookupError): # empty encoding "" + parse((('a',),), {}, '(' + f + ')', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be tuple of length 1, not 0"): + parse(((),), {}, '(' + f + ')', ['a']) + with self.assertRaisesRegex(TypeError, + "argument 1 must be sequence of length 1, not 0"): + parse(([],), {}, '(' + f + ')', ['a']) + + @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') + def test_gh_119213(self): + rc, out, err = script_helper.assert_python_ok("-c", """if True: + from test import support + script = '''if True: + import _testinternalcapi + _testinternalcapi.gh_119213_getargs(spam='eggs') + ''' + config = dict( + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + use_main_obmalloc=False, + gil=2, + check_multi_interp_extensions=True, + ) + rc = support.run_in_subinterp_with_config(script, **config) + assert rc == 0 + + # The crash is different if the interpreter was not destroyed first. + #interpid = _testinternalcapi.create_interpreter() + #rc = _testinternalcapi.exec_interpreter(interpid, script) + #assert rc == 0 + """) + self.assertEqual(rc, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_hash.py b/Lib/test/test_capi/test_hash.py new file mode 100644 index 00000000000..f553ffb0d90 --- /dev/null +++ b/Lib/test/test_capi/test_hash.py @@ -0,0 +1,93 @@ +import sys +import unittest +from test.support import import_helper +_testcapi = import_helper.import_module('_testcapi') + + +SIZEOF_VOID_P = _testcapi.SIZEOF_VOID_P +SIZEOF_PY_HASH_T = SIZEOF_VOID_P + + +class CAPITest(unittest.TestCase): + def test_hash_getfuncdef(self): + # Test PyHash_GetFuncDef() + hash_getfuncdef = _testcapi.hash_getfuncdef + func_def = hash_getfuncdef() + + match func_def.name: + case "fnv": + self.assertEqual(func_def.hash_bits, 8 * SIZEOF_PY_HASH_T) + self.assertEqual(func_def.seed_bits, 16 * SIZEOF_PY_HASH_T) + case "siphash13": + self.assertEqual(func_def.hash_bits, 64) + self.assertEqual(func_def.seed_bits, 128) + case "siphash24": + self.assertEqual(func_def.hash_bits, 64) + self.assertEqual(func_def.seed_bits, 128) + case _: + self.fail(f"unknown function name: {func_def.name!r}") + + # compare with sys.hash_info + hash_info = sys.hash_info + self.assertEqual(func_def.name, hash_info.algorithm) + self.assertEqual(func_def.hash_bits, hash_info.hash_bits) + self.assertEqual(func_def.seed_bits, hash_info.seed_bits) + + def test_hash_pointer(self): + # Test Py_HashPointer() + hash_pointer = _testcapi.hash_pointer + + UHASH_T_MASK = ((2 ** (8 * SIZEOF_PY_HASH_T)) - 1) + HASH_T_MAX = (2 ** (8 * SIZEOF_PY_HASH_T - 1) - 1) + + def python_hash_pointer(x): + # Py_HashPointer() rotates the pointer bits by 4 bits to the right + x = (x >> 4) | ((x & 15) << (8 * SIZEOF_VOID_P - 4)) + + # Convert unsigned uintptr_t (Py_uhash_t) to signed Py_hash_t + if HASH_T_MAX < x: + x = (~x) + 1 + x &= UHASH_T_MASK + x = (~x) + 1 + return x + + if SIZEOF_VOID_P == 8: + values = ( + 0xABCDEF1234567890, + 0x1234567890ABCDEF, + 0xFEE4ABEDD1CECA5E, + ) + else: + values = ( + 0x12345678, + 0x1234ABCD, + 0xDEADCAFE, + ) + + for value in values: + expected = python_hash_pointer(value) + with self.subTest(value=value): + self.assertEqual(hash_pointer(value), expected, + f"hash_pointer({value:x}) = " + f"{hash_pointer(value):x} != {expected:x}") + + # Py_HashPointer(NULL) returns 0 + self.assertEqual(hash_pointer(0), 0) + + # Py_HashPointer((void*)(uintptr_t)-1) doesn't return -1 but -2 + VOID_P_MAX = -1 & (2 ** (8 * SIZEOF_VOID_P) - 1) + self.assertEqual(hash_pointer(VOID_P_MAX), -2) + + def test_hash_buffer(self): + hash_buffer = _testcapi.hash_buffer + + def check(data): + self.assertEqual(hash_buffer(data), hash(data)) + + check(b'') + check(b'abc') + check(b'x' * 1024) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py new file mode 100644 index 00000000000..660e8a0e789 --- /dev/null +++ b/Lib/test/test_capi/test_immortal.py @@ -0,0 +1,42 @@ +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') + + +class TestUnstableCAPI(unittest.TestCase): + def test_immortal(self): + # Not extensive + known_immortals = (True, False, None, 0, ()) + for immortal in known_immortals: + with self.subTest(immortal=immortal): + self.assertTrue(_testcapi.is_immortal(immortal)) + + # Some arbitrary mutable objects + non_immortals = (object(), self, [object()]) + for non_immortal in non_immortals: + with self.subTest(non_immortal=non_immortal): + self.assertFalse(_testcapi.is_immortal(non_immortal)) + + # CRASHES _testcapi.is_immortal(NULL) + + +class TestInternalCAPI(unittest.TestCase): + + def test_immortal_builtins(self): + for obj in range(-5, 256): + self.assertTrue(_testinternalcapi.is_static_immortal(obj)) + self.assertTrue(_testinternalcapi.is_static_immortal(None)) + self.assertTrue(_testinternalcapi.is_static_immortal(False)) + self.assertTrue(_testinternalcapi.is_static_immortal(True)) + self.assertTrue(_testinternalcapi.is_static_immortal(...)) + self.assertTrue(_testinternalcapi.is_static_immortal(())) + for obj in range(300, 400): + self.assertFalse(_testinternalcapi.is_static_immortal(obj)) + for obj in ([], {}, set()): + self.assertFalse(_testinternalcapi.is_static_immortal(obj)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_import.py b/Lib/test/test_capi/test_import.py new file mode 100644 index 00000000000..25136624ca4 --- /dev/null +++ b/Lib/test/test_capi/test_import.py @@ -0,0 +1,381 @@ +import importlib.util +import os.path +import sys +import types +import unittest +from test.support import os_helper +from test.support import import_helper +from test.support.warnings_helper import check_warnings + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +NULL = None + + +class ImportTests(unittest.TestCase): + def test_getmagicnumber(self): + # Test PyImport_GetMagicNumber() + magic = _testlimitedcapi.PyImport_GetMagicNumber() + self.assertEqual(magic, + int.from_bytes(importlib.util.MAGIC_NUMBER, 'little')) + + def test_getmagictag(self): + # Test PyImport_GetMagicTag() + tag = _testlimitedcapi.PyImport_GetMagicTag() + self.assertEqual(tag, sys.implementation.cache_tag) + + def test_getmoduledict(self): + # Test PyImport_GetModuleDict() + modules = _testlimitedcapi.PyImport_GetModuleDict() + self.assertIs(modules, sys.modules) + + def check_import_loaded_module(self, import_module): + for name in ('os', 'sys', 'test', 'unittest'): + with self.subTest(name=name): + self.assertIn(name, sys.modules) + old_module = sys.modules[name] + module = import_module(name) + self.assertIsInstance(module, types.ModuleType) + self.assertIs(module, old_module) + + def check_import_fresh_module(self, import_module): + old_modules = dict(sys.modules) + try: + for name in ('colorsys', 'math'): + with self.subTest(name=name): + sys.modules.pop(name, None) + module = import_module(name) + self.assertIsInstance(module, types.ModuleType) + self.assertIs(module, sys.modules[name]) + self.assertEqual(module.__name__, name) + finally: + sys.modules.clear() + sys.modules.update(old_modules) + + def test_getmodule(self): + # Test PyImport_GetModule() + getmodule = _testlimitedcapi.PyImport_GetModule + self.check_import_loaded_module(getmodule) + + nonexistent = 'nonexistent' + self.assertNotIn(nonexistent, sys.modules) + self.assertIs(getmodule(nonexistent), KeyError) + self.assertIs(getmodule(''), KeyError) + self.assertIs(getmodule(object()), KeyError) + + self.assertRaises(TypeError, getmodule, []) # unhashable + # CRASHES getmodule(NULL) + + def check_addmodule(self, add_module, accept_nonstr=False): + # create a new module + names = ['nonexistent'] + if accept_nonstr: + names.append(b'\xff') # non-UTF-8 + # PyImport_AddModuleObject() accepts non-string names + names.append(tuple(['hashable non-string'])) + for name in names: + with self.subTest(name=name): + self.assertNotIn(name, sys.modules) + try: + module = add_module(name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, name) + self.assertIs(module, sys.modules[name]) + finally: + sys.modules.pop(name, None) + + # get an existing module + self.check_import_loaded_module(add_module) + + def test_addmoduleobject(self): + # Test PyImport_AddModuleObject() + addmoduleobject = _testlimitedcapi.PyImport_AddModuleObject + self.check_addmodule(addmoduleobject, accept_nonstr=True) + + self.assertRaises(TypeError, addmoduleobject, []) # unhashable + # CRASHES addmoduleobject(NULL) + + def test_addmodule(self): + # Test PyImport_AddModule() + addmodule = _testlimitedcapi.PyImport_AddModule + self.check_addmodule(addmodule) + + self.assertRaises(UnicodeDecodeError, addmodule, b'\xff') + # CRASHES addmodule(NULL) + + def test_addmoduleref(self): + # Test PyImport_AddModuleRef() + addmoduleref = _testlimitedcapi.PyImport_AddModuleRef + self.check_addmodule(addmoduleref) + + self.assertRaises(UnicodeDecodeError, addmoduleref, b'\xff') + # CRASHES addmoduleref(NULL) + + def check_import_func(self, import_module): + self.check_import_loaded_module(import_module) + self.check_import_fresh_module(import_module) + self.assertRaises(ModuleNotFoundError, import_module, 'nonexistent') + self.assertRaises(ValueError, import_module, '') + + def test_import(self): + # Test PyImport_Import() + import_ = _testlimitedcapi.PyImport_Import + self.check_import_func(import_) + + self.assertRaises(TypeError, import_, b'os') + self.assertRaises(SystemError, import_, NULL) + + def test_importmodule(self): + # Test PyImport_ImportModule() + importmodule = _testlimitedcapi.PyImport_ImportModule + self.check_import_func(importmodule) + + self.assertRaises(UnicodeDecodeError, importmodule, b'\xff') + # CRASHES importmodule(NULL) + + def test_importmodulenoblock(self): + # Test deprecated PyImport_ImportModuleNoBlock() + importmodulenoblock = _testlimitedcapi.PyImport_ImportModuleNoBlock + with check_warnings(('', DeprecationWarning)): + self.check_import_func(importmodulenoblock) + self.assertRaises(UnicodeDecodeError, importmodulenoblock, b'\xff') + + # CRASHES importmodulenoblock(NULL) + + def check_frozen_import(self, import_frozen_module): + # Importing a frozen module executes its code, so start by unloading + # the module to execute the code in a new (temporary) module. + old_zipimport = sys.modules.pop('zipimport') + try: + self.assertEqual(import_frozen_module('zipimport'), 1) + + # import zipimport again + self.assertEqual(import_frozen_module('zipimport'), 1) + finally: + sys.modules['zipimport'] = old_zipimport + + # not a frozen module + self.assertEqual(import_frozen_module('sys'), 0) + self.assertEqual(import_frozen_module('nonexistent'), 0) + self.assertEqual(import_frozen_module(''), 0) + + def test_importfrozenmodule(self): + # Test PyImport_ImportFrozenModule() + importfrozenmodule = _testlimitedcapi.PyImport_ImportFrozenModule + self.check_frozen_import(importfrozenmodule) + + self.assertRaises(UnicodeDecodeError, importfrozenmodule, b'\xff') + # CRASHES importfrozenmodule(NULL) + + def test_importfrozenmoduleobject(self): + # Test PyImport_ImportFrozenModuleObject() + importfrozenmoduleobject = _testlimitedcapi.PyImport_ImportFrozenModuleObject + self.check_frozen_import(importfrozenmoduleobject) + self.assertEqual(importfrozenmoduleobject(b'zipimport'), 0) + self.assertEqual(importfrozenmoduleobject(NULL), 0) + + def test_importmoduleex(self): + # Test PyImport_ImportModuleEx() + importmoduleex = _testlimitedcapi.PyImport_ImportModuleEx + self.check_import_func(lambda name: importmoduleex(name, NULL, NULL, NULL)) + + self.assertRaises(ModuleNotFoundError, importmoduleex, 'nonexistent', NULL, NULL, NULL) + self.assertRaises(ValueError, importmoduleex, '', NULL, NULL, NULL) + self.assertRaises(UnicodeDecodeError, importmoduleex, b'\xff', NULL, NULL, NULL) + # CRASHES importmoduleex(NULL, NULL, NULL, NULL) + + def check_importmodulelevel(self, importmodulelevel): + self.check_import_func(lambda name: importmodulelevel(name, NULL, NULL, NULL, 0)) + + self.assertRaises(ModuleNotFoundError, importmodulelevel, 'nonexistent', NULL, NULL, NULL, 0) + self.assertRaises(ValueError, importmodulelevel, '', NULL, NULL, NULL, 0) + + if __package__: + self.assertIs(importmodulelevel('test_import', globals(), NULL, NULL, 1), + sys.modules['test.test_capi.test_import']) + self.assertIs(importmodulelevel('test_capi', globals(), NULL, NULL, 2), + sys.modules['test.test_capi']) + self.assertRaises(ValueError, importmodulelevel, 'os', NULL, NULL, NULL, -1) + with self.assertWarns(ImportWarning): + self.assertRaises(KeyError, importmodulelevel, 'test_import', {}, NULL, NULL, 1) + self.assertRaises(TypeError, importmodulelevel, 'test_import', [], NULL, NULL, 1) + + def test_importmodulelevel(self): + # Test PyImport_ImportModuleLevel() + importmodulelevel = _testlimitedcapi.PyImport_ImportModuleLevel + self.check_importmodulelevel(importmodulelevel) + + self.assertRaises(UnicodeDecodeError, importmodulelevel, b'\xff', NULL, NULL, NULL, 0) + # CRASHES importmodulelevel(NULL, NULL, NULL, NULL, 0) + + def test_importmodulelevelobject(self): + # Test PyImport_ImportModuleLevelObject() + importmodulelevel = _testlimitedcapi.PyImport_ImportModuleLevelObject + self.check_importmodulelevel(importmodulelevel) + + self.assertRaises(TypeError, importmodulelevel, b'os', NULL, NULL, NULL, 0) + self.assertRaises(ValueError, importmodulelevel, NULL, NULL, NULL, NULL, 0) + + def check_executecodemodule(self, execute_code, *args): + name = 'test_import_executecode' + try: + # Create a temporary module where the code will be executed + self.assertNotIn(name, sys.modules) + module = _testlimitedcapi.PyImport_AddModuleRef(name) + self.assertNotHasAttr(module, 'attr') + + # Execute the code + code = compile('attr = 1', '', 'exec') + module2 = execute_code(name, code, *args) + self.assertIs(module2, module) + + # Check the function side effects + self.assertEqual(module.attr, 1) + finally: + sys.modules.pop(name, None) + return module.__spec__.origin + + def test_executecodemodule(self): + # Test PyImport_ExecCodeModule() + execcodemodule = _testlimitedcapi.PyImport_ExecCodeModule + self.check_executecodemodule(execcodemodule) + + code = compile('attr = 1', '', 'exec') + self.assertRaises(UnicodeDecodeError, execcodemodule, b'\xff', code) + # CRASHES execcodemodule(NULL, code) + # CRASHES execcodemodule(name, NULL) + + def test_executecodemoduleex(self): + # Test PyImport_ExecCodeModuleEx() + execcodemoduleex = _testlimitedcapi.PyImport_ExecCodeModuleEx + + # Test NULL path (it should not crash) + self.check_executecodemodule(execcodemoduleex, NULL) + + # Test non-NULL path + pathname = b'pathname' + origin = self.check_executecodemodule(execcodemoduleex, pathname) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + pathname = os_helper.TESTFN_UNDECODABLE + if pathname: + origin = self.check_executecodemodule(execcodemoduleex, pathname) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + code = compile('attr = 1', '', 'exec') + self.assertRaises(UnicodeDecodeError, execcodemoduleex, b'\xff', code, NULL) + # CRASHES execcodemoduleex(NULL, code, NULL) + # CRASHES execcodemoduleex(name, NULL, NULL) + + def check_executecode_pathnames(self, execute_code_func, object=False): + # Test non-NULL pathname and NULL cpathname + + # Test NULL paths (it should not crash) + self.check_executecodemodule(execute_code_func, NULL, NULL) + + pathname = 'pathname' + origin = self.check_executecodemodule(execute_code_func, pathname, NULL) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + origin = self.check_executecodemodule(execute_code_func, NULL, pathname) + if not object: + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + pathname = os_helper.TESTFN_UNDECODABLE + if pathname: + if object: + pathname = os.fsdecode(pathname) + origin = self.check_executecodemodule(execute_code_func, pathname, NULL) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + self.check_executecodemodule(execute_code_func, NULL, pathname) + + # Test NULL pathname and non-NULL cpathname + pyc_filename = importlib.util.cache_from_source(__file__) + py_filename = importlib.util.source_from_cache(pyc_filename) + origin = self.check_executecodemodule(execute_code_func, NULL, pyc_filename) + if not object: + self.assertEqual(origin, py_filename) + + def test_executecodemodulewithpathnames(self): + # Test PyImport_ExecCodeModuleWithPathnames() + execute_code_func = _testlimitedcapi.PyImport_ExecCodeModuleWithPathnames + self.check_executecode_pathnames(execute_code_func) + + code = compile('attr = 1', '', 'exec') + self.assertRaises(UnicodeDecodeError, execute_code_func, b'\xff', code, NULL, NULL) + # CRASHES execute_code_func(NULL, code, NULL, NULL) + # CRASHES execute_code_func(name, NULL, NULL, NULL) + + def test_executecodemoduleobject(self): + # Test PyImport_ExecCodeModuleObject() + execute_code_func = _testlimitedcapi.PyImport_ExecCodeModuleObject + self.check_executecode_pathnames(execute_code_func, object=True) + + code = compile('attr = 1', '', 'exec') + self.assertRaises(TypeError, execute_code_func, [], code, NULL, NULL) + nonstring = tuple(['hashable non-string']) + self.assertRaises(AttributeError, execute_code_func, nonstring, code, NULL, NULL) + sys.modules.pop(nonstring, None) + # CRASHES execute_code_func(NULL, code, NULL, NULL) + # CRASHES execute_code_func(name, NULL, NULL, NULL) + + def check_importmoduleattr(self, importmoduleattr): + self.assertIs(importmoduleattr('sys', 'argv'), sys.argv) + self.assertIs(importmoduleattr('types', 'ModuleType'), types.ModuleType) + + # module name containing a dot + attr = importmoduleattr('email.message', 'Message') + from email.message import Message + self.assertIs(attr, Message) + + with self.assertRaises(ImportError): + # nonexistent module + importmoduleattr('nonexistentmodule', 'attr') + with self.assertRaises(AttributeError): + # nonexistent attribute + importmoduleattr('sys', 'nonexistentattr') + with self.assertRaises(AttributeError): + # attribute name containing a dot + importmoduleattr('sys', 'implementation.name') + + def test_importmoduleattr(self): + # Test PyImport_ImportModuleAttr() + importmoduleattr = _testcapi.PyImport_ImportModuleAttr + self.check_importmoduleattr(importmoduleattr) + + # Invalid module name type + for mod_name in (object(), 123, b'bytes'): + with self.subTest(mod_name=mod_name): + with self.assertRaises(TypeError): + importmoduleattr(mod_name, "attr") + + # Invalid attribute name type + for attr_name in (object(), 123, b'bytes'): + with self.subTest(attr_name=attr_name): + with self.assertRaises(TypeError): + importmoduleattr("sys", attr_name) + + with self.assertRaises(SystemError): + importmoduleattr(NULL, "argv") + # CRASHES importmoduleattr("sys", NULL) + + def test_importmoduleattrstring(self): + # Test PyImport_ImportModuleAttrString() + importmoduleattr = _testcapi.PyImport_ImportModuleAttrString + self.check_importmoduleattr(importmoduleattr) + + with self.assertRaises(UnicodeDecodeError): + importmoduleattr(b"sys\xff", "argv") + with self.assertRaises(UnicodeDecodeError): + importmoduleattr("sys", b"argv\xff") + + # CRASHES importmoduleattr(NULL, "argv") + # CRASHES importmoduleattr("sys", NULL) + + # TODO: test PyImport_GetImporter() + # TODO: test PyImport_ReloadModule() + # TODO: test PyImport_ExtendInittab() + # PyImport_AppendInittab() is tested by test_embed + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py new file mode 100644 index 00000000000..b95b8ba960b --- /dev/null +++ b/Lib/test/test_capi/test_list.py @@ -0,0 +1,359 @@ +import unittest +from test.support import import_helper +from collections import UserList +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None +PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN +PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX + +class ListSubclass(list): + pass + + +class DelAppend: + def __init__(self, lst, item): + self.lst = lst + self.item = item + + def __del__(self): + self.lst.append(self.item) + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyList_Check() + check = _testlimitedcapi.list_check + self.assertTrue(check([1, 2])) + self.assertTrue(check([])) + self.assertTrue(check(ListSubclass([1, 2]))) + self.assertFalse(check({1: 2})) + self.assertFalse(check((1, 2))) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + + def test_list_check_exact(self): + # Test PyList_CheckExact() + check = _testlimitedcapi.list_check_exact + self.assertTrue(check([1])) + self.assertTrue(check([])) + self.assertFalse(check(ListSubclass([1]))) + self.assertFalse(check(UserList([1, 2]))) + self.assertFalse(check({1: 2})) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_list_new(self): + # Test PyList_New() + list_new = _testlimitedcapi.list_new + lst = list_new(0) + self.assertEqual(lst, []) + self.assertIs(type(lst), list) + lst2 = list_new(0) + self.assertIsNot(lst2, lst) + self.assertRaises(SystemError, list_new, NULL) + self.assertRaises(SystemError, list_new, -1) + + def test_list_size(self): + # Test PyList_Size() + size = _testlimitedcapi.list_size + self.assertEqual(size([]), 0) + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size(ListSubclass([1, 2])), 2) + self.assertRaises(SystemError, size, UserList()) + self.assertRaises(SystemError, size, {}) + self.assertRaises(SystemError, size, 23) + self.assertRaises(SystemError, size, object()) + # CRASHES size(NULL) + + def test_list_get_size(self): + # Test PyList_GET_SIZE() + size = _testcapi.list_get_size + self.assertEqual(size([]), 0) + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size(ListSubclass([1, 2])), 2) + # CRASHES size(object()) + # CRASHES size(23) + # CRASHES size({}) + # CRASHES size(UserList()) + # CRASHES size(NULL) + + def check_list_get_item(self, getitem, exctype): + # Common test cases for PyList_GetItem() and PyList_GetItemRef() + lst = [1, 2, 3] + self.assertEqual(getitem(lst, 0), 1) + self.assertEqual(getitem(lst, 2), 3) + self.assertRaises(IndexError, getitem, lst, 3) + self.assertRaises(IndexError, getitem, lst, -1) + self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) + self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) + self.assertRaises(exctype, getitem, 42, 1) + self.assertRaises(exctype, getitem, (1, 2, 3), 1) + self.assertRaises(exctype, getitem, {1: 2}, 1) + # CRASHES getitem(NULL, 1) + + def test_list_getitem(self): + # Test PyList_GetItem() + self.check_list_get_item(_testlimitedcapi.list_getitem, SystemError) + + def test_list_get_item_ref(self): + # Test PyList_GetItemRef() + self.check_list_get_item(_testlimitedcapi.list_get_item_ref, TypeError) + + def test_list_get_item(self): + # Test PyList_GET_ITEM() + get_item = _testcapi.list_get_item + lst = [1, 2, [1, 2, 3]] + self.assertEqual(get_item(lst, 0), 1) + self.assertEqual(get_item(lst, 2), [1, 2, 3]) + + # CRASHES for out of index: get_item(lst, 3) + # CRASHES for get_item(lst, PY_SSIZE_T_MIN) + # CRASHES for get_item(lst, PY_SSIZE_T_MAX) + # CRASHES get_item(21, 2) + # CRASHES get_item(NULL, 1) + + def test_list_setitem(self): + # Test PyList_SetItem() + setitem = _testlimitedcapi.list_setitem + lst = [1, 2, 3] + setitem(lst, 0, 10) + self.assertEqual(lst, [10, 2, 3]) + setitem(lst, 2, 12) + self.assertEqual(lst, [10, 2, 12]) + self.assertRaises(IndexError, setitem, lst, 3 , 5) + self.assertRaises(IndexError, setitem, lst, -1, 5) + self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MIN, 5) + self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MAX, 5) + self.assertRaises(SystemError, setitem, (1, 2, 3), 1, 5) + self.assertRaises(SystemError, setitem, {1: 2}, 1, 5) + + # CRASHES setitem(NULL, 'a', 5) + + def test_list_set_item(self): + # Test PyList_SET_ITEM() + set_item = _testcapi.list_set_item + lst = [1, 2, 3] + set_item(lst, 1, 10) + set_item(lst, 2, [1, 2, 3]) + self.assertEqual(lst, [1, 10, [1, 2, 3]]) + + # CRASHES for set_item([1], -1, 5) + # CRASHES for set_item([1], PY_SSIZE_T_MIN, 5) + # CRASHES for set_item([1], PY_SSIZE_T_MAX, 5) + # CRASHES for set_item([], 0, 1) + # CRASHES for set_item(NULL, 0, 1) + + + def test_list_insert(self): + # Test PyList_Insert() + insert = _testlimitedcapi.list_insert + lst = [1, 2, 3] + insert(lst, 0, 23) + self.assertEqual(lst, [23, 1, 2, 3]) + insert(lst, -1, 22) + self.assertEqual(lst, [23, 1, 2, 22, 3]) + insert(lst, PY_SSIZE_T_MIN, 1) + self.assertEqual(lst[0], 1) + insert(lst, len(lst), 123) + self.assertEqual(lst[-1], 123) + insert(lst, len(lst)-1, 124) + self.assertEqual(lst[-2], 124) + insert(lst, PY_SSIZE_T_MAX, 223) + self.assertEqual(lst[-1], 223) + + self.assertRaises(SystemError, insert, (1, 2, 3), 1, 5) + self.assertRaises(SystemError, insert, {1: 2}, 1, 5) + + # CRASHES insert(NULL, 1, 5) + + def test_list_append(self): + # Test PyList_Append() + append = _testlimitedcapi.list_append + lst = [1, 2, 3] + append(lst, 10) + self.assertEqual(lst, [1, 2, 3, 10]) + append(lst, [4, 5]) + self.assertEqual(lst, [1, 2, 3, 10, [4, 5]]) + self.assertRaises(SystemError, append, lst, NULL) + self.assertRaises(SystemError, append, (), 0) + self.assertRaises(SystemError, append, 42, 0) + # CRASHES append(NULL, 0) + + def test_list_getslice(self): + # Test PyList_GetSlice() + getslice = _testlimitedcapi.list_getslice + lst = [1, 2, 3] + + # empty + self.assertEqual(getslice(lst, PY_SSIZE_T_MIN, 0), []) + self.assertEqual(getslice(lst, -1, 0), []) + self.assertEqual(getslice(lst, 3, PY_SSIZE_T_MAX), []) + + # slice + self.assertEqual(getslice(lst, 1, 3), [2, 3]) + + # whole + self.assertEqual(getslice(lst, 0, len(lst)), lst) + self.assertEqual(getslice(lst, 0, 100), lst) + self.assertEqual(getslice(lst, -100, 100), lst) + + self.assertRaises(SystemError, getslice, (1, 2, 3), 0, 0) + self.assertRaises(SystemError, getslice, 'abc', 0, 0) + self.assertRaises(SystemError, getslice, 42, 0, 0) + + # CRASHES getslice(NULL, 0, 0) + + def test_list_setslice(self): + # Test PyList_SetSlice() + list_setslice = _testlimitedcapi.list_setslice + def set_slice(lst, low, high, value): + lst = lst.copy() + self.assertEqual(list_setslice(lst, low, high, value), 0) + return lst + + # insert items + self.assertEqual(set_slice([], 0, 0, list("abc")), list("abc")) + self.assertEqual(set_slice([], PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, list("abc")), list("abc")) + self.assertEqual(set_slice([], PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, list("abc")), list("abc")) + lst = list("abc") + self.assertEqual(set_slice(lst, 0, 0, ["X"]), list("Xabc")) + self.assertEqual(set_slice(lst, 1, 1, list("XY")), list("aXYbc")) + self.assertEqual(set_slice(lst, len(lst), len(lst), ["X"]), list("abcX")) + # self.assertEqual(set_slice(lst, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, ["X"]), list("abcX")) + + # replace items + lst = list("abc") + self.assertEqual(set_slice(lst, -100, 1, list("X")), list("Xbc")) + self.assertEqual(set_slice(lst, 1, 2, list("X")), list("aXc")) + self.assertEqual(set_slice(lst, 1, 3, list("XY")), list("aXY")) + self.assertEqual(set_slice(lst, 0, 3, list("XYZ")), list("XYZ")) + + # delete items + lst = list("abcdef") + self.assertEqual(set_slice(lst, 0, len(lst), []), []) + self.assertEqual(set_slice(lst, -100, 100, []), []) + self.assertEqual(set_slice(lst, 1, 5, []), list("af")) + self.assertEqual(set_slice(lst, 3, len(lst), []), list("abc")) + + # delete items with NULL + lst = list("abcdef") + self.assertEqual(set_slice(lst, 0, len(lst), NULL), []) + self.assertEqual(set_slice(lst, 3, len(lst), NULL), list("abc")) + + self.assertRaises(SystemError, list_setslice, (), 0, 0, []) + self.assertRaises(SystemError, list_setslice, 42, 0, 0, []) + + # Item finalizer modify the list (clear the list) + lst = [] + lst.append(DelAppend(lst, 'zombie')) + self.assertEqual(list_setslice(lst, 0, len(lst), NULL), 0) + self.assertEqual(lst, ['zombie']) + + # Item finalizer modify the list (remove an list item) + lst = [] + lst.append(DelAppend(lst, 'zombie')) + lst.extend("abc") + self.assertEqual(list_setslice(lst, 0, 1, NULL), 0) + self.assertEqual(lst, ['a', 'b', 'c', 'zombie']) + + # CRASHES setslice(NULL, 0, 0, []) + + def test_list_sort(self): + # Test PyList_Sort() + sort = _testlimitedcapi.list_sort + lst = [4, 6, 7, 3, 1, 5, 9, 2, 0, 8] + sort(lst) + self.assertEqual(lst, list(range(10))) + + lst2 = ListSubclass([4, 6, 7, 3, 1, 5, 9, 2, 0, 8]) + sort(lst2) + self.assertEqual(lst2, list(range(10))) + + self.assertRaises(SystemError, sort, ()) + self.assertRaises(SystemError, sort, object()) + self.assertRaises(SystemError, sort, NULL) + + + def test_list_reverse(self): + # Test PyList_Reverse() + reverse = _testlimitedcapi.list_reverse + def list_reverse(lst): + self.assertEqual(reverse(lst), 0) + return lst + + self.assertEqual(list_reverse([]), []) + self.assertEqual(list_reverse([2, 5, 10]), [10, 5, 2]) + self.assertEqual(list_reverse(list_reverse([2, 5, 10])), [2, 5, 10]) + + self.assertRaises(SystemError, reverse, ()) + self.assertRaises(SystemError, reverse, object()) + self.assertRaises(SystemError, reverse, NULL) + + def test_list_astuple(self): + # Test PyList_AsTuple() + astuple = _testlimitedcapi.list_astuple + self.assertEqual(astuple([]), ()) + self.assertEqual(astuple([[]]), ([],)) + self.assertEqual(astuple([2, 5, 10]), (2, 5, 10)) + + self.assertRaises(SystemError, astuple, ()) + self.assertRaises(SystemError, astuple, object()) + self.assertRaises(SystemError, astuple, NULL) + + def test_list_clear(self): + # Test PyList_Clear() + list_clear = _testcapi.list_clear + + lst = [1, 2, 3] + self.assertEqual(list_clear(lst), 0) + self.assertEqual(lst, []) + + lst = [] + self.assertEqual(list_clear(lst), 0) + self.assertEqual(lst, []) + + self.assertRaises(SystemError, list_clear, ()) + self.assertRaises(SystemError, list_clear, object()) + + # Item finalizer modify the list + lst = [] + lst.append(DelAppend(lst, 'zombie')) + list_clear(lst) + self.assertEqual(lst, ['zombie']) + + # CRASHES list_clear(NULL) + + def test_list_extend(self): + # Test PyList_Extend() + list_extend = _testcapi.list_extend + + for other_type in (list, tuple, str, iter): + lst = list("ab") + arg = other_type("def") + self.assertEqual(list_extend(lst, arg), 0) + self.assertEqual(lst, list("abdef")) + + # PyList_Extend(lst, lst) + lst = list("abc") + self.assertEqual(list_extend(lst, lst), 0) + self.assertEqual(lst, list("abcabc")) + + self.assertRaises(TypeError, list_extend, [], object()) + self.assertRaises(SystemError, list_extend, (), list("abc")) + + # CRASHES list_extend(NULL, []) + # CRASHES list_extend([], NULL) + + def test_uninitialized_list_repr(self): + lst = _testlimitedcapi.list_new(3) + self.assertEqual(repr(lst), '[, , ]') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py new file mode 100644 index 00000000000..fc0454b71cb --- /dev/null +++ b/Lib/test/test_capi/test_long.py @@ -0,0 +1,818 @@ +import unittest +import sys +import test.support as support + +from test.support import import_helper + +# Skip this test if the _testcapi and _testlimitedcapi modules isn't available. +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None + + +class IntSubclass(int): + pass + +class Index: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + +# use __index__(), not __int__() +class MyIndexAndInt: + def __index__(self): + return 10 + def __int__(self): + return 22 + + +class LongTests(unittest.TestCase): + + def test_compact(self): + for n in { + # Edge cases + *(2**n for n in range(66)), + *(-2**n for n in range(66)), + *(2**n - 1 for n in range(66)), + *(-2**n + 1 for n in range(66)), + # Essentially random + *(37**n for n in range(14)), + *(-37**n for n in range(14)), + }: + with self.subTest(n=n): + is_compact, value = _testcapi.call_long_compact_api(n) + if is_compact: + self.assertEqual(n, value) + + def test_compact_known(self): + # Sanity-check some implementation details (we don't guarantee + # that these are/aren't compact) + self.assertEqual(_testcapi.call_long_compact_api(-1), (True, -1)) + self.assertEqual(_testcapi.call_long_compact_api(0), (True, 0)) + self.assertEqual(_testcapi.call_long_compact_api(256), (True, 256)) + self.assertEqual(_testcapi.call_long_compact_api(sys.maxsize), + (False, -1)) + + def test_long_check(self): + # Test PyLong_Check() + check = _testlimitedcapi.pylong_check + self.assertTrue(check(1)) + self.assertTrue(check(123456789012345678901234567890)) + self.assertTrue(check(-1)) + self.assertTrue(check(True)) + self.assertTrue(check(IntSubclass(1))) + self.assertFalse(check(1.0)) + self.assertFalse(check(object())) + # CRASHES check(NULL) + + def test_long_checkexact(self): + # Test PyLong_CheckExact() + check = _testlimitedcapi.pylong_checkexact + self.assertTrue(check(1)) + self.assertTrue(check(123456789012345678901234567890)) + self.assertTrue(check(-1)) + self.assertFalse(check(True)) + self.assertFalse(check(IntSubclass(1))) + self.assertFalse(check(1.0)) + self.assertFalse(check(object())) + # CRASHES check(NULL) + + def test_long_fromdouble(self): + # Test PyLong_FromDouble() + fromdouble = _testlimitedcapi.pylong_fromdouble + float_max = sys.float_info.max + for value in (5.0, 5.1, 5.9, -5.1, -5.9, 0.0, -0.0, float_max, -float_max): + with self.subTest(value=value): + self.assertEqual(fromdouble(value), int(value)) + self.assertRaises(OverflowError, fromdouble, float('inf')) + self.assertRaises(OverflowError, fromdouble, float('-inf')) + self.assertRaises(ValueError, fromdouble, float('nan')) + + def test_long_fromvoidptr(self): + # Test PyLong_FromVoidPtr() + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr + obj = object() + x = fromvoidptr(obj) + y = fromvoidptr(NULL) + self.assertIsInstance(x, int) + self.assertGreaterEqual(x, 0) + self.assertIsInstance(y, int) + self.assertEqual(y, 0) + self.assertNotEqual(x, y) + + def test_long_fromstring(self): + # Test PyLong_FromString() + fromstring = _testlimitedcapi.pylong_fromstring + self.assertEqual(fromstring(b'123', 10), (123, 3)) + self.assertEqual(fromstring(b'cafe', 16), (0xcafe, 4)) + self.assertEqual(fromstring(b'xyz', 36), (44027, 3)) + self.assertEqual(fromstring(b'123', 0), (123, 3)) + self.assertEqual(fromstring(b'0xcafe', 0), (0xcafe, 6)) + self.assertRaises(ValueError, fromstring, b'cafe', 0) + self.assertEqual(fromstring(b'-123', 10), (-123, 4)) + self.assertEqual(fromstring(b' -123 ', 10), (-123, 6)) + self.assertEqual(fromstring(b'1_23', 10), (123, 4)) + self.assertRaises(ValueError, fromstring, b'- 123', 10) + self.assertRaises(ValueError, fromstring, b'', 10) + + self.assertRaises(ValueError, fromstring, b'123', 1) + self.assertRaises(ValueError, fromstring, b'123', -1) + self.assertRaises(ValueError, fromstring, b'123', 37) + + self.assertRaises(ValueError, fromstring, '١٢٣٤٥٦٧٨٩٠'.encode(), 0) + self.assertRaises(ValueError, fromstring, '١٢٣٤٥٦٧٨٩٠'.encode(), 16) + + self.assertEqual(fromstring(b'123\x00', 0), (123, 3)) + self.assertEqual(fromstring(b'123\x00456', 0), (123, 3)) + self.assertEqual(fromstring(b'123\x00', 16), (0x123, 3)) + self.assertEqual(fromstring(b'123\x00456', 16), (0x123, 3)) + + # CRASHES fromstring(NULL, 0) + # CRASHES fromstring(NULL, 16) + + def test_long_fromunicodeobject(self): + # Test PyLong_FromUnicodeObject() + fromunicodeobject = _testcapi.pylong_fromunicodeobject + self.assertEqual(fromunicodeobject('123', 10), 123) + self.assertEqual(fromunicodeobject('cafe', 16), 0xcafe) + self.assertEqual(fromunicodeobject('xyz', 36), 44027) + self.assertEqual(fromunicodeobject('123', 0), 123) + self.assertEqual(fromunicodeobject('0xcafe', 0), 0xcafe) + self.assertRaises(ValueError, fromunicodeobject, 'cafe', 0) + self.assertEqual(fromunicodeobject('-123', 10), -123) + self.assertEqual(fromunicodeobject(' -123 ', 10), -123) + self.assertEqual(fromunicodeobject('1_23', 10), 123) + self.assertRaises(ValueError, fromunicodeobject, '- 123', 10) + self.assertRaises(ValueError, fromunicodeobject, '', 10) + + self.assertRaises(ValueError, fromunicodeobject, '123', 1) + self.assertRaises(ValueError, fromunicodeobject, '123', -1) + self.assertRaises(ValueError, fromunicodeobject, '123', 37) + + self.assertEqual(fromunicodeobject('١٢٣٤٥٦٧٨٩٠', 0), 1234567890) + self.assertEqual(fromunicodeobject('١٢٣٤٥٦٧٨٩٠', 16), 0x1234567890) + + self.assertRaises(ValueError, fromunicodeobject, '123\x00', 0) + self.assertRaises(ValueError, fromunicodeobject, '123\x00456', 0) + self.assertRaises(ValueError, fromunicodeobject, '123\x00', 16) + self.assertRaises(ValueError, fromunicodeobject, '123\x00456', 16) + + # CRASHES fromunicodeobject(NULL, 0) + # CRASHES fromunicodeobject(NULL, 16) + + def check_long_asint(self, func, min_val, max_val, *, + use_index=True, + mask=False, + negative_value_error=OverflowError): + # round trip (object -> C integer -> object) + values = (0, 1, 512, 1234, max_val) + if min_val < 0: + values += (-1, -512, -1234, min_val) + for value in values: + with self.subTest(value=value): + self.assertEqual(func(value), value) + self.assertEqual(func(IntSubclass(value)), value) + if use_index: + self.assertEqual(func(Index(value)), value) + + if use_index: + self.assertEqual(func(MyIndexAndInt()), 10) + else: + self.assertRaises(TypeError, func, Index(42)) + self.assertRaises(TypeError, func, MyIndexAndInt()) + + if mask: + self.assertEqual(func(min_val - 1), max_val) + self.assertEqual(func(max_val + 1), min_val) + self.assertEqual(func(-1 << 1000), 0) + self.assertEqual(func(1 << 1000), 0) + else: + self.assertRaises(negative_value_error, func, min_val - 1) + self.assertRaises(negative_value_error, func, -1 << 1000) + self.assertRaises(OverflowError, func, max_val + 1) + self.assertRaises(OverflowError, func, 1 << 1000) + self.assertRaises(TypeError, func, 1.0) + self.assertRaises(TypeError, func, b'2') + self.assertRaises(TypeError, func, '3') + self.assertRaises(SystemError, func, NULL) + + def check_long_asintandoverflow(self, func, min_val, max_val): + # round trip (object -> C integer -> object) + for value in (min_val, max_val, -1, 0, 1, 1234): + with self.subTest(value=value): + self.assertEqual(func(value), (value, 0)) + self.assertEqual(func(IntSubclass(value)), (value, 0)) + self.assertEqual(func(Index(value)), (value, 0)) + + self.assertEqual(func(MyIndexAndInt()), (10, 0)) + + self.assertEqual(func(min_val - 1), (-1, -1)) + self.assertEqual(func(max_val + 1), (-1, +1)) + self.assertRaises(SystemError, func, None) + self.assertRaises(TypeError, func, 1.0) + + def test_long_asint(self): + # Test PyLong_AsInt() + PyLong_AsInt = _testlimitedcapi.PyLong_AsInt + from _testcapi import INT_MIN, INT_MAX + self.check_long_asint(PyLong_AsInt, INT_MIN, INT_MAX) + + def test_long_aslong(self): + # Test PyLong_AsLong() and PyLong_FromLong() + aslong = _testlimitedcapi.pylong_aslong + from _testcapi import LONG_MIN, LONG_MAX + self.check_long_asint(aslong, LONG_MIN, LONG_MAX) + + def test_long_aslongandoverflow(self): + # Test PyLong_AsLongAndOverflow() + aslongandoverflow = _testlimitedcapi.pylong_aslongandoverflow + from _testcapi import LONG_MIN, LONG_MAX + self.check_long_asintandoverflow(aslongandoverflow, LONG_MIN, LONG_MAX) + + def test_long_asunsignedlong(self): + # Test PyLong_AsUnsignedLong() and PyLong_FromUnsignedLong() + asunsignedlong = _testlimitedcapi.pylong_asunsignedlong + from _testcapi import ULONG_MAX + self.check_long_asint(asunsignedlong, 0, ULONG_MAX, + use_index=False) + + def test_long_asunsignedlongmask(self): + # Test PyLong_AsUnsignedLongMask() + asunsignedlongmask = _testlimitedcapi.pylong_asunsignedlongmask + from _testcapi import ULONG_MAX + self.check_long_asint(asunsignedlongmask, 0, ULONG_MAX, mask=True) + + def test_long_aslonglong(self): + # Test PyLong_AsLongLong() and PyLong_FromLongLong() + aslonglong = _testlimitedcapi.pylong_aslonglong + from _testcapi import LLONG_MIN, LLONG_MAX + self.check_long_asint(aslonglong, LLONG_MIN, LLONG_MAX) + + def test_long_aslonglongandoverflow(self): + # Test PyLong_AsLongLongAndOverflow() + aslonglongandoverflow = _testlimitedcapi.pylong_aslonglongandoverflow + from _testcapi import LLONG_MIN, LLONG_MAX + self.check_long_asintandoverflow(aslonglongandoverflow, LLONG_MIN, LLONG_MAX) + + def test_long_asunsignedlonglong(self): + # Test PyLong_AsUnsignedLongLong() and PyLong_FromUnsignedLongLong() + asunsignedlonglong = _testlimitedcapi.pylong_asunsignedlonglong + from _testcapi import ULLONG_MAX + self.check_long_asint(asunsignedlonglong, 0, ULLONG_MAX, use_index=False) + + def test_long_asunsignedlonglongmask(self): + # Test PyLong_AsUnsignedLongLongMask() + asunsignedlonglongmask = _testlimitedcapi.pylong_asunsignedlonglongmask + from _testcapi import ULLONG_MAX + self.check_long_asint(asunsignedlonglongmask, 0, ULLONG_MAX, mask=True) + + def test_long_as_ssize_t(self): + # Test PyLong_AsSsize_t() and PyLong_FromSsize_t() + as_ssize_t = _testlimitedcapi.pylong_as_ssize_t + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + self.check_long_asint(as_ssize_t, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, + use_index=False) + + def test_long_as_size_t(self): + # Test PyLong_AsSize_t() and PyLong_FromSize_t() + as_size_t = _testlimitedcapi.pylong_as_size_t + from _testcapi import SIZE_MAX + self.check_long_asint(as_size_t, 0, SIZE_MAX, use_index=False) + + def test_long_asdouble(self): + # Test PyLong_AsDouble() + asdouble = _testlimitedcapi.pylong_asdouble + MAX = int(sys.float_info.max) + for value in (-MAX, MAX, -1, 0, 1, 1234): + with self.subTest(value=value): + self.assertEqual(asdouble(value), float(value)) + self.assertIsInstance(asdouble(value), float) + + self.assertEqual(asdouble(IntSubclass(42)), 42.0) + self.assertRaises(TypeError, asdouble, Index(42)) + self.assertRaises(TypeError, asdouble, MyIndexAndInt()) + + self.assertRaises(OverflowError, asdouble, 2 * MAX) + self.assertRaises(OverflowError, asdouble, -2 * MAX) + self.assertRaises(TypeError, asdouble, 1.0) + self.assertRaises(TypeError, asdouble, b'2') + self.assertRaises(TypeError, asdouble, '3') + self.assertRaises(SystemError, asdouble, NULL) + + def test_long_asvoidptr(self): + # Test PyLong_AsVoidPtr() + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr + asvoidptr = _testlimitedcapi.pylong_asvoidptr + obj = object() + x = fromvoidptr(obj) + y = fromvoidptr(NULL) + self.assertIs(asvoidptr(x), obj) + self.assertIs(asvoidptr(y), NULL) + self.assertIs(asvoidptr(IntSubclass(x)), obj) + + # negative values + M = (1 << _testcapi.SIZEOF_VOID_P * 8) + if x >= M//2: + self.assertIs(asvoidptr(x - M), obj) + if y >= M//2: + self.assertIs(asvoidptr(y - M), NULL) + + self.assertRaises(TypeError, asvoidptr, Index(x)) + self.assertRaises(TypeError, asvoidptr, object()) + self.assertRaises(OverflowError, asvoidptr, 2**1000) + self.assertRaises(OverflowError, asvoidptr, -2**1000) + # CRASHES asvoidptr(NULL) + + def _test_long_aspid(self, aspid): + # Test PyLong_AsPid() + from _testcapi import SIZEOF_PID_T + bits = 8 * SIZEOF_PID_T + PID_T_MIN = -2**(bits-1) + PID_T_MAX = 2**(bits-1) - 1 + self.check_long_asint(aspid, PID_T_MIN, PID_T_MAX) + + def test_long_aspid(self): + self._test_long_aspid(_testcapi.pylong_aspid) + + def test_long_aspid_limited(self): + self._test_long_aspid(_testlimitedcapi.pylong_aspid) + + @support.bigmemtest(2**32, memuse=0.35) + def test_long_asnativebytes_huge(self, size): + asnativebytes = _testcapi.pylong_asnativebytes + v = 1 << size + buffer = bytearray(size * 2 // 15 + 10) + r = asnativebytes(v, buffer, 0, -1) + self.assertEqual(r, size // 8 + 1) + self.assertEqual(buffer.count(0), len(buffer)) + r = asnativebytes(v, buffer, len(buffer), -1) + self.assertEqual(r, size // 8 + 1) + self.assertEqual(buffer.count(0), len(buffer) - 1) + + def test_long_asnativebytes(self): + import math + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 + MAX_USIZE = 2 ** (SZ * 8) - 1 + if support.verbose: + print(f"SIZEOF_SIZE={SZ}\n{MAX_SSIZE=:016X}\n{MAX_USIZE=:016X}") + + # These tests check that the requested buffer size is correct. + # This matches our current implementation: We only specify that the + # return value is a size *sufficient* to hold the result when queried + # using n_bytes=0. If our implementation changes, feel free to update + # the expectations here -- or loosen them to be range checks. + # (i.e. 0 *could* be stored in 1 byte and 512 in 2) + for v, expect in [ + (0, SZ), + (512, SZ), + (-512, SZ), + (MAX_SSIZE, SZ), + (MAX_USIZE, SZ + 1), + (-MAX_SSIZE, SZ), + (-MAX_USIZE, SZ + 1), + (2**255-1, 32), + (-(2**255-1), 32), + (2**255, 33), + (-(2**255), 33), # if you ask, we'll say 33, but 32 would do + (2**256-1, 33), + (-(2**256-1), 33), + (2**256, 33), + (-(2**256), 33), + ]: + with self.subTest(f"sizeof-{v:X}"): + buffer = bytearray(b"\x5a") + self.assertEqual(expect, asnativebytes(v, buffer, 0, -1), + "PyLong_AsNativeBytes(v, , 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + # Also check via the __index__ path. + # We pass Py_ASNATIVEBYTES_NATIVE_ENDIAN | ALLOW_INDEX + self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, 3 | 16), + "PyLong_AsNativeBytes(Index(v), , 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + + # Test that we populate n=2 bytes but do not overwrite more. + buffer = bytearray(b"\x99"*3) + self.assertEqual(2, asnativebytes(4, buffer, 2, 0), # BE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 0) // BE") + self.assertEqual(buffer, b"\x00\x04\x99") + self.assertEqual(2, asnativebytes(4, buffer, 2, 1), # LE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 1) // LE") + self.assertEqual(buffer, b"\x04\x00\x99") + + # We request as many bytes as `expect_be` contains, and always check + # the result (both big and little endian). We check the return value + # independently, since the buffer should always be filled correctly even + # if we need more bytes + for v, expect_be, expect_n in [ + (0, b'\x00', 1), + (0, b'\x00' * 2, 2), + (0, b'\x00' * 8, min(8, SZ)), + (1, b'\x01', 1), + (1, b'\x00' * 10 + b'\x01', min(11, SZ)), + (42, b'\x2a', 1), + (42, b'\x00' * 10 + b'\x2a', min(11, SZ)), + (-1, b'\xff', 1), + (-1, b'\xff' * 10, min(11, SZ)), + (-42, b'\xd6', 1), + (-42, b'\xff' * 10 + b'\xd6', min(11, SZ)), + # Extracts 255 into a single byte, but requests 2 + # (this is currently a special case, and "should" request SZ) + (255, b'\xff', 2), + (255, b'\x00\xff', 2), + (256, b'\x01\x00', 2), + (0x80, b'\x00' * 7 + b'\x80', min(8, SZ)), + # Extracts successfully (unsigned), but requests 9 bytes + (2**63, b'\x80' + b'\x00' * 7, 9), + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + # Extracts into 8 bytes, but if you provide 9 we'll say 9 + (-2**63, b'\x80' + b'\x00' * 7, 8), + (-2**63, b'\xff\x80' + b'\x00' * 7, 9), + + (2**255-1, b'\x7f' + b'\xff' * 31, 32), + (-(2**255-1), b'\x80' + b'\x00' * 30 + b'\x01', 32), + # Request extra bytes, but result says we only needed 32 + (-(2**255-1), b'\xff\x80' + b'\x00' * 30 + b'\x01', 32), + (-(2**255-1), b'\xff\xff\x80' + b'\x00' * 30 + b'\x01', 32), + + # Extracting 256 bits of integer will request 33 bytes, but still + # copy as many bits as possible into the buffer. So we *can* copy + # into a 32-byte buffer, though negative number may be unrecoverable + (2**256-1, b'\xff' * 32, 33), + (2**256-1, b'\x00' + b'\xff' * 32, 33), + (-(2**256-1), b'\x00' * 31 + b'\x01', 33), + (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 33), + (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 33), + # However, -2**255 precisely will extract into 32 bytes and return + # success. For bigger buffers, it will still succeed, but will + # return 33 + (-(2**255), b'\x80' + b'\x00' * 31, 32), + (-(2**255), b'\xff\x80' + b'\x00' * 31, 33), + + # The classic "Windows HRESULT as negative number" case + # HRESULT hr; + # PyLong_AsNativeBytes(<-2147467259>, &hr, sizeof(HRESULT), -1) + # assert(hr == E_FAIL) + (-2147467259, b'\x80\x00\x40\x05', 4), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes"): + n = len(expect_be) + # Fill the buffer with dummy data to ensure all bytes + # are overwritten. + buffer = bytearray(b"\xa5"*n) + expect_le = expect_be[::-1] + + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 0), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_be, buffer[:n], "") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 1), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_le, buffer[:n], "") + + # Test cases that do not request size for a sign bit when we pass the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag + for v, expect_be, expect_n in [ + (255, b'\xff', 1), + # We pass a 2 byte buffer so it just uses the whole thing + (255, b'\x00\xff', 2), + + (2**63, b'\x80' + b'\x00' * 7, 8), + # We pass a 9 byte buffer so it uses the whole thing + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + + (2**256-1, b'\xff' * 32, 32), + # We pass a 33 byte buffer so it uses the whole thing + (2**256-1, b'\x00' + b'\xff' * 32, 33), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes-unsigned"): + n = len(expect_be) + buffer = bytearray(b"\xa5"*n) + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 4), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 5), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + + # Ensure Py_ASNATIVEBYTES_REJECT_NEGATIVE raises on negative value + with self.assertRaises(ValueError): + asnativebytes(-1, buffer, 0, 8) + + # Ensure omitting Py_ASNATIVEBYTES_ALLOW_INDEX raises on __index__ value + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, -1) + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, 3) + + # Check a few error conditions. These are validated in code, but are + # unspecified in docs, so if we make changes to the implementation, it's + # fine to just update these tests rather than preserve the behaviour. + with self.assertRaises(TypeError): + asnativebytes('not a number', buffer, 0, -1) + + def test_long_asnativebytes_fuzz(self): + import math + from random import Random + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + + rng = Random() + # Allocate bigger buffer than actual values are going to be + buffer = bytearray(260) + + for _ in range(1000): + n = rng.randrange(1, 256) + bytes_be = bytes([ + # Ensure the most significant byte is nonzero + rng.randrange(1, 256), + *[rng.randrange(256) for _ in range(n - 1)] + ]) + bytes_le = bytes_be[::-1] + v = int.from_bytes(bytes_le, 'little') + + expect_1 = expect_2 = (SZ, n) + if bytes_be[0] & 0x80: + # All values are positive, so if MSB is set, expect extra bit + # when we request the size or have a large enough buffer + expect_1 = (SZ, n + 1) + # When passing Py_ASNATIVEBYTES_UNSIGNED_BUFFER, we expect the + # return to be exactly the right size. + expect_2 = (n,) + + try: + actual = asnativebytes(v, buffer, 0, -1) + self.assertIn(actual, expect_1) + + actual = asnativebytes(v, buffer, len(buffer), 0) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_be, buffer[-n:]) + + actual = asnativebytes(v, buffer, len(buffer), 1) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_le, buffer[:n]) + + actual = asnativebytes(v, buffer, n, 4) + self.assertIn(actual, expect_2, bytes_be.hex()) + actual = asnativebytes(v, buffer, n, 5) + self.assertIn(actual, expect_2, bytes_be.hex()) + except AssertionError as ex: + value_hex = ''.join(reversed([ + f'{b:02X}{"" if i % 8 else "_"}' + for i, b in enumerate(bytes_le, start=1) + ])).strip('_') + if support.verbose: + print() + print(n, 'bytes') + print('hex =', value_hex) + print('int =', v) + raise + raise AssertionError(f"Value: 0x{value_hex}") from ex + + def test_long_fromnativebytes(self): + import math + from _testcapi import ( + pylong_fromnativebytes as fromnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 + MAX_USIZE = 2 ** (SZ * 8) - 1 + + for v_be, expect_s, expect_u in [ + (b'\x00', 0, 0), + (b'\x01', 1, 1), + (b'\xff', -1, 255), + (b'\x00\xff', 255, 255), + (b'\xff\xff', -1, 65535), + ]: + with self.subTest(f"{expect_s}-{expect_u:X}-{len(v_be)}bytes"): + n = len(v_be) + v_le = v_be[::-1] + + self.assertEqual(expect_s, fromnativebytes(v_be, n, 0, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_s, fromnativebytes(v_le, n, 1, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_be, n, 0, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_le, n, 1, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + + # Check native endian when the result would be the same either + # way and we can test it. + if v_be == v_le: + self.assertEqual(expect_s, fromnativebytes(v_be, n, -1, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + self.assertEqual(expect_u, fromnativebytes(v_be, n, -1, 0), + f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + + # Swap the unsigned request for tests and use the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag instead + self.assertEqual(expect_u, fromnativebytes(v_be, n, 4, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + + def test_long_getsign(self): + # Test PyLong_GetSign() + getsign = _testcapi.pylong_getsign + self.assertEqual(getsign(1), 1) + self.assertEqual(getsign(123456), 1) + self.assertEqual(getsign(-2), -1) + self.assertEqual(getsign(0), 0) + self.assertEqual(getsign(True), 1) + self.assertEqual(getsign(IntSubclass(-11)), -1) + self.assertEqual(getsign(False), 0) + + self.assertRaises(TypeError, getsign, 1.0) + self.assertRaises(TypeError, getsign, Index(123)) + + # CRASHES getsign(NULL) + + def test_long_ispositive(self): + # Test PyLong_IsPositive() + ispositive = _testcapi.pylong_ispositive + self.assertEqual(ispositive(1), 1) + self.assertEqual(ispositive(123), 1) + self.assertEqual(ispositive(-1), 0) + self.assertEqual(ispositive(0), 0) + self.assertEqual(ispositive(True), 1) + self.assertEqual(ispositive(False), 0) + self.assertEqual(ispositive(IntSubclass(-1)), 0) + self.assertRaises(TypeError, ispositive, 1.0) + self.assertRaises(TypeError, ispositive, Index(123)) + + # CRASHES ispositive(NULL) + + def test_long_isnegative(self): + # Test PyLong_IsNegative() + isnegative = _testcapi.pylong_isnegative + self.assertEqual(isnegative(1), 0) + self.assertEqual(isnegative(123), 0) + self.assertEqual(isnegative(-1), 1) + self.assertEqual(isnegative(0), 0) + self.assertEqual(isnegative(True), 0) + self.assertEqual(isnegative(False), 0) + self.assertEqual(isnegative(IntSubclass(-1)), 1) + self.assertRaises(TypeError, isnegative, 1.0) + self.assertRaises(TypeError, isnegative, Index(123)) + + # CRASHES isnegative(NULL) + + def test_long_iszero(self): + # Test PyLong_IsZero() + iszero = _testcapi.pylong_iszero + self.assertEqual(iszero(1), 0) + self.assertEqual(iszero(-1), 0) + self.assertEqual(iszero(0), 1) + self.assertEqual(iszero(True), 0) + self.assertEqual(iszero(False), 1) + self.assertEqual(iszero(IntSubclass(-1)), 0) + self.assertEqual(iszero(IntSubclass(0)), 1) + self.assertRaises(TypeError, iszero, 1.0) + self.assertRaises(TypeError, iszero, Index(123)) + + # CRASHES iszero(NULL) + + def test_long_asint32(self): + # Test PyLong_AsInt32() and PyLong_FromInt32() + to_int32 = _testlimitedcapi.pylong_asint32 + from _testcapi import INT32_MIN, INT32_MAX + self.check_long_asint(to_int32, INT32_MIN, INT32_MAX) + + def test_long_asint64(self): + # Test PyLong_AsInt64() and PyLong_FromInt64() + as_int64 = _testlimitedcapi.pylong_asint64 + from _testcapi import INT64_MIN, INT64_MAX + self.check_long_asint(as_int64, INT64_MIN, INT64_MAX) + + def test_long_asuint32(self): + # Test PyLong_AsUInt32() and PyLong_FromUInt32() + as_uint32 = _testlimitedcapi.pylong_asuint32 + from _testcapi import UINT32_MAX + self.check_long_asint(as_uint32, 0, UINT32_MAX, + negative_value_error=ValueError) + + def test_long_asuint64(self): + # Test PyLong_AsUInt64() and PyLong_FromUInt64() + as_uint64 = _testlimitedcapi.pylong_asuint64 + from _testcapi import UINT64_MAX + self.check_long_asint(as_uint64, 0, UINT64_MAX, + negative_value_error=ValueError) + + def test_long_layout(self): + # Test PyLong_GetNativeLayout() + int_info = sys.int_info + layout = _testcapi.get_pylong_layout() + expected = { + 'bits_per_digit': int_info.bits_per_digit, + 'digit_size': int_info.sizeof_digit, + 'digits_order': -1, + 'digit_endianness': -1 if sys.byteorder == 'little' else 1, + } + self.assertEqual(layout, expected) + + def test_long_export(self): + # Test PyLong_Export() + layout = _testcapi.get_pylong_layout() + base = 2 ** layout['bits_per_digit'] + + pylong_export = _testcapi.pylong_export + + # value fits into int64_t + self.assertEqual(pylong_export(0), 0) + self.assertEqual(pylong_export(123), 123) + self.assertEqual(pylong_export(-123), -123) + self.assertEqual(pylong_export(IntSubclass(123)), 123) + + # use an array, doesn't fit into int64_t + self.assertEqual(pylong_export(base**10 * 2 + 1), + (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + self.assertEqual(pylong_export(-(base**10 * 2 + 1)), + (1, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + self.assertEqual(pylong_export(IntSubclass(base**10 * 2 + 1)), + (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + + self.assertRaises(TypeError, pylong_export, 1.0) + self.assertRaises(TypeError, pylong_export, 0+1j) + self.assertRaises(TypeError, pylong_export, "abc") + + def test_longwriter_create(self): + # Test PyLongWriter_Create() + layout = _testcapi.get_pylong_layout() + base = 2 ** layout['bits_per_digit'] + + pylongwriter_create = _testcapi.pylongwriter_create + self.assertRaises(ValueError, pylongwriter_create, 0, []) + self.assertRaises(ValueError, pylongwriter_create, -123, []) + self.assertEqual(pylongwriter_create(0, [0]), 0) + self.assertEqual(pylongwriter_create(0, [123]), 123) + self.assertEqual(pylongwriter_create(1, [123]), -123) + self.assertEqual(pylongwriter_create(1, [1, 2]), + -(base * 2 + 1)) + self.assertEqual(pylongwriter_create(0, [1, 2, 3]), + base**2 * 3 + base * 2 + 1) + max_digit = base - 1 + self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit]), + base**2 * max_digit + base * max_digit + max_digit) + + # normalize + self.assertEqual(pylongwriter_create(0, [123, 0, 0]), 123) + + # test singletons + normalize + for num in (-2, 0, 1, 5, 42, 100): + self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0]), + num) + + def to_digits(num): + digits = [] + while True: + num, digit = divmod(num, base) + digits.append(digit) + if not num: + break + return digits + + # round trip: Python int -> export -> Python int + pylong_export = _testcapi.pylong_export + numbers = [*range(0, 10), 12345, 0xdeadbeef, 2**100, 2**100-1] + numbers.extend(-num for num in list(numbers)) + for num in numbers: + with self.subTest(num=num): + data = pylong_export(num) + if isinstance(data, tuple): + negative, digits = data + else: + value = data + negative = int(value < 0) + digits = to_digits(abs(value)) + self.assertEqual(pylongwriter_create(negative, digits), num, + (negative, digits)) + + def test_bug_143050(self): + with support.adjust_int_max_str_digits(0): + # Bug coming from using _pylong.int_from_string(), that + # currently requires > 6000 decimal digits. + int('-' + '0' * 7000, 10) + _testcapi.test_immortal_small_ints() + # Test also nonzero small int + int('-' + '0' * 7000 + '123', 10) + _testcapi.test_immortal_small_ints() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py new file mode 100644 index 00000000000..5035b2b4829 --- /dev/null +++ b/Lib/test/test_capi/test_mem.py @@ -0,0 +1,183 @@ +import re +import textwrap +import unittest + + +from test import support +from test.support import import_helper, requires_subprocess +from test.support.script_helper import assert_python_failure, assert_python_ok + + +# Skip this test if the _testcapi and _testinternalcapi extensions are not +# available. +_testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') + +@requires_subprocess() +class PyMemDebugTests(unittest.TestCase): + PYTHONMALLOC = 'debug' + # '0x04c06e0' or '04C06E0' + PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' + + def check(self, code): + with support.SuppressCrashReport(): + out = assert_python_failure( + '-c', code, + PYTHONMALLOC=self.PYTHONMALLOC, + # FreeBSD: instruct jemalloc to not fill freed() memory + # with junk byte 0x5a, see JEMALLOC(3) + MALLOC_CONF="junk:false", + ) + stderr = out.err + return stderr.decode('ascii', 'replace') + + def test_buffer_overflow(self): + out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" + r" at tail\+0: 0x78 \*\*\* OUCH\n" + r" at tail\+1: 0xfd\n" + r" at tail\+2: 0xfd\n" + r" .*\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" + r" Data at p: cd cd cd .*\n" + r"\n" + r"Enable tracemalloc to get the memory block allocation traceback\n" + r"\n" + r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") + regex = regex.format(ptr=self.PTR_REGEX) + regex = re.compile(regex, flags=re.DOTALL) + self.assertRegex(out, regex) + + def test_api_misuse(self): + out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" + r" Data at p: cd cd cd .*\n" + r"\n" + r"Enable tracemalloc to get the memory block allocation traceback\n" + r"\n" + r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") + regex = regex.format(ptr=self.PTR_REGEX) + self.assertRegex(out, regex) + + def check_malloc_without_gil(self, code): + out = self.check(code) + if not support.Py_GIL_DISABLED: + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without holding the GIL') + else: + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without an active thread state. ' + 'Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?') + self.assertIn(expected, out) + + def test_pymem_malloc_without_gil(self): + # Debug hooks must raise an error if PyMem_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def test_pyobject_malloc_without_gil(self): + # Debug hooks must raise an error if PyObject_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def check_pyobject_is_freed(self, func_name): + code = textwrap.dedent(f''' + import gc, os, sys, _testinternalcapi + # Disable the GC to avoid crash on GC collection + gc.disable() + _testinternalcapi.{func_name}() + # Exit immediately to avoid a crash while deallocating + # the invalid object + os._exit(0) + ''') + assert_python_ok( + '-c', code, + PYTHONMALLOC=self.PYTHONMALLOC, + MALLOC_CONF="junk:false", + ) + + def test_pyobject_null_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_null_is_freed') + + def test_pyobject_uninitialized_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') + + def test_pyobject_forbidden_bytes_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') + + def test_pyobject_freed_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_freed_is_freed') + + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_set_nomemory(self): + code = """if 1: + import _testcapi + + class C(): pass + + # The first loop tests both functions and that remove_mem_hooks() + # can be called twice in a row. The second loop checks a call to + # set_nomemory() after a call to remove_mem_hooks(). The third + # loop checks the start and stop arguments of set_nomemory(). + for outer_cnt in range(1, 4): + start = 10 * outer_cnt + for j in range(100): + if j == 0: + if outer_cnt != 3: + _testcapi.set_nomemory(start) + else: + _testcapi.set_nomemory(start, start + 1) + try: + C() + except MemoryError as e: + if outer_cnt != 3: + _testcapi.remove_mem_hooks() + print('MemoryError', outer_cnt, j) + _testcapi.remove_mem_hooks() + break + """ + rc, out, err = assert_python_ok('-c', code) + lines = out.splitlines() + for i, line in enumerate(lines, 1): + self.assertIn(b'MemoryError', out) + *_, count = line.split(b' ') + count = int(count) + self.assertLessEqual(count, i*10) + self.assertGreaterEqual(count, i*10-4) + + +# free-threading requires mimalloc (not malloc) +@support.requires_gil_enabled() +class PyMemMallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'malloc_debug' + + +@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') +class PyMemPymallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'pymalloc_debug' + + +@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc') +class PyMemMimallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'mimalloc_debug' + + +@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') +class PyMemDefaultTests(PyMemDebugTests): + # test default allocator of Python compiled in debug mode + PYTHONMALLOC = '' + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py new file mode 100644 index 00000000000..e71c1f6f554 --- /dev/null +++ b/Lib/test/test_capi/test_misc.py @@ -0,0 +1,2922 @@ +# Run the _testcapi module tests (tests for the Python/C API): by defn, +# these are all functions _testcapi exports whose name begins with 'test_'. + +import _thread +from collections import deque +import contextlib +import importlib.machinery +import importlib.util +import json +import os +import pickle +import queue +import random +import sys +import textwrap +import threading +import time +import types +import unittest +import weakref +import operator +from test import support +from test.support import MISSING_C_DOCSTRINGS +from test.support import import_helper +from test.support import threading_helper +from test.support import warnings_helper +from test.support import requires_limited_api +from test.support import expected_failure_if_gil_disabled +from test.support import Py_GIL_DISABLED +from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end +try: + import _posixsubprocess +except ImportError: + _posixsubprocess = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + +from _testcapi import HeapCTypeSubclass, HeapCTypeSubclassWithFinalizer + +import _testlimitedcapi +import _testinternalcapi + + +NULL = None + +def decode_stderr(err): + return err.decode('utf-8', 'replace').replace('\r', '') + + +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(_interpreters is None, + 'subinterpreters required')(meth) + + +def testfunction(self): + """some doc""" + return self + + +class InstanceMethod: + id = _testcapi.instancemethod(id) + testfunction = _testcapi.instancemethod(testfunction) + + +CURRENT_THREAD_REGEX = r'Current thread.*:\n' if not support.Py_GIL_DISABLED else r'Stack .*:\n' + + +@support.force_not_colorized_test_class +class CAPITest(unittest.TestCase): + + def test_instancemethod(self): + inst = InstanceMethod() + self.assertEqual(id(inst), inst.id()) + self.assertTrue(inst.testfunction() is inst) + self.assertEqual(inst.testfunction.__doc__, testfunction.__doc__) + self.assertEqual(InstanceMethod.testfunction.__doc__, testfunction.__doc__) + + InstanceMethod.testfunction.attribute = "test" + self.assertEqual(testfunction.attribute, "test") + self.assertRaises(AttributeError, setattr, inst.testfunction, "attribute", "test") + + @support.requires_subprocess() + def test_no_FatalError_infinite_loop(self): + code = textwrap.dedent(""" + import _testcapi + from test import support + + with support.SuppressCrashReport(): + _testcapi.crash_no_current_thread() + """) + + run_result, _cmd_line = run_python_until_end('-c', code) + _rc, out, err = run_result + self.assertEqual(out, b'') + # This used to cause an infinite loop. + if not support.Py_GIL_DISABLED: + msg = ("Fatal Python error: PyThreadState_Get: " + "the function must be called with the GIL held, " + "after Python initialization and before Python finalization, " + "but the GIL is released " + "(the current Python thread state is NULL)").encode() + else: + msg = ("Fatal Python error: PyThreadState_Get: " + "the function must be called with an active thread state, " + "after Python initialization and before Python finalization, " + "but it was called without an active thread state. " + "Are you trying to call the C API inside of a Py_BEGIN_ALLOW_THREADS block?").encode() + self.assertStartsWith(err.rstrip(), msg) + + def test_memoryview_from_NULL_pointer(self): + self.assertRaises(ValueError, _testcapi.make_memoryview_from_NULL_pointer) + + @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') + def test_seq_bytes_to_charp_array(self): + # Issue #15732: crash in _PySequence_BytesToCharpArray() + class Z(object): + def __len__(self): + return 1 + with self.assertRaisesRegex(TypeError, 'indexing'): + _posixsubprocess.fork_exec( + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) + # Issue #15736: overflow in _PySequence_BytesToCharpArray() + class Z(object): + def __len__(self): + return sys.maxsize + def __getitem__(self, i): + return b'x' + self.assertRaises(MemoryError, _posixsubprocess.fork_exec, + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) + + @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') + def test_subprocess_fork_exec(self): + class Z(object): + def __len__(self): + return 1 + + # Issue #15738: crash in subprocess_fork_exec() + self.assertRaises(TypeError, _posixsubprocess.fork_exec, + Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_docstring_signature_parsing(self): + + self.assertEqual(_testcapi.no_docstring.__doc__, None) + self.assertEqual(_testcapi.no_docstring.__text_signature__, None) + + self.assertEqual(_testcapi.docstring_empty.__doc__, None) + self.assertEqual(_testcapi.docstring_empty.__text_signature__, None) + + self.assertEqual(_testcapi.docstring_no_signature.__doc__, + "This docstring has no signature.") + self.assertEqual(_testcapi.docstring_no_signature.__text_signature__, None) + + self.assertEqual(_testcapi.docstring_with_invalid_signature.__doc__, + "docstring_with_invalid_signature($module, /, boo)\n" + "\n" + "This docstring has an invalid signature." + ) + self.assertEqual(_testcapi.docstring_with_invalid_signature.__text_signature__, None) + + self.assertEqual(_testcapi.docstring_with_invalid_signature2.__doc__, + "docstring_with_invalid_signature2($module, /, boo)\n" + "\n" + "--\n" + "\n" + "This docstring also has an invalid signature." + ) + self.assertEqual(_testcapi.docstring_with_invalid_signature2.__text_signature__, None) + + self.assertEqual(_testcapi.docstring_with_signature.__doc__, + "This docstring has a valid signature.") + self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "($module, /, sig)") + + self.assertEqual(_testcapi.docstring_with_signature_but_no_doc.__doc__, None) + self.assertEqual(_testcapi.docstring_with_signature_but_no_doc.__text_signature__, + "($module, /, sig)") + + self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__doc__, + "\nThis docstring has a valid signature and some extra newlines.") + self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__text_signature__, + "($module, /, parameter)") + + def test_c_type_with_matrix_multiplication(self): + M = _testcapi.matmulType + m1 = M() + m2 = M() + self.assertEqual(m1 @ m2, ("matmul", m1, m2)) + self.assertEqual(m1 @ 42, ("matmul", m1, 42)) + self.assertEqual(42 @ m1, ("matmul", 42, m1)) + o = m1 + o @= m2 + self.assertEqual(o, ("imatmul", m1, m2)) + o = m1 + o @= 42 + self.assertEqual(o, ("imatmul", m1, 42)) + o = 42 + o @= m1 + self.assertEqual(o, ("matmul", 42, m1)) + + def test_c_type_with_ipow(self): + # When the __ipow__ method of a type was implemented in C, using the + # modulo param would cause segfaults. + o = _testcapi.ipowType() + self.assertEqual(o.__ipow__(1), (1, None)) + self.assertEqual(o.__ipow__(2, 2), (2, 2)) + + def test_return_null_without_error(self): + # Issue #23571: A function must not return NULL without setting an + # error + if support.Py_DEBUG: + code = textwrap.dedent(""" + import _testcapi + from test import support + + with support.SuppressCrashReport(): + _testcapi.return_null_without_error() + """) + rc, out, err = assert_python_failure('-c', code) + err = decode_stderr(err) + self.assertRegex(err, + r'Fatal Python error: _Py_CheckFunctionResult: ' + r'a function returned NULL without setting an exception\n' + r'Python runtime state: initialized\n' + r'SystemError: ' + r'returned NULL without setting an exception\n' + r'\n' + + CURRENT_THREAD_REGEX + + r' File .*", line 6 in \n') + else: + with self.assertRaises(SystemError) as cm: + _testcapi.return_null_without_error() + self.assertRegex(str(cm.exception), + 'return_null_without_error.* ' + 'returned NULL without setting an exception') + + def test_return_result_with_error(self): + # Issue #23571: A function must not return a result with an error set + if support.Py_DEBUG: + code = textwrap.dedent(""" + import _testcapi + from test import support + + with support.SuppressCrashReport(): + _testcapi.return_result_with_error() + """) + rc, out, err = assert_python_failure('-c', code) + err = decode_stderr(err) + self.assertRegex(err, + r'Fatal Python error: _Py_CheckFunctionResult: ' + r'a function returned a result with an exception set\n' + r'Python runtime state: initialized\n' + r'ValueError\n' + r'\n' + r'The above exception was the direct cause ' + r'of the following exception:\n' + r'\n' + r'SystemError: ' + r'returned a result with an exception set\n' + r'\n' + + CURRENT_THREAD_REGEX + + r' File .*, line 6 in \n') + else: + with self.assertRaises(SystemError) as cm: + _testcapi.return_result_with_error() + self.assertRegex(str(cm.exception), + 'return_result_with_error.* ' + 'returned a result with an exception set') + + def test_getitem_with_error(self): + # Test _Py_CheckSlotResult(). Raise an exception and then calls + # PyObject_GetItem(): check that the assertion catches the bug. + # PyObject_GetItem() must not be called with an exception set. + code = textwrap.dedent(""" + import _testcapi + from test import support + + with support.SuppressCrashReport(): + _testcapi.getitem_with_error({1: 2}, 1) + """) + rc, out, err = assert_python_failure('-c', code) + err = decode_stderr(err) + if 'SystemError: ' not in err: + self.assertRegex(err, + r'Fatal Python error: _Py_CheckSlotResult: ' + r'Slot __getitem__ of type dict succeeded ' + r'with an exception set\n' + r'Python runtime state: initialized\n' + r'ValueError: bug\n' + r'\n' + + CURRENT_THREAD_REGEX + + r' File .*, line 6 in \n' + r'\n' + r'Extension modules: _testcapi \(total: 1\)\n') + else: + # Python built with NDEBUG macro defined: + # test _Py_CheckFunctionResult() instead. + self.assertIn('returned a result with an exception set', err) + + def test_buildvalue(self): + # Test Py_BuildValue() with object arguments + buildvalue = _testcapi.py_buildvalue + self.assertEqual(buildvalue(''), None) + self.assertEqual(buildvalue('()'), ()) + self.assertEqual(buildvalue('[]'), []) + self.assertEqual(buildvalue('{}'), {}) + self.assertEqual(buildvalue('()[]{}'), ((), [], {})) + self.assertEqual(buildvalue('O', 1), 1) + self.assertEqual(buildvalue('(O)', 1), (1,)) + self.assertEqual(buildvalue('[O]', 1), [1]) + self.assertRaises(SystemError, buildvalue, '{O}', 1) + self.assertEqual(buildvalue('OO', 1, 2), (1, 2)) + self.assertEqual(buildvalue('(OO)', 1, 2), (1, 2)) + self.assertEqual(buildvalue('[OO]', 1, 2), [1, 2]) + self.assertEqual(buildvalue('{OO}', 1, 2), {1: 2}) + self.assertEqual(buildvalue('{OOOO}', 1, 2, 3, 4), {1: 2, 3: 4}) + self.assertEqual(buildvalue('((O))', 1), ((1,),)) + self.assertEqual(buildvalue('((OO))', 1, 2), ((1, 2),)) + + self.assertEqual(buildvalue(' \t,:'), None) + self.assertEqual(buildvalue('O,', 1), 1) + self.assertEqual(buildvalue(' O ', 1), 1) + self.assertEqual(buildvalue('\tO\t', 1), 1) + self.assertEqual(buildvalue('O,O', 1, 2), (1, 2)) + self.assertEqual(buildvalue('O, O', 1, 2), (1, 2)) + self.assertEqual(buildvalue('O,\tO', 1, 2), (1, 2)) + self.assertEqual(buildvalue('O O', 1, 2), (1, 2)) + self.assertEqual(buildvalue('O\tO', 1, 2), (1, 2)) + self.assertEqual(buildvalue('(O,O)', 1, 2), (1, 2)) + self.assertEqual(buildvalue('(O, O,)', 1, 2), (1, 2)) + self.assertEqual(buildvalue(' ( O O ) ', 1, 2), (1, 2)) + self.assertEqual(buildvalue('\t(\tO\tO\t)\t', 1, 2), (1, 2)) + self.assertEqual(buildvalue('[O,O]', 1, 2), [1, 2]) + self.assertEqual(buildvalue('[O, O,]', 1, 2), [1, 2]) + self.assertEqual(buildvalue(' [ O O ] ', 1, 2), [1, 2]) + self.assertEqual(buildvalue(' [\tO\tO\t] ', 1, 2), [1, 2]) + self.assertEqual(buildvalue('{O:O}', 1, 2), {1: 2}) + self.assertEqual(buildvalue('{O:O,O:O}', 1, 2, 3, 4), {1: 2, 3: 4}) + self.assertEqual(buildvalue('{O: O, O: O,}', 1, 2, 3, 4), {1: 2, 3: 4}) + self.assertEqual(buildvalue(' { O O O O } ', 1, 2, 3, 4), {1: 2, 3: 4}) + self.assertEqual(buildvalue('\t{\tO\tO\tO\tO\t}\t', 1, 2, 3, 4), {1: 2, 3: 4}) + + self.assertRaises(SystemError, buildvalue, 'O', NULL) + self.assertRaises(SystemError, buildvalue, '(O)', NULL) + self.assertRaises(SystemError, buildvalue, '[O]', NULL) + self.assertRaises(SystemError, buildvalue, '{O}', NULL) + self.assertRaises(SystemError, buildvalue, 'OO', 1, NULL) + self.assertRaises(SystemError, buildvalue, 'OO', NULL, 2) + self.assertRaises(SystemError, buildvalue, '(OO)', 1, NULL) + self.assertRaises(SystemError, buildvalue, '(OO)', NULL, 2) + self.assertRaises(SystemError, buildvalue, '[OO]', 1, NULL) + self.assertRaises(SystemError, buildvalue, '[OO]', NULL, 2) + self.assertRaises(SystemError, buildvalue, '{OO}', 1, NULL) + self.assertRaises(SystemError, buildvalue, '{OO}', NULL, 2) + + def test_buildvalue_ints(self): + # Test Py_BuildValue() with integer arguments + buildvalue = _testcapi.py_buildvalue_ints + from _testcapi import SHRT_MIN, SHRT_MAX, USHRT_MAX, INT_MIN, INT_MAX, UINT_MAX + self.assertEqual(buildvalue('i', INT_MAX), INT_MAX) + self.assertEqual(buildvalue('i', INT_MIN), INT_MIN) + self.assertEqual(buildvalue('I', UINT_MAX), UINT_MAX) + + self.assertEqual(buildvalue('h', SHRT_MAX), SHRT_MAX) + self.assertEqual(buildvalue('h', SHRT_MIN), SHRT_MIN) + self.assertEqual(buildvalue('H', USHRT_MAX), USHRT_MAX) + + self.assertEqual(buildvalue('b', 127), 127) + self.assertEqual(buildvalue('b', -128), -128) + self.assertEqual(buildvalue('B', 255), 255) + + self.assertEqual(buildvalue('c', ord('A')), b'A') + self.assertEqual(buildvalue('c', 255), b'\xff') + self.assertEqual(buildvalue('c', 256), b'\x00') + self.assertEqual(buildvalue('c', -1), b'\xff') + + self.assertEqual(buildvalue('C', 255), chr(255)) + self.assertEqual(buildvalue('C', 256), chr(256)) + self.assertEqual(buildvalue('C', sys.maxunicode), chr(sys.maxunicode)) + self.assertRaises(ValueError, buildvalue, 'C', -1) + self.assertRaises(ValueError, buildvalue, 'C', sys.maxunicode+1) + + # gh-84489 + self.assertRaises(ValueError, buildvalue, '(C )i', -1, 2) + self.assertRaises(ValueError, buildvalue, '[C ]i', -1, 2) + self.assertRaises(ValueError, buildvalue, '{Ci }i', -1, 2, 3) + + def test_buildvalue_N(self): + _testcapi.test_buildvalue_N() + + def test_trashcan_subclass(self): + # bpo-35983: Check that the trashcan mechanism for "list" is NOT + # activated when its tp_dealloc is being called by a subclass + from _testcapi import MyList + L = None + for i in range(100): + L = MyList((L,)) + + @support.requires_resource('cpu') + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + def test_trashcan_python_class1(self): + self.do_test_trashcan_python_class(list) + + @support.requires_resource('cpu') + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + def test_trashcan_python_class2(self): + from _testcapi import MyList + self.do_test_trashcan_python_class(MyList) + + def do_test_trashcan_python_class(self, base): + # Check that the trashcan mechanism works properly for a Python + # subclass of a class using the trashcan (this specific test assumes + # that the base class "base" behaves like list) + class PyList(base): + # Count the number of PyList instances to verify that there is + # no memory leak + num = 0 + def __init__(self, *args): + __class__.num += 1 + super().__init__(*args) + def __del__(self): + __class__.num -= 1 + + for parity in (0, 1): + L = None + # We need in the order of 2**20 iterations here such that a + # typical 8MB stack would overflow without the trashcan. + for i in range(2**20): + L = PyList((L,)) + L.attr = i + if parity: + # Add one additional nesting layer + L = (L,) + self.assertGreater(PyList.num, 0) + del L + self.assertEqual(PyList.num, 0) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_heap_ctype_doc_and_text_signature(self): + self.assertEqual(_testcapi.HeapDocCType.__doc__, "somedoc") + self.assertEqual(_testcapi.HeapDocCType.__text_signature__, "(arg1, arg2)") + + def test_null_type_doc(self): + self.assertEqual(_testcapi.NullTpDocType.__doc__, None) + + def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self): + class HeapGcCTypeSubclass(_testcapi.HeapGcCType): + def __init__(self): + self.value2 = 20 + super().__init__() + + subclass_instance = HeapGcCTypeSubclass() + type_refcnt = sys.getrefcount(HeapGcCTypeSubclass) + + # Test that subclass instance was fully created + self.assertEqual(subclass_instance.value, 10) + self.assertEqual(subclass_instance.value2, 20) + + # Test that the type reference count is only decremented once + del subclass_instance + self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapGcCTypeSubclass)) + + def test_subclass_of_heap_gc_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): + class A(_testcapi.HeapGcCType): + def __init__(self): + self.value2 = 20 + super().__init__() + + class B(A): + def __init__(self): + super().__init__() + + def __del__(self): + self.__class__ = A + A.refcnt_in_del = sys.getrefcount(A) + B.refcnt_in_del = sys.getrefcount(B) + + subclass_instance = B() + type_refcnt = sys.getrefcount(B) + new_type_refcnt = sys.getrefcount(A) + + # Test that subclass instance was fully created + self.assertEqual(subclass_instance.value, 10) + self.assertEqual(subclass_instance.value2, 20) + + del subclass_instance + + # Test that setting __class__ modified the reference counts of the types + if support.Py_DEBUG: + # gh-89373: In debug mode, _Py_Dealloc() keeps a strong reference + # to the type while calling tp_dealloc() + self.assertEqual(type_refcnt, B.refcnt_in_del) + else: + self.assertEqual(type_refcnt - 1, B.refcnt_in_del) + self.assertEqual(new_type_refcnt + 1, A.refcnt_in_del) + + # Test that the original type already has decreased its refcnt + self.assertEqual(type_refcnt - 1, sys.getrefcount(B)) + + # Test that subtype_dealloc decref the newly assigned __class__ only once + self.assertEqual(new_type_refcnt, sys.getrefcount(A)) + + def test_heaptype_with_dict(self): + for cls in ( + _testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict, + ): + with self.subTest(cls=cls): + inst = cls() + inst.foo = 42 + self.assertEqual(inst.foo, 42) + self.assertEqual(inst.dictobj, inst.__dict__) + self.assertEqual(inst.dictobj, {"foo": 42}) + + inst = cls() + self.assertEqual({}, inst.__dict__) + + def test_heaptype_with_managed_dict(self): + inst = _testcapi.HeapCTypeWithManagedDict() + inst.foo = 42 + self.assertEqual(inst.foo, 42) + self.assertEqual(inst.__dict__, {"foo": 42}) + + inst = _testcapi.HeapCTypeWithManagedDict() + self.assertEqual({}, inst.__dict__) + + a = _testcapi.HeapCTypeWithManagedDict() + b = _testcapi.HeapCTypeWithManagedDict() + a.b = b + b.a = a + del a, b + + def test_sublclassing_managed_dict(self): + + class C(_testcapi.HeapCTypeWithManagedDict): + pass + + i = C() + i.spam = i + del i + + def test_heaptype_with_negative_dict(self): + inst = _testcapi.HeapCTypeWithNegativeDict() + inst.foo = 42 + self.assertEqual(inst.foo, 42) + self.assertEqual(inst.dictobj, inst.__dict__) + self.assertEqual(inst.dictobj, {"foo": 42}) + + inst = _testcapi.HeapCTypeWithNegativeDict() + self.assertEqual({}, inst.__dict__) + + def test_heaptype_with_weakref(self): + for cls in ( + _testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref, + ): + with self.subTest(cls=cls): + inst = cls() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + self.assertEqual(inst.weakreflist, ref) + + def test_heaptype_with_managed_weakref(self): + inst = _testcapi.HeapCTypeWithManagedWeakref() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + + def test_sublclassing_managed_weakref(self): + + class C(_testcapi.HeapCTypeWithManagedWeakref): + pass + + inst = C() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + + def test_sublclassing_managed_both(self): + + class C1(_testcapi.HeapCTypeWithManagedWeakref, _testcapi.HeapCTypeWithManagedDict): + pass + + class C2(_testcapi.HeapCTypeWithManagedDict, _testcapi.HeapCTypeWithManagedWeakref): + pass + + for cls in (C1, C2): + inst = cls() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + inst.spam = inst + del inst + ref = weakref.ref(cls()) + self.assertIs(ref(), None) + + def test_heaptype_with_buffer(self): + inst = _testcapi.HeapCTypeWithBuffer() + b = bytes(inst) + self.assertEqual(b, b"1234") + + def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self): + subclass_instance = _testcapi.HeapCTypeSubclass() + type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass) + + # Test that subclass instance was fully created + self.assertEqual(subclass_instance.value, 10) + self.assertEqual(subclass_instance.value2, 20) + + # Test that the type reference count is only decremented once + del subclass_instance + self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclass)) + + def test_c_subclass_of_heap_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): + subclass_instance = HeapCTypeSubclassWithFinalizer() + type_refcnt = sys.getrefcount(HeapCTypeSubclassWithFinalizer) + new_type_refcnt = sys.getrefcount(HeapCTypeSubclass) + + # Test that subclass instance was fully created + self.assertEqual(subclass_instance.value, 10) + self.assertEqual(subclass_instance.value2, 20) + + # The tp_finalize slot will set __class__ to HeapCTypeSubclass + del subclass_instance + + # Test that setting __class__ modified the reference counts of the types + # + # This is highly sensitive to implementation details and may break in the future. + # + # We expect the refcount on the old type, HeapCTypeSubclassWithFinalizer, to + # remain the same: the finalizer gets a strong reference (+1) when it gets the + # type from the module and setting __class__ decrements the refcount (-1). + # + # We expect the refcount on the new type, HeapCTypeSubclass, to increase by 2: + # the finalizer get a strong reference (+1) when it gets the type from the + # module and setting __class__ increments the refcount (+1). + expected_type_refcnt = type_refcnt + expected_new_type_refcnt = new_type_refcnt + 2 + + if not Py_GIL_DISABLED: + # In default builds the result returned from sys.getrefcount + # includes a temporary reference that is created by the interpreter + # when it pushes its argument on the operand stack. This temporary + # reference is not included in the result returned by Py_REFCNT, which + # is used in the finalizer. + # + # In free-threaded builds the result returned from sys.getrefcount + # does not include the temporary reference. Types use deferred + # refcounting and the interpreter will not create a new reference + # for deferred values on the operand stack. + expected_type_refcnt -= 1 + expected_new_type_refcnt -= 1 + + if support.Py_DEBUG: + # gh-89373: In debug mode, _Py_Dealloc() keeps a strong reference + # to the type while calling tp_dealloc() + expected_type_refcnt += 1 + + self.assertEqual(expected_type_refcnt, HeapCTypeSubclassWithFinalizer.refcnt_in_del) + self.assertEqual(expected_new_type_refcnt, HeapCTypeSubclass.refcnt_in_del) + + # Test that the original type already has decreased its refcnt + self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapCTypeSubclassWithFinalizer)) + + # Test that subtype_dealloc decref the newly assigned __class__ only once + self.assertEqual(new_type_refcnt, sys.getrefcount(HeapCTypeSubclass)) + + def test_heaptype_with_setattro(self): + obj = _testcapi.HeapCTypeSetattr() + self.assertEqual(obj.pvalue, 10) + obj.value = 12 + self.assertEqual(obj.pvalue, 12) + del obj.value + self.assertEqual(obj.pvalue, 0) + + def test_heaptype_with_custom_metaclass(self): + metaclass = _testcapi.HeapCTypeMetaclass + self.assertIsSubclass(metaclass, type) + + # Class creation from C + t = _testcapi.pytype_fromspec_meta(metaclass) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, "HeapCTypeViaMetaclass") + self.assertIs(type(t), metaclass) + + # Class creation from Python + t = metaclass("PyClassViaMetaclass", (), {}) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, "PyClassViaMetaclass") + + def test_heaptype_with_custom_metaclass_null_new(self): + metaclass = _testcapi.HeapCTypeMetaclassNullNew + + self.assertIsSubclass(metaclass, type) + + # Class creation from C + t = _testcapi.pytype_fromspec_meta(metaclass) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, "HeapCTypeViaMetaclass") + self.assertIs(type(t), metaclass) + + # Class creation from Python + with self.assertRaisesRegex(TypeError, "cannot create .* instances"): + metaclass("PyClassViaMetaclass", (), {}) + + def test_heaptype_with_custom_metaclass_custom_new(self): + metaclass = _testcapi.HeapCTypeMetaclassCustomNew + + self.assertIsSubclass(_testcapi.HeapCTypeMetaclassCustomNew, type) + + msg = "Metaclasses with custom tp_new are not supported." + with self.assertRaisesRegex(TypeError, msg): + t = _testcapi.pytype_fromspec_meta(metaclass) + + def test_heaptype_base_with_custom_metaclass(self): + metaclass = _testcapi.HeapCTypeMetaclassCustomNew + + class Base(metaclass=metaclass): + pass + + # Class creation from C + msg = "Metaclasses with custom tp_new are not supported." + with self.assertRaisesRegex(TypeError, msg): + sub = _testcapi.make_type_with_base(Base) + + def test_heaptype_with_tp_vectorcall(self): + tp = _testcapi.HeapCTypeVectorcall + v0 = tp.__new__(tp) + v0.__init__() + v1 = tp() + self.assertEqual(v0.value, 2) + self.assertEqual(v1.value, 1) + + def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): + for weakref_cls in (_testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref): + for dict_cls in (_testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict): + with self.subTest(weakref_cls=weakref_cls, dict_cls=dict_cls): + + with self.assertRaises(TypeError): + class Both1(weakref_cls, dict_cls): + pass + with self.assertRaises(TypeError): + class Both2(dict_cls, weakref_cls): + pass + + def test_multiple_inheritance_ctypes_with_weakref_or_dict_and_other_builtin(self): + for dict_cls in (_testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict): + for weakref_cls in (_testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref): + with self.subTest(dict_cls=dict_cls, weakref_cls=weakref_cls): + + with self.assertRaises(TypeError): + class C1(dict_cls, list): + pass + + with self.assertRaises(TypeError): + class C2(weakref_cls, list): + pass + + class C3(_testcapi.HeapCTypeWithManagedDict, list): + pass + class C4(_testcapi.HeapCTypeWithManagedWeakref, list): + pass + + inst = C3() + inst.append(0) + str(inst.__dict__) + + inst = C4() + inst.append(0) + str(inst.__weakref__) + + for cls in (_testcapi.HeapCTypeWithManagedDict, + _testcapi.HeapCTypeWithManagedWeakref): + for cls2 in (dict_cls, weakref_cls): + class S(cls, cls2): + pass + class B1(C3, cls): + pass + class B2(C4, cls): + pass + + def test_pytype_fromspec_with_repeated_slots(self): + for variant in range(2): + with self.subTest(variant=variant): + with self.assertRaises(SystemError): + _testcapi.create_type_from_repeated_slots(variant) + + def test_immutable_type_with_mutable_base(self): + class MutableBase: ... + + with self.assertRaisesRegex(TypeError, 'Creating immutable type'): + _testcapi.make_immutable_type_with_base(MutableBase) + + def test_pynumber_tobase(self): + from _testcapi import pynumber_tobase + small_number = 123 + large_number = 2**64 + class IDX: + def __init__(self, val): + self.val = val + def __index__(self): + return self.val + + test_cases = ((2, '0b1111011', '0b10000000000000000000000000000000000000000000000000000000000000000'), + (8, '0o173', '0o2000000000000000000000'), + (10, '123', '18446744073709551616'), + (16, '0x7b', '0x10000000000000000')) + for base, small_target, large_target in test_cases: + with self.subTest(base=base, st=small_target, lt=large_target): + # Test for small number + self.assertEqual(pynumber_tobase(small_number, base), small_target) + self.assertEqual(pynumber_tobase(-small_number, base), '-' + small_target) + self.assertEqual(pynumber_tobase(IDX(small_number), base), small_target) + # Test for large number(out of range of a longlong,i.e.[-2**63, 2**63-1]) + self.assertEqual(pynumber_tobase(large_number, base), large_target) + self.assertEqual(pynumber_tobase(-large_number, base), '-' + large_target) + self.assertEqual(pynumber_tobase(IDX(large_number), base), large_target) + self.assertRaises(TypeError, pynumber_tobase, IDX(123.0), 10) + self.assertRaises(TypeError, pynumber_tobase, IDX('123'), 10) + self.assertRaises(TypeError, pynumber_tobase, 123.0, 10) + self.assertRaises(TypeError, pynumber_tobase, '123', 10) + self.assertRaises(SystemError, pynumber_tobase, 123, 0) + + def test_pyobject_repr_from_null(self): + s = _testcapi.pyobject_repr_from_null() + self.assertEqual(s, '') + + def test_pyobject_str_from_null(self): + s = _testcapi.pyobject_str_from_null() + self.assertEqual(s, '') + + def test_pyobject_bytes_from_null(self): + s = _testcapi.pyobject_bytes_from_null() + self.assertEqual(s, b'') + + def test_Py_CompileString(self): + # Check that Py_CompileString respects the coding cookie + _compile = _testcapi.Py_CompileString + code = b"# -*- coding: latin1 -*-\nprint('\xc2\xa4')\n" + result = _compile(code) + expected = compile(code, "", "exec") + self.assertEqual(result.co_consts, expected.co_consts) + + def test_export_symbols(self): + # bpo-44133: Ensure that the "Py_FrozenMain" and + # "PyThread_get_thread_native_id" symbols are exported by the Python + # (directly by the binary, or via by the Python dynamic library). + ctypes = import_helper.import_module('ctypes') + names = [] + + # Test if the PY_HAVE_THREAD_NATIVE_ID macro is defined + if hasattr(_thread, 'get_native_id'): + names.append('PyThread_get_thread_native_id') + + # Python/frozenmain.c fails to build on Windows when the symbols are + # missing: + # - PyWinFreeze_ExeInit + # - PyWinFreeze_ExeTerm + # - PyInitFrozenExtensions + if os.name != 'nt': + names.append('Py_FrozenMain') + + for name in names: + self.assertHasAttr(ctypes.pythonapi, name) + + def test_clear_managed_dict(self): + + class C: + def __init__(self): + self.a = 1 + + c = C() + _testcapi.clear_managed_dict(c) + self.assertEqual(c.__dict__, {}) + c = C() + self.assertEqual(c.__dict__, {'a':1}) + _testcapi.clear_managed_dict(c) + self.assertEqual(c.__dict__, {}) + + def test_unstable_gc_new_with_extra_data(self): + class Data(_testcapi.ObjExtraData): + __slots__ = ('x', 'y') + + d = Data() + d.x = 10 + d.y = 20 + d.extra = 30 + self.assertEqual(d.x, 10) + self.assertEqual(d.y, 20) + self.assertEqual(d.extra, 30) + del d.extra + self.assertIsNone(d.extra) + + def test_gen_get_code(self): + def genf(): yield + gen = genf() + self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + + def test_tp_bases_slot(self): + cls = _testcapi.HeapCTypeWithBasesSlot + self.assertEqual(cls.__bases__, (int,)) + self.assertEqual(cls.__base__, int) + + def test_tp_bases_slot_none(self): + self.assertRaisesRegex( + SystemError, + "Py_tp_bases is not a tuple", + _testcapi.create_heapctype_with_none_bases_slot + ) + + +@requires_limited_api +class TestHeapTypeRelative(unittest.TestCase): + """Test API for extending opaque types (PEP 697)""" + + @requires_limited_api + def test_heaptype_relative_sizes(self): + # Test subclassing using "relative" basicsize, see PEP 697 + def check(extra_base_size, extra_size): + Base, Sub, instance, data_ptr, data_offset, data_size = ( + _testlimitedcapi.make_sized_heaptypes( + extra_base_size, -extra_size)) + + # no alignment shenanigans when inheriting directly + if extra_size == 0: + self.assertEqual(Base.__basicsize__, Sub.__basicsize__) + self.assertEqual(data_size, 0) + + else: + # The following offsets should be in increasing order: + offsets = [ + (0, 'start of object'), + (Base.__basicsize__, 'end of base data'), + (data_offset, 'subclass data'), + (data_offset + extra_size, 'end of requested subcls data'), + (data_offset + data_size, 'end of reserved subcls data'), + (Sub.__basicsize__, 'end of object'), + ] + ordered_offsets = sorted(offsets, key=operator.itemgetter(0)) + self.assertEqual( + offsets, ordered_offsets, + msg=f'Offsets not in expected order, got: {ordered_offsets}') + + # end of reserved subcls data == end of object + self.assertEqual(Sub.__basicsize__, data_offset + data_size) + + # we don't reserve (requested + alignment) or more data + self.assertLess(data_size - extra_size, + _testlimitedcapi.ALIGNOF_MAX_ALIGN_T) + + # The offsets/sizes we calculated should be aligned. + self.assertEqual(data_offset % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_size % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) + + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_base_size in sizes: + for extra_size in sizes: + args = dict(extra_base_size=extra_base_size, + extra_size=extra_size) + with self.subTest(**args): + check(**args) + + def test_HeapCCollection(self): + """Make sure HeapCCollection works properly by itself""" + collection = _testcapi.HeapCCollection(1, 2, 3) + self.assertEqual(list(collection), [1, 2, 3]) + + def test_heaptype_inherit_itemsize(self): + """Test HeapCCollection subclasses work properly""" + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_size in sizes: + with self.subTest(extra_size=extra_size): + Sub = _testlimitedcapi.subclass_var_heaptype( + _testcapi.HeapCCollection, -extra_size, 0, 0) + collection = Sub(1, 2, 3) + collection.set_data_to_3s() + + self.assertEqual(list(collection), [1, 2, 3]) + mem = collection.get_data() + self.assertGreaterEqual(len(mem), extra_size) + self.assertTrue(set(mem) <= {3}, f'got {mem!r}') + + def test_heaptype_invalid_inheritance(self): + with self.assertRaises(SystemError, + msg="Cannot extend variable-size class without " + + "Py_TPFLAGS_ITEMS_AT_END"): + _testlimitedcapi.subclass_heaptype(int, -8, 0) + + def test_heaptype_relative_members(self): + """Test HeapCCollection subclasses work properly""" + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_base_size in sizes: + for extra_size in sizes: + for offset in sizes: + with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size, offset=offset): + if offset < extra_size: + Sub = _testlimitedcapi.make_heaptype_with_member( + extra_base_size, -extra_size, offset, True) + Base = Sub.mro()[1] + instance = Sub() + self.assertEqual(instance.memb, instance.get_memb()) + instance.set_memb(13) + self.assertEqual(instance.memb, instance.get_memb()) + self.assertEqual(instance.get_memb(), 13) + instance.memb = 14 + self.assertEqual(instance.memb, instance.get_memb()) + self.assertEqual(instance.get_memb(), 14) + self.assertGreaterEqual(instance.get_memb_offset(), Base.__basicsize__) + self.assertLess(instance.get_memb_offset(), Sub.__basicsize__) + with self.assertRaises(SystemError): + instance.get_memb_relative() + with self.assertRaises(SystemError): + instance.set_memb_relative(0) + else: + with self.assertRaises(SystemError): + Sub = _testlimitedcapi.make_heaptype_with_member( + extra_base_size, -extra_size, offset, True) + with self.assertRaises(SystemError): + Sub = _testlimitedcapi.make_heaptype_with_member( + extra_base_size, extra_size, offset, True) + with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size): + with self.assertRaises(SystemError): + Sub = _testlimitedcapi.make_heaptype_with_member( + extra_base_size, -extra_size, -1, True) + + def test_heaptype_relative_members_errors(self): + with self.assertRaisesRegex( + SystemError, + r"With Py_RELATIVE_OFFSET, basicsize must be negative"): + _testlimitedcapi.make_heaptype_with_member(0, 1234, 0, True) + with self.assertRaisesRegex( + SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + _testlimitedcapi.make_heaptype_with_member(0, -8, 1234, True) + with self.assertRaisesRegex( + SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + _testlimitedcapi.make_heaptype_with_member(0, -8, -1, True) + + Sub = _testlimitedcapi.make_heaptype_with_member(0, -8, 0, True) + instance = Sub() + with self.assertRaisesRegex( + SystemError, r"PyMember_GetOne used with Py_RELATIVE_OFFSET"): + instance.get_memb_relative() + with self.assertRaisesRegex( + SystemError, r"PyMember_SetOne used with Py_RELATIVE_OFFSET"): + instance.set_memb_relative(0) + + def test_heaptype_relative_special_members_errors(self): + for member_name in "__vectorcalloffset__", "__dictoffset__", "__weaklistoffset__": + with self.subTest(member_name=member_name): + with self.assertRaisesRegex( + SystemError, + r"With Py_RELATIVE_OFFSET, basicsize must be negative."): + _testlimitedcapi.make_heaptype_with_member( + basicsize=sys.getsizeof(object()) + 100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"Member offset out of range \(0\.\.-basicsize\)"): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-8, + add_relative_flag=True, + member_name=member_name, + member_offset=-1, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"type of %s must be Py_T_PYSSIZET" % member_name): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"flags for %s must be " % member_name): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=0, + ) + + def test_pyobject_getitemdata_error(self): + """Test PyObject_GetItemData fails on unsupported types""" + with self.assertRaises(TypeError): + # None is not variable-length + _testcapi.pyobject_getitemdata(None) + with self.assertRaises(TypeError): + # int is variable-length, but doesn't have the + # Py_TPFLAGS_ITEMS_AT_END layout (and flag) + _testcapi.pyobject_getitemdata(0) + + +class TestPendingCalls(unittest.TestCase): + + # See the comment in ceval.c (at the "handle_eval_breaker" label) + # about when pending calls get run. This is especially relevant + # here for creating deterministic tests. + + def main_pendingcalls_submit(self, l, n): + def callback(): + #this function can be interrupted by thread switching so let's + #use an atomic operation + l.append(None) + + for i in range(n): + time.sleep(random.random()*0.02) #0.01 secs on average + #try submitting callback until successful. + #rely on regular interrupt to flush queue if we are + #unsuccessful. + while True: + if _testcapi._pending_threadfunc(callback): + break + + def pendingcalls_submit(self, l, n, *, main=True, ensure=False): + def callback(): + #this function can be interrupted by thread switching so let's + #use an atomic operation + l.append(None) + + if main: + return _testcapi._pending_threadfunc(callback, n, + blocking=False, + ensure_added=ensure) + else: + return _testinternalcapi.pending_threadfunc(callback, n, + blocking=False, + ensure_added=ensure) + + def pendingcalls_wait(self, l, numadded, context = None): + #now, stick around until l[0] has grown to 10 + count = 0 + while len(l) != numadded: + #this busy loop is where we expect to be interrupted to + #run our callbacks. Note that some callbacks are only run on the + #main thread + if False and support.verbose: + print("(%i)"%(len(l),),) + for i in range(1000): + a = i*i + if context and not context.event.is_set(): + continue + count += 1 + self.assertTrue(count < 10000, + "timeout waiting for %i callbacks, got %i"%(numadded, len(l))) + if False and support.verbose: + print("(%i)"%(len(l),)) + + @threading_helper.requires_working_threading() + def test_main_pendingcalls_threaded(self): + + #do every callback on a separate thread + n = 32 #total callbacks + threads = [] + class foo(object):pass + context = foo() + context.l = [] + context.n = 2 #submits per thread + context.nThreads = n // context.n + context.nFinished = 0 + context.lock = threading.Lock() + context.event = threading.Event() + + threads = [threading.Thread(target=self.main_pendingcalls_thread, + args=(context,)) + for i in range(context.nThreads)] + with threading_helper.start_threads(threads): + self.pendingcalls_wait(context.l, n, context) + + def main_pendingcalls_thread(self, context): + try: + self.main_pendingcalls_submit(context.l, context.n) + finally: + with context.lock: + context.nFinished += 1 + nFinished = context.nFinished + if False and support.verbose: + print("finished threads: ", nFinished) + if nFinished == context.nThreads: + context.event.set() + + def test_main_pendingcalls_non_threaded(self): + #again, just using the main thread, likely they will all be dispatched at + #once. It is ok to ask for too many, because we loop until we find a slot. + #the loop can be interrupted to dispatch. + #there are only 32 dispatch slots, so we go for twice that! + l = [] + n = 64 + self.main_pendingcalls_submit(l, n) + self.pendingcalls_wait(l, n) + + def test_max_pending(self): + with self.subTest('main-only'): + maxpending = 32 + + l = [] + added = self.pendingcalls_submit(l, 1, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, 1) + + l = [] + added = self.pendingcalls_submit(l, maxpending, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + l = [] + added = self.pendingcalls_submit(l, maxpending+1, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + with self.subTest('not main-only'): + # Per-interpreter pending calls has a much higher limit + # on how many may be pending at a time. + maxpending = 300 + + l = [] + added = self.pendingcalls_submit(l, 1, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, 1) + + l = [] + added = self.pendingcalls_submit(l, maxpending, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + l = [] + added = self.pendingcalls_submit(l, maxpending+1, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + class PendingTask(types.SimpleNamespace): + + _add_pending = _testinternalcapi.pending_threadfunc + + def __init__(self, req, taskid=None, notify_done=None): + self.id = taskid + self.req = req + self.notify_done = notify_done + + self.creator_tid = threading.get_ident() + self.requester_tid = None + self.runner_tid = None + self.result = None + + def run(self): + assert self.result is None + self.runner_tid = threading.get_ident() + self._run() + if self.notify_done is not None: + self.notify_done() + + def _run(self): + self.result = self.req + + def run_in_pending_call(self, worker_tids): + assert self._add_pending is _testinternalcapi.pending_threadfunc + self.requester_tid = threading.get_ident() + def callback(): + assert self.result is None + # It can be tricky to control which thread handles + # the eval breaker, so we take a naive approach to + # make sure. + if threading.get_ident() not in worker_tids: + self._add_pending(callback, ensure_added=True) + return + self.run() + self._add_pending(callback, ensure_added=True) + + def create_thread(self, worker_tids): + return threading.Thread( + target=self.run_in_pending_call, + args=(worker_tids,), + ) + + def wait_for_result(self): + while self.result is None: + time.sleep(0.01) + + @threading_helper.requires_working_threading() + def test_subthreads_can_handle_pending_calls(self): + payload = 'Spam spam spam spam. Lovely spam! Wonderful spam!' + + task = self.PendingTask(payload) + def do_the_work(): + tid = threading.get_ident() + t = task.create_thread({tid}) + with threading_helper.start_threads([t]): + task.wait_for_result() + t = threading.Thread(target=do_the_work) + with threading_helper.start_threads([t]): + pass + + self.assertEqual(task.result, payload) + + @threading_helper.requires_working_threading() + def test_many_subthreads_can_handle_pending_calls(self): + main_tid = threading.get_ident() + self.assertEqual(threading.main_thread().ident, main_tid) + + # We can't use queue.Queue since it isn't reentrant relative + # to pending calls. + _queue = deque() + _active = deque() + _done_lock = threading.Lock() + def queue_put(task): + _queue.append(task) + _active.append(True) + def queue_get(): + try: + task = _queue.popleft() + except IndexError: + raise queue.Empty + return task + def queue_task_done(): + _active.pop() + if not _active: + try: + _done_lock.release() + except RuntimeError: + assert not _done_lock.locked() + def queue_empty(): + return not _queue + def queue_join(): + _done_lock.acquire() + _done_lock.release() + + tasks = [] + for i in range(20): + task = self.PendingTask( + req=f'request {i}', + taskid=i, + notify_done=queue_task_done, + ) + tasks.append(task) + queue_put(task) + # This will be released once all the tasks have finished. + _done_lock.acquire() + + def add_tasks(worker_tids): + while True: + if done: + return + try: + task = queue_get() + except queue.Empty: + break + task.run_in_pending_call(worker_tids) + + done = False + def run_tasks(): + while not queue_empty(): + if done: + return + time.sleep(0.01) + # Give the worker a chance to handle any remaining pending calls. + while not done: + time.sleep(0.01) + + # Start the workers and wait for them to finish. + worker_threads = [threading.Thread(target=run_tasks) + for _ in range(3)] + with threading_helper.start_threads(worker_threads): + try: + # Add a pending call for each task. + worker_tids = [t.ident for t in worker_threads] + threads = [threading.Thread(target=add_tasks, args=(worker_tids,)) + for _ in range(3)] + with threading_helper.start_threads(threads): + try: + pass + except BaseException: + done = True + raise # re-raise + # Wait for the pending calls to finish. + queue_join() + # Notify the workers that they can stop. + done = True + except BaseException: + done = True + raise # re-raise + runner_tids = [t.runner_tid for t in tasks] + + self.assertNotIn(main_tid, runner_tids) + for task in tasks: + with self.subTest(f'task {task.id}'): + self.assertNotEqual(task.requester_tid, main_tid) + self.assertNotEqual(task.requester_tid, task.runner_tid) + self.assertNotIn(task.requester_tid, runner_tids) + + @requires_subinterpreters + @support.skip_if_sanitizer("gh-129824: race on assign_version_tag", thread=True) + def test_isolated_subinterpreter(self): + # We exercise the most important permutations. + + # This test relies on pending calls getting called + # (eval breaker tripped) at each loop iteration + # and at each call. + + maxtext = 250 + main_interpid = 0 + interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) + _interpreters.run_string(interpid, f"""if True: + import json + import os + import threading + import time + import _testinternalcapi + from test.support import threading_helper + """) + + def create_pipe(): + r, w = os.pipe() + self.addCleanup(lambda: os.close(r)) + self.addCleanup(lambda: os.close(w)) + return r, w + + with self.subTest('add in main, run in subinterpreter'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + timeout = time.time() + 30 # seconds + + def do_work(): + _interpreters.run_string(interpid, f"""if True: + # Wait until this interp has handled the pending call. + waiting = False + done = False + def wait(os_read=os.read): + global done, waiting + waiting = True + os_read({r_done}, 1) + done = True + t = threading.Thread(target=wait) + with threading_helper.start_threads([t]): + while not waiting: + pass + os.write({w_ready}, b'\\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > {timeout}: + raise Exception('timed out!') + """) + t = threading.Thread(target=do_work) + with threading_helper.start_threads([t]): + os.read(r_ready, 1) + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify(interpid) + # Signal the subinterpreter to stop. + os.write(w_done, b'\0') + + self.assertEqual(actual, int(interpid)) + + with self.subTest('add in main, run in subinterpreter sub-thread'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + timeout = time.time() + 30 # seconds + + def do_work(): + _interpreters.run_string(interpid, f"""if True: + waiting = False + done = False + def subthread(): + while not waiting: + pass + os.write({w_ready}, b'\\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > {timeout}: + raise Exception('timed out!') + t = threading.Thread(target=subthread) + with threading_helper.start_threads([t]): + # Wait until this interp has handled the pending call. + waiting = True + os.read({r_done}, 1) + done = True + """) + t = threading.Thread(target=do_work) + with threading_helper.start_threads([t]): + os.read(r_ready, 1) + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify(interpid) + # Signal the subinterpreter to stop. + os.write(w_done, b'\0') + + self.assertEqual(actual, int(interpid)) + + with self.subTest('add in subinterpreter, run in main'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + r_data, w_data= create_pipe() + timeout = time.time() + 30 # seconds + + def add_job(): + os.read(r_ready, 1) + _interpreters.run_string(interpid, f"""if True: + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify({main_interpid}) + # Signal the subinterpreter to stop. + os.write({w_done}, b'\\0') + os.write({w_data}, actual.to_bytes(1, 'little')) + """) + # Wait until this interp has handled the pending call. + waiting = False + done = False + def wait(os_read=os.read): + nonlocal done, waiting + waiting = True + os_read(r_done, 1) + done = True + t1 = threading.Thread(target=add_job) + t2 = threading.Thread(target=wait) + with threading_helper.start_threads([t1, t2]): + while not waiting: + pass + os.write(w_ready, b'\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > timeout: + raise Exception('timed out!') + text = os.read(r_data, 1) + actual = int.from_bytes(text, 'little') + + self.assertEqual(actual, int(main_interpid)) + + with self.subTest('add in subinterpreter, run in sub-thread'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + r_data, w_data= create_pipe() + timeout = time.time() + 30 # seconds + + def add_job(): + os.read(r_ready, 1) + _interpreters.run_string(interpid, f"""if True: + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify({main_interpid}) + # Signal the subinterpreter to stop. + os.write({w_done}, b'\\0') + os.write({w_data}, actual.to_bytes(1, 'little')) + """) + # Wait until this interp has handled the pending call. + waiting = False + done = False + def wait(os_read=os.read): + nonlocal done, waiting + waiting = True + os_read(r_done, 1) + done = True + def subthread(): + while not waiting: + pass + os.write(w_ready, b'\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > timeout: + raise Exception('timed out!') + t1 = threading.Thread(target=add_job) + t2 = threading.Thread(target=wait) + t3 = threading.Thread(target=subthread) + with threading_helper.start_threads([t1, t2, t3]): + pass + text = os.read(r_data, 1) + actual = int.from_bytes(text, 'little') + + self.assertEqual(actual, int(main_interpid)) + + # XXX We can't use the rest until gh-105716 is fixed. + return + + with self.subTest('add in subinterpreter, run in subinterpreter sub-thread'): + r_ready, w_ready = create_pipe() + r_done, w_done= create_pipe() + r_data, w_data= create_pipe() + timeout = time.time() + 30 # seconds + + def do_work(): + _interpreters.run_string(interpid, f"""if True: + waiting = False + done = False + def subthread(): + while not waiting: + pass + os.write({w_ready}, b'\\0') + # Loop to trigger the eval breaker. + while not done: + time.sleep(0.01) + if time.time() > {timeout}: + raise Exception('timed out!') + t = threading.Thread(target=subthread) + with threading_helper.start_threads([t]): + # Wait until this interp has handled the pending call. + waiting = True + os.read({r_done}, 1) + done = True + """) + t = threading.Thread(target=do_work) + #with threading_helper.start_threads([t]): + t.start() + if True: + os.read(r_ready, 1) + _interpreters.run_string(interpid, f"""if True: + # Add the pending call and wait for it to finish. + actual = _testinternalcapi.pending_identify({interpid}) + # Signal the subinterpreter to stop. + os.write({w_done}, b'\\0') + os.write({w_data}, actual.to_bytes(1, 'little')) + """) + t.join() + text = os.read(r_data, 1) + actual = int.from_bytes(text, 'little') + + self.assertEqual(actual, int(interpid)) + + +class SubinterpreterTest(unittest.TestCase): + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_subinterps(self): + import builtins + r, w = os.pipe() + code = """if 1: + import sys, builtins, pickle + with open({:d}, "wb") as f: + pickle.dump(id(sys.modules), f) + pickle.dump(id(builtins), f) + """.format(w) + with open(r, "rb") as f: + ret = support.run_in_subinterp(code) + self.assertEqual(ret, 0) + self.assertNotEqual(pickle.load(f), id(sys.modules)) + self.assertNotEqual(pickle.load(f), id(builtins)) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_subinterps_recent_language_features(self): + r, w = os.pipe() + code = """if 1: + import pickle + with open({:d}, "wb") as f: + + @(lambda x:x) # Py 3.9 + def noop(x): return x + + a = (b := f'1{{2}}3') + noop('x') # Py 3.8 (:=) / 3.6 (f'') + + async def foo(arg): return await arg # Py 3.5 + + pickle.dump(dict(a=a, b=b), f) + """.format(w) + + with open(r, "rb") as f: + ret = support.run_in_subinterp(code) + self.assertEqual(ret, 0) + self.assertEqual(pickle.load(f), {'a': '123x', 'b': '123'}) + + # _testcapi cannot be imported in a subinterpreter on a Free Threaded build + @support.requires_gil_enabled() + def test_py_config_isoloated_per_interpreter(self): + # A config change in one interpreter must not leak to out to others. + # + # This test could verify ANY config value, it just happens to have been + # written around the time of int_max_str_digits. Refactoring is okay. + code = """if 1: + import sys, _testcapi + + # Any config value would do, this happens to be the one being + # double checked at the time this test was written. + _testcapi.config_set('int_max_str_digits', 55555) + sub_value = _testcapi.config_get('int_max_str_digits') + assert sub_value == 55555, sub_value + """ + before_config = _testcapi.config_get('int_max_str_digits') + assert before_config != 55555 + self.assertEqual(support.run_in_subinterp(code), 0, + 'subinterp code failure, check stderr.') + after_config = _testcapi.config_get('int_max_str_digits') + self.assertIsNot( + before_config, after_config, + "Expected get_config() to return a new dict on each call") + self.assertEqual(before_config, after_config, + "CAUTION: Tests executed after this may be " + "running under an altered config.") + # try:...finally: calling set_config(before_config) not done + # as that results in sys.argv, sys.path, and sys.warnoptions + # "being modified by test_capi" per test.regrtest. So if this + # test fails, assume that the environment in this process may + # be altered and suspect. + + @requires_subinterpreters + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_configured_settings(self): + """ + The config with which an interpreter is created corresponds + 1-to-1 with the new interpreter's settings. This test verifies + that they match. + """ + + OBMALLOC = 1<<5 + EXTENSIONS = 1<<8 + THREADS = 1<<10 + DAEMON_THREADS = 1<<11 + FORK = 1<<15 + EXEC = 1<<16 + ALL_FLAGS = (OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS + | EXTENSIONS); + + features = [ + 'obmalloc', + 'fork', + 'exec', + 'threads', + 'daemon_threads', + 'extensions', + 'own_gil', + ] + kwlist = [f'allow_{n}' for n in features] + kwlist[0] = 'use_main_obmalloc' + kwlist[-2] = 'check_multi_interp_extensions' + kwlist[-1] = 'own_gil' + + expected_to_work = { + (True, True, True, True, True, True, True): + (ALL_FLAGS, True), + (True, False, False, False, False, False, False): + (OBMALLOC, False), + (False, False, False, True, False, True, False): + (THREADS | EXTENSIONS, False), + } + + expected_to_fail = { + (False, False, False, False, False, False, False), + } + + # gh-117649: The free-threaded build does not currently allow + # setting check_multi_interp_extensions to False. + if Py_GIL_DISABLED: + for config in list(expected_to_work.keys()): + kwargs = dict(zip(kwlist, config)) + if not kwargs['check_multi_interp_extensions']: + del expected_to_work[config] + expected_to_fail.add(config) + + # expected to work + for config, expected in expected_to_work.items(): + kwargs = dict(zip(kwlist, config)) + exp_flags, exp_gil = expected + expected = { + 'feature_flags': exp_flags, + 'own_gil': exp_gil, + } + with self.subTest(config): + r, w = os.pipe() + script = textwrap.dedent(f''' + import _testinternalcapi, json, os + settings = _testinternalcapi.get_interp_settings() + with os.fdopen({w}, "w") as stdin: + json.dump(settings, stdin) + ''') + with os.fdopen(r) as stdout: + ret = support.run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + out = stdout.read() + settings = json.loads(out) + + self.assertEqual(settings, expected) + + # expected to fail + for config in expected_to_fail: + kwargs = dict(zip(kwlist, config)) + with self.subTest(config): + script = textwrap.dedent(f''' + import _testinternalcapi + _testinternalcapi.get_interp_settings() + raise NotImplementedError('unreachable') + ''') + with self.assertRaises(_interpreters.InterpreterError): + support.run_in_subinterp_with_config(script, **kwargs) + + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + # gh-117649: The free-threaded build does not currently allow overriding + # the check_multi_interp_extensions setting. + @expected_failure_if_gil_disabled() + def test_overridden_setting_extensions_subinterp_check(self): + """ + PyInterpreterConfig.check_multi_interp_extensions can be overridden + with PyInterpreterState.override_multi_interp_extensions_check. + This verifies that the override works but does not modify + the underlying setting. + """ + + OBMALLOC = 1<<5 + EXTENSIONS = 1<<8 + THREADS = 1<<10 + DAEMON_THREADS = 1<<11 + FORK = 1<<15 + EXEC = 1<<16 + BASE_FLAGS = OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS + base_kwargs = { + 'use_main_obmalloc': True, + 'allow_fork': True, + 'allow_exec': True, + 'allow_threads': True, + 'allow_daemon_threads': True, + 'own_gil': False, + } + + def check(enabled, override): + kwargs = dict( + base_kwargs, + check_multi_interp_extensions=enabled, + ) + flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS + settings = { + 'feature_flags': flags, + 'own_gil': False, + } + + expected = { + 'requested': override, + 'override__initial': 0, + 'override_after': override, + 'override_restored': 0, + # The override should not affect the config or settings. + 'settings__initial': settings, + 'settings_after': settings, + 'settings_restored': settings, + # These are the most likely values to be wrong. + 'allowed__initial': not enabled, + 'allowed_after': not ((override > 0) if override else enabled), + 'allowed_restored': not enabled, + } + + r, w = os.pipe() + if Py_GIL_DISABLED: + # gh-117649: The test fails before `w` is closed + self.addCleanup(os.close, w) + script = textwrap.dedent(f''' + from test.test_capi.check_config import run_singlephase_check + run_singlephase_check({override}, {w}) + ''') + with os.fdopen(r) as stdout: + ret = support.run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + out = stdout.read() + results = json.loads(out) + + self.assertEqual(results, expected) + + self.maxDiff = None + + # setting: check disabled + with self.subTest('config: check disabled; override: disabled'): + check(False, -1) + with self.subTest('config: check disabled; override: use config'): + check(False, 0) + with self.subTest('config: check disabled; override: enabled'): + check(False, 1) + + # setting: check enabled + with self.subTest('config: check enabled; override: disabled'): + check(True, -1) + with self.subTest('config: check enabled; override: use config'): + check(True, 0) + with self.subTest('config: check enabled; override: enabled'): + check(True, 1) + + def test_mutate_exception(self): + """ + Exceptions saved in global module state get shared between + individual module instances. This test checks whether or not + a change in one interpreter's module gets reflected into the + other ones. + """ + import binascii + + support.run_in_subinterp("import binascii; binascii.Error.foobar = 'foobar'") + + self.assertNotHasAttr(binascii.Error, "foobar") + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + # gh-117649: The free-threaded build does not currently support sharing + # extension module state between interpreters. + @expected_failure_if_gil_disabled() + def test_module_state_shared_in_global(self): + """ + bpo-44050: Extension module state should be shared between interpreters + when it doesn't support sub-interpreters. + """ + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + + script = textwrap.dedent(f""" + import importlib.machinery + import importlib.util + import os + + fullname = '_test_module_state_shared' + origin = importlib.util.find_spec('_testmultiphase').origin + loader = importlib.machinery.{loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + attr_id = str(id(module.Error)).encode() + + os.write({w}, attr_id) + """) + exec(script) + main_attr_id = os.read(r, 100) + + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + subinterp_attr_id = os.read(r, 100) + self.assertEqual(main_attr_id, subinterp_attr_id) + + +@requires_subinterpreters +class InterpreterConfigTests(unittest.TestCase): + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + def iter_all_configs(self): + for use_main_obmalloc in (True, False): + for allow_fork in (True, False): + for allow_exec in (True, False): + for allow_threads in (True, False): + for allow_daemon in (True, False): + for checkext in (True, False): + for gil in ('shared', 'own', 'default'): + yield types.SimpleNamespace( + use_main_obmalloc=use_main_obmalloc, + allow_fork=allow_fork, + allow_exec=allow_exec, + allow_threads=allow_threads, + allow_daemon_threads=allow_daemon, + check_multi_interp_extensions=checkext, + gil=gil, + ) + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def test_predefined_config(self): + def check(name, expected): + expected = self.supported[expected] + args = (name,) if name else () + + config1 = _interpreters.new_config(*args) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(*args) + self.assert_ns_equal(config2, expected) + self.assertIsNot(config2, expected) + self.assertIsNot(config2, config1) + + with self.subTest('default'): + check(None, 'isolated') + + for name in self.supported: + with self.subTest(name): + check(name, name) + + def test_update_from_dict(self): + for name, vanilla in self.supported.items(): + with self.subTest(f'noop ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'change all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in self.gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in self.gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('unsupported field'): + for name in self.supported: + with self.assertRaises(ValueError): + _interpreters.new_config(name, spam=True) + + # Bad values for bool fields. + for field, value in vars(self.supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'unsupported value ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'unsupported value(gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'unsupported value (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_interp_init(self): + questionable = [ + # strange + dict( + allow_fork=True, + allow_exec=False, + ), + dict( + gil='shared', + use_main_obmalloc=False, + ), + # risky + dict( + allow_fork=True, + allow_threads=True, + ), + # ought to be invalid? + dict( + allow_threads=False, + allow_daemon_threads=True, + ), + dict( + gil='own', + use_main_obmalloc=True, + ), + ] + invalid = [ + dict( + use_main_obmalloc=False, + check_multi_interp_extensions=False + ), + ] + if Py_GIL_DISABLED: + invalid.append(dict(check_multi_interp_extensions=False)) + def match(config, override_cases): + ns = vars(config) + for overrides in override_cases: + if dict(ns, **overrides) == ns: + return True + return False + + def check(config): + script = 'pass' + rc = _testinternalcapi.run_in_subinterp_with_config(script, config) + self.assertEqual(rc, 0) + + for config in self.iter_all_configs(): + if config.gil == 'default': + continue + if match(config, invalid): + with self.subTest(f'invalid: {config}'): + with self.assertRaises(_interpreters.InterpreterError): + check(config) + elif match(config, questionable): + with self.subTest(f'questionable: {config}'): + check(config) + else: + with self.subTest(f'valid: {config}'): + check(config) + + def test_get_config(self): + @contextlib.contextmanager + def new_interp(config): + interpid = _interpreters.create(config, reqrefs=False) + try: + yield interpid + finally: + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + if Py_GIL_DISABLED: + expected.check_multi_interp_extensions = False + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + with new_interp('isolated') as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + with new_interp('legacy') as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config( + 'empty', + use_main_obmalloc=True, + gil='shared', + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + ) + with new_interp(orig) as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + +@requires_subinterpreters +class InterpreterIDTests(unittest.TestCase): + + def add_interp_cleanup(self, interpid): + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + + def new_interpreter(self): + id = _interpreters.create() + self.add_interp_cleanup(id) + return id + + def test_conversion_int(self): + convert = _testinternalcapi.normalize_interp_id + interpid = convert(10) + self.assertEqual(interpid, 10) + + def test_conversion_coerced(self): + convert = _testinternalcapi.normalize_interp_id + class MyInt(str): + def __index__(self): + return 10 + interpid = convert(MyInt()) + self.assertEqual(interpid, 10) + + def test_conversion_from_interpreter(self): + convert = _testinternalcapi.normalize_interp_id + interpid = self.new_interpreter() + converted = convert(interpid) + self.assertEqual(converted, interpid) + + def test_conversion_bad(self): + convert = _testinternalcapi.normalize_interp_id + + for badid in [ + object(), + 10.0, + '10', + b'10', + ]: + with self.subTest(f'bad: {badid!r}'): + with self.assertRaises(TypeError): + convert(badid) + + badid = -1 + with self.subTest(f'bad: {badid!r}'): + with self.assertRaises(ValueError): + convert(badid) + + badid = 2**64 + with self.subTest(f'bad: {badid!r}'): + with self.assertRaises(OverflowError): + convert(badid) + + def test_lookup_exists(self): + interpid = self.new_interpreter() + self.assertTrue( + _testinternalcapi.interpreter_exists(interpid)) + + def test_lookup_does_not_exist(self): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) + + def test_lookup_destroyed(self): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) + + def get_refcount_helpers(self): + return ( + _testinternalcapi.get_interpreter_refcount, + (lambda id: _interpreters.incref(id, implieslink=False)), + _interpreters.decref, + ) + + def test_linked_lifecycle_does_not_exist(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + with self.subTest('never existed'): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + with self.subTest('destroyed'): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + def test_linked_lifecycle_initial(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount, _, _ = self.get_refcount_helpers() + + # A new interpreter will start out not linked, with a refcount of 0. + interpid = self.new_interpreter() + linked = is_linked(interpid) + refcount = get_refcount(interpid) + + self.assertFalse(linked) + self.assertEqual(refcount, 0) + + def test_linked_lifecycle_never_linked(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + # Incref will not automatically link it. + incref(interpid) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + # It isn't linked so it isn't destroyed. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) + + def test_linked_lifecycle_link_unlink(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + + interpid = self.new_interpreter() + + # Linking at refcount 0 does not destroy the interpreter. + link(interpid) + self.assertTrue( + exists(interpid)) + self.assertTrue( + is_linked(interpid)) + + # Unlinking at refcount 0 does not destroy the interpreter. + unlink(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + + def test_linked_lifecycle_link_incref_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + # Linking it will not change the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) + + # Decref with a refcount of 0 is not allowed. + incref(interpid) + self.assertEqual( + 1, get_refcount(interpid)) + + # When linked, decref back to 0 destroys the interpreter. + decref(interpid) + self.assertFalse( + exists(interpid)) + + def test_linked_lifecycle_incref_link(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount, incref, _ = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + incref(interpid) + self.assertEqual( + 1, get_refcount(interpid)) + + # Linking it will not reset the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + def test_linked_lifecycle_link_incref_unlink_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + link(interpid) + self.assertTrue( + is_linked(interpid)) + + incref(interpid) + self.assertEqual( + 1, get_refcount(interpid)) + + # Unlinking it will not change the refcount. + unlink(interpid) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + # Unlinked: decref back to 0 does not destroys the interpreter. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) + + +class TestStaticTypes(unittest.TestCase): + + _has_run = False + + @classmethod + def setUpClass(cls): + # The tests here don't play nice with our approach to refleak + # detection, so we bail out in that case. + if cls._has_run: + raise unittest.SkipTest('these tests do not support re-running') + cls._has_run = True + + @contextlib.contextmanager + def basic_static_type(self, *args): + cls = _testcapi.get_basic_static_type(*args) + yield cls + + def test_pytype_ready_always_sets_tp_type(self): + # The point of this test is to prevent something like + # https://github.com/python/cpython/issues/104614 + # from happening again. + + # First check when tp_base/tp_bases is *not* set before PyType_Ready(). + with self.basic_static_type() as cls: + self.assertIs(cls.__base__, object); + self.assertEqual(cls.__bases__, (object,)); + self.assertIs(type(cls), type(object)); + + # Then check when we *do* set tp_base/tp_bases first. + with self.basic_static_type(object) as cls: + self.assertIs(cls.__base__, object); + self.assertEqual(cls.__bases__, (object,)); + self.assertIs(type(cls), type(object)); + + +class TestThreadState(unittest.TestCase): + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_thread_state(self): + # some extra thread-state tests driven via _testcapi + def target(): + idents = [] + + def callback(): + idents.append(threading.get_ident()) + + _testcapi._test_thread_state(callback) + a = b = callback + time.sleep(1) + # Check our main thread is in the list exactly 3 times. + self.assertEqual(idents.count(threading.get_ident()), 3, + "Couldn't find main thread correctly in the list") + + target() + t = threading.Thread(target=target) + t.start() + t.join() + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_thread_gilstate_in_clear(self): + # See https://github.com/python/cpython/issues/119585 + class C: + def __del__(self): + _testcapi.gilstate_ensure_release() + + # Thread-local variables are destroyed in `PyThreadState_Clear()`. + local_var = threading.local() + + def callback(): + local_var.x = C() + + _testcapi._test_thread_state(callback) + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_gilstate_ensure_no_deadlock(self): + # See https://github.com/python/cpython/issues/96071 + code = textwrap.dedent(""" + import _testcapi + + def callback(): + print('callback called') + + _testcapi._test_thread_state(callback) + """) + ret = assert_python_ok('-X', 'tracemalloc', '-c', code) + self.assertIn(b'callback called', ret.out) + + def test_gilstate_matches_current(self): + _testcapi.test_current_tstate_matches() + + +def get_test_funcs(mod, exclude_prefix=None): + funcs = {} + for name in dir(mod): + if not name.startswith('test_'): + continue + if exclude_prefix is not None and name.startswith(exclude_prefix): + continue + funcs[name] = getattr(mod, name) + return funcs + + +class Test_testcapi(unittest.TestCase): + locals().update(get_test_funcs(_testcapi)) + + # Suppress warning from PyUnicode_FromUnicode(). + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_widechar(self): + _testlimitedcapi.test_widechar() + + def test_version_api_data(self): + self.assertEqual(_testcapi.Py_Version, sys.hexversion) + + +class Test_testlimitedcapi(unittest.TestCase): + locals().update(get_test_funcs(_testlimitedcapi)) + + +class Test_testinternalcapi(unittest.TestCase): + locals().update(get_test_funcs(_testinternalcapi, + exclude_prefix='test_lock_')) + + +@threading_helper.requires_working_threading() +class Test_PyLock(unittest.TestCase): + locals().update((name, getattr(_testinternalcapi, name)) + for name in dir(_testinternalcapi) + if name.startswith('test_lock_')) + + +@unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") +class Test_ModuleStateAccess(unittest.TestCase): + """Test access to module start (PEP 573)""" + + # The C part of the tests lives in _testmultiphase, in a module called + # _testmultiphase_meth_state_access. + # This module has multi-phase initialization, unlike _testcapi. + + def setUp(self): + fullname = '_testmultiphase_meth_state_access' # XXX + origin = importlib.util.find_spec('_testmultiphase').origin + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = importlib.machinery.AppleFrameworkLoader(fullname, origin) + else: + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + self.module = module + + def test_subclass_get_module(self): + """PyType_GetModule for defining_class""" + class StateAccessType_Subclass(self.module.StateAccessType): + pass + + instance = StateAccessType_Subclass() + self.assertIs(instance.get_defining_module(), self.module) + + def test_subclass_get_module_with_super(self): + class StateAccessType_Subclass(self.module.StateAccessType): + def get_defining_module(self): + return super().get_defining_module() + + instance = StateAccessType_Subclass() + self.assertIs(instance.get_defining_module(), self.module) + + def test_state_access(self): + """Checks methods defined with and without argument clinic + + This tests a no-arg method (get_count) and a method with + both a positional and keyword argument. + """ + + a = self.module.StateAccessType() + b = self.module.StateAccessType() + + methods = { + 'clinic': a.increment_count_clinic, + 'noclinic': a.increment_count_noclinic, + } + + for name, increment_count in methods.items(): + with self.subTest(name): + self.assertEqual(a.get_count(), b.get_count()) + self.assertEqual(a.get_count(), 0) + + increment_count() + self.assertEqual(a.get_count(), b.get_count()) + self.assertEqual(a.get_count(), 1) + + increment_count(3) + self.assertEqual(a.get_count(), b.get_count()) + self.assertEqual(a.get_count(), 4) + + increment_count(-2, twice=True) + self.assertEqual(a.get_count(), b.get_count()) + self.assertEqual(a.get_count(), 0) + + with self.assertRaises(TypeError): + increment_count(thrice=3) + + with self.assertRaises(TypeError): + increment_count(1, 2, 3) + + def test_get_module_bad_def(self): + # PyType_GetModuleByDef fails gracefully if it doesn't + # find what it's looking for. + # see bpo-46433 + instance = self.module.StateAccessType() + with self.assertRaises(TypeError): + instance.getmodulebydef_bad_def() + + def test_get_module_static_in_mro(self): + # Here, the class PyType_GetModuleByDef is looking for + # appears in the MRO after a static type (Exception). + # see bpo-46433 + class Subclass(BaseException, self.module.StateAccessType): + pass + self.assertIs(Subclass().get_defining_module(), self.module) + + +class TestInternalFrameApi(unittest.TestCase): + + @staticmethod + def func(): + return sys._getframe() + + def test_code(self): + frame = self.func() + code = _testinternalcapi.iframe_getcode(frame) + self.assertIs(code, self.func.__code__) + + def test_lasti(self): + frame = self.func() + lasti = _testinternalcapi.iframe_getlasti(frame) + self.assertGreater(lasti, 0) + self.assertLess(lasti, len(self.func.__code__.co_code)) + + def test_line(self): + frame = self.func() + line = _testinternalcapi.iframe_getline(frame) + firstline = self.func.__code__.co_firstlineno + self.assertEqual(line, firstline + 2) + + +SUFFICIENT_TO_DEOPT_AND_SPECIALIZE = 100 + +class Test_Pep523API(unittest.TestCase): + + def do_test(self, func, names): + actual_calls = [] + start = SUFFICIENT_TO_DEOPT_AND_SPECIALIZE + count = start + SUFFICIENT_TO_DEOPT_AND_SPECIALIZE + try: + for i in range(count): + if i == start: + _testinternalcapi.set_eval_frame_record(actual_calls) + func() + finally: + _testinternalcapi.set_eval_frame_default() + expected_calls = names * SUFFICIENT_TO_DEOPT_AND_SPECIALIZE + self.assertEqual(len(expected_calls), len(actual_calls)) + for expected, actual in zip(expected_calls, actual_calls, strict=True): + self.assertEqual(expected, actual) + + def test_inlined_binary_subscr(self): + class C: + def __getitem__(self, other): + return None + def func(): + C()[42] + names = ["func", "__getitem__"] + self.do_test(func, names) + + def test_inlined_call(self): + def inner(x=42): + pass + def func(): + inner() + inner(42) + names = ["func", "inner", "inner"] + self.do_test(func, names) + + def test_inlined_call_function_ex(self): + def inner(x): + pass + def func(): + inner(*[42]) + names = ["func", "inner"] + self.do_test(func, names) + + def test_inlined_for_iter(self): + def gen(): + yield 42 + def func(): + for _ in gen(): + pass + names = ["func", "gen", "gen", "gen"] + self.do_test(func, names) + + def test_inlined_load_attr(self): + class C: + @property + def a(self): + return 42 + class D: + def __getattribute__(self, name): + return 42 + def func(): + C().a + D().a + names = ["func", "a", "__getattribute__"] + self.do_test(func, names) + + def test_inlined_send(self): + def inner(): + yield 42 + def outer(): + yield from inner() + def func(): + list(outer()) + names = ["func", "outer", "outer", "inner", "inner", "outer", "inner"] + self.do_test(func, names) + + +@unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') +class TestPyThreadId(unittest.TestCase): + def test_py_thread_id(self): + # gh-112535: Test _Py_ThreadId(): make sure that thread identifiers + # in a few threads are unique + py_thread_id = _testinternalcapi.py_thread_id + short_sleep = 0.010 + + class GetThreadId(threading.Thread): + def __init__(self): + super().__init__() + self.get_lock = threading.Lock() + self.get_lock.acquire() + self.started_lock = threading.Event() + self.py_tid = None + + def run(self): + self.started_lock.set() + self.get_lock.acquire() + self.py_tid = py_thread_id() + time.sleep(short_sleep) + self.py_tid2 = py_thread_id() + + nthread = 5 + threads = [GetThreadId() for _ in range(nthread)] + + # first make run sure that all threads are running + for thread in threads: + thread.start() + for thread in threads: + thread.started_lock.wait() + + # call _Py_ThreadId() in the main thread + py_thread_ids = [py_thread_id()] + + # now call _Py_ThreadId() in each thread + for thread in threads: + thread.get_lock.release() + + # call _Py_ThreadId() in each thread and wait until threads complete + for thread in threads: + thread.join() + py_thread_ids.append(thread.py_tid) + # _PyThread_Id() should not change for a given thread. + # For example, it should remain the same after a short sleep. + self.assertEqual(thread.py_tid2, thread.py_tid) + + # make sure that all _Py_ThreadId() are unique + for tid in py_thread_ids: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + self.assertEqual(len(set(py_thread_ids)), len(py_thread_ids), + py_thread_ids) + +class TestVersions(unittest.TestCase): + full_cases = ( + (3, 4, 1, 0xA, 2, 0x030401a2), + (3, 10, 0, 0xF, 0, 0x030a00f0), + (0x103, 0x10B, 0xFF00, -1, 0xF0, 0x030b00f0), # test masking + ) + xy_cases = ( + (3, 4, 0x03040000), + (3, 10, 0x030a0000), + (0x103, 0x10B, 0x030b0000), # test masking + ) + + def test_pack_full_version(self): + for *args, expected in self.full_cases: + with self.subTest(hexversion=hex(expected)): + result = _testlimitedcapi.pack_full_version(*args) + self.assertEqual(result, expected) + + def test_pack_version(self): + for *args, expected in self.xy_cases: + with self.subTest(hexversion=hex(expected)): + result = _testlimitedcapi.pack_version(*args) + self.assertEqual(result, expected) + + def test_pack_full_version_ctypes(self): + ctypes = import_helper.import_module('ctypes') + ctypes_func = ctypes.pythonapi.Py_PACK_FULL_VERSION + ctypes_func.restype = ctypes.c_uint32 + ctypes_func.argtypes = [ctypes.c_int] * 5 + for *args, expected in self.full_cases: + with self.subTest(hexversion=hex(expected)): + result = ctypes_func(*args) + self.assertEqual(result, expected) + + def test_pack_version_ctypes(self): + ctypes = import_helper.import_module('ctypes') + ctypes_func = ctypes.pythonapi.Py_PACK_VERSION + ctypes_func.restype = ctypes.c_uint32 + ctypes_func.argtypes = [ctypes.c_int] * 2 + for *args, expected in self.xy_cases: + with self.subTest(hexversion=hex(expected)): + result = ctypes_func(*args) + self.assertEqual(result, expected) + + +class TestCEval(unittest.TestCase): + def test_ceval_decref(self): + code = textwrap.dedent(""" + import _testcapi + _testcapi.toggle_reftrace_printer(True) + l1 = [] + l2 = [] + del l1 + del l2 + _testcapi.toggle_reftrace_printer(False) + """) + _, out, _ = assert_python_ok("-c", code) + lines = out.decode("utf-8").splitlines() + self.assertEqual(lines.count("CREATE list"), 2) + self.assertEqual(lines.count("DESTROY list"), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_number.py b/Lib/test/test_capi/test_number.py new file mode 100644 index 00000000000..bdd8868529f --- /dev/null +++ b/Lib/test/test_capi/test_number.py @@ -0,0 +1,352 @@ +import itertools +import operator +import unittest +import warnings + +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +from _testcapi import PY_SSIZE_T_MAX, PY_SSIZE_T_MIN + +try: + from _testbuffer import ndarray +except ImportError: + ndarray = None + +NULL = None + +class BadDescr: + def __get__(self, obj, objtype=None): + raise RuntimeError + +class WithDunder: + def _meth(self, *args): + if self.val: + return self.val + if self.exc: + raise self.exc + @classmethod + def with_val(cls, val): + obj = super().__new__(cls) + obj.val = val + obj.exc = None + setattr(cls, cls.methname, cls._meth) + return obj + + @classmethod + def with_exc(cls, exc): + obj = super().__new__(cls) + obj.val = None + obj.exc = exc + setattr(cls, cls.methname, cls._meth) + return obj + +class HasBadAttr: + def __new__(cls): + obj = super().__new__(cls) + setattr(cls, cls.methname, BadDescr()) + return obj + + +class IndexLike(WithDunder): + methname = '__index__' + +class IntLike(WithDunder): + methname = '__int__' + +class FloatLike(WithDunder): + methname = '__float__' + + +def subclassof(base): + return type(base.__name__ + 'Subclass', (base,), {}) + + +class SomeError(Exception): + pass + +class OtherError(Exception): + pass + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyNumber_Check() + check = _testcapi.number_check + + self.assertTrue(check(1)) + self.assertTrue(check(IndexLike.with_val(1))) + self.assertTrue(check(IntLike.with_val(99))) + self.assertTrue(check(0.5)) + self.assertTrue(check(FloatLike.with_val(4.25))) + self.assertTrue(check(1+2j)) + + self.assertFalse(check([])) + self.assertFalse(check("abc")) + self.assertFalse(check(object())) + self.assertFalse(check(NULL)) + + def test_unary_ops(self): + methmap = {'__neg__': _testcapi.number_negative, # PyNumber_Negative() + '__pos__': _testcapi.number_positive, # PyNumber_Positive() + '__abs__': _testcapi.number_absolute, # PyNumber_Absolute() + '__invert__': _testcapi.number_invert} # PyNumber_Invert() + + for name, func in methmap.items(): + # Generic object, has no tp_as_number structure + self.assertRaises(TypeError, func, object()) + + # C-API function accepts NULL + self.assertRaises(SystemError, func, NULL) + + # Behave as corresponding unary operation + op = getattr(operator, name) + for x in [0, 42, -1, 3.14, 1+2j]: + try: + op(x) + except TypeError: + self.assertRaises(TypeError, func, x) + else: + self.assertEqual(func(x), op(x)) + + def test_binary_ops(self): + methmap = {'__add__': _testcapi.number_add, # PyNumber_Add() + '__sub__': _testcapi.number_subtract, # PyNumber_Subtract() + '__mul__': _testcapi.number_multiply, # PyNumber_Multiply() + '__matmul__': _testcapi.number_matrixmultiply, # PyNumber_MatrixMultiply() + '__floordiv__': _testcapi.number_floordivide, # PyNumber_FloorDivide() + '__truediv__': _testcapi.number_truedivide, # PyNumber_TrueDivide() + '__mod__': _testcapi.number_remainder, # PyNumber_Remainder() + '__divmod__': _testcapi.number_divmod, # PyNumber_Divmod() + '__lshift__': _testcapi.number_lshift, # PyNumber_Lshift() + '__rshift__': _testcapi.number_rshift, # PyNumber_Rshift() + '__and__': _testcapi.number_and, # PyNumber_And() + '__xor__': _testcapi.number_xor, # PyNumber_Xor() + '__or__': _testcapi.number_or, # PyNumber_Or() + '__pow__': _testcapi.number_power, # PyNumber_Power() + '__iadd__': _testcapi.number_inplaceadd, # PyNumber_InPlaceAdd() + '__isub__': _testcapi.number_inplacesubtract, # PyNumber_InPlaceSubtract() + '__imul__': _testcapi.number_inplacemultiply, # PyNumber_InPlaceMultiply() + '__imatmul__': _testcapi.number_inplacematrixmultiply, # PyNumber_InPlaceMatrixMultiply() + '__ifloordiv__': _testcapi.number_inplacefloordivide, # PyNumber_InPlaceFloorDivide() + '__itruediv__': _testcapi.number_inplacetruedivide, # PyNumber_InPlaceTrueDivide() + '__imod__': _testcapi.number_inplaceremainder, # PyNumber_InPlaceRemainder() + '__ilshift__': _testcapi.number_inplacelshift, # PyNumber_InPlaceLshift() + '__irshift__': _testcapi.number_inplacershift, # PyNumber_InPlaceRshift() + '__iand__': _testcapi.number_inplaceand, # PyNumber_InPlaceAnd() + '__ixor__': _testcapi.number_inplacexor, # PyNumber_InPlaceXor() + '__ior__': _testcapi.number_inplaceor, # PyNumber_InPlaceOr() + '__ipow__': _testcapi.number_inplacepower, # PyNumber_InPlacePower() + } + + for name, func in methmap.items(): + cases = [0, 42, 3.14, -1, 123, 1+2j] + + # Generic object, has no tp_as_number structure + for x in cases: + self.assertRaises(TypeError, func, object(), x) + self.assertRaises(TypeError, func, x, object()) + + # Behave as corresponding binary operation + op = getattr(operator, name, divmod) + for x, y in itertools.combinations(cases, 2): + try: + op(x, y) + except (TypeError, ValueError, ZeroDivisionError) as exc: + self.assertRaises(exc.__class__, func, x, y) + else: + self.assertEqual(func(x, y), op(x, y)) + + # CRASHES func(NULL, object()) + # CRASHES func(object(), NULL) + + @unittest.skipIf(ndarray is None, "needs _testbuffer") + def test_misc_add(self): + # PyNumber_Add(), PyNumber_InPlaceAdd() + add = _testcapi.number_add + inplaceadd = _testcapi.number_inplaceadd + + # test sq_concat/sq_inplace_concat slots + a, b, r = [1, 2], [3, 4], [1, 2, 3, 4] + self.assertEqual(add(a, b), r) + self.assertEqual(a, [1, 2]) + self.assertRaises(TypeError, add, ndarray([1], (1,)), 2) + a, b, r = [1, 2], [3, 4], [1, 2, 3, 4] + self.assertEqual(inplaceadd(a, b), r) + self.assertEqual(a, r) + self.assertRaises(TypeError, inplaceadd, ndarray([1], (1,)), 2) + + @unittest.skipIf(ndarray is None, "needs _testbuffer") + def test_misc_multiply(self): + # PyNumber_Multiply(), PyNumber_InPlaceMultiply() + multiply = _testcapi.number_multiply + inplacemultiply = _testcapi.number_inplacemultiply + + # test sq_repeat/sq_inplace_repeat slots + a, b, r = [1], 2, [1, 1] + self.assertEqual(multiply(a, b), r) + self.assertEqual((a, b), ([1], 2)) + self.assertEqual(multiply(b, a), r) + self.assertEqual((a, b), ([1], 2)) + self.assertEqual(multiply([1], -1), []) + self.assertRaises(TypeError, multiply, ndarray([1], (1,)), 2) + self.assertRaises(TypeError, multiply, [1], 0.5) + self.assertRaises(OverflowError, multiply, [1], PY_SSIZE_T_MAX + 1) + self.assertRaises(MemoryError, multiply, [1, 2], PY_SSIZE_T_MAX//2 + 1) + a, b, r = [1], 2, [1, 1] + self.assertEqual(inplacemultiply(a, b), r) + self.assertEqual((a, b), (r, 2)) + a = [1] + self.assertEqual(inplacemultiply(b, a), r) + self.assertEqual((a, b), ([1], 2)) + self.assertRaises(TypeError, inplacemultiply, ndarray([1], (1,)), 2) + self.assertRaises(OverflowError, inplacemultiply, [1], PY_SSIZE_T_MAX + 1) + self.assertRaises(MemoryError, inplacemultiply, [1, 2], PY_SSIZE_T_MAX//2 + 1) + + def test_misc_power(self): + # PyNumber_Power(), PyNumber_InPlacePower() + power = _testcapi.number_power + inplacepower = _testcapi.number_inplacepower + + class HasPow(WithDunder): + methname = '__pow__' + + # ternary op + self.assertEqual(power(4, 11, 5), pow(4, 11, 5)) + self.assertRaises(TypeError, power, 4, 11, 1.25) + self.assertRaises(TypeError, power, 4, 11, HasPow.with_val(NotImplemented)) + self.assertRaises(TypeError, power, 4, 11, object()) + self.assertEqual(inplacepower(4, 11, 5), pow(4, 11, 5)) + self.assertRaises(TypeError, inplacepower, 4, 11, 1.25) + self.assertRaises(TypeError, inplacepower, 4, 11, object()) + + class X: + def __pow__(*args): + return args + + x = X() + self.assertEqual(power(x, 11), (x, 11)) + self.assertEqual(inplacepower(x, 11), (x, 11)) + self.assertEqual(power(x, 11, 5), (x, 11, 5)) + self.assertEqual(inplacepower(x, 11, 5), (x, 11, 5)) + + class X: + def __rpow__(*args): + return args + + x = X() + self.assertEqual(power(4, x), (x, 4)) + self.assertEqual(inplacepower(4, x), (x, 4)) + self.assertEqual(power(4, x, 5), (x, 4, 5)) + self.assertEqual(inplacepower(4, x, 5), (x, 4, 5)) + + class X: + def __ipow__(*args): + return args + + x = X() + self.assertEqual(inplacepower(x, 11), (x, 11)) + # XXX: In-place power doesn't pass the third arg to __ipow__. + self.assertEqual(inplacepower(x, 11, 5), (x, 11)) + + def test_long(self): + # Test PyNumber_Long() + long = _testcapi.number_long + + self.assertEqual(long(42), 42) + self.assertEqual(long(1.25), 1) + self.assertEqual(long("42"), 42) + self.assertEqual(long(b"42"), 42) + self.assertEqual(long(bytearray(b"42")), 42) + self.assertEqual(long(memoryview(b"42")), 42) + self.assertEqual(long(IndexLike.with_val(99)), 99) + self.assertEqual(long(IntLike.with_val(99)), 99) + + self.assertRaises(TypeError, long, IntLike.with_val(1.0)) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, long, IntLike.with_val(True)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(long(IntLike.with_val(True)), 1) + self.assertRaises(RuntimeError, long, IntLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, long, 1j) + self.assertRaises(TypeError, long, object()) + self.assertRaises(SystemError, long, NULL) + + def test_float(self): + # Test PyNumber_Float() + float_ = _testcapi.number_float + + self.assertEqual(float_(1.25), 1.25) + self.assertEqual(float_(123), 123.) + self.assertEqual(float_("1.25"), 1.25) + + self.assertEqual(float_(FloatLike.with_val(4.25)), 4.25) + self.assertEqual(float_(IndexLike.with_val(99)), 99.0) + self.assertEqual(float_(IndexLike.with_val(-1)), -1.0) + + self.assertRaises(TypeError, float_, FloatLike.with_val(687)) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, float_, FloatLike.with_val(subclassof(float)(4.25))) + with self.assertWarns(DeprecationWarning): + self.assertEqual(float_(FloatLike.with_val(subclassof(float)(4.25))), 4.25) + self.assertRaises(RuntimeError, float_, FloatLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, float_, IndexLike.with_val(1.25)) + self.assertRaises(OverflowError, float_, IndexLike.with_val(2**2000)) + + self.assertRaises(TypeError, float_, 1j) + self.assertRaises(TypeError, float_, object()) + self.assertRaises(SystemError, float_, NULL) + + def test_index(self): + # Test PyNumber_Index() + index = _testcapi.number_index + + self.assertEqual(index(11), 11) + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, index, IndexLike.with_val(True)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(index(IndexLike.with_val(True)), 1) + self.assertRaises(TypeError, index, IndexLike.with_val(1.0)) + self.assertRaises(RuntimeError, index, IndexLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, index, 1.25) + self.assertRaises(TypeError, index, "42") + self.assertRaises(TypeError, index, object()) + self.assertRaises(SystemError, index, NULL) + + def test_tobase(self): + # Test PyNumber_ToBase() + tobase = _testcapi.number_tobase + + self.assertEqual(tobase(10, 2), bin(10)) + self.assertEqual(tobase(11, 8), oct(11)) + self.assertEqual(tobase(16, 10), str(16)) + self.assertEqual(tobase(13, 16), hex(13)) + + self.assertRaises(SystemError, tobase, NULL, 2) + self.assertRaises(SystemError, tobase, 2, 3) + self.assertRaises(TypeError, tobase, 1.25, 2) + self.assertRaises(TypeError, tobase, "42", 2) + + def test_asssizet(self): + # Test PyNumber_AsSsize_t() + asssizet = _testcapi.number_asssizet + + for n in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + self.assertEqual(asssizet(n, OverflowError), n) + self.assertEqual(asssizet(PY_SSIZE_T_MAX+10, NULL), PY_SSIZE_T_MAX) + self.assertEqual(asssizet(PY_SSIZE_T_MIN-10, NULL), PY_SSIZE_T_MIN) + + self.assertRaises(OverflowError, asssizet, PY_SSIZE_T_MAX + 10, OverflowError) + self.assertRaises(RuntimeError, asssizet, PY_SSIZE_T_MAX + 10, RuntimeError) + self.assertRaises(SystemError, asssizet, NULL, TypeError) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py new file mode 100644 index 00000000000..807b62c1986 --- /dev/null +++ b/Lib/test/test_capi/test_object.py @@ -0,0 +1,258 @@ +import enum +import sys +import textwrap +import unittest +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import threading_helper +from test.support.script_helper import assert_python_failure + + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') + + +class Constant(enum.IntEnum): + Py_CONSTANT_NONE = 0 + Py_CONSTANT_FALSE = 1 + Py_CONSTANT_TRUE = 2 + Py_CONSTANT_ELLIPSIS = 3 + Py_CONSTANT_NOT_IMPLEMENTED = 4 + Py_CONSTANT_ZERO = 5 + Py_CONSTANT_ONE = 6 + Py_CONSTANT_EMPTY_STR = 7 + Py_CONSTANT_EMPTY_BYTES = 8 + Py_CONSTANT_EMPTY_TUPLE = 9 + + INVALID_CONSTANT = Py_CONSTANT_EMPTY_TUPLE + 1 + + +class GetConstantTest(unittest.TestCase): + def check_get_constant(self, get_constant): + self.assertIs(get_constant(Constant.Py_CONSTANT_NONE), None) + self.assertIs(get_constant(Constant.Py_CONSTANT_FALSE), False) + self.assertIs(get_constant(Constant.Py_CONSTANT_TRUE), True) + self.assertIs(get_constant(Constant.Py_CONSTANT_ELLIPSIS), Ellipsis) + self.assertIs(get_constant(Constant.Py_CONSTANT_NOT_IMPLEMENTED), NotImplemented) + + for constant_id, constant_type, value in ( + (Constant.Py_CONSTANT_ZERO, int, 0), + (Constant.Py_CONSTANT_ONE, int, 1), + (Constant.Py_CONSTANT_EMPTY_STR, str, ""), + (Constant.Py_CONSTANT_EMPTY_BYTES, bytes, b""), + (Constant.Py_CONSTANT_EMPTY_TUPLE, tuple, ()), + ): + with self.subTest(constant_id=constant_id): + obj = get_constant(constant_id) + self.assertEqual(type(obj), constant_type, obj) + self.assertEqual(obj, value) + + with self.assertRaises(SystemError): + get_constant(Constant.INVALID_CONSTANT) + + def test_get_constant(self): + self.check_get_constant(_testlimitedcapi.get_constant) + + def test_get_constant_borrowed(self): + self.check_get_constant(_testlimitedcapi.get_constant_borrowed) + + +class PrintTest(unittest.TestCase): + def testPyObjectPrintObject(self): + + class PrintableObject: + + def __repr__(self): + return "spam spam spam" + + def __str__(self): + return "egg egg egg" + + obj = PrintableObject() + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.call_pyobject_print(obj, output_filename, False) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), repr(obj)) + + # Test str printing + _testcapi.call_pyobject_print(obj, output_filename, True) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), str(obj)) + + def testPyObjectPrintNULL(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.pyobject_print_null(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), '') + + def testPyObjectPrintNoRefObject(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + correct_output = _testcapi.pyobject_print_noref_object(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), correct_output) + + def testPyObjectPrintOSError(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + open(output_filename, "w+").close() + with self.assertRaises(OSError): + _testcapi.pyobject_print_os_error(output_filename) + + +class ClearWeakRefsNoCallbacksTest(unittest.TestCase): + """Test PyUnstable_Object_ClearWeakRefsNoCallbacks""" + def test_ClearWeakRefsNoCallbacks(self): + """Ensure PyUnstable_Object_ClearWeakRefsNoCallbacks works""" + import weakref + import gc + class C: + pass + obj = C() + messages = [] + ref = weakref.ref(obj, lambda: messages.append("don't add this")) + self.assertIs(ref(), obj) + self.assertFalse(messages) + _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) + self.assertIsNone(ref()) + gc.collect() + self.assertFalse(messages) + + def test_ClearWeakRefsNoCallbacks_no_weakref_support(self): + """Don't fail on objects that don't support weakrefs""" + import weakref + obj = object() + with self.assertRaises(TypeError): + ref = weakref.ref(obj) + _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) + + +@threading_helper.requires_working_threading() +class EnableDeferredRefcountingTest(unittest.TestCase): + """Test PyUnstable_Object_EnableDeferredRefcount""" + @support.requires_resource("cpu") + def test_enable_deferred_refcount(self): + from threading import Thread + + self.assertEqual(_testcapi.pyobject_enable_deferred_refcount("not tracked"), 0) + foo = [] + self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(foo), int(support.Py_GIL_DISABLED)) + + # Make sure reference counting works on foo now + self.assertEqual(foo, []) + if support.Py_GIL_DISABLED: + self.assertTrue(_testinternalcapi.has_deferred_refcount(foo)) + + # Make sure that PyUnstable_Object_EnableDeferredRefcount is thread safe + def silly_func(obj): + self.assertIn( + _testcapi.pyobject_enable_deferred_refcount(obj), + (0, 1) + ) + + silly_list = [1, 2, 3] + threads = [ + Thread(target=silly_func, args=(silly_list,)) for _ in range(4) + ] + + with threading_helper.start_threads(threads): + for i in range(10): + silly_list.append(i) + + if support.Py_GIL_DISABLED: + self.assertTrue(_testinternalcapi.has_deferred_refcount(silly_list)) + + +class IsUniquelyReferencedTest(unittest.TestCase): + """Test PyUnstable_Object_IsUniquelyReferenced""" + def test_is_uniquely_referenced(self): + self.assertTrue(_testcapi.is_uniquely_referenced(object())) + self.assertTrue(_testcapi.is_uniquely_referenced([])) + # Immortals + self.assertFalse(_testcapi.is_uniquely_referenced("spanish inquisition")) + self.assertFalse(_testcapi.is_uniquely_referenced(42)) + # CRASHES is_uniquely_referenced(NULL) + +class CAPITest(unittest.TestCase): + def check_negative_refcount(self, code): + # bpo-35059: Check that Py_DECREF() reports the correct filename + # when calling _Py_NegativeRefcount() to abort Python. + code = textwrap.dedent(code) + rc, out, err = assert_python_failure('-c', code) + self.assertRegex(err, + br'object\.c:[0-9]+: ' + br'_Py_NegativeRefcount: Assertion failed: ' + br'object has negative ref count') + + @unittest.skipUnless(hasattr(_testcapi, 'negative_refcount'), + 'need _testcapi.negative_refcount()') + def test_negative_refcount(self): + code = """ + import _testcapi + from test import support + + with support.SuppressCrashReport(): + _testcapi.negative_refcount() + """ + self.check_negative_refcount(code) + + @unittest.skipUnless(hasattr(_testcapi, 'decref_freed_object'), + 'need _testcapi.decref_freed_object()') + @support.skip_if_sanitizer("use after free on purpose", + address=True, memory=True, ub=True) + def test_decref_freed_object(self): + code = """ + import _testcapi + from test import support + + with support.SuppressCrashReport(): + _testcapi.decref_freed_object() + """ + self.check_negative_refcount(code) + + @support.requires_resource('cpu') + def test_decref_delayed(self): + # gh-130519: Test that _PyObject_XDecRefDelayed() and QSBR code path + # handles destructors that are possibly re-entrant or trigger a GC. + import gc + + class MyObj: + def __del__(self): + gc.collect() + + for _ in range(1000): + obj = MyObj() + _testinternalcapi.incref_decref_delayed(obj) + + def test_is_unique_temporary(self): + self.assertTrue(_testcapi.pyobject_is_unique_temporary(object())) + obj = object() + self.assertFalse(_testcapi.pyobject_is_unique_temporary(obj)) + + def func(x): + # This relies on the LOAD_FAST_BORROW optimization (gh-130704) + self.assertEqual(sys.getrefcount(x), 1) + self.assertFalse(_testcapi.pyobject_is_unique_temporary(x)) + + func(object()) + + # Test that a newly created object in C is not considered + # a uniquely referenced temporary, because it's not on the stack. + # gh-142586: do the test in a loop over a list to test for handling + # tagged ints on the stack. + for i in [0, 1, 2]: + self.assertFalse(_testcapi.pyobject_is_unique_temporary_new_object()) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py new file mode 100644 index 00000000000..be3f8895db5 --- /dev/null +++ b/Lib/test/test_capi/test_opt.py @@ -0,0 +1,2021 @@ +import contextlib +import itertools +import sys +import textwrap +import unittest +import gc +import os + +import _opcode + +from test.support import (script_helper, requires_specialization, + import_helper, Py_GIL_DISABLED, requires_jit_enabled, + reset_code) + +_testinternalcapi = import_helper.import_module("_testinternalcapi") + +from _testinternalcapi import TIER2_THRESHOLD + + +@contextlib.contextmanager +def clear_executors(func): + # Clear executors in func before and after running a block + reset_code(func) + try: + yield + finally: + reset_code(func) + + +def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + +def iter_opnames(ex): + for item in ex: + yield item[0] + + +def get_opnames(ex): + return list(iter_opnames(ex)) + + +@requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +@requires_jit_enabled +class TestExecutorInvalidation(unittest.TestCase): + + def test_invalidate_object(self): + # Generate a new set of functions at each call + ns = {} + func_src = "\n".join( + f""" + def f{n}(): + for _ in range({TIER2_THRESHOLD}): + pass + """ for n in range(5) + ) + exec(textwrap.dedent(func_src), ns, ns) + funcs = [ ns[f'f{n}'] for n in range(5)] + objects = [object() for _ in range(5)] + + for f in funcs: + f() + executors = [get_first_executor(f) for f in funcs] + # Set things up so each executor depends on the objects + # with an equal or lower index. + for i, exe in enumerate(executors): + self.assertTrue(exe.is_valid()) + for obj in objects[:i+1]: + _testinternalcapi.add_executor_dependency(exe, obj) + self.assertTrue(exe.is_valid()) + # Assert that the correct executors are invalidated + # and check that nothing crashes when we invalidate + # an executor multiple times. + for i in (4,3,2,1,0): + _testinternalcapi.invalidate_executors(objects[i]) + for exe in executors[i:]: + self.assertFalse(exe.is_valid()) + for exe in executors[:i]: + self.assertTrue(exe.is_valid()) + + def test_uop_optimizer_invalidation(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(f""" + def f(): + for i in range({TIER2_THRESHOLD}): + pass + """), ns, ns) + f = ns['f'] + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + _testinternalcapi.invalidate_executors(f.__code__) + self.assertFalse(exe.is_valid()) + + def test_sys__clear_internal_caches(self): + def f(): + for _ in range(TIER2_THRESHOLD): + pass + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + sys._clear_internal_caches() + self.assertFalse(exe.is_valid()) + exe = get_first_executor(f) + self.assertIsNone(exe) + + +@requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +@requires_jit_enabled +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") +class TestUops(unittest.TestCase): + + def test_basic_loop(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_JUMP_TO_TOP", uops) + self.assertIn("_LOAD_FAST_BORROW_0", uops) + + def test_extended_arg(self): + "Check EXTENDED_ARG handling in superblock creation" + ns = {} + exec(textwrap.dedent(f""" + def many_vars(): + # 260 vars, so z9 should have index 259 + a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 + b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 + c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 + d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 + e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 + f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 + g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 + h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 + i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 + j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 + k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 + l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 + m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 + n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 + o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 + p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 + q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 + s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 + t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 + u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 + v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 + w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 + x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 + y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = {TIER2_THRESHOLD} + while z9 > 0: + z9 = z9 - 1 + +z9 + """), ns, ns) + many_vars = ns["many_vars"] + + ex = get_first_executor(many_vars) + self.assertIsNone(ex) + many_vars() + + ex = get_first_executor(many_vars) + self.assertIsNotNone(ex) + self.assertTrue(any((opcode, oparg, operand) == ("_LOAD_FAST_BORROW", 259, 0) + for opcode, oparg, _, operand in list(ex))) + + def test_unspecialized_unpack(self): + # An example of an unspecialized opcode + def testfunc(x): + i = 0 + while i < x: + i += 1 + a, b = {1: 2, 3: 3} + assert a == 1 and b == 3 + i = 0 + while i < x: + i += 1 + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_UNPACK_SEQUENCE", uops) + + def test_pop_jump_if_false(self): + def testfunc(n): + i = 0 + while i < n: + i += 1 + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_IS_TRUE_POP", uops) + + def test_pop_jump_if_none(self): + def testfunc(a): + for x in a: + if x is None: + x = 0 + + testfunc(range(TIER2_THRESHOLD)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) + + def test_pop_jump_if_not_none(self): + def testfunc(a): + for x in a: + x = None + if x is not None: + x = 0 + + testfunc(range(TIER2_THRESHOLD)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) + + def test_pop_jump_if_true(self): + def testfunc(n): + i = 0 + while not i >= n: + i += 1 + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_IS_FALSE_POP", uops) + + def test_jump_backward(self): + def testfunc(n): + i = 0 + while i < n: + i += 1 + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_JUMP_TO_TOP", uops) + + def test_jump_forward(self): + def testfunc(n): + a = 0 + while a < n: + if a < 0: + a = -a + else: + a = +a + a += 1 + return a + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # Since there is no JUMP_FORWARD instruction, + # look for indirect evidence: the += operator + self.assertIn("_BINARY_OP_ADD_INT", uops) + + def test_for_iter_range(self): + def testfunc(n): + total = 0 + for i in range(n): + total += i + return total + + total = testfunc(TIER2_THRESHOLD) + self.assertEqual(total, sum(range(TIER2_THRESHOLD))) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = get_opnames(ex) + self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_list(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + a = list(range(TIER2_THRESHOLD)) + total = testfunc(a) + self.assertEqual(total, sum(a)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = get_opnames(ex) + self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_tuple(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + a = tuple(range(TIER2_THRESHOLD)) + total = testfunc(a) + self.assertEqual(total, sum(a)) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = get_opnames(ex) + self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_list_edge_case(self): + def testfunc(it): + for x in it: + pass + + a = [1, 2, 3] + it = iter(a) + testfunc(it) + a.append(4) + with self.assertRaises(StopIteration): + next(it) + + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + + def test_branch_taken(self): + def testfunc(n): + for i in range(n): + if i < 0: + i = 0 + else: + i = 1 + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_IS_FALSE_POP", uops) + + def test_for_iter_tier_two(self): + class MyIter: + def __init__(self, n): + self.n = n + def __iter__(self): + return self + def __next__(self): + self.n -= 1 + if self.n < 0: + raise StopIteration + return self.n + + def testfunc(n, m): + x = 0 + for i in range(m): + for j in MyIter(n): + x += 1000*i + j + return x + + x = testfunc(TIER2_THRESHOLD, TIER2_THRESHOLD) + + self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * TIER2_THRESHOLD * 1001) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_FOR_ITER_TIER_TWO", uops) + + def test_confidence_score(self): + def testfunc(n): + bits = 0 + for i in range(n): + if i & 0x01: + bits += 1 + if i & 0x02: + bits += 1 + if i&0x04: + bits += 1 + if i&0x08: + bits += 1 + if i&0x10: + bits += 1 + return bits + + x = testfunc(TIER2_THRESHOLD * 2) + + self.assertEqual(x, TIER2_THRESHOLD * 5) + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + ops = list(iter_opnames(ex)) + #Since branch is 50/50 the trace could go either way. + count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") + self.assertLessEqual(count, 2) + + +@requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +@requires_jit_enabled +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") +class TestUopsOptimization(unittest.TestCase): + + def _run_with_optimizer(self, testfunc, arg): + res = testfunc(arg) + + ex = get_first_executor(testfunc) + return res, ex + + + def test_int_type_propagation(self): + def testfunc(loops): + num = 0 + for i in range(loops): + x = num + num + a = x + 1 + num += 1 + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 2 + 1) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] + guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_tos_int_count), 1) + self.assertLessEqual(len(guard_nos_int_count), 1) + + def test_int_type_propagation_through_frame(self): + def double(x): + return x + x + def testfunc(loops): + num = 0 + for i in range(loops): + x = num + num + a = double(x) + num += 1 + return a + + res = testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 4) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] + guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_tos_int_count), 1) + self.assertLessEqual(len(guard_nos_int_count), 1) + + def test_int_type_propagation_from_frame(self): + def double(x): + return x + x + def testfunc(loops): + num = 0 + for i in range(loops): + a = double(num) + x = a + a + num += 1 + return x + + res = testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 4) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] + guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + self.assertGreaterEqual(len(binop_count), 3) + self.assertLessEqual(len(guard_tos_int_count), 1) + self.assertLessEqual(len(guard_nos_int_count), 1) + + def test_int_impure_region(self): + def testfunc(loops): + num = 0 + while num < loops: + x = num + num + y = 1 + x // 2 + a = x + y + num += 1 + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + self.assertGreaterEqual(len(binop_count), 3) + + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertNotIn("_CHECK_PEP_523", uops) + + def test_int_type_propagate_through_range(self): + def testfunc(n): + + for i in range(n): + x = i + i + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 2) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_TOS_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + + def test_int_value_numbering(self): + def testfunc(n): + + y = 1 + for i in range(n): + x = y + z = x + a = z + b = a + res = x + z + a + b + return res + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 4) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_TOS_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + guard_tos_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] + self.assertEqual(len(guard_tos_count), 1) + + def test_comprehension(self): + def testfunc(n): + for _ in range(n): + return [i for i in range(n)] + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, list(range(TIER2_THRESHOLD))) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_BINARY_OP_ADD_INT", uops) + + def test_call_py_exact_args_disappearing(self): + def dummy(x): + return x+1 + + def testfunc(n): + for i in range(n): + dummy(i) + + # Trigger specialization + testfunc(8) + del dummy + gc.collect() + + def dummy(x): + return x + 2 + testfunc(32) + + ex = get_first_executor(testfunc) + # Honestly as long as it doesn't crash it's fine. + # Whether we get an executor or not is non-deterministic, + # because it's decided by when the function is freed. + # This test is a little implementation specific. + + def test_promote_globals_to_constants(self): + + result = script_helper.run_python_until_end('-c', textwrap.dedent(""" + import _testinternalcapi + import opcode + import _opcode + + def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + def get_opnames(ex): + return {item[0] for item in ex} + + def testfunc(n): + for i in range(n): + x = range(i) + return x + + testfunc(_testinternalcapi.TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + assert ex is not None + uops = get_opnames(ex) + assert "_LOAD_GLOBAL_BUILTINS" not in uops + assert "_LOAD_CONST_INLINE_BORROW" in uops + """), PYTHON_JIT="1") + self.assertEqual(result[0].rc, 0, result) + + def test_float_add_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a + 0.25 + a = a + 0.25 + a = a + 0.25 + a = a + 0.25 + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertAlmostEqual(res, TIER2_THRESHOLD + 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] + guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + self.assertLessEqual(len(guard_tos_float_count), 1) + self.assertLessEqual(len(guard_nos_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + + def test_float_subtract_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a - 0.25 + a = a - 0.25 + a = a - 0.25 + a = a - 0.25 + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertAlmostEqual(res, -TIER2_THRESHOLD + 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] + guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + self.assertLessEqual(len(guard_tos_float_count), 1) + self.assertLessEqual(len(guard_nos_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + + def test_float_multiply_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a * 1.0 + a = a * 1.0 + a = a * 1.0 + a = a * 1.0 + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertAlmostEqual(res, 1.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] + guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + self.assertLessEqual(len(guard_tos_float_count), 1) + self.assertLessEqual(len(guard_nos_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + + def test_add_unicode_propagation(self): + def testfunc(n): + a = "" + for _ in range(n): + a + a + a + a + a + a + a + a + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, "") + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_tos_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_UNICODE"] + guard_nos_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_UNICODE"] + self.assertLessEqual(len(guard_tos_unicode_count), 1) + self.assertLessEqual(len(guard_nos_unicode_count), 1) + self.assertIn("_BINARY_OP_ADD_UNICODE", uops) + + def test_compare_op_type_propagation_float(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] + guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + self.assertLessEqual(len(guard_tos_float_count), 1) + self.assertLessEqual(len(guard_nos_float_count), 1) + self.assertIn("_COMPARE_OP_FLOAT", uops) + + def test_compare_op_type_propagation_int(self): + def testfunc(n): + a = 1 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] + guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + self.assertLessEqual(len(guard_tos_int_count), 1) + self.assertLessEqual(len(guard_nos_int_count), 1) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_int_partial(self): + def testfunc(n): + a = 1 + for _ in range(n): + if a > 2: + x = 0 + if a < 2: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] + self.assertLessEqual(len(guard_nos_int_count), 1) + self.assertEqual(len(guard_tos_int_count), 0) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_float_partial(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + if a > 2.0: + x = 0 + if a < 2.0: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] + self.assertLessEqual(len(guard_nos_float_count), 1) + self.assertEqual(len(guard_tos_float_count), 0) + self.assertIn("_COMPARE_OP_FLOAT", uops) + + def test_compare_op_type_propagation_unicode(self): + def testfunc(n): + a = "" + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_tos_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_UNICODE"] + guard_nos_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_UNICODE"] + self.assertLessEqual(len(guard_tos_unicode_count), 1) + self.assertLessEqual(len(guard_nos_unicode_count), 1) + self.assertIn("_COMPARE_OP_STR", uops) + + def test_type_inconsistency(self): + ns = {} + src = textwrap.dedent(""" + def testfunc(n): + for i in range(n): + x = _test_global + _test_global + """) + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) + self.assertIsNone(ex) + ns['_test_global'] = 1 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_TOS_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + # Try again, but between the runs, set the global to a float. + # This should result in no executor the second time. + ns = {} + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) + self.assertIsNone(ex) + ns['_test_global'] = 3.14 + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) + self.assertIsNone(ex) + + def test_combine_stack_space_checks_sequential(self): + def dummy12(x): + return x - 1 + def dummy13(y): + z = y + 2 + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(7) + c, d = dummy13(9) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 26) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # sequential calls: max(12, 13) == 13 + largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_nested(self): + def dummy12(x): + return x + 3 + def dummy15(y): + z = dummy12(y) + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b, c = dummy15(2) + a += b + c + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 7) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # nested calls: 15 + 12 == 27 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy12.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls(self): + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(5) + c, d, e = dummy18(2) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 25) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(12, 18 + max(12, 13)) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls_different_order(self): + # same as `several_calls` but with top-level calls reversed + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + c, d, e = dummy18(2) + b = dummy12(5) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 25) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(18 + max(12, 13), 12) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_complex(self): + def dummy0(x): + return x + def dummy1(x): + return dummy0(x) + def dummy2(x): + return dummy1(x) + def dummy3(x): + return dummy0(x) + def dummy4(x): + y = dummy0(x) + return dummy3(y) + def dummy5(x): + return dummy2(x) + def dummy6(x): + y = dummy5(x) + z = dummy0(y) + return dummy4(z) + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy5(1) + c = dummy0(1) + d = dummy6(1) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 3) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 15) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 15) + + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy6.__code__) + + _testinternalcapi.get_co_framesize(dummy5.__code__) + + _testinternalcapi.get_co_framesize(dummy2.__code__) + + _testinternalcapi.get_co_framesize(dummy1.__code__) + + _testinternalcapi.get_co_framesize(dummy0.__code__) + ) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_large_framesize(self): + # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is + # actually doing its job. Note that the resulting trace hits + # UOP_MAX_TRACE_LENGTH, but since all _CHECK_STACK_SPACEs happen early, this + # test is still meaningful. + repetitions = 10000 + ns = {} + header = """ + def dummy_large(a0): + """ + body = "".join([f""" + a{n+1} = a{n} + 1 + """ for n in range(repetitions)]) + return_ = f""" + return a{repetitions-1} + """ + exec(textwrap.dedent(header + body + return_), ns, ns) + dummy_large = ns['dummy_large'] + + # this is something like: + # + # def dummy_large(a0): + # a1 = a0 + 1 + # a2 = a1 + 1 + # .... + # a9999 = a9998 + 1 + # return a9999 + + def dummy15(z): + y = dummy_large(z) + return y + 3 + + def testfunc(n): + b = 0 + for _ in range(n): + b += dummy15(7) + return b + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * (repetitions + 9)) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + + # this hits a different case during trace projection in refcount test runs only, + # so we need to account for both possibilities + self.assertIn(uop_names.count("_CHECK_STACK_SPACE"), [0, 1]) + if uop_names.count("_CHECK_STACK_SPACE") == 0: + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy_large.__code__) + ) + else: + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_recursion(self): + def dummy15(x): + while x > 0: + return dummy15(x - 1) + return 42 + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy15(n) + return a + + recursion_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(TIER2_THRESHOLD + recursion_limit) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + finally: + sys.setrecursionlimit(recursion_limit) + self.assertEqual(res, TIER2_THRESHOLD * 42) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 1) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_many_nested(self): + # overflow the trace_stack + def dummy_a(x): + return x + def dummy_b(x): + return dummy_a(x) + def dummy_c(x): + return dummy_b(x) + def dummy_d(x): + return dummy_c(x) + def dummy_e(x): + return dummy_d(x) + def dummy_f(x): + return dummy_e(x) + def dummy_g(x): + return dummy_f(x) + def dummy_h(x): + return dummy_g(x) + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy_h(n) + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 32 * 32) + self.assertIsNone(ex) + + def test_return_generator(self): + def gen(): + yield None + def testfunc(n): + for i in range(n): + gen() + return i + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD - 1) + self.assertIsNotNone(ex) + self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) + + @unittest.skip("Tracing into generators currently isn't supported.") + def test_for_iter_gen(self): + def gen(n): + for i in range(n): + yield i + def testfunc(n): + g = gen(n) + s = 0 + for i in g: + s += i + return s + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, sum(range(TIER2_THRESHOLD))) + self.assertIsNotNone(ex) + self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex)) + + def test_modified_local_is_seen_by_optimized_code(self): + l = sys._getframe().f_locals + a = 1 + s = 0 + for j in range(1 << 10): + a + a + l["xa"[j >> 9]] = 1.0 + s += a + self.assertIs(type(a), float) + self.assertIs(type(s), float) + self.assertEqual(s, 1024.0) + + def test_guard_type_version_removed(self): + def thing(a): + x = 0 + for _ in range(TIER2_THRESHOLD): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 2) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_removed_inlined(self): + """ + Verify that the guard type version if we have an inlined function + """ + + def fn(): + pass + + def thing(a): + x = 0 + for _ in range(TIER2_THRESHOLD): + x += a.attr + fn() + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 2) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_removed_invalidation(self): + + def thing(a): + x = 0 + for i in range(TIER2_THRESHOLD * 2 + 1): + x += a.attr + # The first TIER2_THRESHOLD iterations we set the attribute on + # this dummy class, which shouldn't trigger the type watcher. + # Note that the code needs to be in this weird form so it's + # optimized inline without any control flow: + setattr((Bar, Foo)[i == TIER2_THRESHOLD + 1], "attr", 2) + x += a.attr + return x + + class Foo: + attr = 1 + + class Bar: + pass + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 6 + 1) + call = opnames.index("_CALL_BUILTIN_FAST") + load_attr_top = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", 0, call) + load_attr_bottom = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", call) + self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) + self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2) + + def test_guard_type_version_removed_escaping(self): + + def thing(a): + x = 0 + for i in range(TIER2_THRESHOLD): + x += a.attr + # eval should be escaping + eval("None") + x += a.attr + return x + + class Foo: + attr = 1 + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 2) + call = opnames.index("_CALL_BUILTIN_FAST_WITH_KEYWORDS") + load_attr_top = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", 0, call) + load_attr_bottom = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", call) + self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) + self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2) + + def test_guard_type_version_executor_invalidated(self): + """ + Verify that the executor is invalided on a type change. + """ + + def thing(a): + x = 0 + for i in range(TIER2_THRESHOLD): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + self.assertEqual(res, TIER2_THRESHOLD * 2) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1) + self.assertTrue(ex.is_valid()) + Foo.attr = 0 + self.assertFalse(ex.is_valid()) + + def test_type_version_doesnt_segfault(self): + """ + Tests that setting a type version doesn't cause a segfault when later looking at the stack. + """ + + # Minimized from mdp.py benchmark + + class A: + def __init__(self): + self.attr = {} + + def method(self, arg): + self.attr[arg] = None + + def fn(a): + for _ in range(100): + (_ for _ in []) + (_ for _ in [a.method(None)]) + + fn(A()) + + def test_func_guards_removed_or_reduced(self): + def testfunc(n): + for i in range(n): + # Only works on functions promoted to constants + global_identity(i) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + # Strength reduced version + self.assertIn("_CHECK_FUNCTION_VERSION_INLINE", uops) + self.assertNotIn("_CHECK_FUNCTION_VERSION", uops) + # Removed guard + self.assertNotIn("_CHECK_FUNCTION_EXACT_ARGS", uops) + + def test_jit_error_pops(self): + """ + Tests that the correct number of pops are inserted into the + exit stub + """ + items = 17 * [None] + [[]] + with self.assertRaises(TypeError): + {item for item in items} + + def test_power_type_depends_on_input_values(self): + template = textwrap.dedent(""" + import _testinternalcapi + + L, R, X, Y = {l}, {r}, {x}, {y} + + def check(actual: complex, expected: complex) -> None: + assert actual == expected, (actual, expected) + assert type(actual) is type(expected), (actual, expected) + + def f(l: complex, r: complex) -> None: + expected_local_local = pow(l, r) + pow(l, r) + expected_const_local = pow(L, r) + pow(L, r) + expected_local_const = pow(l, R) + pow(l, R) + expected_const_const = pow(L, R) + pow(L, R) + for _ in range(_testinternalcapi.TIER2_THRESHOLD): + # Narrow types: + l + l, r + r + # The powers produce results, and the addition is unguarded: + check(l ** r + l ** r, expected_local_local) + check(L ** r + L ** r, expected_const_local) + check(l ** R + l ** R, expected_local_const) + check(L ** R + L ** R, expected_const_const) + + # JIT for one pair of values... + f(L, R) + # ...then run with another: + f(X, Y) + """) + interesting = [ + (1, 1), # int ** int -> int + (1, -1), # int ** int -> float + (1.0, 1), # float ** int -> float + (1, 1.0), # int ** float -> float + (-1, 0.5), # int ** float -> complex + (1.0, 1.0), # float ** float -> float + (-1.0, 0.5), # float ** float -> complex + ] + for (l, r), (x, y) in itertools.product(interesting, repeat=2): + s = template.format(l=l, r=r, x=x, y=y) + with self.subTest(l=l, r=r, x=x, y=y): + script_helper.assert_python_ok("-c", s) + + def test_symbols_flow_through_tuples(self): + def testfunc(n): + for _ in range(n): + a = 1 + b = 2 + t = a, b + x, y = t + r = x + y + return r + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 3) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GUARD_TOS_INT", uops) + + def test_decref_escapes(self): + class Convert9999ToNone: + def __del__(self): + ns = sys._getframe(1).f_locals + if ns["i"] == _testinternalcapi.TIER2_THRESHOLD: + ns["i"] = None + + def crash_addition(): + try: + for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + n = Convert9999ToNone() + i + i # Remove guards for i. + n = None # Change i. + i + i # This crashed when we didn't treat DECREF as escaping (gh-124483) + except TypeError: + pass + + crash_addition() + + def test_narrow_type_to_constant_bool_false(self): + def f(n): + trace = [] + for i in range(n): + # false is always False, but we can only prove that it's a bool: + false = i == TIER2_THRESHOLD + trace.append("A") + if not false: # Kept. + trace.append("B") + if not false: # Removed! + trace.append("C") + trace.append("D") + if false: # Removed! + trace.append("X") + trace.append("E") + trace.append("F") + if false: # Removed! + trace.append("X") + trace.append("G") + return trace + + trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # Only one guard remains: + self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 1) + self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 0) + # But all of the appends we care about are still there: + self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + + def test_narrow_type_to_constant_bool_true(self): + def f(n): + trace = [] + for i in range(n): + # true always True, but we can only prove that it's a bool: + true = i != TIER2_THRESHOLD + trace.append("A") + if true: # Kept. + trace.append("B") + if not true: # Removed! + trace.append("X") + trace.append("C") + if true: # Removed! + trace.append("D") + trace.append("E") + trace.append("F") + if not true: # Removed! + trace.append("X") + trace.append("G") + return trace + + trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # Only one guard remains: + self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 0) + self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 1) + # But all of the appends we care about are still there: + self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + + def test_narrow_type_to_constant_int_zero(self): + def f(n): + trace = [] + for i in range(n): + # zero is always (int) 0, but we can only prove that it's a integer: + false = i == TIER2_THRESHOLD # this will always be false, while hopefully still fooling optimizer improvements + zero = false + 0 # this should always set the variable zero equal to 0 + trace.append("A") + if not zero: # Kept. + trace.append("B") + if not zero: # Removed! + trace.append("C") + trace.append("D") + if zero: # Removed! + trace.append("X") + trace.append("E") + trace.append("F") + if zero: # Removed! + trace.append("X") + trace.append("G") + return trace + + trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # Only one guard remains: + self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 1) + self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 0) + # But all of the appends we care about are still there: + self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + + def test_narrow_type_to_constant_str_empty(self): + def f(n): + trace = [] + for i in range(n): + # Hopefully the optimizer can't guess what the value is. + # empty is always "", but we can only prove that it's a string: + false = i == TIER2_THRESHOLD + empty = "X"[:false] + trace.append("A") + if not empty: # Kept. + trace.append("B") + if not empty: # Removed! + trace.append("C") + trace.append("D") + if empty: # Removed! + trace.append("X") + trace.append("E") + trace.append("F") + if empty: # Removed! + trace.append("X") + trace.append("G") + return trace + + trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # Only one guard remains: + self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 1) + self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 0) + # But all of the appends we care about are still there: + self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + + def test_compare_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = 10 + b = 10 + if a == b: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_COMPARE_OP_INT", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + + def test_to_bool_bool_contains_op_set(self): + """ + Test that _TO_BOOL_BOOL is removed from code like: + + res = foo in some_set + if res: + .... + + """ + def testfunc(n): + x = 0 + s = {1, 2, 3} + for _ in range(n): + a = 2 + in_set = a in s + if in_set: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CONTAINS_OP_SET", uops) + self.assertNotIn("_TO_BOOL_BOOL", uops) + + def test_to_bool_bool_contains_op_dict(self): + """ + Test that _TO_BOOL_BOOL is removed from code like: + + res = foo in some_dict + if res: + .... + + """ + def testfunc(n): + x = 0 + s = {1: 1, 2: 2, 3: 3} + for _ in range(n): + a = 2 + in_dict = a in s + if in_dict: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CONTAINS_OP_DICT", uops) + self.assertNotIn("_TO_BOOL_BOOL", uops) + + + def test_remove_guard_for_known_type_str(self): + def f(n): + for i in range(n): + false = i == TIER2_THRESHOLD + empty = "X"[:false] + empty += "" # Make JIT realize this is a string. + if empty: + return 1 + return 0 + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_TO_BOOL_STR", uops) + self.assertNotIn("_GUARD_TOS_UNICODE", uops) + + def test_remove_guard_for_known_type_dict(self): + def f(n): + x = 0 + for _ in range(n): + d = {} + d["Spam"] = 1 # unguarded! + x += d["Spam"] # ...unguarded! + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_GUARD_NOS_DICT"), 0) + self.assertEqual(uops.count("_STORE_SUBSCR_DICT"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_DICT"), 1) + + def test_remove_guard_for_known_type_list(self): + def f(n): + x = 0 + for _ in range(n): + l = [0] + l[0] = 1 # unguarded! + [a] = l # ...unguarded! + b = l[0] # ...unguarded! + if l: # ...unguarded! + x += a + b + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 2 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_GUARD_NOS_LIST"), 0) + self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT"), 1) + self.assertEqual(uops.count("_GUARD_TOS_LIST"), 0) + self.assertEqual(uops.count("_UNPACK_SEQUENCE_LIST"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT"), 1) + self.assertEqual(uops.count("_TO_BOOL_LIST"), 1) + + def test_remove_guard_for_known_type_set(self): + def f(n): + x = 0 + for _ in range(n): + x += "Spam" in {"Spam"} # Unguarded! + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_TOS_ANY_SET", uops) + self.assertIn("_CONTAINS_OP_SET", uops) + + def test_remove_guard_for_known_type_tuple(self): + def f(n): + x = 0 + for _ in range(n): + t = (1, 2, (3, (4,))) + t_0, t_1, (t_2_0, t_2_1) = t # Unguarded! + t_2_1_0 = t_2_1[0] # Unguarded! + x += t_0 + t_1 + t_2_0 + t_2_1_0 + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 10 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_TOS_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) + self.assertNotIn("_GUARD_NOS_TUPLE", uops) + self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) + + def test_binary_subcsr_str_int_narrows_to_str(self): + def testfunc(n): + x = [] + s = "foo" + for _ in range(n): + y = s[0] # _BINARY_OP_SUBSCR_STR_INT + z = "bar" + y # (_GUARD_TOS_UNICODE) + _BINARY_OP_ADD_UNICODE + x.append(z) + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, ["barf"] * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBSCR_STR_INT", uops) + # _BINARY_OP_SUBSCR_STR_INT narrows the result to 'str' so + # the unicode guard before _BINARY_OP_ADD_UNICODE is removed. + self.assertNotIn("_GUARD_TOS_UNICODE", uops) + self.assertIn("_BINARY_OP_ADD_UNICODE", uops) + + def test_call_type_1(self): + def testfunc(n): + x = 0 + for _ in range(n): + x += type(42) is int + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_TYPE_1", uops) + self.assertNotIn("_GUARD_NOS_NULL", uops) + self.assertNotIn("_GUARD_CALLABLE_TYPE_1", uops) + + def test_call_type_1_result_is_const(self): + def testfunc(n): + x = 0 + for _ in range(n): + t = type(42) + if t is not None: # guard is removed + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_TYPE_1", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) + + def test_call_str_1(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = str(42) + if y == '42': + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_STR_1", uops) + self.assertNotIn("_GUARD_NOS_NULL", uops) + self.assertNotIn("_GUARD_CALLABLE_STR_1", uops) + + def test_call_str_1_result_is_str(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = str(42) + 'foo' + if y == '42foo': + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_STR_1", uops) + self.assertIn("_BINARY_OP_ADD_UNICODE", uops) + self.assertNotIn("_GUARD_NOS_UNICODE", uops) + self.assertNotIn("_GUARD_TOS_UNICODE", uops) + + def test_call_str_1_result_is_const_for_str_input(self): + # Test a special case where the argument of str(arg) + # is known to be a string. The information about the + # argument being a string should be propagated to the + # result of str(arg). + def testfunc(n): + x = 0 + for _ in range(n): + y = str('foo') # string argument + if y: # _TO_BOOL_STR + _GUARD_IS_TRUE_POP are removed + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_STR_1", uops) + self.assertNotIn("_TO_BOOL_STR", uops) + self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + + def test_call_tuple_1(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = tuple([1, 2]) # _CALL_TUPLE_1 + if y == (1, 2): + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_TUPLE_1", uops) + self.assertNotIn("_GUARD_NOS_NULL", uops) + self.assertNotIn("_GUARD_CALLABLE_TUPLE_1", uops) + + def test_call_tuple_1_result_is_tuple(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = tuple([1, 2]) # _CALL_TUPLE_1 + if y[0] == 1: # _BINARY_OP_SUBSCR_TUPLE_INT + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_TUPLE_1", uops) + self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) + self.assertNotIn("_GUARD_NOS_TUPLE", uops) + + def test_call_tuple_1_result_propagates_for_tuple_input(self): + # Test a special case where the argument of tuple(arg) + # is known to be a tuple. The information about the + # argument being a tuple should be propagated to the + # result of tuple(arg). + def testfunc(n): + x = 0 + for _ in range(n): + y = tuple((1, 2)) # tuple argument + a, _ = y # _UNPACK_SEQUENCE_TWO_TUPLE + if a == 1: # _COMPARE_OP_INT + _GUARD_IS_TRUE_POP are removed + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_TUPLE_1", uops) + self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) + self.assertNotIn("_COMPARE_OP_INT", uops) + self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + + def test_call_len(self): + def testfunc(n): + a = [1, 2, 3, 4] + for _ in range(n): + _ = len(a) - 1 + + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_NOS_NULL", uops) + self.assertNotIn("_GUARD_CALLABLE_LEN", uops) + self.assertIn("_CALL_LEN", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GUARD_TOS_INT", uops) + + def test_binary_op_subscr_tuple_int(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = (1, 2) + if y[0] == 1: # _COMPARE_OP_INT + _GUARD_IS_TRUE_POP are removed + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) + self.assertNotIn("_COMPARE_OP_INT", uops) + self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + + def test_attr_promotion_failure(self): + # We're not testing for any specific uops here, just + # testing it doesn't crash. + script_helper.assert_python_ok('-c', textwrap.dedent(""" + import _testinternalcapi + import _opcode + import email + + def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + def testfunc(n): + for _ in range(n): + email.jit_testing = None + prompt = email.jit_testing + del email.jit_testing + + + testfunc(_testinternalcapi.TIER2_THRESHOLD) + ex = get_first_executor(testfunc) + assert ex is not None + """)) + + def test_interpreter_finalization_with_generator_alive(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + t = tuple(range(%d)) + def simple_for(): + for x in t: + x + + def gen(): + try: + yield + except: + simple_for() + + sys.settrace(lambda *args: None) + simple_for() + g = gen() + next(g) + """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) + + def test_next_instr_for_exception_handler_set(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + for i in range(TIER2_THRESHOLD + 3): + try: + undefined_variable(i) + except Exception: + pass + + f() + + def test_next_instr_for_exception_handler_set_lasts_instr(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + a_list = [] + for _ in range(TIER2_THRESHOLD + 3): + try: + a_list[""] = 0 + except Exception: + pass + + +def global_identity(x): + return x + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_pyatomic.py b/Lib/test/test_capi/test_pyatomic.py new file mode 100644 index 00000000000..846d6d50c25 --- /dev/null +++ b/Lib/test/test_capi/test_pyatomic.py @@ -0,0 +1,15 @@ +import unittest +from test.support import import_helper + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + +class PyAtomicTests(unittest.TestCase): + pass + +for name in sorted(dir(_testcapi)): + if name.startswith('test_atomic'): + setattr(PyAtomicTests, name, getattr(_testcapi, name)) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py new file mode 100644 index 00000000000..894f66b437a --- /dev/null +++ b/Lib/test/test_capi/test_run.py @@ -0,0 +1,118 @@ +import os +import unittest +from collections import UserDict +from test.support import import_helper +from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE + +NULL = None +_testcapi = import_helper.import_module('_testcapi') +Py_single_input = _testcapi.Py_single_input +Py_file_input = _testcapi.Py_file_input +Py_eval_input = _testcapi.Py_eval_input + + +class DictSubclass(dict): + pass + + +class CAPITest(unittest.TestCase): + # TODO: Test the following functions: + # + # PyRun_SimpleStringFlags + # PyRun_AnyFileExFlags + # PyRun_SimpleFileExFlags + # PyRun_InteractiveOneFlags + # PyRun_InteractiveOneObject + # PyRun_InteractiveLoopFlags + # PyRun_String (may be a macro) + # PyRun_AnyFile (may be a macro) + # PyRun_AnyFileEx (may be a macro) + # PyRun_AnyFileFlags (may be a macro) + # PyRun_SimpleString (may be a macro) + # PyRun_SimpleFile (may be a macro) + # PyRun_SimpleFileEx (may be a macro) + # PyRun_InteractiveOne (may be a macro) + # PyRun_InteractiveLoop (may be a macro) + # PyRun_File (may be a macro) + # PyRun_FileEx (may be a macro) + # PyRun_FileFlags (may be a macro) + + def test_run_stringflags(self): + # Test PyRun_StringFlags(). + # XXX: fopen() uses different path encoding than Python on Windows. + def run(s, *args): + return _testcapi.run_stringflags(s, Py_file_input, *args) + source = b'a\n' + + self.assertIsNone(run(b'a\n', dict(a=1))) + self.assertIsNone(run(b'a\n', dict(a=1), {})) + self.assertIsNone(run(b'a\n', {}, dict(a=1))) + self.assertIsNone(run(b'a\n', {}, UserDict(a=1))) + + self.assertRaises(NameError, run, b'a\n', {}) + self.assertRaises(NameError, run, b'a\n', {}, {}) + self.assertRaises(TypeError, run, b'a\n', dict(a=1), []) + self.assertRaises(TypeError, run, b'a\n', dict(a=1), 1) + + self.assertIsNone(run(b'a\n', DictSubclass(a=1))) + self.assertIsNone(run(b'a\n', DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, b'a\n', DictSubclass()) + + self.assertIsNone(run(b'\xc3\xa4\n', {'\xe4': 1})) + self.assertRaises(SyntaxError, run, b'\xe4\n', {}) + + self.assertRaises(SystemError, run, b'a\n', NULL) + self.assertRaises(SystemError, run, b'a\n', NULL, {}) + self.assertRaises(SystemError, run, b'a\n', NULL, dict(a=1)) + self.assertRaises(SystemError, run, b'a\n', UserDict()) + self.assertRaises(SystemError, run, b'a\n', UserDict(), {}) + self.assertRaises(SystemError, run, b'a\n', UserDict(), dict(a=1)) + + # CRASHES run(NULL, {}) + + def test_run_fileexflags(self): + # Test PyRun_FileExFlags(). + filename = os.fsencode(TESTFN if os.name != 'nt' else TESTFN_ASCII) + with open(filename, 'wb') as fp: + fp.write(b'a\n') + self.addCleanup(unlink, filename) + def run(*args): + return _testcapi.run_fileexflags(filename, Py_file_input, *args) + + self.assertIsNone(run(dict(a=1))) + self.assertIsNone(run(dict(a=1), {})) + self.assertIsNone(run({}, dict(a=1))) + self.assertIsNone(run({}, UserDict(a=1))) + self.assertIsNone(run(dict(a=1), {}, 1)) # closeit = True + + self.assertRaises(NameError, run, {}) + self.assertRaises(NameError, run, {}, {}) + self.assertRaises(TypeError, run, dict(a=1), []) + self.assertRaises(TypeError, run, dict(a=1), 1) + + self.assertIsNone(run(DictSubclass(a=1))) + self.assertIsNone(run(DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, DictSubclass()) + + self.assertRaises(SystemError, run, NULL) + self.assertRaises(SystemError, run, NULL, {}) + self.assertRaises(SystemError, run, NULL, dict(a=1)) + self.assertRaises(SystemError, run, UserDict()) + self.assertRaises(SystemError, run, UserDict(), {}) + self.assertRaises(SystemError, run, UserDict(), dict(a=1)) + + @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') + @unittest.skipIf(os.name == 'nt', 'does not work on Windows') + def test_run_fileexflags_with_undecodable_filename(self): + run = _testcapi.run_fileexflags + try: + with open(TESTFN_UNDECODABLE, 'wb') as fp: + fp.write(b'a\n') + self.addCleanup(unlink, TESTFN_UNDECODABLE) + except OSError: + self.skipTest('undecodable paths are not supported') + self.assertIsNone(run(TESTFN_UNDECODABLE, Py_file_input, dict(a=1))) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_capi/test_set.py b/Lib/test/test_capi/test_set.py new file mode 100644 index 00000000000..62d90a3f943 --- /dev/null +++ b/Lib/test/test_capi/test_set.py @@ -0,0 +1,271 @@ +import unittest + +from test.support import import_helper + +# Skip this test if the _testcapi, _testlimitedcapi or _testinternalcapi +# modules aren't available. +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') + +class set_subclass(set): + pass + +class frozenset_subclass(frozenset): + pass + + +class BaseSetTests: + def assertImmutable(self, action, *args): + self.assertRaises(SystemError, action, frozenset(), *args) + self.assertRaises(SystemError, action, frozenset({1}), *args) + self.assertRaises(SystemError, action, frozenset_subclass(), *args) + self.assertRaises(SystemError, action, frozenset_subclass({1}), *args) + + +class TestSetCAPI(BaseSetTests, unittest.TestCase): + def test_set_check(self): + check = _testlimitedcapi.set_check + self.assertTrue(check(set())) + self.assertTrue(check({1, 2})) + self.assertFalse(check(frozenset())) + self.assertTrue(check(set_subclass())) + self.assertFalse(check(frozenset_subclass())) + self.assertFalse(check(object())) + # CRASHES: check(NULL) + + def test_set_check_exact(self): + check = _testlimitedcapi.set_checkexact + self.assertTrue(check(set())) + self.assertTrue(check({1, 2})) + self.assertFalse(check(frozenset())) + self.assertFalse(check(set_subclass())) + self.assertFalse(check(frozenset_subclass())) + self.assertFalse(check(object())) + # CRASHES: check(NULL) + + def test_frozenset_check(self): + check = _testlimitedcapi.frozenset_check + self.assertFalse(check(set())) + self.assertTrue(check(frozenset())) + self.assertTrue(check(frozenset({1, 2}))) + self.assertFalse(check(set_subclass())) + self.assertTrue(check(frozenset_subclass())) + self.assertFalse(check(object())) + # CRASHES: check(NULL) + + def test_frozenset_check_exact(self): + check = _testlimitedcapi.frozenset_checkexact + self.assertFalse(check(set())) + self.assertTrue(check(frozenset())) + self.assertTrue(check(frozenset({1, 2}))) + self.assertFalse(check(set_subclass())) + self.assertFalse(check(frozenset_subclass())) + self.assertFalse(check(object())) + # CRASHES: check(NULL) + + def test_anyset_check(self): + check = _testlimitedcapi.anyset_check + self.assertTrue(check(set())) + self.assertTrue(check({1, 2})) + self.assertTrue(check(frozenset())) + self.assertTrue(check(frozenset({1, 2}))) + self.assertTrue(check(set_subclass())) + self.assertTrue(check(frozenset_subclass())) + self.assertFalse(check(object())) + # CRASHES: check(NULL) + + def test_anyset_check_exact(self): + check = _testlimitedcapi.anyset_checkexact + self.assertTrue(check(set())) + self.assertTrue(check({1, 2})) + self.assertTrue(check(frozenset())) + self.assertTrue(check(frozenset({1, 2}))) + self.assertFalse(check(set_subclass())) + self.assertFalse(check(frozenset_subclass())) + self.assertFalse(check(object())) + # CRASHES: check(NULL) + + def test_set_new(self): + set_new = _testlimitedcapi.set_new + self.assertEqual(set_new().__class__, set) + self.assertEqual(set_new(), set()) + self.assertEqual(set_new((1, 1, 2)), {1, 2}) + self.assertEqual(set_new([1, 1, 2]), {1, 2}) + with self.assertRaisesRegex(TypeError, 'object is not iterable'): + set_new(object()) + with self.assertRaisesRegex(TypeError, 'object is not iterable'): + set_new(1) + with self.assertRaisesRegex(TypeError, "unhashable type: 'dict'"): + set_new((1, {})) + + def test_frozenset_new(self): + frozenset_new = _testlimitedcapi.frozenset_new + self.assertEqual(frozenset_new().__class__, frozenset) + self.assertEqual(frozenset_new(), frozenset()) + self.assertEqual(frozenset_new((1, 1, 2)), frozenset({1, 2})) + self.assertEqual(frozenset_new([1, 1, 2]), frozenset({1, 2})) + with self.assertRaisesRegex(TypeError, 'object is not iterable'): + frozenset_new(object()) + with self.assertRaisesRegex(TypeError, 'object is not iterable'): + frozenset_new(1) + with self.assertRaisesRegex(TypeError, "unhashable type: 'dict'"): + frozenset_new((1, {})) + + def test_set_size(self): + get_size = _testlimitedcapi.set_size + self.assertEqual(get_size(set()), 0) + self.assertEqual(get_size(frozenset()), 0) + self.assertEqual(get_size({1, 1, 2}), 2) + self.assertEqual(get_size(frozenset({1, 1, 2})), 2) + self.assertEqual(get_size(set_subclass((1, 2, 3))), 3) + self.assertEqual(get_size(frozenset_subclass((1, 2, 3))), 3) + with self.assertRaises(SystemError): + get_size(object()) + # CRASHES: get_size(NULL) + + def test_set_get_size(self): + get_size = _testcapi.set_get_size + self.assertEqual(get_size(set()), 0) + self.assertEqual(get_size(frozenset()), 0) + self.assertEqual(get_size({1, 1, 2}), 2) + self.assertEqual(get_size(frozenset({1, 1, 2})), 2) + self.assertEqual(get_size(set_subclass((1, 2, 3))), 3) + self.assertEqual(get_size(frozenset_subclass((1, 2, 3))), 3) + # CRASHES: get_size(NULL) + # CRASHES: get_size(object()) + + def test_set_contains(self): + contains = _testlimitedcapi.set_contains + for cls in (set, frozenset, set_subclass, frozenset_subclass): + with self.subTest(cls=cls): + instance = cls((1, 2)) + self.assertTrue(contains(instance, 1)) + self.assertFalse(contains(instance, 'missing')) + with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): + contains(instance, []) + # CRASHES: contains(instance, NULL) + # CRASHES: contains(NULL, object()) + # CRASHES: contains(NULL, NULL) + + def test_add(self): + add = _testlimitedcapi.set_add + for cls in (set, set_subclass): + with self.subTest(cls=cls): + instance = cls((1, 2)) + self.assertEqual(add(instance, 1), 0) + self.assertEqual(instance, {1, 2}) + self.assertEqual(add(instance, 3), 0) + self.assertEqual(instance, {1, 2, 3}) + with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): + add(instance, []) + with self.assertRaises(SystemError): + add(object(), 1) + self.assertImmutable(add, 1) + # CRASHES: add(NULL, object()) + # CRASHES: add(instance, NULL) + # CRASHES: add(NULL, NULL) + + def test_discard(self): + discard = _testlimitedcapi.set_discard + for cls in (set, set_subclass): + with self.subTest(cls=cls): + instance = cls((1, 2)) + self.assertEqual(discard(instance, 3), 0) + self.assertEqual(instance, {1, 2}) + self.assertEqual(discard(instance, 1), 1) + self.assertEqual(instance, {2}) + self.assertEqual(discard(instance, 2), 1) + self.assertEqual(instance, set()) + self.assertEqual(discard(instance, 2), 0) + self.assertEqual(instance, set()) + with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): + discard(instance, []) + with self.assertRaises(SystemError): + discard(object(), 1) + self.assertImmutable(discard, 1) + # CRASHES: discard(NULL, object()) + # CRASHES: discard(instance, NULL) + # CRASHES: discard(NULL, NULL) + + def test_pop(self): + pop = _testlimitedcapi.set_pop + orig = (1, 2) + for cls in (set, set_subclass): + with self.subTest(cls=cls): + instance = cls(orig) + self.assertIn(pop(instance), orig) + self.assertEqual(len(instance), 1) + self.assertIn(pop(instance), orig) + self.assertEqual(len(instance), 0) + with self.assertRaises(KeyError): + pop(instance) + with self.assertRaises(SystemError): + pop(object()) + self.assertImmutable(pop) + # CRASHES: pop(NULL) + + def test_clear(self): + clear = _testlimitedcapi.set_clear + for cls in (set, set_subclass): + with self.subTest(cls=cls): + instance = cls((1, 2)) + self.assertEqual(clear(instance), 0) + self.assertEqual(instance, set()) + self.assertEqual(clear(instance), 0) + self.assertEqual(instance, set()) + with self.assertRaises(SystemError): + clear(object()) + self.assertImmutable(clear) + # CRASHES: clear(NULL) + + +class TestInternalCAPI(BaseSetTests, unittest.TestCase): + def test_set_update(self): + update = _testinternalcapi.set_update + for cls in (set, set_subclass): + for it in ('ab', ('a', 'b'), ['a', 'b'], + set('ab'), set_subclass('ab'), + frozenset('ab'), frozenset_subclass('ab')): + with self.subTest(cls=cls, it=it): + instance = cls() + self.assertEqual(update(instance, it), 0) + self.assertEqual(instance, {'a', 'b'}) + instance = cls(it) + self.assertEqual(update(instance, it), 0) + self.assertEqual(instance, {'a', 'b'}) + with self.assertRaisesRegex(TypeError, 'object is not iterable'): + update(cls(), 1) + with self.assertRaisesRegex(TypeError, "unhashable type: 'dict'"): + update(cls(), [{}]) + with self.assertRaises(SystemError): + update(object(), 'ab') + self.assertImmutable(update, 'ab') + # CRASHES: update(NULL, object()) + # CRASHES: update(instance, NULL) + # CRASHES: update(NULL, NULL) + + def test_set_next_entry(self): + set_next = _testinternalcapi.set_next_entry + for cls in (set, set_subclass, frozenset, frozenset_subclass): + with self.subTest(cls=cls): + instance = cls('abc') + pos = 0 + items = [] + while True: + res = set_next(instance, pos) + if res is None: + break + rc, pos, hash_, item = res + items.append(item) + self.assertEqual(rc, 1) + self.assertIn(item, instance) + self.assertEqual(hash(item), hash_) + self.assertEqual(items, list(instance)) + with self.assertRaises(SystemError): + set_next(object(), 0) + # CRASHES: set_next(NULL, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py new file mode 100644 index 00000000000..f14ad9a9a5f --- /dev/null +++ b/Lib/test/test_capi/test_structmembers.py @@ -0,0 +1,186 @@ +import unittest +from test.support import import_helper + +# Skip this test if the _testcapi module isn't available. +import_helper.import_module('_testcapi') +from _testcapi import (_test_structmembersType_OldAPI, + _test_structmembersType_NewAPI, + CHAR_MAX, CHAR_MIN, UCHAR_MAX, + SHRT_MAX, SHRT_MIN, USHRT_MAX, + INT_MAX, INT_MIN, UINT_MAX, + LONG_MAX, LONG_MIN, ULONG_MAX, + LLONG_MAX, LLONG_MIN, ULLONG_MAX, + PY_SSIZE_T_MAX, PY_SSIZE_T_MIN, + ) + + +class Index: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + +# There are two classes: one using and another using +# `Py_`-prefixed API. They should behave the same in Python + +def _make_test_object(cls): + return cls(False, # T_BOOL + 1, # T_BYTE + 2, # T_UBYTE + 3, # T_SHORT + 4, # T_USHORT + 5, # T_INT + 6, # T_UINT + 7, # T_LONG + 8, # T_ULONG + 23, # T_PYSSIZET + 9.99999,# T_FLOAT + 10.1010101010, # T_DOUBLE + "hi", # T_STRING_INPLACE + 12, # T_LONGLONG + 13, # T_ULONGLONG + b"c", # T_CHAR + ) + + +class ReadWriteTests: + def setUp(self): + self.ts = _make_test_object(self.cls) + + def _test_write(self, name, value, expected=None): + if expected is None: + expected = value + ts = self.ts + setattr(ts, name, value) + self.assertEqual(getattr(ts, name), expected) + + def _test_warn(self, name, value, expected=None): + ts = self.ts + self.assertWarns(RuntimeWarning, setattr, ts, name, value) + if expected is not None: + self.assertEqual(getattr(ts, name), expected) + + def _test_overflow(self, name, value): + ts = self.ts + self.assertRaises(OverflowError, setattr, ts, name, value) + + def _test_int_range(self, name, minval, maxval, *, hardlimit=None, + indexlimit=None): + if hardlimit is None: + hardlimit = (minval, maxval) + ts = self.ts + self._test_write(name, minval) + self._test_write(name, maxval) + hardminval, hardmaxval = hardlimit + self._test_overflow(name, hardminval-1) + self._test_overflow(name, hardmaxval+1) + self._test_overflow(name, 2**1000) + self._test_overflow(name, -2**1000) + if hardminval < minval: + self._test_warn(name, hardminval) + self._test_warn(name, minval-1, maxval) + if maxval < hardmaxval: + self._test_warn(name, maxval+1, minval) + self._test_warn(name, hardmaxval) + + if indexlimit is False: + self.assertRaises(TypeError, setattr, ts, name, Index(minval)) + self.assertRaises(TypeError, setattr, ts, name, Index(maxval)) + else: + self._test_write(name, Index(minval), minval) + self._test_write(name, Index(maxval), maxval) + self._test_overflow(name, Index(hardminval-1)) + self._test_overflow(name, Index(hardmaxval+1)) + self._test_overflow(name, Index(2**1000)) + self._test_overflow(name, Index(-2**1000)) + if hardminval < minval: + self._test_warn(name, Index(hardminval)) + self._test_warn(name, Index(minval-1), maxval) + if maxval < hardmaxval: + self._test_warn(name, Index(maxval+1), minval) + self._test_warn(name, Index(hardmaxval)) + + def test_bool(self): + ts = self.ts + ts.T_BOOL = True + self.assertIs(ts.T_BOOL, True) + ts.T_BOOL = False + self.assertIs(ts.T_BOOL, False) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 1) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 0) + self.assertRaises(TypeError, setattr, ts, 'T_BOOL', None) + + def test_byte(self): + self._test_int_range('T_BYTE', CHAR_MIN, CHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UBYTE', 0, UCHAR_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + + def test_short(self): + self._test_int_range('T_SHORT', SHRT_MIN, SHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_USHORT', 0, USHRT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + + def test_int(self): + self._test_int_range('T_INT', INT_MIN, INT_MAX, + hardlimit=(LONG_MIN, LONG_MAX)) + self._test_int_range('T_UINT', 0, UINT_MAX, + hardlimit=(LONG_MIN, ULONG_MAX)) + + def test_long(self): + self._test_int_range('T_LONG', LONG_MIN, LONG_MAX) + self._test_int_range('T_ULONG', 0, ULONG_MAX, + hardlimit=(LONG_MIN, ULONG_MAX)) + + def test_py_ssize_t(self): + self._test_int_range('T_PYSSIZET', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, indexlimit=False) + + def test_longlong(self): + self._test_int_range('T_LONGLONG', LLONG_MIN, LLONG_MAX) + self._test_int_range('T_ULONGLONG', 0, ULLONG_MAX, + hardlimit=(LONG_MIN, ULLONG_MAX)) + + def test_bad_assignments(self): + ts = self.ts + integer_attributes = [ + 'T_BOOL', + 'T_BYTE', 'T_UBYTE', + 'T_SHORT', 'T_USHORT', + 'T_INT', 'T_UINT', + 'T_LONG', 'T_ULONG', + 'T_LONGLONG', 'T_ULONGLONG', + 'T_PYSSIZET' + ] + + # issue8014: this produced 'bad argument to internal function' + # internal error + for nonint in None, 3.2j, "full of eels", {}, []: + for attr in integer_attributes: + self.assertRaises(TypeError, setattr, ts, attr, nonint) + + def test_inplace_string(self): + ts = self.ts + self.assertEqual(ts.T_STRING_INPLACE, "hi") + self.assertRaises(TypeError, setattr, ts, "T_STRING_INPLACE", "s") + self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + + def test_char(self): + ts = self.ts + self.assertEqual(ts.T_CHAR, "c") + ts.T_CHAR = "z" + self.assertEqual(ts.T_CHAR, "z") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", "") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", b"a") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", bytearray(b"b")) + self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + +class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): + cls = _test_structmembersType_OldAPI + +class ReadWriteTests_NewAPI(ReadWriteTests, unittest.TestCase): + cls = _test_structmembersType_NewAPI + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py new file mode 100644 index 00000000000..d3a9b378e77 --- /dev/null +++ b/Lib/test/test_capi/test_sys.py @@ -0,0 +1,154 @@ +import unittest +import contextlib +import sys +from test import support +from test.support import import_helper + +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None + +NULL = None + +class CAPITest(unittest.TestCase): + # TODO: Test the following functions: + # + # PySys_Audit() + # PySys_AuditTuple() + + maxDiff = None + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getobject(self): + # Test PySys_GetObject() + getobject = _testlimitedcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + with support.catch_unraisable_exception() as cm: + self.assertIs(getobject(b'\xff'), AttributeError) + self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) + self.assertRegex(str(cm.unraisable.exc_value), + "'utf-8' codec can't decode") + # CRASHES getobject(NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_setobject(self): + # Test PySys_SetObject() + setobject = _testlimitedcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertNotHasAttr(sys, 'newattr') + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertNotHasAttr(sys, '\U0001f40d') + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getxoptions(self): + # Test PySys_GetXOptions() + getxoptions = _testlimitedcapi.sys_getxoptions + + self.assertIs(getxoptions(), sys._xoptions) + + xoptions = sys._xoptions + try: + sys._xoptions = 'non-dict' + self.assertEqual(getxoptions(), {}) + self.assertIs(getxoptions(), sys._xoptions) + + del sys._xoptions + self.assertEqual(getxoptions(), {}) + self.assertIs(getxoptions(), sys._xoptions) + finally: + sys._xoptions = xoptions + self.assertIs(getxoptions(), sys._xoptions) + + def _test_sys_formatstream(self, funname, streamname): + import_helper.import_module('ctypes') + from ctypes import pythonapi, c_char_p, py_object + func = getattr(pythonapi, funname) + func.argtypes = (c_char_p,) + + # Supports plain C types. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world')) + self.assertEqual(stream.getvalue(), 'Hello, world!') + + # Supports Python objects. + with support.captured_output(streamname) as stream: + func(b'Hello, %R!', py_object('world')) + self.assertEqual(stream.getvalue(), "Hello, 'world'!") + + # The total length is not limited. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*200)) + self.assertEqual(stream.getvalue(), 'Hello, ' + 'world'*200 + '!') + + def test_sys_formatstdout(self): + # Test PySys_FormatStdout() + self._test_sys_formatstream('PySys_FormatStdout', 'stdout') + + def test_sys_formatstderr(self): + # Test PySys_FormatStderr() + self._test_sys_formatstream('PySys_FormatStderr', 'stderr') + + def _test_sys_writestream(self, funname, streamname): + import_helper.import_module('ctypes') + from ctypes import pythonapi, c_char_p + func = getattr(pythonapi, funname) + func.argtypes = (c_char_p,) + + # Supports plain C types. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world')) + self.assertEqual(stream.getvalue(), 'Hello, world!') + + # There is a limit on the total length. + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*100)) + self.assertEqual(stream.getvalue(), 'Hello, ' + 'world'*100 + '!') + with support.captured_output(streamname) as stream: + func(b'Hello, %s!', c_char_p(b'world'*200)) + out = stream.getvalue() + self.assertEqual(out[:20], 'Hello, worldworldwor') + self.assertEqual(out[-13:], '... truncated') + self.assertGreater(len(out), 1000) + + def test_sys_writestdout(self): + # Test PySys_WriteStdout() + self._test_sys_writestream('PySys_WriteStdout', 'stdout') + + def test_sys_writestderr(self): + # Test PySys_WriteStderr() + self._test_sys_writestream('PySys_WriteStderr', 'stderr') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_time.py b/Lib/test/test_capi/test_time.py new file mode 100644 index 00000000000..989c158818c --- /dev/null +++ b/Lib/test/test_capi/test_time.py @@ -0,0 +1,78 @@ +import time +import unittest +from test.support import import_helper +_testcapi = import_helper.import_module('_testcapi') + + +PyTime_MIN = _testcapi.PyTime_MIN +PyTime_MAX = _testcapi.PyTime_MAX +SEC_TO_NS = 10 ** 9 +DAY_TO_SEC = (24 * 60 * 60) +# Worst clock resolution: maximum delta between two clock reads. +CLOCK_RES = 0.050 + + +class CAPITest(unittest.TestCase): + def test_min_max(self): + # PyTime_t is just int64_t + self.assertEqual(PyTime_MIN, -2**63) + self.assertEqual(PyTime_MAX, 2**63 - 1) + + def test_assecondsdouble(self): + # Test PyTime_AsSecondsDouble() + def ns_to_sec(ns): + if abs(ns) % SEC_TO_NS == 0: + return float(ns // SEC_TO_NS) + else: + return float(ns) / SEC_TO_NS + + seconds = ( + 0, + 1, + DAY_TO_SEC, + 365 * DAY_TO_SEC, + ) + values = { + PyTime_MIN, + PyTime_MIN + 1, + PyTime_MAX - 1, + PyTime_MAX, + } + for second in seconds: + ns = second * SEC_TO_NS + values.add(ns) + # test nanosecond before/after to test rounding + values.add(ns - 1) + values.add(ns + 1) + for ns in list(values): + if (-ns) > PyTime_MAX: + continue + values.add(-ns) + for ns in sorted(values): + with self.subTest(ns=ns): + self.assertEqual(_testcapi.PyTime_AsSecondsDouble(ns), + ns_to_sec(ns)) + + def check_clock(self, c_func, py_func): + t1 = c_func() + t2 = py_func() + self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) + + def test_monotonic(self): + # Test PyTime_Monotonic() and PyTime_MonotonicRaw() + self.check_clock(_testcapi.PyTime_Monotonic, time.monotonic) + self.check_clock(_testcapi.PyTime_MonotonicRaw, time.monotonic) + + def test_perf_counter(self): + # Test PyTime_PerfCounter() and PyTime_PerfCounterRaw() + self.check_clock(_testcapi.PyTime_PerfCounter, time.perf_counter) + self.check_clock(_testcapi.PyTime_PerfCounterRaw, time.perf_counter) + + def test_time(self): + # Test PyTime_Time() and PyTime_TimeRaw() + self.check_clock(_testcapi.PyTime_Time, time.time) + self.check_clock(_testcapi.PyTime_TimeRaw, time.time) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py new file mode 100644 index 00000000000..3683c008dbc --- /dev/null +++ b/Lib/test/test_capi/test_tuple.py @@ -0,0 +1,301 @@ +import unittest +import gc +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None +PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN +PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX + +class TupleSubclass(tuple): + pass + + +class CAPITest(unittest.TestCase): + def _not_tracked(self, t): + self.assertFalse(gc.is_tracked(t), t) + + def _tracked(self, t): + self.assertTrue(gc.is_tracked(t), t) + + def test_check(self): + # Test PyTuple_Check() + check = _testlimitedcapi.tuple_check + + self.assertTrue(check((1, 2))) + self.assertTrue(check(())) + self.assertTrue(check(TupleSubclass((1, 2)))) + self.assertFalse(check({1: 2})) + self.assertFalse(check([1, 2])) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_tuple_checkexact(self): + # Test PyTuple_CheckExact() + check = _testlimitedcapi.tuple_checkexact + + self.assertTrue(check((1, 2))) + self.assertTrue(check(())) + self.assertFalse(check(TupleSubclass((1, 2)))) + self.assertFalse(check({1: 2})) + self.assertFalse(check([1, 2])) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_tuple_new(self): + # Test PyTuple_New() + tuple_new = _testlimitedcapi.tuple_new + size = _testlimitedcapi.tuple_size + checknull = _testcapi._check_tuple_item_is_NULL + + tup1 = tuple_new(0) + self.assertEqual(tup1, ()) + self.assertEqual(size(tup1), 0) + self.assertIs(type(tup1), tuple) + self._not_tracked(tup1) + + tup2 = tuple_new(1) + self.assertIs(type(tup2), tuple) + self.assertEqual(size(tup2), 1) + self.assertIsNot(tup2, tup1) + self.assertTrue(checknull(tup2, 0)) + self._tracked(tup2) + + self.assertRaises(SystemError, tuple_new, -1) + self.assertRaises(SystemError, tuple_new, PY_SSIZE_T_MIN) + self.assertRaises(MemoryError, tuple_new, PY_SSIZE_T_MAX) + + def test_tuple_pack(self): + # Test PyTuple_Pack() + pack = _testlimitedcapi.tuple_pack + + self.assertEqual(pack(0), ()) + self.assertEqual(pack(1, [1]), ([1],)) + self.assertEqual(pack(2, [1], [2]), ([1], [2])) + + self._tracked(pack(1, [1])) + self._tracked(pack(2, [1], b'abc')) + self._not_tracked(pack(2, 42, b'abc')) + + self.assertRaises(SystemError, pack, PY_SSIZE_T_MIN) + self.assertRaises(SystemError, pack, -1) + self.assertRaises(MemoryError, pack, PY_SSIZE_T_MAX) + + # CRASHES pack(1, NULL) + # CRASHES pack(2, [1]) + + def test_tuple_size(self): + # Test PyTuple_Size() + size = _testlimitedcapi.tuple_size + + self.assertEqual(size(()), 0) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size(TupleSubclass((1, 2))), 2) + + self.assertRaises(SystemError, size, []) + self.assertRaises(SystemError, size, 42) + self.assertRaises(SystemError, size, object()) + + # CRASHES size(NULL) + + def test_tuple_get_size(self): + # Test PyTuple_GET_SIZE() + size = _testcapi.tuple_get_size + + self.assertEqual(size(()), 0) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size(TupleSubclass((1, 2))), 2) + + def test_tuple_getitem(self): + # Test PyTuple_GetItem() + getitem = _testlimitedcapi.tuple_getitem + + tup = ([1], [2], [3]) + self.assertEqual(getitem(tup, 0), [1]) + self.assertEqual(getitem(tup, 2), [3]) + + tup2 = TupleSubclass(([1], [2], [3])) + self.assertEqual(getitem(tup2, 0), [1]) + self.assertEqual(getitem(tup2, 2), [3]) + + self.assertRaises(IndexError, getitem, tup, PY_SSIZE_T_MIN) + self.assertRaises(IndexError, getitem, tup, -1) + self.assertRaises(IndexError, getitem, tup, len(tup)) + self.assertRaises(IndexError, getitem, tup, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, getitem, [1, 2, 3], 1) + self.assertRaises(SystemError, getitem, 42, 1) + + # CRASHES getitem(NULL, 0) + + def test_tuple_get_item(self): + # Test PyTuple_GET_ITEM() + get_item = _testcapi.tuple_get_item + + tup = ([1], [2], [3]) + self.assertEqual(get_item(tup, 0), [1]) + self.assertEqual(get_item(tup, 2), [3]) + + tup2 = TupleSubclass(([1], [2], [3])) + self.assertEqual(get_item(tup2, 0), [1]) + self.assertEqual(get_item(tup2, 2), [3]) + + # CRASHES get_item(NULL, 0) + + def test_tuple_getslice(self): + # Test PyTuple_GetSlice() + getslice = _testlimitedcapi.tuple_getslice + + # empty + tup = ([1], [2], [3]) + self.assertEqual(getslice(tup, PY_SSIZE_T_MIN, 0), ()) + self.assertEqual(getslice(tup, -1, 0), ()) + self.assertEqual(getslice(tup, 3, PY_SSIZE_T_MAX), ()) + self.assertEqual(getslice(tup, 1, 1), ()) + self.assertEqual(getslice(tup, 2, 1), ()) + tup = TupleSubclass(([1], [2], [3])) + self.assertEqual(getslice(tup, PY_SSIZE_T_MIN, 0), ()) + self.assertEqual(getslice(tup, -1, 0), ()) + self.assertEqual(getslice(tup, 3, PY_SSIZE_T_MAX), ()) + self.assertEqual(getslice(tup, 1, 1), ()) + self.assertEqual(getslice(tup, 2, 1), ()) + + # slice + tup = ([1], [2], [3], [4]) + self.assertEqual(getslice(tup, 1, 3), ([2], [3])) + tup = TupleSubclass(([1], [2], [3], [4])) + self.assertEqual(getslice(tup, 1, 3), ([2], [3])) + + # whole + tup = ([1], [2], [3]) + self.assertEqual(getslice(tup, 0, 3), tup) + self.assertEqual(getslice(tup, 0, 100), tup) + self.assertEqual(getslice(tup, -100, 100), tup) + tup = TupleSubclass(([1], [2], [3])) + self.assertEqual(getslice(tup, 0, 3), tup) + self.assertEqual(getslice(tup, 0, 100), tup) + self.assertEqual(getslice(tup, -100, 100), tup) + + self.assertRaises(SystemError, getslice, [[1], [2], [3]], 0, 0) + self.assertRaises(SystemError, getslice, 42, 0, 0) + + # CRASHES getslice(NULL, 0, 0) + + def test_tuple_setitem(self): + # Test PyTuple_SetItem() + setitem = _testlimitedcapi.tuple_setitem + checknull = _testcapi._check_tuple_item_is_NULL + + tup = ([1], [2]) + self.assertEqual(setitem(tup, 0, []), ([], [2])) + self.assertEqual(setitem(tup, 1, []), ([1], [])) + + tup2 = setitem(tup, 1, NULL) + self.assertTrue(checknull(tup2, 1)) + + tup2 = TupleSubclass(([1], [2])) + self.assertRaises(SystemError, setitem, tup2, 0, []) + + self.assertRaises(IndexError, setitem, tup, PY_SSIZE_T_MIN, []) + self.assertRaises(IndexError, setitem, tup, -1, []) + self.assertRaises(IndexError, setitem, tup, len(tup), []) + self.assertRaises(IndexError, setitem, tup, PY_SSIZE_T_MAX, []) + self.assertRaises(SystemError, setitem, [1], 0, []) + self.assertRaises(SystemError, setitem, 42, 0, []) + + # CRASHES setitem(NULL, 0, []) + + def test_tuple_set_item(self): + # Test PyTuple_SET_ITEM() + set_item = _testcapi.tuple_set_item + checknull = _testcapi._check_tuple_item_is_NULL + + tup = ([1], [2]) + self.assertEqual(set_item(tup, 0, []), ([], [2])) + self.assertEqual(set_item(tup, 1, []), ([1], [])) + + tup2 = set_item(tup, 1, NULL) + self.assertTrue(checknull(tup2, 1)) + + tup2 = TupleSubclass(([1], [2])) + self.assertIs(set_item(tup2, 0, []), tup2) + self.assertEqual(tup2, ([], [2])) + + # CRASHES set_item(tup, -1, []) + # CRASHES set_item(tup, len(tup), []) + # CRASHES set_item([1], 0, []) + # CRASHES set_item(NULL, 0, []) + + def test__tuple_resize(self): + # Test _PyTuple_Resize() + resize = _testcapi._tuple_resize + checknull = _testcapi._check_tuple_item_is_NULL + + a = () + b = resize(a, 0, False) + self.assertEqual(len(a), 0) + self.assertEqual(len(b), 0) + b = resize(a, 2, False) + self.assertEqual(len(a), 0) + self.assertEqual(len(b), 2) + self.assertTrue(checknull(b, 0)) + self.assertTrue(checknull(b, 1)) + + a = ([1], [2], [3]) + b = resize(a, 3) + self.assertEqual(b, a) + b = resize(a, 2) + self.assertEqual(b, a[:2]) + b = resize(a, 5) + self.assertEqual(len(b), 5) + self.assertEqual(b[:3], a) + self.assertTrue(checknull(b, 3)) + self.assertTrue(checknull(b, 4)) + + a = () + self.assertRaises(MemoryError, resize, a, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, resize, a, -1) + self.assertRaises(SystemError, resize, a, PY_SSIZE_T_MIN) + # refcount > 1 + a = (1, 2, 3) + self.assertRaises(SystemError, resize, a, 3, False) + self.assertRaises(SystemError, resize, a, 0, False) + # non-tuple + self.assertRaises(SystemError, resize, [1, 2, 3], 0, False) + self.assertRaises(SystemError, resize, NULL, 0, False) + + def test_bug_59313(self): + # Before 3.14, the C-API function PySequence_Tuple + # would create incomplete tuples which were visible to + # the cycle GC, and this test would crash the interpeter. + TAG = object() + tuples = [] + + def referrer_tuples(): + return [x for x in gc.get_referrers(TAG) + if isinstance(x, tuple)] + + def my_iter(): + nonlocal tuples + yield TAG # 'tag' gets stored in the result tuple + tuples += referrer_tuples() + for x in range(10): + tuples += referrer_tuples() + # Prior to 3.13 would raise a SystemError when the tuple needs to be resized + yield x + + self.assertEqual(tuple(my_iter()), (TAG, *range(10))) + self.assertEqual(tuples, []) + + def test_uninitialized_tuple_repr(self): + tup = _testlimitedcapi.tuple_new(3) + self.assertEqual(repr(tup), '(, , )') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py new file mode 100644 index 00000000000..15fb4a93e2a --- /dev/null +++ b/Lib/test/test_capi/test_type.py @@ -0,0 +1,276 @@ +from test.support import import_helper, Py_GIL_DISABLED, refleak_helper +import unittest + +_testcapi = import_helper.import_module('_testcapi') + + +class BuiltinStaticTypesTests(unittest.TestCase): + + TYPES = [ + object, + type, + int, + str, + dict, + type(None), + bool, + BaseException, + Exception, + Warning, + DeprecationWarning, # Warning subclass + ] + + def test_tp_bases_is_set(self): + # PyTypeObject.tp_bases is documented as public API. + # See https://github.com/python/cpython/issues/105020. + for typeobj in self.TYPES: + with self.subTest(typeobj): + bases = _testcapi.type_get_tp_bases(typeobj) + self.assertIsNot(bases, None) + + def test_tp_mro_is_set(self): + # PyTypeObject.tp_bases is documented as public API. + # See https://github.com/python/cpython/issues/105020. + for typeobj in self.TYPES: + with self.subTest(typeobj): + mro = _testcapi.type_get_tp_mro(typeobj) + self.assertIsNot(mro, None) + + +class TypeTests(unittest.TestCase): + def test_get_type_name(self): + class MyType: + pass + + from _testcapi import ( + get_type_name, get_type_qualname, + get_type_fullyqualname, get_type_module_name) + + from collections import OrderedDict + ht = _testcapi.get_heaptype_for_name() + for cls, fullname, modname, qualname, name in ( + (int, + 'int', + 'builtins', + 'int', + 'int'), + (OrderedDict, + 'collections.OrderedDict', + 'collections', + 'OrderedDict', + 'OrderedDict'), + (ht, + '_testcapi.HeapTypeNameType', + '_testcapi', + 'HeapTypeNameType', + 'HeapTypeNameType'), + (MyType, + f'{__name__}.TypeTests.test_get_type_name..MyType', + __name__, + 'TypeTests.test_get_type_name..MyType', + 'MyType'), + ): + with self.subTest(cls=repr(cls)): + self.assertEqual(get_type_fullyqualname(cls), fullname) + self.assertEqual(get_type_module_name(cls), modname) + self.assertEqual(get_type_qualname(cls), qualname) + self.assertEqual(get_type_name(cls), name) + + # override __module__ + ht.__module__ = 'test_module' + self.assertEqual(get_type_fullyqualname(ht), 'test_module.HeapTypeNameType') + self.assertEqual(get_type_module_name(ht), 'test_module') + self.assertEqual(get_type_qualname(ht), 'HeapTypeNameType') + self.assertEqual(get_type_name(ht), 'HeapTypeNameType') + + # override __name__ and __qualname__ + MyType.__name__ = 'my_name' + MyType.__qualname__ = 'my_qualname' + self.assertEqual(get_type_fullyqualname(MyType), f'{__name__}.my_qualname') + self.assertEqual(get_type_module_name(MyType), __name__) + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # override also __module__ + MyType.__module__ = 'my_module' + self.assertEqual(get_type_fullyqualname(MyType), 'my_module.my_qualname') + self.assertEqual(get_type_module_name(MyType), 'my_module') + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # PyType_GetFullyQualifiedName() ignores the module if it's "builtins" + # or "__main__" of it is not a string + MyType.__module__ = 'builtins' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = '__main__' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = 123 + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + + def test_get_base_by_token(self): + def get_base_by_token(src, key, comparable=True): + def run(use_mro): + find_first = _testcapi.pytype_getbasebytoken + ret1, result = find_first(src, key, use_mro, True) + ret2, no_result = find_first(src, key, use_mro, False) + self.assertIn(ret1, (0, 1)) + self.assertEqual(ret1, result is not None) + self.assertEqual(ret1, ret2) + self.assertIsNone(no_result) + return result + + found_in_mro = run(True) + found_in_bases = run(False) + if comparable: + self.assertIs(found_in_mro, found_in_bases) + return found_in_mro + return found_in_mro, found_in_bases + + create_type = _testcapi.create_type_with_token + get_token = _testcapi.get_tp_token + + Py_TP_USE_SPEC = _testcapi.Py_TP_USE_SPEC + self.assertEqual(Py_TP_USE_SPEC, 0) + + A1 = create_type('_testcapi.A1', Py_TP_USE_SPEC) + self.assertTrue(get_token(A1) != Py_TP_USE_SPEC) + + B1 = create_type('_testcapi.B1', id(self)) + self.assertTrue(get_token(B1) == id(self)) + + tokenA1 = get_token(A1) + # find A1 from A1 + found = get_base_by_token(A1, tokenA1) + self.assertIs(found, A1) + + # no token in static types + STATIC = type(1) + self.assertEqual(get_token(STATIC), 0) + found = get_base_by_token(STATIC, tokenA1) + self.assertIs(found, None) + + # no token in pure subtypes + class A2(A1): pass + self.assertEqual(get_token(A2), 0) + # find A1 + class Z(STATIC, B1, A2): pass + found = get_base_by_token(Z, tokenA1) + self.assertIs(found, A1) + + # searching for NULL token is an error + with self.assertRaises(SystemError): + get_base_by_token(Z, 0) + with self.assertRaises(SystemError): + get_base_by_token(STATIC, 0) + + # share the token with A1 + C1 = create_type('_testcapi.C1', tokenA1) + self.assertTrue(get_token(C1) == tokenA1) + + # find C1 first by shared token + class Z(C1, A2): pass + found = get_base_by_token(Z, tokenA1) + self.assertIs(found, C1) + # B1 not found + found = get_base_by_token(Z, get_token(B1)) + self.assertIs(found, None) + + with self.assertRaises(TypeError): + _testcapi.pytype_getbasebytoken( + 'not a type', id(self), True, False) + + def test_get_module_by_def(self): + heaptype = _testcapi.create_type_with_token('_testcapi.H', 0) + mod = _testcapi.pytype_getmodulebydef(heaptype) + self.assertIs(mod, _testcapi) + + class H1(heaptype): pass + mod = _testcapi.pytype_getmodulebydef(H1) + self.assertIs(mod, _testcapi) + + with self.assertRaises(TypeError): + _testcapi.pytype_getmodulebydef(int) + + class H2(int): pass + with self.assertRaises(TypeError): + _testcapi.pytype_getmodulebydef(H2) + + def test_freeze(self): + # test PyType_Freeze() + type_freeze = _testcapi.type_freeze + + # simple case, no inherante + class MyType: + pass + MyType.attr = "mutable" + + type_freeze(MyType) + err_msg = "cannot set 'attr' attribute of immutable type 'MyType'" + with self.assertRaisesRegex(TypeError, err_msg): + # the class is now immutable + MyType.attr = "immutable" + + # test MRO: PyType_Freeze() requires base classes to be immutable + class A: pass + class B: pass + class C(B): pass + class D(A, C): pass + + self.assertEqual(D.mro(), [D, A, C, B, object]) + with self.assertRaises(TypeError): + type_freeze(D) + + type_freeze(A) + type_freeze(B) + type_freeze(C) + # all parent classes are now immutable, so D can be made immutable + # as well + type_freeze(D) + + @unittest.skipIf( + Py_GIL_DISABLED and refleak_helper.hunting_for_refleaks(), + "Specialization failure triggers gh-127773") + def test_freeze_meta(self): + """test PyType_Freeze() with overridden MRO""" + type_freeze = _testcapi.type_freeze + + class Base: + value = 1 + + class Meta(type): + def mro(cls): + return (cls, Base, object) + + class FreezeThis(metaclass=Meta): + """This has `Base` in the MRO, but not tp_bases""" + + self.assertEqual(FreezeThis.value, 1) + + with self.assertRaises(TypeError): + type_freeze(FreezeThis) + + Base.value = 2 + self.assertEqual(FreezeThis.value, 2) + + type_freeze(Base) + with self.assertRaises(TypeError): + Base.value = 3 + type_freeze(FreezeThis) + self.assertEqual(FreezeThis.value, 2) + + def test_manual_heap_type(self): + # gh-128923: test that a manually allocated and initailized heap type + # works correctly + ManualHeapType = _testcapi.ManualHeapType + for i in range(100): + self.assertIsInstance(ManualHeapType(), ManualHeapType) + + def test_extension_managed_dict_type(self): + ManagedDictType = _testcapi.ManagedDictType + obj = ManagedDictType() + obj.foo = 42 + self.assertEqual(obj.foo, 42) + self.assertEqual(obj.__dict__, {'foo': 42}) + obj.__dict__ = {'bar': 3} + self.assertEqual(obj.__dict__, {'bar': 3}) + self.assertEqual(obj.bar, 3) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py new file mode 100644 index 00000000000..7e5f4c9dac9 --- /dev/null +++ b/Lib/test/test_capi/test_unicode.py @@ -0,0 +1,2039 @@ +import unittest +import sys +from test import support +from test.support import threading_helper + +try: + import _testcapi + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX +except ImportError: + _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None +try: + import ctypes +except ImportError: + ctypes = None + + +NULL = None + +class Str(str): + pass + + +class CAPITest(unittest.TestCase): + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_new(self): + """Test PyUnicode_New()""" + from _testcapi import unicode_new as new + + for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff: + self.assertEqual(new(0, maxchar), '') + self.assertEqual(new(5, maxchar), chr(maxchar)*5) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX, maxchar) + self.assertEqual(new(0, 0x110000), '') + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x4f60) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x4f60) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x1f600) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x1f600) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4, 0x1f600) + self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4+1, 0x1f600) + self.assertRaises(SystemError, new, 5, 0x110000) + self.assertRaises(SystemError, new, -1, 0) + self.assertRaises(SystemError, new, PY_SSIZE_T_MIN, 0) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fill(self): + """Test PyUnicode_Fill()""" + from _testcapi import unicode_fill as fill + + strings = [ + # all strings have exactly 5 characters + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + chars = [0x78, 0xa9, 0x20ac, 0x1f638] + + for idx, fill_char in enumerate(chars): + # wide -> narrow: exceed maxchar limitation + for to in strings[:idx]: + self.assertRaises(ValueError, fill, to, 0, 0, fill_char) + for to in strings[idx:]: + for start in [*range(7), PY_SSIZE_T_MAX]: + for length in [*range(-1, 7 - start), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + filled = max(min(length, 5 - start), 0) + if filled == 5 and to != strings[idx]: + # narrow -> wide + # Tests omitted since this creates invalid strings. + continue + expected = to[:start] + chr(fill_char) * filled + to[start + filled:] + self.assertEqual(fill(to, start, length, fill_char), + (expected, filled)) + + s = strings[0] + self.assertRaises(IndexError, fill, s, -1, 0, 0x78) + self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78) + self.assertRaises(ValueError, fill, s, 0, 0, 0x110000) + self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78) + self.assertRaises(SystemError, fill, [], 0, 0, 0x78) + # CRASHES fill(s, 0, NULL, 0, 0) + # CRASHES fill(NULL, 0, 0, 0x78) + # TODO: Test PyUnicode_Fill() with non-modifiable unicode. + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_writechar(self): + """Test PyUnicode_WriteChar()""" + from _testlimitedcapi import unicode_writechar as writechar + + strings = [ + # one string for every kind + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + # one character for every kind + out of range code + chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000] + for i, s in enumerate(strings): + for j, c in enumerate(chars): + if j <= i: + self.assertEqual(writechar(s, 1, c), + (s[:1] + chr(c) + s[2:], 0)) + else: + self.assertRaises(ValueError, writechar, s, 1, c) + + self.assertRaises(IndexError, writechar, 'abc', 3, 0x78) + self.assertRaises(IndexError, writechar, 'abc', -1, 0x78) + self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MAX, 0x78) + self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MIN, 0x78) + self.assertRaises(TypeError, writechar, b'abc', 0, 0x78) + self.assertRaises(TypeError, writechar, [], 0, 0x78) + # CRASHES writechar(NULL, 0, 0x78) + # TODO: Test PyUnicode_WriteChar() with non-modifiable and legacy + # unicode. + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_resize(self): + """Test PyUnicode_Resize()""" + from _testlimitedcapi import unicode_resize as resize + + strings = [ + # all strings have exactly 3 characters + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for s in strings: + self.assertEqual(resize(s, 3), (s, 0)) + self.assertEqual(resize(s, 2), (s[:2], 0)) + self.assertEqual(resize(s, 4), (s + '\0', 0)) + self.assertEqual(resize(s, 10), (s + '\0'*7, 0)) + self.assertEqual(resize(s, 0), ('', 0)) + self.assertRaises(MemoryError, resize, s, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, resize, s, -1) + self.assertRaises(SystemError, resize, s, PY_SSIZE_T_MIN) + self.assertRaises(SystemError, resize, b'abc', 0) + self.assertRaises(SystemError, resize, [], 0) + self.assertRaises(SystemError, resize, NULL, 0) + # TODO: Test PyUnicode_Resize() with non-modifiable and legacy unicode + # and with NULL as the address. + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_append(self): + """Test PyUnicode_Append()""" + from _testlimitedcapi import unicode_append as append + + strings = [ + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for left in strings: + left = left[::-1] + for right in strings: + expected = left + right + self.assertEqual(append(left, right), expected) + + self.assertRaises(SystemError, append, 'abc', b'abc') + self.assertRaises(SystemError, append, b'abc', 'abc') + self.assertRaises(SystemError, append, b'abc', b'abc') + self.assertRaises(SystemError, append, 'abc', []) + self.assertRaises(SystemError, append, [], 'abc') + self.assertRaises(SystemError, append, [], []) + self.assertRaises(SystemError, append, NULL, 'abc') + self.assertRaises(SystemError, append, 'abc', NULL) + # TODO: Test PyUnicode_Append() with modifiable unicode + # and with NULL as the address. + # TODO: Check reference counts. + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_appendanddel(self): + """Test PyUnicode_AppendAndDel()""" + from _testlimitedcapi import unicode_appendanddel as appendanddel + + strings = [ + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for left in strings: + left = left[::-1] + for right in strings: + self.assertEqual(appendanddel(left, right), left + right) + + self.assertRaises(SystemError, appendanddel, 'abc', b'abc') + self.assertRaises(SystemError, appendanddel, b'abc', 'abc') + self.assertRaises(SystemError, appendanddel, b'abc', b'abc') + self.assertRaises(SystemError, appendanddel, 'abc', []) + self.assertRaises(SystemError, appendanddel, [], 'abc') + self.assertRaises(SystemError, appendanddel, [], []) + self.assertRaises(SystemError, appendanddel, NULL, 'abc') + self.assertRaises(SystemError, appendanddel, 'abc', NULL) + # TODO: Test PyUnicode_AppendAndDel() with modifiable unicode + # and with NULL as the address. + # TODO: Check reference counts. + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_fromstringandsize(self): + """Test PyUnicode_FromStringAndSize()""" + from _testlimitedcapi import unicode_fromstringandsize as fromstringandsize + + self.assertEqual(fromstringandsize(b'abc'), 'abc') + self.assertEqual(fromstringandsize(b'abc', 2), 'ab') + self.assertEqual(fromstringandsize(b'abc\0def'), 'abc\0def') + self.assertEqual(fromstringandsize(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') + self.assertEqual(fromstringandsize(b'\xe4\xbd\xa0'), '\u4f60') + self.assertEqual(fromstringandsize(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xc2\xa1', 1) + self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xa1', 1) + self.assertEqual(fromstringandsize(b'', 0), '') + self.assertEqual(fromstringandsize(NULL, 0), '') + + self.assertRaises(MemoryError, fromstringandsize, b'abc', PY_SSIZE_T_MAX) + self.assertRaises(SystemError, fromstringandsize, b'abc', -1) + self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) + self.assertRaises(SystemError, fromstringandsize, NULL, -1) + self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) + self.assertRaises(SystemError, fromstringandsize, NULL, 3) + self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_fromstring(self): + """Test PyUnicode_FromString()""" + from _testlimitedcapi import unicode_fromstring as fromstring + + self.assertEqual(fromstring(b'abc'), 'abc') + self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') + self.assertEqual(fromstring(b'\xe4\xbd\xa0'), '\u4f60') + self.assertEqual(fromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, fromstring, b'\xc2') + self.assertRaises(UnicodeDecodeError, fromstring, b'\xa1') + self.assertEqual(fromstring(b''), '') + + # CRASHES fromstring(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromkindanddata(self): + """Test PyUnicode_FromKindAndData()""" + from _testcapi import unicode_fromkindanddata as fromkindanddata + + strings = [ + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + enc1 = 'latin1' + for s in strings[:2]: + self.assertEqual(fromkindanddata(1, s.encode(enc1)), s) + enc2 = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' + for s in strings[:3]: + self.assertEqual(fromkindanddata(2, s.encode(enc2)), s) + enc4 = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + for s in strings: + self.assertEqual(fromkindanddata(4, s.encode(enc4)), s) + self.assertEqual(fromkindanddata(2, '\U0001f600'.encode(enc2)), + '\ud83d\ude00') + for kind in 1, 2, 4: + self.assertEqual(fromkindanddata(kind, b''), '') + self.assertEqual(fromkindanddata(kind, b'\0'*kind), '\0') + self.assertEqual(fromkindanddata(kind, NULL, 0), '') + + for kind in -1, 0, 3, 5, 8: + self.assertRaises(SystemError, fromkindanddata, kind, b'') + self.assertRaises(ValueError, fromkindanddata, 1, b'abc', -1) + self.assertRaises(ValueError, fromkindanddata, 1, b'abc', PY_SSIZE_T_MIN) + self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1) + self.assertRaises(ValueError, fromkindanddata, 1, NULL, PY_SSIZE_T_MIN) + # CRASHES fromkindanddata(1, NULL, 1) + # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_substring(self): + """Test PyUnicode_Substring()""" + from _testlimitedcapi import unicode_substring as substring + + strings = [ + 'ab', 'ab\xa1\xa2', + 'ab\xa1\xa2\u4f60\u597d', + 'ab\xa1\xa2\u4f60\u597d\U0001f600\U0001f601' + ] + for s in strings: + for start in [*range(0, len(s) + 2), PY_SSIZE_T_MAX]: + for end in [*range(max(start-1, 0), len(s) + 2), PY_SSIZE_T_MAX]: + self.assertEqual(substring(s, start, end), s[start:end]) + + self.assertRaises(IndexError, substring, 'abc', -1, 0) + self.assertRaises(IndexError, substring, 'abc', PY_SSIZE_T_MIN, 0) + self.assertRaises(IndexError, substring, 'abc', 0, -1) + self.assertRaises(IndexError, substring, 'abc', 0, PY_SSIZE_T_MIN) + # CRASHES substring(b'abc', 0, 0) + # CRASHES substring([], 0, 0) + # CRASHES substring(NULL, 0, 0) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_getlength(self): + """Test PyUnicode_GetLength()""" + from _testlimitedcapi import unicode_getlength as getlength + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + self.assertEqual(getlength(s), len(s)) + + self.assertRaises(TypeError, getlength, b'abc') + self.assertRaises(TypeError, getlength, []) + # CRASHES getlength(NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_readchar(self): + """Test PyUnicode_ReadChar()""" + from _testlimitedcapi import unicode_readchar as readchar + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + for i, c in enumerate(s): + self.assertEqual(readchar(s, i), ord(c)) + self.assertRaises(IndexError, readchar, s, len(s)) + self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MAX) + self.assertRaises(IndexError, readchar, s, -1) + self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MIN) + + self.assertRaises(TypeError, readchar, b'abc', 0) + self.assertRaises(TypeError, readchar, [], 0) + # CRASHES readchar(NULL, 0) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_fromobject(self): + """Test PyUnicode_FromObject()""" + from _testlimitedcapi import unicode_fromobject as fromobject + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + self.assertEqual(fromobject(s), s) + o = Str(s) + s2 = fromobject(o) + self.assertEqual(s2, s) + self.assertIs(type(s2), str) + self.assertIsNot(s2, s) + + self.assertRaises(TypeError, fromobject, b'abc') + self.assertRaises(TypeError, fromobject, []) + # CRASHES fromobject(NULL) + + @unittest.skipIf(ctypes is None, 'need ctypes') + def test_from_format(self): + """Test PyUnicode_FromFormat()""" + # Length modifiers "j" and "t" are not tested here because ctypes does + # not expose types for intmax_t and ptrdiff_t. + # _testlimitedcapi.test_string_from_format() has a wider coverage of all + # formats. + from ctypes import ( + c_char_p, + pythonapi, py_object, sizeof, + c_int, c_long, c_longlong, c_ssize_t, + c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p, + c_wchar, c_wchar_p) + name = "PyUnicode_FromFormat" + _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) + _PyUnicode_FromFormat.restype = py_object + + def PyUnicode_FromFormat(format, *args): + cargs = tuple( + py_object(arg) if isinstance(arg, str) else arg + for arg in args) + return _PyUnicode_FromFormat(format, *cargs) + + def check_format(expected, format, *args): + text = PyUnicode_FromFormat(format, *args) + self.assertEqual(expected, text) + + # ascii format, non-ascii argument + check_format('ascii\x7f=unicode\xe9', + b'ascii\x7f=%U', 'unicode\xe9') + + # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() + # raises an error + self.assertRaisesRegex(ValueError, + r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' + 'string, got a non-ASCII byte: 0xe9$', + PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') + + # test "%c" + check_format('\uabcd', + b'%c', c_int(0xabcd)) + check_format('\U0010ffff', + b'%c', c_int(0x10ffff)) + with self.assertRaises(OverflowError): + PyUnicode_FromFormat(b'%c', c_int(0x110000)) + # Issue #18183 + check_format('\U00010000\U00100000', + b'%c%c', c_int(0x10000), c_int(0x100000)) + + # test "%" + check_format('%', + b'%%') + check_format('%s', + b'%%s') + check_format('[%]', + b'[%%]') + check_format('%abc', + b'%%%s', b'abc') + + # truncated string + check_format('abc', + b'%.3s', b'abcdef') + check_format('abc[', + b'%.6s', 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\u20ac', + b'%.7s', 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\ufffd', + b'%.5s', b'abc[\xff]') + check_format('abc[', + b'%.6s', b'abc[\xe2\x82]') + check_format('abc[\ufffd]', + b'%.7s', b'abc[\xe2\x82]') + check_format('abc[\ufffd', + b'%.7s', b'abc[\xe2\x82\0') + check_format(' abc[', + b'%10.6s', 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\u20ac', + b'%10.7s', 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\ufffd', + b'%10.5s', b'abc[\xff]') + check_format(' abc[', + b'%10.6s', b'abc[\xe2\x82]') + check_format(' abc[\ufffd]', + b'%10.7s', b'abc[\xe2\x82]') + + check_format("'\\u20acABC'", + b'%A', '\u20acABC') + check_format("'\\u20", + b'%.5A', '\u20acABCDEF') + check_format("'\u20acABC'", + b'%R', '\u20acABC') + check_format("'\u20acA", + b'%.3R', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3S', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3U', '\u20acABCDEF') + + check_format('\u20acAB', + b'%.3V', '\u20acABCDEF', None) + check_format('abc[', + b'%.6V', None, 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\u20ac', + b'%.7V', None, 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\ufffd', + b'%.5V', None, b'abc[\xff]') + check_format('abc[', + b'%.6V', None, b'abc[\xe2\x82]') + check_format('abc[\ufffd]', + b'%.7V', None, b'abc[\xe2\x82]') + check_format(' abc[', + b'%10.6V', None, 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\u20ac', + b'%10.7V', None, 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\ufffd', + b'%10.5V', None, b'abc[\xff]') + check_format(' abc[', + b'%10.6V', None, b'abc[\xe2\x82]') + check_format(' abc[\ufffd]', + b'%10.7V', None, b'abc[\xe2\x82]') + check_format(' abc[\ufffd', + b'%10.7V', None, b'abc[\xe2\x82\0') + + # following tests comes from #7330 + # test width modifier and precision modifier with %S + check_format("repr= abc", + b'repr=%5S', 'abc') + check_format("repr=ab", + b'repr=%.2S', 'abc') + check_format("repr= ab", + b'repr=%5.2S', 'abc') + + # test width modifier and precision modifier with %R + check_format("repr= 'abc'", + b'repr=%8R', 'abc') + check_format("repr='ab", + b'repr=%.3R', 'abc') + check_format("repr= 'ab", + b'repr=%5.3R', 'abc') + + # test width modifier and precision modifier with %A + check_format("repr= 'abc'", + b'repr=%8A', 'abc') + check_format("repr='ab", + b'repr=%.3A', 'abc') + check_format("repr= 'ab", + b'repr=%5.3A', 'abc') + + # test width modifier and precision modifier with %s + check_format("repr= abc", + b'repr=%5s', b'abc') + check_format("repr=ab", + b'repr=%.2s', b'abc') + check_format("repr= ab", + b'repr=%5.2s', b'abc') + + # test width modifier and precision modifier with %U + check_format("repr= abc", + b'repr=%5U', 'abc') + check_format("repr=ab", + b'repr=%.2U', 'abc') + check_format("repr= ab", + b'repr=%5.2U', 'abc') + + # test width modifier and precision modifier with %V + check_format("repr= abc", + b'repr=%5V', 'abc', b'123') + check_format("repr=ab", + b'repr=%.2V', 'abc', b'123') + check_format("repr= ab", + b'repr=%5.2V', 'abc', b'123') + check_format("repr= 123", + b'repr=%5V', None, b'123') + check_format("repr=12", + b'repr=%.2V', None, b'123') + check_format("repr= 12", + b'repr=%5.2V', None, b'123') + + # test integer formats (%i, %d, %u, %o, %x, %X) + check_format('010', + b'%03i', c_int(10)) + check_format('0010', + b'%0.4i', c_int(10)) + for conv, signed, value, expected in [ + (b'i', True, -123, '-123'), + (b'd', True, -123, '-123'), + (b'u', False, 123, '123'), + (b'o', False, 0o123, '123'), + (b'x', False, 0xabc, 'abc'), + (b'X', False, 0xabc, 'ABC'), + ]: + for mod, ctype in [ + (b'', c_int if signed else c_uint), + (b'l', c_long if signed else c_ulong), + (b'll', c_longlong if signed else c_ulonglong), + (b'z', c_ssize_t if signed else c_size_t), + ]: + with self.subTest(format=b'%' + mod + conv): + check_format(expected, + b'%' + mod + conv, ctype(value)) + + # test long output + min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) + max_longlong = -min_longlong - 1 + check_format(str(min_longlong), + b'%lld', c_longlong(min_longlong)) + check_format(str(max_longlong), + b'%lld', c_longlong(max_longlong)) + max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 + check_format(str(max_ulonglong), + b'%llu', c_ulonglong(max_ulonglong)) + PyUnicode_FromFormat(b'%p', c_void_p(-1)) + + # test padding (width and/or precision) + check_format('123', b'%2i', c_int(123)) + check_format(' 123', b'%10i', c_int(123)) + check_format('0000000123', b'%010i', c_int(123)) + check_format('123 ', b'%-10i', c_int(123)) + check_format('123 ', b'%-010i', c_int(123)) + check_format('123', b'%.2i', c_int(123)) + check_format('0000123', b'%.7i', c_int(123)) + check_format(' 123', b'%10.2i', c_int(123)) + check_format(' 0000123', b'%10.7i', c_int(123)) + check_format('0000000123', b'%010.7i', c_int(123)) + check_format('0000123 ', b'%-10.7i', c_int(123)) + check_format('0000123 ', b'%-010.7i', c_int(123)) + + check_format('-123', b'%2i', c_int(-123)) + check_format(' -123', b'%10i', c_int(-123)) + check_format('-000000123', b'%010i', c_int(-123)) + check_format('-123 ', b'%-10i', c_int(-123)) + check_format('-123 ', b'%-010i', c_int(-123)) + check_format('-123', b'%.2i', c_int(-123)) + check_format('-0000123', b'%.7i', c_int(-123)) + check_format(' -123', b'%10.2i', c_int(-123)) + check_format(' -0000123', b'%10.7i', c_int(-123)) + check_format('-000000123', b'%010.7i', c_int(-123)) + check_format('-0000123 ', b'%-10.7i', c_int(-123)) + check_format('-0000123 ', b'%-010.7i', c_int(-123)) + + check_format('123', b'%2u', c_uint(123)) + check_format(' 123', b'%10u', c_uint(123)) + check_format('0000000123', b'%010u', c_uint(123)) + check_format('123 ', b'%-10u', c_uint(123)) + check_format('123 ', b'%-010u', c_uint(123)) + check_format('123', b'%.2u', c_uint(123)) + check_format('0000123', b'%.7u', c_uint(123)) + check_format(' 123', b'%10.2u', c_uint(123)) + check_format(' 0000123', b'%10.7u', c_uint(123)) + check_format('0000000123', b'%010.7u', c_uint(123)) + check_format('0000123 ', b'%-10.7u', c_uint(123)) + check_format('0000123 ', b'%-010.7u', c_uint(123)) + + check_format('123', b'%2o', c_uint(0o123)) + check_format(' 123', b'%10o', c_uint(0o123)) + check_format('0000000123', b'%010o', c_uint(0o123)) + check_format('123 ', b'%-10o', c_uint(0o123)) + check_format('123 ', b'%-010o', c_uint(0o123)) + check_format('123', b'%.2o', c_uint(0o123)) + check_format('0000123', b'%.7o', c_uint(0o123)) + check_format(' 123', b'%10.2o', c_uint(0o123)) + check_format(' 0000123', b'%10.7o', c_uint(0o123)) + check_format('0000000123', b'%010.7o', c_uint(0o123)) + check_format('0000123 ', b'%-10.7o', c_uint(0o123)) + check_format('0000123 ', b'%-010.7o', c_uint(0o123)) + + check_format('abc', b'%2x', c_uint(0xabc)) + check_format(' abc', b'%10x', c_uint(0xabc)) + check_format('0000000abc', b'%010x', c_uint(0xabc)) + check_format('abc ', b'%-10x', c_uint(0xabc)) + check_format('abc ', b'%-010x', c_uint(0xabc)) + check_format('abc', b'%.2x', c_uint(0xabc)) + check_format('0000abc', b'%.7x', c_uint(0xabc)) + check_format(' abc', b'%10.2x', c_uint(0xabc)) + check_format(' 0000abc', b'%10.7x', c_uint(0xabc)) + check_format('0000000abc', b'%010.7x', c_uint(0xabc)) + check_format('0000abc ', b'%-10.7x', c_uint(0xabc)) + check_format('0000abc ', b'%-010.7x', c_uint(0xabc)) + + check_format('ABC', b'%2X', c_uint(0xabc)) + check_format(' ABC', b'%10X', c_uint(0xabc)) + check_format('0000000ABC', b'%010X', c_uint(0xabc)) + check_format('ABC ', b'%-10X', c_uint(0xabc)) + check_format('ABC ', b'%-010X', c_uint(0xabc)) + check_format('ABC', b'%.2X', c_uint(0xabc)) + check_format('0000ABC', b'%.7X', c_uint(0xabc)) + check_format(' ABC', b'%10.2X', c_uint(0xabc)) + check_format(' 0000ABC', b'%10.7X', c_uint(0xabc)) + check_format('0000000ABC', b'%010.7X', c_uint(0xabc)) + check_format('0000ABC ', b'%-10.7X', c_uint(0xabc)) + check_format('0000ABC ', b'%-010.7X', c_uint(0xabc)) + + # test %A + check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", + b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') + + # test %V + check_format('abc', + b'%V', 'abc', b'xyz') + check_format('xyz', + b'%V', None, b'xyz') + + # test %ls + check_format('abc', b'%ls', c_wchar_p('abc')) + check_format('\u4eba\u6c11', b'%ls', c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb+\U0001f40d', + b'%ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format(' ab', b'%5.2ls', c_wchar_p('abc')) + check_format(' \u4eba\u6c11', b'%5ls', c_wchar_p('\u4eba\u6c11')) + check_format(' \U0001f4bb+\U0001f40d', + b'%5ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\u4eba', b'%.1ls', c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', + b'%.1ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', + b'%.2ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + + # test %lV + check_format('abc', + b'%lV', 'abc', c_wchar_p('xyz')) + check_format('xyz', + b'%lV', None, c_wchar_p('xyz')) + check_format('\u4eba\u6c11', + b'%lV', None, c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb+\U0001f40d', + b'%lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format(' ab', + b'%5.2lV', None, c_wchar_p('abc')) + check_format(' \u4eba\u6c11', + b'%5lV', None, c_wchar_p('\u4eba\u6c11')) + check_format(' \U0001f4bb+\U0001f40d', + b'%5lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\u4eba', + b'%.1lV', None, c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', + b'%.1lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', + b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + + # test %T + check_format('type: str', + b'type: %T', py_object("abc")) + check_format(f'type: st', + b'type: %.2T', py_object("abc")) + check_format(f'type: str', + b'type: %10T', py_object("abc")) + + class LocalType: + pass + obj = LocalType() + fullname = f'{__name__}.{LocalType.__qualname__}' + check_format(f'type: {fullname}', + b'type: %T', py_object(obj)) + fullname_alt = f'{__name__}:{LocalType.__qualname__}' + check_format(f'type: {fullname_alt}', + b'type: %#T', py_object(obj)) + + # test %N + check_format('type: str', + b'type: %N', py_object(str)) + check_format(f'type: st', + b'type: %.2N', py_object(str)) + check_format(f'type: str', + b'type: %10N', py_object(str)) + + check_format(f'type: {fullname}', + b'type: %N', py_object(type(obj))) + check_format(f'type: {fullname_alt}', + b'type: %#N', py_object(type(obj))) + with self.assertRaisesRegex(TypeError, "%N argument must be a type"): + check_format('type: str', + b'type: %N', py_object("abc")) + + # test variable width and precision + check_format(' abc', b'%*s', c_int(5), b'abc') + check_format('ab', b'%.*s', c_int(2), b'abc') + check_format(' ab', b'%*.*s', c_int(5), c_int(2), b'abc') + check_format(' abc', b'%*U', c_int(5), 'abc') + check_format('ab', b'%.*U', c_int(2), 'abc') + check_format(' ab', b'%*.*U', c_int(5), c_int(2), 'abc') + check_format(' ab', b'%*.*V', c_int(5), c_int(2), None, b'abc') + check_format(' ab', b'%*.*lV', c_int(5), c_int(2), + None, c_wchar_p('abc')) + check_format(' 123', b'%*i', c_int(8), c_int(123)) + check_format('00123', b'%.*i', c_int(5), c_int(123)) + check_format(' 00123', b'%*.*i', c_int(8), c_int(5), c_int(123)) + + # test %p + # We cannot test the exact result, + # because it returns a hex representation of a C pointer, + # which is going to be different each time. But, we can test the format. + p_format_regex = r'^0x[a-zA-Z0-9]{3,}$' + p_format1 = PyUnicode_FromFormat(b'%p', 'abc') + self.assertIsInstance(p_format1, str) + self.assertRegex(p_format1, p_format_regex) + + p_format2 = PyUnicode_FromFormat(b'%p %p', '123456', b'xyz') + self.assertIsInstance(p_format2, str) + self.assertRegex(p_format2, + r'0x[a-zA-Z0-9]{3,} 0x[a-zA-Z0-9]{3,}') + + # Extra args are ignored: + p_format3 = PyUnicode_FromFormat(b'%p', '123456', None, b'xyz') + self.assertIsInstance(p_format3, str) + self.assertRegex(p_format3, p_format_regex) + + # Test string decode from parameter of %s using utf-8. + # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of + # '\u4eba\u6c11' + check_format('repr=\u4eba\u6c11', + b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') + + #Test replace error handler. + check_format('repr=abc\ufffd', + b'repr=%V', None, b'abc\xff') + + # Issue #33817: empty strings + check_format('', + b'') + check_format('', + b'%s', b'') + + # test invalid format strings. these tests are just here + # to check for crashes and should not be considered as specifications + for fmt in (b'%', b'%0', b'%01', b'%.', b'%.1', + b'%0%s', b'%1%s', b'%.%s', b'%.1%s', b'%1abc', + b'%l', b'%ll', b'%z', b'%lls', b'%zs'): + with self.subTest(fmt=fmt): + self.assertRaisesRegex(SystemError, 'invalid format string', + PyUnicode_FromFormat, fmt, b'abc') + self.assertRaisesRegex(SystemError, 'invalid format string', + PyUnicode_FromFormat, b'%+i', c_int(10)) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_interninplace(self): + """Test PyUnicode_InternInPlace()""" + from _testlimitedcapi import unicode_interninplace as interninplace + + s = b'abc'.decode() + r = interninplace(s) + self.assertEqual(r, 'abc') + + # CRASHES interninplace(b'abc') + # CRASHES interninplace(NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_internfromstring(self): + """Test PyUnicode_InternFromString()""" + from _testlimitedcapi import unicode_internfromstring as internfromstring + + self.assertEqual(internfromstring(b'abc'), 'abc') + self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, internfromstring, b'\xc2') + self.assertRaises(UnicodeDecodeError, internfromstring, b'\xa1') + self.assertEqual(internfromstring(b''), '') + + # CRASHES internfromstring(NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_fromwidechar(self): + """Test PyUnicode_FromWideChar()""" + from _testlimitedcapi import unicode_fromwidechar as fromwidechar + from _testcapi import SIZEOF_WCHAR_T + + if SIZEOF_WCHAR_T == 2: + encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' + elif SIZEOF_WCHAR_T == 4: + encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + + for s in '', 'abc', '\xa1\xa2', '\u4f60', '\U0001f600': + b = s.encode(encoding) + self.assertEqual(fromwidechar(b), s) + self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) + for s in '\ud83d', '\ude00': + b = s.encode(encoding, 'surrogatepass') + self.assertEqual(fromwidechar(b), s) + self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) + + self.assertEqual(fromwidechar('abc'.encode(encoding), 2), 'ab') + if SIZEOF_WCHAR_T == 2: + self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d') + + self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX) + self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2) + self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN) + self.assertEqual(fromwidechar(NULL, 0), '') + self.assertRaises(SystemError, fromwidechar, NULL, 1) + self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, fromwidechar, NULL, -1) + self.assertRaises(SystemError, fromwidechar, NULL, -2) + self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_aswidechar(self): + """Test PyUnicode_AsWideChar()""" + from _testlimitedcapi import unicode_aswidechar + from _testlimitedcapi import unicode_aswidechar_null + from _testcapi import SIZEOF_WCHAR_T + + wchar, size = unicode_aswidechar('abcdef', 2) + self.assertEqual(size, 2) + self.assertEqual(wchar, 'ab') + + wchar, size = unicode_aswidechar('abc', 3) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc') + self.assertEqual(unicode_aswidechar_null('abc', 10), 4) + self.assertEqual(unicode_aswidechar_null('abc', 0), 4) + + wchar, size = unicode_aswidechar('abc', 4) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidechar('abc', 10) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidechar('abc\0def', 20) + self.assertEqual(size, 7) + self.assertEqual(wchar, 'abc\0def\0') + self.assertEqual(unicode_aswidechar_null('abc\0def', 20), 8) + + nonbmp = chr(0x10ffff) + if SIZEOF_WCHAR_T == 2: + nchar = 2 + else: # SIZEOF_WCHAR_T == 4 + nchar = 1 + wchar, size = unicode_aswidechar(nonbmp, 10) + self.assertEqual(size, nchar) + self.assertEqual(wchar, nonbmp + '\0') + self.assertEqual(unicode_aswidechar_null(nonbmp, 10), nchar + 1) + + self.assertRaises(TypeError, unicode_aswidechar, b'abc', 10) + self.assertRaises(TypeError, unicode_aswidechar, [], 10) + self.assertRaises(SystemError, unicode_aswidechar, NULL, 10) + self.assertRaises(TypeError, unicode_aswidechar_null, b'abc', 10) + self.assertRaises(TypeError, unicode_aswidechar_null, [], 10) + self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_aswidecharstring(self): + """Test PyUnicode_AsWideCharString()""" + from _testlimitedcapi import unicode_aswidecharstring + from _testlimitedcapi import unicode_aswidecharstring_null + from _testcapi import SIZEOF_WCHAR_T + + wchar, size = unicode_aswidecharstring('abc') + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + self.assertEqual(unicode_aswidecharstring_null('abc'), 'abc') + + wchar, size = unicode_aswidecharstring('abc\0def') + self.assertEqual(size, 7) + self.assertEqual(wchar, 'abc\0def\0') + self.assertRaises(ValueError, unicode_aswidecharstring_null, 'abc\0def') + + nonbmp = chr(0x10ffff) + if SIZEOF_WCHAR_T == 2: + nchar = 2 + else: # SIZEOF_WCHAR_T == 4 + nchar = 1 + wchar, size = unicode_aswidecharstring(nonbmp) + self.assertEqual(size, nchar) + self.assertEqual(wchar, nonbmp + '\0') + self.assertEqual(unicode_aswidecharstring_null(nonbmp), nonbmp) + + self.assertRaises(TypeError, unicode_aswidecharstring, b'abc') + self.assertRaises(TypeError, unicode_aswidecharstring, []) + self.assertRaises(SystemError, unicode_aswidecharstring, NULL) + self.assertRaises(TypeError, unicode_aswidecharstring_null, b'abc') + self.assertRaises(TypeError, unicode_aswidecharstring_null, []) + self.assertRaises(SystemError, unicode_aswidecharstring_null, NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_asucs4(self): + """Test PyUnicode_AsUCS4()""" + from _testcapi import unicode_asucs4 + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + l = len(s) + self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') + self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') + self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) + self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) + s = '\0'.join([s, s]) + self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') + self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') + + # CRASHES unicode_asucs4(b'abc', 1, 0) + # CRASHES unicode_asucs4(b'abc', 1, 1) + # CRASHES unicode_asucs4([], 1, 1) + # CRASHES unicode_asucs4(NULL, 1, 0) + # CRASHES unicode_asucs4(NULL, 1, 1) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_asucs4copy(self): + """Test PyUnicode_AsUCS4Copy()""" + from _testcapi import unicode_asucs4copy as asucs4copy + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + self.assertEqual(asucs4copy(s), s+'\0') + s = '\0'.join([s, s]) + self.assertEqual(asucs4copy(s), s+'\0') + + # CRASHES asucs4copy(b'abc') + # CRASHES asucs4copy([]) + # CRASHES asucs4copy(NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_fromordinal(self): + """Test PyUnicode_FromOrdinal()""" + from _testlimitedcapi import unicode_fromordinal as fromordinal + + self.assertEqual(fromordinal(0x61), 'a') + self.assertEqual(fromordinal(0x20ac), '\u20ac') + self.assertEqual(fromordinal(0x1f600), '\U0001f600') + + self.assertRaises(ValueError, fromordinal, 0x110000) + self.assertRaises(ValueError, fromordinal, -1) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_asutf8(self): + """Test PyUnicode_AsUTF8()""" + from _testcapi import unicode_asutf8 + + self.assertEqual(unicode_asutf8('abc', 4), b'abc\0') + self.assertEqual(unicode_asutf8('абв', 7), b'\xd0\xb0\xd0\xb1\xd0\xb2\0') + self.assertEqual(unicode_asutf8('\U0001f600', 5), b'\xf0\x9f\x98\x80\0') + self.assertEqual(unicode_asutf8('abc\0def', 8), b'abc\0def\0') + + self.assertRaises(UnicodeEncodeError, unicode_asutf8, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8, [], 0) + # CRASHES unicode_asutf8(NULL, 0) + + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @threading_helper.requires_working_threading() + def test_asutf8_race(self): + """Test that there's no race condition in PyUnicode_AsUTF8()""" + unicode_asutf8 = _testcapi.unicode_asutf8 + from threading import Thread + + data = "😊" + + def worker(): + for _ in range(1000): + self.assertEqual(unicode_asutf8(data, 5), b'\xf0\x9f\x98\x8a\0') + + threads = [Thread(target=worker) for _ in range(10)] + with threading_helper.start_threads(threads): + pass + + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_asutf8andsize(self): + """Test PyUnicode_AsUTF8AndSize()""" + from _testlimitedcapi import unicode_asutf8andsize + from _testlimitedcapi import unicode_asutf8andsize_null + + self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) + self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) + self.assertEqual(unicode_asutf8andsize('\U0001f600', 5), (b'\xf0\x9f\x98\x80\0', 4)) + self.assertEqual(unicode_asutf8andsize('abc\0def', 8), (b'abc\0def\0', 7)) + self.assertEqual(unicode_asutf8andsize_null('abc', 4), b'abc\0') + self.assertEqual(unicode_asutf8andsize_null('abc\0def', 8), b'abc\0def\0') + + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8andsize, [], 0) + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize_null, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8andsize_null, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8andsize_null, [], 0) + # CRASHES unicode_asutf8andsize(NULL, 0) + # CRASHES unicode_asutf8andsize_null(NULL, 0) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_getdefaultencoding(self): + """Test PyUnicode_GetDefaultEncoding()""" + from _testlimitedcapi import unicode_getdefaultencoding as getdefaultencoding + + self.assertEqual(getdefaultencoding(), b'utf-8') + + @support.cpython_only + @unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') + def test_transform_decimal_and_space(self): + """Test _PyUnicode_TransformDecimalAndSpaceToASCII()""" + from _testinternalcapi import _PyUnicode_TransformDecimalAndSpaceToASCII as transform_decimal + + self.assertEqual(transform_decimal('123'), + '123') + self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), + '3.14') + self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), + " 3.14 ") + self.assertEqual(transform_decimal('12\u20ac3'), + '12?') + self.assertEqual(transform_decimal(''), '') + + self.assertRaises(SystemError, transform_decimal, b'123') + self.assertRaises(SystemError, transform_decimal, []) + # CRASHES transform_decimal(NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_concat(self): + """Test PyUnicode_Concat()""" + from _testlimitedcapi import unicode_concat as concat + + self.assertEqual(concat('abc', 'def'), 'abcdef') + self.assertEqual(concat('abc', 'где'), 'abcгде') + self.assertEqual(concat('абв', 'def'), 'абвdef') + self.assertEqual(concat('абв', 'где'), 'абвгде') + self.assertEqual(concat('a\0b', 'c\0d'), 'a\0bc\0d') + + self.assertRaises(TypeError, concat, b'abc', 'def') + self.assertRaises(TypeError, concat, 'abc', b'def') + self.assertRaises(TypeError, concat, b'abc', b'def') + self.assertRaises(TypeError, concat, [], 'def') + self.assertRaises(TypeError, concat, 'abc', []) + self.assertRaises(TypeError, concat, [], []) + # CRASHES concat(NULL, 'def') + # CRASHES concat('abc', NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_split(self): + """Test PyUnicode_Split()""" + from _testlimitedcapi import unicode_split as split + + self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) + self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d']) + self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MAX), + ['a', 'b', 'c', 'd']) + self.assertEqual(split('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd']) + self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MIN), + ['a', 'b', 'c', 'd']) + self.assertEqual(split('a|b|c|d', '\u20ac'), ['a|b|c|d']) + self.assertEqual(split('a||b|c||d', '||'), ['a', 'b|c', 'd']) + self.assertEqual(split('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) + self.assertEqual(split('абабагаламага', 'а'), + ['', 'б', 'б', 'г', 'л', 'м', 'г', '']) + self.assertEqual(split(' a\tb\nc\rd\ve\f', NULL), + ['a', 'b', 'c', 'd', 'e']) + self.assertEqual(split('a\x85b\xa0c\u1680d\u2000e', NULL), + ['a', 'b', 'c', 'd', 'e']) + + self.assertRaises(ValueError, split, 'a|b|c|d', '') + self.assertRaises(TypeError, split, 'a|b|c|d', ord('|')) + self.assertRaises(TypeError, split, [], '|') + # CRASHES split(NULL, '|') + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_rsplit(self): + """Test PyUnicode_RSplit()""" + from _testlimitedcapi import unicode_rsplit as rsplit + + self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MAX), + ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MIN), + ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '\u20ac'), ['a|b|c|d']) + self.assertEqual(rsplit('a||b|c||d', '||'), ['a', 'b|c', 'd']) + self.assertEqual(rsplit('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) + self.assertEqual(rsplit('абабагаламага', 'а'), + ['', 'б', 'б', 'г', 'л', 'м', 'г', '']) + self.assertEqual(rsplit('aжbжcжd', 'ж'), ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit(' a\tb\nc\rd\ve\f', NULL), + ['a', 'b', 'c', 'd', 'e']) + self.assertEqual(rsplit('a\x85b\xa0c\u1680d\u2000e', NULL), + ['a', 'b', 'c', 'd', 'e']) + + self.assertRaises(ValueError, rsplit, 'a|b|c|d', '') + self.assertRaises(TypeError, rsplit, 'a|b|c|d', ord('|')) + self.assertRaises(TypeError, rsplit, [], '|') + # CRASHES rsplit(NULL, '|') + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_partition(self): + """Test PyUnicode_Partition()""" + from _testlimitedcapi import unicode_partition as partition + + self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c')) + self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c')) + self.assertEqual(partition('а|б|в', '|'), ('а', '|', 'б|в')) + self.assertEqual(partition('кабан', 'а'), ('к', 'а', 'бан')) + self.assertEqual(partition('aжbжc', 'ж'), ('a', 'ж', 'bжc')) + + self.assertRaises(ValueError, partition, 'a|b|c', '') + self.assertRaises(TypeError, partition, b'a|b|c', '|') + self.assertRaises(TypeError, partition, 'a|b|c', b'|') + self.assertRaises(TypeError, partition, 'a|b|c', ord('|')) + self.assertRaises(TypeError, partition, [], '|') + # CRASHES partition(NULL, '|') + # CRASHES partition('a|b|c', NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_rpartition(self): + """Test PyUnicode_RPartition()""" + from _testlimitedcapi import unicode_rpartition as rpartition + + self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c')) + self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c')) + self.assertEqual(rpartition('а|б|в', '|'), ('а|б', '|', 'в')) + self.assertEqual(rpartition('кабан', 'а'), ('каб', 'а', 'н')) + self.assertEqual(rpartition('aжbжc', 'ж'), ('aжb', 'ж', 'c')) + + self.assertRaises(ValueError, rpartition, 'a|b|c', '') + self.assertRaises(TypeError, rpartition, b'a|b|c', '|') + self.assertRaises(TypeError, rpartition, 'a|b|c', b'|') + self.assertRaises(TypeError, rpartition, 'a|b|c', ord('|')) + self.assertRaises(TypeError, rpartition, [], '|') + # CRASHES rpartition(NULL, '|') + # CRASHES rpartition('a|b|c', NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_splitlines(self): + """Test PyUnicode_SplitLines()""" + from _testlimitedcapi import unicode_splitlines as splitlines + + self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd']) + self.assertEqual(splitlines('a\nb\rc\r\nd', True), + ['a\n', 'b\r', 'c\r\n', 'd']) + self.assertEqual(splitlines('a\x85b\u2028c\u2029d'), + ['a', 'b', 'c', 'd']) + self.assertEqual(splitlines('a\x85b\u2028c\u2029d', True), + ['a\x85', 'b\u2028', 'c\u2029', 'd']) + self.assertEqual(splitlines('а\nб\rв\r\nг'), ['а', 'б', 'в', 'г']) + + self.assertRaises(TypeError, splitlines, b'a\nb\rc\r\nd') + # CRASHES splitlines(NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_translate(self): + """Test PyUnicode_Translate()""" + from _testlimitedcapi import unicode_translate as translate + + self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d') + self.assertEqual(translate('абвг', {ord('а'): 'А', ord('б'): ord('Б'), ord('в'): '<>'}), 'АБ<>г') + self.assertEqual(translate('abc', {}), 'abc') + self.assertEqual(translate('abc', []), 'abc') + self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}) + self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}, 'strict') + self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo') + self.assertEqual(translate('abc', {ord('b'): None}, 'ignore'), 'ac') + self.assertEqual(translate('abc', {ord('b'): None}, 'replace'), 'a\ufffdc') + self.assertEqual(translate('abc', {ord('b'): None}, 'backslashreplace'), r'a\x62c') + # XXX Other error handlers do not support UnicodeTranslateError + self.assertRaises(TypeError, translate, b'abc', []) + self.assertRaises(TypeError, translate, 123, []) + self.assertRaises(TypeError, translate, 'abc', {ord('a'): b'A'}) + self.assertRaises(TypeError, translate, 'abc', 123) + self.assertRaises(TypeError, translate, 'abc', NULL) + self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo') + # CRASHES translate(NULL, []) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_join(self): + """Test PyUnicode_Join()""" + from _testlimitedcapi import unicode_join as join + self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c') + self.assertEqual(join('|', ['a', '', 'c']), 'a||c') + self.assertEqual(join('', ['a', 'b', 'c']), 'abc') + self.assertEqual(join(NULL, ['a', 'b', 'c']), 'a b c') + self.assertEqual(join('|', ['а', 'б', 'в']), 'а|б|в') + self.assertEqual(join('ж', ['а', 'б', 'в']), 'ажбжв') + self.assertRaises(TypeError, join, b'|', ['a', 'b', 'c']) + self.assertRaises(TypeError, join, '|', [b'a', b'b', b'c']) + self.assertRaises(TypeError, join, NULL, [b'a', b'b', b'c']) + self.assertRaises(TypeError, join, '|', b'123') + self.assertRaises(TypeError, join, '|', 123) + self.assertRaises(SystemError, join, '|', NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_count(self): + """Test PyUnicode_Count()""" + from _testlimitedcapi import unicode_count + + for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": + for i, ch in enumerate(str): + self.assertEqual(unicode_count(str, ch, 0, len(str)), 1) + + str = "!>_= end + self.assertEqual(unicode_count(str, '!', 0, 0), 0) + self.assertEqual(unicode_count(str, '!', len(str), 0), 0) + # negative + self.assertEqual(unicode_count(str, '!', -len(str), -1), 1) + self.assertEqual(unicode_count(str, '!', -len(str)-1, -1), 1) + self.assertEqual(unicode_count(str, '!', PY_SSIZE_T_MIN, -1), 1) + # bad arguments + self.assertRaises(TypeError, unicode_count, str, b'!', 0, len(str)) + self.assertRaises(TypeError, unicode_count, b"!>__= end + self.assertEqual(find(str, '!', 0, 0, 1), -1) + self.assertEqual(find(str, '!', 0, 0, -1), -1) + self.assertEqual(find(str, '!', len(str), 0, 1), -1) + self.assertEqual(find(str, '!', len(str), 0, -1), -1) + # negative + self.assertEqual(find(str, '!', -len(str), -1, 1), 0) + self.assertEqual(find(str, '!', -len(str), -1, -1), 0) + self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, 1), 0) + self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, -1), 0) + self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0) + self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4) + # bad arguments + self.assertRaises(TypeError, find, str, b'!', 0, len(str), 1) + self.assertRaises(TypeError, find, b"!>__= end + self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) + self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, -1), -1) + self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) + self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, -1), -1) + # negative + self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, 1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, -1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4) + # bad arguments + # CRASHES unicode_findchar(b"!>_'), '<>br<>c<>d<>br<>') + self.assertEqual(replace(str, 'abra', '='), '=cad=') + self.assertEqual(replace(str, 'a', '=', 2), '=br=cadabra') + self.assertEqual(replace(str, 'a', '=', 0), str) + self.assertEqual(replace(str, 'a', '=', PY_SSIZE_T_MAX), '=br=c=d=br=') + self.assertEqual(replace(str, 'a', '=', -1), '=br=c=d=br=') + self.assertEqual(replace(str, 'a', '=', PY_SSIZE_T_MIN), '=br=c=d=br=') + self.assertEqual(replace(str, 'z', '='), str) + self.assertEqual(replace(str, '', '='), '=a=b=r=a=c=a=d=a=b=r=a=') + self.assertEqual(replace(str, 'a', 'ж'), 'жbrжcжdжbrж') + self.assertEqual(replace('абабагаламага', 'а', '='), '=б=б=г=л=м=г=') + self.assertEqual(replace('Баден-Баден', 'Баден', 'Baden'), 'Baden-Baden') + # bad arguments + self.assertRaises(TypeError, replace, 'a', 'a', b'=') + self.assertRaises(TypeError, replace, 'a', b'a', '=') + self.assertRaises(TypeError, replace, b'a', 'a', '=') + self.assertRaises(TypeError, replace, 'a', 'a', ord('=')) + self.assertRaises(TypeError, replace, 'a', ord('a'), '=') + self.assertRaises(TypeError, replace, [], 'a', '=') + # CRASHES replace('a', 'a', NULL) + # CRASHES replace('a', NULL, '=') + # CRASHES replace(NULL, 'a', '=') + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_compare(self): + """Test PyUnicode_Compare()""" + from _testlimitedcapi import unicode_compare as compare + + self.assertEqual(compare('abc', 'abc'), 0) + self.assertEqual(compare('abc', 'def'), -1) + self.assertEqual(compare('def', 'abc'), 1) + self.assertEqual(compare('abc', 'abc\0def'), -1) + self.assertEqual(compare('abc\0def', 'abc\0def'), 0) + self.assertEqual(compare('абв', 'abc'), 1) + + self.assertRaises(TypeError, compare, b'abc', 'abc') + self.assertRaises(TypeError, compare, 'abc', b'abc') + self.assertRaises(TypeError, compare, b'abc', b'abc') + self.assertRaises(TypeError, compare, [], 'abc') + self.assertRaises(TypeError, compare, 'abc', []) + self.assertRaises(TypeError, compare, [], []) + # CRASHES compare(NULL, 'abc') + # CRASHES compare('abc', NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_comparewithasciistring(self): + """Test PyUnicode_CompareWithASCIIString()""" + from _testlimitedcapi import unicode_comparewithasciistring as comparewithasciistring + + self.assertEqual(comparewithasciistring('abc', b'abc'), 0) + self.assertEqual(comparewithasciistring('abc', b'def'), -1) + self.assertEqual(comparewithasciistring('def', b'abc'), 1) + self.assertEqual(comparewithasciistring('abc', b'abc\0def'), 0) + self.assertEqual(comparewithasciistring('abc\0def', b'abc\0def'), 1) + self.assertEqual(comparewithasciistring('абв', b'abc'), 1) + + # CRASHES comparewithasciistring(b'abc', b'abc') + # CRASHES comparewithasciistring([], b'abc') + # CRASHES comparewithasciistring(NULL, b'abc') + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_equaltoutf8(self): + # Test PyUnicode_EqualToUTF8() + from _testlimitedcapi import unicode_equaltoutf8 as equaltoutf8 + from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize + + strings = [ + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602', + '\U0010ffff', + ] + for s in strings: + # Call PyUnicode_AsUTF8AndSize() which creates the UTF-8 + # encoded string cached in the Unicode object. + asutf8andsize(s, 0) + b = s.encode() + self.assertEqual(equaltoutf8(s, b), 1) # Use the UTF-8 cache. + s2 = b.decode() # New Unicode object without the UTF-8 cache. + self.assertEqual(equaltoutf8(s2, b), 1) + self.assertEqual(equaltoutf8(s + 'x', b + b'x'), 1) + self.assertEqual(equaltoutf8(s + 'x', b + b'y'), 0) + self.assertEqual(equaltoutf8(s, b + b'\0'), 1) + self.assertEqual(equaltoutf8(s2, b + b'\0'), 1) + self.assertEqual(equaltoutf8(s + '\0', b + b'\0'), 0) + self.assertEqual(equaltoutf8(s + '\0', b), 0) + self.assertEqual(equaltoutf8(s2, b + b'x'), 0) + self.assertEqual(equaltoutf8(s2, b[:-1]), 0) + self.assertEqual(equaltoutf8(s2, b[:-1] + b'x'), 0) + + self.assertEqual(equaltoutf8('', b''), 1) + self.assertEqual(equaltoutf8('', b'\0'), 1) + + # embedded null chars/bytes + self.assertEqual(equaltoutf8('abc', b'abc\0def\0'), 1) + self.assertEqual(equaltoutf8('a\0bc', b'abc'), 0) + self.assertEqual(equaltoutf8('abc', b'a\0bc'), 0) + + # Surrogate characters are always treated as not equal + self.assertEqual(equaltoutf8('\udcfe', + '\udcfe'.encode("utf8", "surrogateescape")), 0) + self.assertEqual(equaltoutf8('\udcfe', + '\udcfe'.encode("utf8", "surrogatepass")), 0) + self.assertEqual(equaltoutf8('\ud801', + '\ud801'.encode("utf8", "surrogatepass")), 0) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_equaltoutf8andsize(self): + # Test PyUnicode_EqualToUTF8AndSize() + from _testlimitedcapi import unicode_equaltoutf8andsize as equaltoutf8andsize + from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize + + strings = [ + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602', + '\U0010ffff', + ] + for s in strings: + # Call PyUnicode_AsUTF8AndSize() which creates the UTF-8 + # encoded string cached in the Unicode object. + asutf8andsize(s, 0) + b = s.encode() + self.assertEqual(equaltoutf8andsize(s, b), 1) # Use the UTF-8 cache. + s2 = b.decode() # New Unicode object without the UTF-8 cache. + self.assertEqual(equaltoutf8andsize(s2, b), 1) + self.assertEqual(equaltoutf8andsize(s + 'x', b + b'x'), 1) + self.assertEqual(equaltoutf8andsize(s + 'x', b + b'y'), 0) + self.assertEqual(equaltoutf8andsize(s, b + b'\0'), 0) + self.assertEqual(equaltoutf8andsize(s2, b + b'\0'), 0) + self.assertEqual(equaltoutf8andsize(s + '\0', b + b'\0'), 1) + self.assertEqual(equaltoutf8andsize(s + '\0', b), 0) + self.assertEqual(equaltoutf8andsize(s2, b + b'x'), 0) + self.assertEqual(equaltoutf8andsize(s2, b[:-1]), 0) + self.assertEqual(equaltoutf8andsize(s2, b[:-1] + b'x'), 0) + # Not null-terminated, + self.assertEqual(equaltoutf8andsize(s, b + b'x', len(b)), 1) + self.assertEqual(equaltoutf8andsize(s2, b + b'x', len(b)), 1) + self.assertEqual(equaltoutf8andsize(s + '\0', b + b'\0x', len(b) + 1), 1) + self.assertEqual(equaltoutf8andsize(s2, b, len(b) - 1), 0) + self.assertEqual(equaltoutf8andsize(s, b, -1), 0) + self.assertEqual(equaltoutf8andsize(s, b, PY_SSIZE_T_MAX), 0) + self.assertEqual(equaltoutf8andsize(s, b, PY_SSIZE_T_MIN), 0) + + self.assertEqual(equaltoutf8andsize('', b''), 1) + self.assertEqual(equaltoutf8andsize('', b'\0'), 0) + self.assertEqual(equaltoutf8andsize('', b'x', 0), 1) + + # embedded null chars/bytes + self.assertEqual(equaltoutf8andsize('abc\0def', b'abc\0def'), 1) + self.assertEqual(equaltoutf8andsize('abc\0def\0', b'abc\0def\0'), 1) + + # Surrogate characters are always treated as not equal + self.assertEqual(equaltoutf8andsize('\udcfe', + '\udcfe'.encode("utf8", "surrogateescape")), 0) + self.assertEqual(equaltoutf8andsize('\udcfe', + '\udcfe'.encode("utf8", "surrogatepass")), 0) + self.assertEqual(equaltoutf8andsize('\ud801', + '\ud801'.encode("utf8", "surrogatepass")), 0) + + def check_not_equal_encoding(text, encoding): + self.assertEqual(equaltoutf8andsize(text, text.encode(encoding)), 0) + self.assertNotEqual(text.encode(encoding), text.encode("utf8")) + + # Strings encoded to other encodings are not equal to expected UTF8-encoding string + check_not_equal_encoding('Stéphane', 'latin1') + check_not_equal_encoding('Stéphane', 'utf-16-le') # embedded null characters + check_not_equal_encoding('北京市', 'gbk') + + # CRASHES equaltoutf8andsize('abc', b'abc', -1) + # CRASHES equaltoutf8andsize(b'abc', b'abc') + # CRASHES equaltoutf8andsize([], b'abc') + # CRASHES equaltoutf8andsize(NULL, b'abc') + # CRASHES equaltoutf8andsize('abc', NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_richcompare(self): + """Test PyUnicode_RichCompare()""" + from _testlimitedcapi import unicode_richcompare as richcompare + + LT, LE, EQ, NE, GT, GE = range(6) + strings = ('abc', 'абв', '\U0001f600', 'abc\0') + for s1 in strings: + for s2 in strings: + self.assertIs(richcompare(s1, s2, LT), s1 < s2) + self.assertIs(richcompare(s1, s2, LE), s1 <= s2) + self.assertIs(richcompare(s1, s2, EQ), s1 == s2) + self.assertIs(richcompare(s1, s2, NE), s1 != s2) + self.assertIs(richcompare(s1, s2, GT), s1 > s2) + self.assertIs(richcompare(s1, s2, GE), s1 >= s2) + + for op in LT, LE, EQ, NE, GT, GE: + self.assertIs(richcompare(b'abc', 'abc', op), NotImplemented) + self.assertIs(richcompare('abc', b'abc', op), NotImplemented) + self.assertIs(richcompare(b'abc', b'abc', op), NotImplemented) + self.assertIs(richcompare([], 'abc', op), NotImplemented) + self.assertIs(richcompare('abc', [], op), NotImplemented) + self.assertIs(richcompare([], [], op), NotImplemented) + + # CRASHES richcompare(NULL, 'abc', op) + # CRASHES richcompare('abc', NULL, op) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_format(self): + """Test PyUnicode_Format()""" + from _testlimitedcapi import unicode_format as format + + self.assertEqual(format('x=%d!', 42), 'x=42!') + self.assertEqual(format('x=%d!', (42,)), 'x=42!') + self.assertEqual(format('x=%d y=%s!', (42, [])), 'x=42 y=[]!') + + self.assertRaises(SystemError, format, 'x=%d!', NULL) + self.assertRaises(SystemError, format, NULL, 42) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_contains(self): + """Test PyUnicode_Contains()""" + from _testlimitedcapi import unicode_contains as contains + + self.assertEqual(contains('abcd', ''), 1) + self.assertEqual(contains('abcd', 'b'), 1) + self.assertEqual(contains('abcd', 'x'), 0) + self.assertEqual(contains('abcd', 'ж'), 0) + self.assertEqual(contains('abcd', '\0'), 0) + self.assertEqual(contains('abc\0def', '\0'), 1) + self.assertEqual(contains('abcd', 'bc'), 1) + + self.assertRaises(TypeError, contains, b'abcd', 'b') + self.assertRaises(TypeError, contains, 'abcd', b'b') + self.assertRaises(TypeError, contains, b'abcd', b'b') + self.assertRaises(TypeError, contains, [], 'b') + self.assertRaises(TypeError, contains, 'abcd', ord('b')) + # CRASHES contains(NULL, 'b') + # CRASHES contains('abcd', NULL) + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_isidentifier(self): + """Test PyUnicode_IsIdentifier()""" + from _testlimitedcapi import unicode_isidentifier as isidentifier + + self.assertEqual(isidentifier("a"), 1) + self.assertEqual(isidentifier("b0"), 1) + self.assertEqual(isidentifier("µ"), 1) + self.assertEqual(isidentifier("𝔘𝔫𝔦𝔠𝔬𝔡𝔢"), 1) + + self.assertEqual(isidentifier(""), 0) + self.assertEqual(isidentifier(" "), 0) + self.assertEqual(isidentifier("["), 0) + self.assertEqual(isidentifier("©"), 0) + self.assertEqual(isidentifier("0"), 0) + self.assertEqual(isidentifier("32M"), 0) + + # CRASHES isidentifier(b"a") + # CRASHES isidentifier([]) + # CRASHES isidentifier(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_copycharacters(self): + """Test PyUnicode_CopyCharacters()""" + from _testcapi import unicode_copycharacters + + strings = [ + # all strings have exactly 5 characters + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + + for idx, from_ in enumerate(strings): + # wide -> narrow: exceed maxchar limitation + for to in strings[:idx]: + self.assertRaises( + SystemError, + unicode_copycharacters, to, 0, from_, 0, 5 + ) + # same kind + for from_start in range(5): + self.assertEqual( + unicode_copycharacters(from_, 0, from_, from_start, 5), + (from_[from_start:from_start+5].ljust(5, '\0'), + 5-from_start) + ) + for to_start in range(5): + self.assertEqual( + unicode_copycharacters(from_, to_start, from_, to_start, 5), + (from_[to_start:to_start+5].rjust(5, '\0'), + 5-to_start) + ) + # narrow -> wide + # Tests omitted since this creates invalid strings. + + s = strings[0] + self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, PY_SSIZE_T_MAX, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, PY_SSIZE_T_MIN, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, PY_SSIZE_T_MAX, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, PY_SSIZE_T_MIN, 5) + self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) + self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, PY_SSIZE_T_MIN) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, [], 0, 0) + # CRASHES unicode_copycharacters(s, 0, NULL, 0, 0) + # TODO: Test PyUnicode_CopyCharacters() with non-unicode and + # non-modifiable unicode as "to". + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_pep393_utf8_caching_bug(self): + # Issue #25709: Problem with string concatenation and utf-8 cache + from _testcapi import getargs_s_hash + for k in 0x24, 0xa4, 0x20ac, 0x1f40d: + s = '' + for i in range(5): + # Due to CPython specific optimization the 's' string can be + # resized in-place. + s += chr(k) + # Parsing with the "s#" format code calls indirectly + # PyUnicode_AsUTF8AndSize() which creates the UTF-8 + # encoded string cached in the Unicode object. + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + # Check that the second call returns the same result + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + + +class PyUnicodeWriterTest(unittest.TestCase): + def create_writer(self, size): + return _testcapi.PyUnicodeWriter(size) + + def test_basic(self): + writer = self.create_writer(100) + + # test PyUnicodeWriter_WriteUTF8() + writer.write_utf8(b'var', -1) + + # test PyUnicodeWriter_WriteChar() + writer.write_char(ord('=')) + + # test PyUnicodeWriter_WriteSubstring() + writer.write_substring("[long]", 1, 5) + # CRASHES writer.write_substring(NULL, 0, 0) + + # test PyUnicodeWriter_WriteStr() + writer.write_str(" value ") + # CRASHES writer.write_str(NULL) + + # test PyUnicodeWriter_WriteRepr() + writer.write_repr("repr") + + self.assertEqual(writer.finish(), + "var=long value 'repr'") + + def test_repr_null(self): + writer = self.create_writer(0) + writer.write_utf8(b'var=', -1) + writer.write_repr(NULL) + self.assertEqual(writer.finish(), + "var=") + + def test_write_char(self): + writer = self.create_writer(0) + writer.write_char(0) + writer.write_char(ord('$')) + writer.write_char(0x20ac) + writer.write_char(0x10_ffff) + self.assertRaises(ValueError, writer.write_char, 0x11_0000) + self.assertRaises(ValueError, writer.write_char, 0xFFFF_FFFF) + self.assertEqual(writer.finish(), + "\0$\u20AC\U0010FFFF") + + def test_utf8(self): + writer = self.create_writer(0) + writer.write_utf8(b"ascii", -1) + writer.write_char(ord('-')) + writer.write_utf8(b"latin1=\xC3\xA9", -1) + writer.write_char(ord('-')) + writer.write_utf8(b"euro=\xE2\x82\xAC", -1) + writer.write_char(ord('.')) + writer.write_utf8(NULL, 0) + # CRASHES writer.write_utf8(NULL, 1) + # CRASHES writer.write_utf8(NULL, -1) + self.assertEqual(writer.finish(), + "ascii-latin1=\xE9-euro=\u20AC.") + + def test_ascii(self): + writer = self.create_writer(0) + writer.write_ascii(b"Hello ", -1) + writer.write_ascii(b"", 0) + writer.write_ascii(NULL, 0) + # CRASHES writer.write_ascii(NULL, 1) + # CRASHES writer.write_ascii(NULL, -1) + writer.write_ascii(b"Python! ", 6) + self.assertEqual(writer.finish(), "Hello Python") + + def test_invalid_utf8(self): + writer = self.create_writer(0) + with self.assertRaises(UnicodeDecodeError): + writer.write_utf8(b"invalid=\xFF", -1) + + def test_recover_utf8_error(self): + # test recovering from PyUnicodeWriter_WriteUTF8() error + writer = self.create_writer(0) + writer.write_utf8(b"value=", -1) + + # write fails with an invalid string + with self.assertRaises(UnicodeDecodeError): + writer.write_utf8(b"invalid\xFF", -1) + with self.assertRaises(UnicodeDecodeError): + s = "truncated\u20AC".encode() + writer.write_utf8(s, len(s) - 1) + + # retry write with a valid string + writer.write_utf8(b"valid", -1) + + self.assertEqual(writer.finish(), + "value=valid") + + def test_decode_utf8(self): + # test PyUnicodeWriter_DecodeUTF8Stateful() + writer = self.create_writer(0) + writer.decodeutf8stateful(b"ign\xFFore", -1, b"ignore") + writer.write_char(ord('-')) + writer.decodeutf8stateful(b"replace\xFF", -1, b"replace") + writer.write_char(ord('-')) + + # incomplete trailing UTF-8 sequence + writer.decodeutf8stateful(b"incomplete\xC3", -1, b"replace") + + writer.decodeutf8stateful(NULL, 0, b"replace") + # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace") + # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace") + with self.assertRaises(UnicodeDecodeError): + writer.decodeutf8stateful(b"default\xFF", -1, NULL) + + self.assertEqual(writer.finish(), + "ignore-replace\uFFFD-incomplete\uFFFD") + + def test_decode_utf8_consumed(self): + # test PyUnicodeWriter_DecodeUTF8Stateful() with consumed + writer = self.create_writer(0) + + # valid string + consumed = writer.decodeutf8stateful(b"text", -1, b"strict", True) + self.assertEqual(consumed, 4) + writer.write_char(ord('-')) + + # non-ASCII + consumed = writer.decodeutf8stateful(b"\xC3\xA9-\xE2\x82\xAC", 6, b"strict", True) + self.assertEqual(consumed, 6) + writer.write_char(ord('-')) + + # invalid UTF-8 (consumed is 0 on error) + with self.assertRaises(UnicodeDecodeError): + writer.decodeutf8stateful(b"invalid\xFF", -1, b"strict", True) + + # ignore error handler + consumed = writer.decodeutf8stateful(b"more\xFF", -1, b"ignore", True) + self.assertEqual(consumed, 5) + writer.write_char(ord('-')) + + # incomplete trailing UTF-8 sequence + consumed = writer.decodeutf8stateful(b"incomplete\xC3", -1, b"ignore", True) + self.assertEqual(consumed, 10) + writer.write_char(ord('-')) + + consumed = writer.decodeutf8stateful(NULL, 0, b"replace", True) + self.assertEqual(consumed, 0) + # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace", True) + # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace", True) + consumed = writer.decodeutf8stateful(b"default\xC3", -1, NULL, True) + self.assertEqual(consumed, 7) + + self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete-default") + + def test_widechar(self): + from _testcapi import SIZEOF_WCHAR_T + + if SIZEOF_WCHAR_T == 2: + encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' + elif SIZEOF_WCHAR_T == 4: + encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + + writer = self.create_writer(0) + writer.write_widechar("latin1=\xE9".encode(encoding)) + writer.write_char(ord("-")) + writer.write_widechar("euro=\u20AC".encode(encoding)) + writer.write_char(ord("-")) + writer.write_widechar("max=\U0010ffff".encode(encoding)) + writer.write_char(ord("-")) + writer.write_widechar("zeroes=".encode(encoding).ljust(SIZEOF_WCHAR_T * 10, b'\0'), + 10) + writer.write_char(ord('.')) + + if SIZEOF_WCHAR_T == 4: + invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else + b'\x00\x11\x00\x00') + with self.assertRaises(ValueError): + writer.write_widechar("invalid=".encode(encoding) + invalid) + writer.write_widechar(b'', -5) + writer.write_widechar(NULL, 0) + # CRASHES writer.write_widechar(NULL, 1) + # CRASHES writer.write_widechar(NULL, -1) + + self.assertEqual(writer.finish(), + "latin1=\xE9-euro=\u20AC-max=\U0010ffff-zeroes=\0\0\0.") + + def test_ucs4(self): + encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + + writer = self.create_writer(0) + writer.write_ucs4("ascii IGNORED".encode(encoding), 5) + writer.write_char(ord("-")) + writer.write_ucs4("latin1=\xe9".encode(encoding)) + writer.write_char(ord("-")) + writer.write_ucs4("euro=\u20ac".encode(encoding)) + writer.write_char(ord("-")) + writer.write_ucs4("max=\U0010ffff".encode(encoding)) + writer.write_char(ord(".")) + self.assertEqual(writer.finish(), + "ascii-latin1=\xE9-euro=\u20AC-max=\U0010ffff.") + + # Test some special characters + writer = self.create_writer(0) + # Lone surrogate character + writer.write_ucs4("lone\uDC80".encode(encoding, 'surrogatepass')) + writer.write_char(ord("-")) + # Surrogate pair + writer.write_ucs4("pair\uD83D\uDC0D".encode(encoding, 'surrogatepass')) + writer.write_char(ord("-")) + writer.write_ucs4("null[\0]".encode(encoding), 7) + invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else + b'\x00\x11\x00\x00') + # CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid) + writer.write_ucs4(NULL, 0) + # CRASHES writer.write_ucs4(NULL, 1) + self.assertEqual(writer.finish(), + "lone\udc80-pair\ud83d\udc0d-null[\x00]") + + # invalid size + writer = self.create_writer(0) + with self.assertRaises(ValueError): + writer.write_ucs4("text".encode(encoding), -1) + self.assertRaises(ValueError, writer.write_ucs4, b'', -1) + self.assertRaises(ValueError, writer.write_ucs4, NULL, -1) + + def test_substring_empty(self): + writer = self.create_writer(0) + writer.write_substring("abc", 1, 1) + self.assertEqual(writer.finish(), '') + + +@unittest.skipIf(ctypes is None, 'need ctypes') +class PyUnicodeWriterFormatTest(unittest.TestCase): + def create_writer(self, size): + return _testcapi.PyUnicodeWriter(size) + + def writer_format(self, writer, *args): + from ctypes import c_char_p, pythonapi, c_int, c_void_p + _PyUnicodeWriter_Format = getattr(pythonapi, "PyUnicodeWriter_Format") + _PyUnicodeWriter_Format.argtypes = (c_void_p, c_char_p,) + _PyUnicodeWriter_Format.restype = c_int + + if _PyUnicodeWriter_Format(writer.get_pointer(), *args) < 0: + raise ValueError("PyUnicodeWriter_Format failed") + + def test_format(self): + from ctypes import c_int + writer = self.create_writer(0) + self.writer_format(writer, b'%s %i', b'abc', c_int(123)) + writer.write_char(ord('.')) + self.assertEqual(writer.finish(), 'abc 123.') + + def test_recover_error(self): + # test recovering from PyUnicodeWriter_Format() error + writer = self.create_writer(0) + self.writer_format(writer, b"%s ", b"Hello") + + # PyUnicodeWriter_Format() fails with an invalid format string + with self.assertRaises(ValueError): + self.writer_format(writer, b"%s\xff", b"World") + + # Retry PyUnicodeWriter_Format() with a valid format string + self.writer_format(writer, b"%s.", b"World") + + self.assertEqual(writer.finish(), 'Hello World.') + + def test_unicode_equal(self): + unicode_equal = _testlimitedcapi.unicode_equal + + def copy(text): + return text.encode().decode() + + self.assertTrue(unicode_equal("", "")) + self.assertTrue(unicode_equal("abc", "abc")) + self.assertTrue(unicode_equal("abc", copy("abc"))) + self.assertTrue(unicode_equal("\u20ac", copy("\u20ac"))) + self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff"))) + + self.assertFalse(unicode_equal("abc", "abcd")) + self.assertFalse(unicode_equal("\u20ac", "\u20ad")) + self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe")) + + # str subclass + self.assertTrue(unicode_equal("abc", Str("abc"))) + self.assertTrue(unicode_equal(Str("abc"), "abc")) + self.assertFalse(unicode_equal("abc", Str("abcd"))) + self.assertFalse(unicode_equal(Str("abc"), "abcd")) + + # invalid type + for invalid_type in (b'bytes', 123, ("tuple",)): + with self.subTest(invalid_type=invalid_type): + with self.assertRaises(TypeError): + unicode_equal("abc", invalid_type) + with self.assertRaises(TypeError): + unicode_equal(invalid_type, "abc") + + # CRASHES unicode_equal("abc", NULL) + # CRASHES unicode_equal(NULL, "abc") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py new file mode 100644 index 00000000000..8644479d83d --- /dev/null +++ b/Lib/test/test_capi/test_watchers.py @@ -0,0 +1,678 @@ +import unittest +import contextvars + +from contextlib import contextmanager, ExitStack +from test.support import ( + catch_unraisable_exception, import_helper, + gc_collect) + + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + + +class TestDictWatchers(unittest.TestCase): + # types of watchers testcapimodule can add: + EVENTS = 0 # appends dict events as strings to global event list + ERROR = 1 # unconditionally sets and signals a RuntimeException + SECOND = 2 # always appends "second" to global event list + + def add_watcher(self, kind=EVENTS): + return _testcapi.add_dict_watcher(kind) + + def clear_watcher(self, watcher_id): + _testcapi.clear_dict_watcher(watcher_id) + + @contextmanager + def watcher(self, kind=EVENTS): + wid = self.add_watcher(kind) + try: + yield wid + finally: + self.clear_watcher(wid) + + def assert_events(self, expected): + actual = _testcapi.get_dict_watcher_events() + self.assertEqual(actual, expected) + + def watch(self, wid, d): + _testcapi.watch_dict(wid, d) + + def unwatch(self, wid, d): + _testcapi.unwatch_dict(wid, d) + + def test_set_new_item(self): + d = {} + with self.watcher() as wid: + self.watch(wid, d) + d["foo"] = "bar" + self.assert_events(["new:foo:bar"]) + + def test_set_existing_item(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + d["foo"] = "baz" + self.assert_events(["mod:foo:baz"]) + + def test_clone(self): + d = {} + d2 = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + d.update(d2) + self.assert_events(["clone"]) + + def test_no_event_if_not_watched(self): + d = {} + with self.watcher() as wid: + d["foo"] = "bar" + self.assert_events([]) + + def test_del(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + del d["foo"] + self.assert_events(["del:foo"]) + + def test_pop(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + d.pop("foo") + self.assert_events(["del:foo"]) + + def test_clear(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + d.clear() + self.assert_events(["clear"]) + + def test_dealloc(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + del d + self.assert_events(["dealloc"]) + + def test_object_dict(self): + class MyObj: pass + o = MyObj() + + with self.watcher() as wid: + self.watch(wid, o.__dict__) + o.foo = "bar" + o.foo = "baz" + del o.foo + self.assert_events(["new:foo:bar", "mod:foo:baz", "del:foo"]) + + with self.watcher() as wid: + self.watch(wid, o.__dict__) + for _ in range(100): + o.foo = "bar" + self.assert_events(["new:foo:bar"] + ["mod:foo:bar"] * 99) + + def test_unwatch(self): + d = {} + with self.watcher() as wid: + self.watch(wid, d) + d["foo"] = "bar" + self.unwatch(wid, d) + d["hmm"] = "baz" + self.assert_events(["new:foo:bar"]) + + def test_error(self): + d = {} + with self.watcher(kind=self.ERROR) as wid: + self.watch(wid, d) + with catch_unraisable_exception() as cm: + d["foo"] = "bar" + self.assertIn( + "Exception ignored in " + "PyDict_EVENT_ADDED watcher callback for ") + print(f"extra_compile_args: {cflags!r}") + + ext = Extension( + module_name, + sources=sources, + extra_compile_args=cflags, + include_dirs=include_dirs, + library_dirs=library_dirs) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py new file mode 100644 index 00000000000..73bb942af7c --- /dev/null +++ b/Lib/test/test_clinic.py @@ -0,0 +1,4594 @@ +# Argument Clinic +# Copyright 2012-2013 by Larry Hastings. +# Licensed to the PSF under a contributor agreement. + +from functools import partial +from test import support, test_tools +from test.support import os_helper +from test.support.os_helper import TESTFN, unlink, rmtree +from textwrap import dedent +from unittest import TestCase +import inspect +import os.path +import re +import sys +import unittest + +test_tools.skip_if_missing('clinic') +with test_tools.imports_under_tool('clinic'): + import libclinic + from libclinic import ClinicError, unspecified, NULL, fail + from libclinic.converters import int_converter, str_converter, self_converter + from libclinic.function import ( + Module, Class, Function, FunctionKind, Parameter, + permute_optional_groups, permute_right_option_groups, + permute_left_option_groups) + import clinic + from libclinic.clanguage import CLanguage + from libclinic.converter import converters, legacy_converters + from libclinic.return_converters import return_converters, int_return_converter + from libclinic.block_parser import Block, BlockParser + from libclinic.codegen import BlockPrinter, Destination + from libclinic.dsl_parser import DSLParser + from libclinic.cli import parse_file, Clinic + + +def repeat_fn(*functions): + def wrapper(test): + def wrapped(self): + for fn in functions: + with self.subTest(fn=fn): + test(self, fn) + return wrapped + return wrapper + +def _make_clinic(*, filename='clinic_tests', limited_capi=False): + clang = CLanguage(filename) + c = Clinic(clang, filename=filename, limited_capi=limited_capi) + c.block_parser = BlockParser('', clang) + return c + + +def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, + strip=True): + """Helper for the parser tests. + + tc: unittest.TestCase; passed self in the wrapper + parser: the clinic parser used for this test case + code: a str with input text (clinic code) + errmsg: the expected error message + filename: str, optional filename + lineno: int, optional line number + """ + code = dedent(code) + if strip: + code = code.strip() + errmsg = re.escape(errmsg) + with tc.assertRaisesRegex(ClinicError, errmsg) as cm: + parser(code) + if filename is not None: + tc.assertEqual(cm.exception.filename, filename) + if lineno is not None: + tc.assertEqual(cm.exception.lineno, lineno) + return cm.exception + + +def restore_dict(converters, old_converters): + converters.clear() + converters.update(old_converters) + + +def save_restore_converters(testcase): + testcase.addCleanup(restore_dict, converters, + converters.copy()) + testcase.addCleanup(restore_dict, legacy_converters, + legacy_converters.copy()) + testcase.addCleanup(restore_dict, return_converters, + return_converters.copy()) + + +class ClinicWholeFileTest(TestCase): + maxDiff = None + + def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): + _expect_failure(self, self.clinic.parse, raw, errmsg, + filename=filename, lineno=lineno) + + def setUp(self): + save_restore_converters(self) + self.clinic = _make_clinic(filename="test.c") + + def test_eol(self): + # regression test: + # clinic's block parser didn't recognize + # the "end line" for the block if it + # didn't end in "\n" (as in, the last) + # byte of the file was '/'. + # so it would spit out an end line for you. + # and since you really already had one, + # the last line of the block got corrupted. + raw = "/*[clinic]\nfoo\n[clinic]*/" + cooked = self.clinic.parse(raw).splitlines() + end_line = cooked[2].rstrip() + # this test is redundant, it's just here explicitly to catch + # the regression test so we don't forget what it looked like + self.assertNotEqual(end_line, "[clinic]*/[clinic]*/") + self.assertEqual(end_line, "[clinic]*/") + + def test_mangled_marker_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: foo]*/ + """ + err = ( + "Mangled Argument Clinic marker line: " + "'/*[clinic end generated code: foo]*/'" + ) + self.expect_failure(raw, err, filename="test.c", lineno=3) + + def test_checksum_mismatch(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ + """ + err = ("Checksum mismatch! " + "Expected '0123456789abcdef', computed 'da39a3ee5e6b4b0d'") + self.expect_failure(raw, err, filename="test.c", lineno=3) + + def test_garbage_after_stop_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/foobarfoobar! + """ + err = "Garbage after stop line: 'foobarfoobar!'" + self.expect_failure(raw, err, filename="test.c", lineno=2) + + def test_whitespace_before_stop_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + """ + err = ( + "Whitespace is not allowed before the stop line: " + "' [clinic start generated code]*/'" + ) + self.expect_failure(raw, err, filename="test.c", lineno=2) + + def test_parse_with_body_prefix(self): + clang = CLanguage(None) + clang.body_prefix = "//" + clang.start_line = "//[{dsl_name} start]" + clang.stop_line = "//[{dsl_name} stop]" + cl = Clinic(clang, filename="test.c", limited_capi=False) + raw = dedent(""" + //[clinic start] + //module test + //[clinic stop] + """).strip() + out = cl.parse(raw) + expected = dedent(""" + //[clinic start] + //module test + // + //[clinic stop] + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=65fab8adff58cf08]*/ + """).lstrip() # Note, lstrip() because of the newline + self.assertEqual(out, expected) + + def test_cpp_monitor_fail_nested_block_comment(self): + raw = """ + /* start + /* nested + */ + */ + """ + err = 'Nested block comment!' + self.expect_failure(raw, err, filename="test.c", lineno=2) + + def test_cpp_monitor_fail_invalid_format_noarg(self): + raw = """ + #if + a() + #endif + """ + err = 'Invalid format for #if line: no argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) + + def test_cpp_monitor_fail_invalid_format_toomanyargs(self): + raw = """ + #ifdef A B + a() + #endif + """ + err = 'Invalid format for #ifdef line: should be exactly one argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) + + def test_cpp_monitor_fail_no_matching_if(self): + raw = '#else' + err = '#else without matching #if / #ifdef / #ifndef!' + self.expect_failure(raw, err, filename="test.c", lineno=1) + + def test_directive_output_unknown_preset(self): + raw = """ + /*[clinic input] + output preset nosuchpreset + [clinic start generated code]*/ + """ + err = "Unknown preset 'nosuchpreset'" + self.expect_failure(raw, err) + + def test_directive_output_cant_pop(self): + raw = """ + /*[clinic input] + output pop + [clinic start generated code]*/ + """ + err = "Can't 'output pop', stack is empty" + self.expect_failure(raw, err) + + def test_directive_output_print(self): + raw = dedent(""" + /*[clinic input] + output print 'I told you once.' + [clinic start generated code]*/ + """) + out = self.clinic.parse(raw) + # The generated output will differ for every run, but we can check that + # it starts with the clinic block, we check that it contains all the + # expected fields, and we check that it contains the checksum line. + self.assertStartsWith(out, dedent(""" + /*[clinic input] + output print 'I told you once.' + [clinic start generated code]*/ + """)) + fields = { + "cpp_endif", + "cpp_if", + "docstring_definition", + "docstring_prototype", + "impl_definition", + "impl_prototype", + "methoddef_define", + "methoddef_ifndef", + "parser_definition", + "parser_prototype", + } + for field in fields: + with self.subTest(field=field): + self.assertIn(field, out) + last_line = out.rstrip().split("\n")[-1] + self.assertStartsWith(last_line, "/*[clinic end generated code: output=") + + def test_directive_wrong_arg_number(self): + raw = dedent(""" + /*[clinic input] + preserve foo bar baz eggs spam ham mushrooms + [clinic start generated code]*/ + """) + err = "takes 1 positional argument but 8 were given" + self.expect_failure(raw, err) + + def test_unknown_destination_command(self): + raw = """ + /*[clinic input] + destination buffer nosuchcommand + [clinic start generated code]*/ + """ + err = "unknown destination command 'nosuchcommand'" + self.expect_failure(raw, err) + + def test_no_access_to_members_in_converter_init(self): + raw = """ + /*[python input] + class Custom_converter(CConverter): + converter = "some_c_function" + def converter_init(self): + self.function.noaccess + [python start generated code]*/ + /*[clinic input] + module test + test.fn + a: Custom + [clinic start generated code]*/ + """ + err = ( + "accessing self.function inside converter_init is disallowed!" + ) + self.expect_failure(raw, err) + + def test_clone_mismatch(self): + err = "'kind' of function and cloned function don't match!" + block = """ + /*[clinic input] + module m + @classmethod + m.f1 + a: object + [clinic start generated code]*/ + /*[clinic input] + @staticmethod + m.f2 = m.f1 + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=9) + + def test_badly_formed_return_annotation(self): + err = "Badly formed annotation for 'm.f': 'Custom'" + block = """ + /*[python input] + class Custom_return_converter(CReturnConverter): + def __init__(self): + raise ValueError("abc") + [python start generated code]*/ + /*[clinic input] + module m + m.f -> Custom + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=8) + + def test_star_after_vararg(self): + err = "'my_test_func' uses '*' more than once." + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + *args: tuple + * + kw_arg: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_vararg_after_star(self): + err = "'my_test_func' uses '*' more than once." + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + * + *args: tuple + kw_arg: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_module_already_got_one(self): + err = "Already defined module 'm'!" + block = """ + /*[clinic input] + module m + module m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_destination_already_got_one(self): + err = "Destination already exists: 'test'" + block = """ + /*[clinic input] + destination test new buffer + destination test new buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_destination_does_not_exist(self): + err = "Destination does not exist: '/dev/null'" + block = """ + /*[clinic input] + output everything /dev/null + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_class_already_got_one(self): + err = "Already defined class 'C'!" + block = """ + /*[clinic input] + class C "" "" + class C "" "" + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_cant_nest_module_inside_class(self): + err = "Can't nest a module inside a class!" + block = """ + /*[clinic input] + class C "" "" + module C.m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_dest_buffer_not_empty_at_eof(self): + expected_warning = ("Destination buffer 'buffer' not empty at " + "end of file, emptying.") + expected_generated = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=1c4668687f5fd002]*/ + + /*[clinic input] + dump buffer + [clinic start generated code]*/ + + PyDoc_VAR(fn__doc__); + + PyDoc_STRVAR(fn__doc__, + "fn($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FN_METHODDEF \\ + {"fn", (PyCFunction)fn, METH_O, fn__doc__}, + + static PyObject * + fn(PyObject *module, PyObject *a) + /*[clinic end generated code: output=be6798b148ab4e53 input=524ce2e021e4eba6]*/ + """) + block = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + """) + with support.captured_stdout() as stdout: + generated = self.clinic.parse(block) + self.assertIn(expected_warning, stdout.getvalue()) + self.assertEqual(generated, expected_generated) + + def test_dest_clear(self): + err = "Can't clear destination 'file': it's not of type 'buffer'" + block = """ + /*[clinic input] + destination file clear + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_directive_set_misuse(self): + err = "unknown variable 'ets'" + block = """ + /*[clinic input] + set ets tse + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_directive_set_prefix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '// ' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "// PyDoc_VAR(fn__doc__);" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_suffix(self): + block = dedent(""" + /*[clinic input] + set line_suffix ' // test' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "PyDoc_VAR(fn__doc__); // test" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_prefix_and_suffix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '{block comment start} ' + set line_suffix ' {block comment end}' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "/* PyDoc_VAR(fn__doc__); */" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_printout(self): + block = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + """) + expected = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + test + /*[clinic end generated code: output=4e1243bd22c66e76 input=898f1a32965d44ca]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, expected) + + def test_directive_preserve_twice(self): + err = "Can't have 'preserve' twice in one block!" + block = """ + /*[clinic input] + preserve + preserve + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_directive_preserve_input(self): + err = "'preserve' only works for blocks that don't produce any output!" + block = """ + /*[clinic input] + preserve + fn + a: object + / + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_directive_preserve_output(self): + block = dedent(""" + /*[clinic input] + output everything buffer + preserve + [clinic start generated code]*/ + // Preserve this + /*[clinic end generated code: output=eaa49677ae4c1f7d input=559b5db18fddae6a]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=524ce2e021e4eba6]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, block) + + def test_directive_output_invalid_command(self): + err = dedent(""" + Invalid command or destination name 'cmd'. Must be one of: + - 'preset' + - 'push' + - 'pop' + - 'print' + - 'everything' + - 'cpp_if' + - 'docstring_prototype' + - 'docstring_definition' + - 'methoddef_define' + - 'impl_prototype' + - 'parser_prototype' + - 'parser_definition' + - 'cpp_endif' + - 'methoddef_ifndef' + - 'impl_definition' + """).strip() + block = """ + /*[clinic input] + output cmd buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_validate_cloned_init(self): + block = """ + /*[clinic input] + class C "void *" "" + C.meth + a: int + [clinic start generated code]*/ + /*[clinic input] + @classmethod + C.__init__ = C.meth + [clinic start generated code]*/ + """ + err = "'__init__' must be a normal method; got 'FunctionKind.CLASS_METHOD'!" + self.expect_failure(block, err, lineno=8) + + def test_validate_cloned_new(self): + block = """ + /*[clinic input] + class C "void *" "" + C.meth + a: int + [clinic start generated code]*/ + /*[clinic input] + C.__new__ = C.meth + [clinic start generated code]*/ + """ + err = "'__new__' must be a class method" + self.expect_failure(block, err, lineno=7) + + def test_no_c_basename_cloned(self): + block = """ + /*[clinic input] + foo2 + [clinic start generated code]*/ + /*[clinic input] + foo as = foo2 + [clinic start generated code]*/ + """ + err = "No C basename provided after 'as' keyword" + self.expect_failure(block, err, lineno=5) + + def test_cloned_with_custom_c_basename(self): + raw = dedent(""" + /*[clinic input] + # Make sure we don't create spurious clinic/ directories. + output everything suppress + foo2 + [clinic start generated code]*/ + + /*[clinic input] + foo as foo1 = foo2 + [clinic start generated code]*/ + """) + self.clinic.parse(raw) + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + self.assertEqual(funcs[1].name, "foo") + self.assertEqual(funcs[1].c_basename, "foo1") + + def test_cloned_with_illegal_c_basename(self): + block = """ + /*[clinic input] + class C "void *" "" + foo1 + [clinic start generated code]*/ + + /*[clinic input] + foo2 as .illegal. = foo1 + [clinic start generated code]*/ + """ + err = "Illegal C basename: '.illegal.'" + self.expect_failure(block, err, lineno=7) + + def test_cloned_forced_text_signature(self): + block = dedent(""" + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + param a + b: object = NULL + / + + docstring + [clinic start generated code]*/ + + /*[clinic input] + dst = src + [clinic start generated code]*/ + """) + self.clinic.parse(block) + self.addCleanup(rmtree, "clinic") + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + + src_docstring_lines = funcs[0].docstring.split("\n") + dst_docstring_lines = funcs[1].docstring.split("\n") + + # Signatures are copied. + self.assertEqual(src_docstring_lines[0], "src($module, a[, b])") + self.assertEqual(dst_docstring_lines[0], "dst($module, a[, b])") + + # Param docstrings are copied. + self.assertIn(" param a", src_docstring_lines) + self.assertIn(" param a", dst_docstring_lines) + + # Docstrings are not copied. + self.assertIn("docstring", src_docstring_lines) + self.assertNotIn("docstring", dst_docstring_lines) + + def test_cloned_forced_text_signature_illegal(self): + block = """ + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + b: object = NULL + / + [clinic start generated code]*/ + + /*[clinic input] + @text_signature "($module, a_override[, b])" + dst = src + [clinic start generated code]*/ + """ + err = "Cannot use @text_signature when cloning a function" + self.expect_failure(block, err, lineno=11) + + def test_ignore_preprocessor_in_comments(self): + for dsl in "clinic", "python": + raw = dedent(f"""\ + /*[{dsl} input] + # CPP directives, valid or not, should be ignored in C comments. + # + [{dsl} start generated code]*/ + """) + self.clinic.parse(raw) + + +class ParseFileUnitTest(TestCase): + def expect_parsing_failure( + self, *, filename, expected_error, verify=True, output=None + ): + errmsg = re.escape(dedent(expected_error).strip()) + with self.assertRaisesRegex(ClinicError, errmsg): + parse_file(filename, limited_capi=False) + + def test_parse_file_no_extension(self) -> None: + self.expect_parsing_failure( + filename="foo", + expected_error="Can't extract file type for file 'foo'" + ) + + def test_parse_file_strange_extension(self) -> None: + filenames_to_errors = { + "foo.rs": "Can't identify file type for file 'foo.rs'", + "foo.hs": "Can't identify file type for file 'foo.hs'", + "foo.js": "Can't identify file type for file 'foo.js'", + } + for filename, errmsg in filenames_to_errors.items(): + with self.subTest(filename=filename): + self.expect_parsing_failure(filename=filename, expected_error=errmsg) + + +class ClinicGroupPermuterTest(TestCase): + def _test(self, l, m, r, output): + computed = permute_optional_groups(l, m, r) + self.assertEqual(output, computed) + + def test_range(self): + self._test([['start']], ['stop'], [['step']], + ( + ('stop',), + ('start', 'stop',), + ('start', 'stop', 'step',), + )) + + def test_add_window(self): + self._test([['x', 'y']], ['ch'], [['attr']], + ( + ('ch',), + ('ch', 'attr'), + ('x', 'y', 'ch',), + ('x', 'y', 'ch', 'attr'), + )) + + def test_ludicrous(self): + self._test([['a1', 'a2', 'a3'], ['b1', 'b2']], ['c1'], [['d1', 'd2'], ['e1', 'e2', 'e3']], + ( + ('c1',), + ('b1', 'b2', 'c1'), + ('b1', 'b2', 'c1', 'd1', 'd2'), + ('a1', 'a2', 'a3', 'b1', 'b2', 'c1'), + ('a1', 'a2', 'a3', 'b1', 'b2', 'c1', 'd1', 'd2'), + ('a1', 'a2', 'a3', 'b1', 'b2', 'c1', 'd1', 'd2', 'e1', 'e2', 'e3'), + )) + + def test_right_only(self): + self._test([], [], [['a'],['b'],['c']], + ( + (), + ('a',), + ('a', 'b'), + ('a', 'b', 'c') + )) + + def test_have_left_options_but_required_is_empty(self): + def fn(): + permute_optional_groups(['a'], [], []) + self.assertRaises(ValueError, fn) + + +class ClinicLinearFormatTest(TestCase): + def _test(self, input, output, **kwargs): + computed = libclinic.linear_format(input, **kwargs) + self.assertEqual(output, computed) + + def test_empty_strings(self): + self._test('', '') + + def test_solo_newline(self): + self._test('\n', '\n') + + def test_no_substitution(self): + self._test(""" + abc + """, """ + abc + """) + + def test_empty_substitution(self): + self._test(""" + abc + {name} + def + """, """ + abc + def + """, name='') + + def test_single_line_substitution(self): + self._test(""" + abc + {name} + def + """, """ + abc + GARGLE + def + """, name='GARGLE') + + def test_multiline_substitution(self): + self._test(""" + abc + {name} + def + """, """ + abc + bingle + bungle + + def + """, name='bingle\nbungle\n') + + def test_text_before_block_marker(self): + regex = re.escape("found before '{marker}'") + with self.assertRaisesRegex(ClinicError, regex): + libclinic.linear_format("no text before marker for you! {marker}", + marker="not allowed!") + + def test_text_after_block_marker(self): + regex = re.escape("found after '{marker}'") + with self.assertRaisesRegex(ClinicError, regex): + libclinic.linear_format("{marker} no text after marker for you!", + marker="not allowed!") + + +class InertParser: + def __init__(self, clinic): + pass + + def parse(self, block): + pass + +class CopyParser: + def __init__(self, clinic): + pass + + def parse(self, block): + block.output = block.input + + +class ClinicBlockParserTest(TestCase): + def _test(self, input, output): + language = CLanguage(None) + + blocks = list(BlockParser(input, language)) + writer = BlockPrinter(language) + for block in blocks: + writer.print_block(block) + output = writer.f.getvalue() + assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input) + + def round_trip(self, input): + return self._test(input, input) + + def test_round_trip_1(self): + self.round_trip(""" + verbatim text here + lah dee dah + """) + def test_round_trip_2(self): + self.round_trip(""" + verbatim text here + lah dee dah +/*[inert] +abc +[inert]*/ +def +/*[inert checksum: 7b18d017f89f61cf17d47f92749ea6930a3f1deb]*/ +xyz +""") + + def _test_clinic(self, input, output): + language = CLanguage(None) + c = Clinic(language, filename="file", limited_capi=False) + c.parsers['inert'] = InertParser(c) + c.parsers['copy'] = CopyParser(c) + computed = c.parse(input) + self.assertEqual(output, computed) + + def test_clinic_1(self): + self._test_clinic(""" + verbatim text here + lah dee dah +/*[copy input] +def +[copy start generated code]*/ +abc +/*[copy end generated code: output=03cfd743661f0797 input=7b18d017f89f61cf]*/ +xyz +""", """ + verbatim text here + lah dee dah +/*[copy input] +def +[copy start generated code]*/ +def +/*[copy end generated code: output=7b18d017f89f61cf input=7b18d017f89f61cf]*/ +xyz +""") + + +class ClinicParserTest(TestCase): + + def parse(self, text): + c = _make_clinic() + parser = DSLParser(c) + block = Block(text) + parser.parse(block) + return block + + def parse_function(self, text, signatures_in_block=2, function_index=1): + block = self.parse(text) + s = block.signatures + self.assertEqual(len(s), signatures_in_block) + assert isinstance(s[0], Module) + assert isinstance(s[function_index], Function) + return s[function_index] + + def expect_failure(self, block, err, *, + filename=None, lineno=None, strip=True): + return _expect_failure(self, self.parse_function, block, err, + filename=filename, lineno=lineno, strip=strip) + + def checkDocstring(self, fn, expected): + self.assertTrue(hasattr(fn, "docstring")) + self.assertEqual(dedent(expected).strip(), + fn.docstring.strip()) + + def test_trivial(self): + parser = DSLParser(_make_clinic()) + block = Block(""" + module os + os.access + """) + parser.parse(block) + module, function = block.signatures + self.assertEqual("access", function.name) + self.assertEqual("os", module.name) + + def test_ignore_line(self): + block = self.parse(dedent(""" + # + module os + os.access + """)) + module, function = block.signatures + self.assertEqual("access", function.name) + self.assertEqual("os", module.name) + + def test_param(self): + function = self.parse_function(""" + module os + os.access + path: int + """) + self.assertEqual("access", function.name) + self.assertEqual(2, len(function.parameters)) + p = function.parameters['path'] + self.assertEqual('path', p.name) + self.assertIsInstance(p.converter, int_converter) + + def test_param_default(self): + function = self.parse_function(""" + module os + os.access + follow_symlinks: bool = True + """) + p = function.parameters['follow_symlinks'] + self.assertEqual(True, p.default) + + def test_param_with_continuations(self): + function = self.parse_function(r""" + module os + os.access + follow_symlinks: \ + bool \ + = \ + True + """) + p = function.parameters['follow_symlinks'] + self.assertEqual(True, p.default) + + def test_param_default_none(self): + function = self.parse_function(r""" + module test + test.func + obj: object = None + str: str(accept={str, NoneType}) = None + buf: Py_buffer(accept={str, buffer, NoneType}) = None + """) + p = function.parameters['obj'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, 'Py_None') + + p = function.parameters['str'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['buf'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, '{NULL, NULL}') + + def test_param_default_null(self): + function = self.parse_function(r""" + module test + test.func + obj: object = NULL + str: str = NULL + buf: Py_buffer = NULL + fsencoded: unicode_fs_encoded = NULL + fsdecoded: unicode_fs_decoded = NULL + """) + p = function.parameters['obj'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['str'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['buf'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, '{NULL, NULL}') + + p = function.parameters['fsencoded'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['fsdecoded'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + def test_param_default_str_literal(self): + function = self.parse_function(r""" + module test + test.func + str: str = ' \t\n\r\v\f\xa0' + buf: Py_buffer(accept={str, buffer}) = ' \t\n\r\v\f\xa0' + """) + p = function.parameters['str'] + self.assertEqual(p.default, ' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, r'" \t\n\r\v\f\u00a0"') + + p = function.parameters['buf'] + self.assertEqual(p.default, ' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, + r'{.buf = " \t\n\r\v\f\302\240", .obj = NULL, .len = 8}') + + def test_param_default_bytes_literal(self): + function = self.parse_function(r""" + module test + test.func + str: str(accept={robuffer}) = b' \t\n\r\v\f\xa0' + buf: Py_buffer = b' \t\n\r\v\f\xa0' + """) + p = function.parameters['str'] + self.assertEqual(p.default, b' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"b' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, r'" \t\n\r\v\f\240"') + + p = function.parameters['buf'] + self.assertEqual(p.default, b' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"b' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, + r'{.buf = " \t\n\r\v\f\240", .obj = NULL, .len = 7}') + + def test_param_default_byte_literal(self): + function = self.parse_function(r""" + module test + test.func + zero: char = b'\0' + one: char = b'\1' + lf: char = b'\n' + nbsp: char = b'\xa0' + """) + p = function.parameters['zero'] + self.assertEqual(p.default, b'\0') + self.assertEqual(p.converter.py_default, r"b'\x00'") + self.assertEqual(p.converter.c_default, r"'\0'") + + p = function.parameters['one'] + self.assertEqual(p.default, b'\1') + self.assertEqual(p.converter.py_default, r"b'\x01'") + self.assertEqual(p.converter.c_default, r"'\001'") + + p = function.parameters['lf'] + self.assertEqual(p.default, b'\n') + self.assertEqual(p.converter.py_default, r"b'\n'") + self.assertEqual(p.converter.c_default, r"'\n'") + + p = function.parameters['nbsp'] + self.assertEqual(p.default, b'\xa0') + self.assertEqual(p.converter.py_default, r"b'\xa0'") + self.assertEqual(p.converter.c_default, r"'\240'") + + def test_param_default_unicode_char(self): + function = self.parse_function(r""" + module test + test.func + zero: int(accept={str}) = '\0' + one: int(accept={str}) = '\1' + lf: int(accept={str}) = '\n' + nbsp: int(accept={str}) = '\xa0' + snake: int(accept={str}) = '\U0001f40d' + """) + p = function.parameters['zero'] + self.assertEqual(p.default, '\0') + self.assertEqual(p.converter.py_default, r"'\x00'") + self.assertEqual(p.converter.c_default, '0') + + p = function.parameters['one'] + self.assertEqual(p.default, '\1') + self.assertEqual(p.converter.py_default, r"'\x01'") + self.assertEqual(p.converter.c_default, '0x01') + + p = function.parameters['lf'] + self.assertEqual(p.default, '\n') + self.assertEqual(p.converter.py_default, r"'\n'") + self.assertEqual(p.converter.c_default, r"'\n'") + + p = function.parameters['nbsp'] + self.assertEqual(p.default, '\xa0') + self.assertEqual(p.converter.py_default, r"'\xa0'") + self.assertEqual(p.converter.c_default, '0xa0') + + p = function.parameters['snake'] + self.assertEqual(p.default, '\U0001f40d') + self.assertEqual(p.converter.py_default, "'\U0001f40d'") + self.assertEqual(p.converter.c_default, '0x1f40d') + + def test_param_default_bool(self): + function = self.parse_function(r""" + module test + test.func + bool: bool = True + intbool: bool(accept={int}) = True + intbool2: bool(accept={int}) = 2 + """) + p = function.parameters['bool'] + self.assertIs(p.default, True) + self.assertEqual(p.converter.py_default, 'True') + self.assertEqual(p.converter.c_default, '1') + + p = function.parameters['intbool'] + self.assertIs(p.default, True) + self.assertEqual(p.converter.py_default, 'True') + self.assertEqual(p.converter.c_default, '1') + + p = function.parameters['intbool2'] + self.assertEqual(p.default, 2) + self.assertEqual(p.converter.py_default, '2') + self.assertEqual(p.converter.c_default, '2') + + def test_param_default_expr_named_constant(self): + function = self.parse_function(""" + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = sys.maxsize + """) + p = function.parameters['follow_symlinks'] + self.assertEqual(sys.maxsize, p.default) + self.assertEqual("MAXSIZE", p.converter.c_default) + + err = ( + "When you specify a named constant ('sys.maxsize') as your default value, " + "you MUST specify a valid c_default." + ) + block = """ + module os + os.access + follow_symlinks: int = sys.maxsize + """ + self.expect_failure(block, err, lineno=2) + + def test_param_with_bizarre_default_fails_correctly(self): + template = """ + module os + os.access + follow_symlinks: int = {default} + """ + err = "Unsupported expression as default value" + for bad_default_value in ( + "{1, 2, 3}", + "3 if bool() else 4", + "[x for x in range(42)]" + ): + with self.subTest(bad_default=bad_default_value): + block = template.format(default=bad_default_value) + self.expect_failure(block, err, lineno=2) + + def test_unspecified_not_allowed_as_default_value(self): + block = """ + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = unspecified + """ + err = "'unspecified' is not a legal default value!" + exc = self.expect_failure(block, err, lineno=2) + self.assertNotIn('Malformed expression given as default value', str(exc)) + + def test_malformed_expression_as_default_value(self): + block = """ + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = 1/0 + """ + err = "Malformed expression given as default value" + self.expect_failure(block, err, lineno=2) + + def test_param_default_expr_binop(self): + err = ( + "When you specify an expression ('a + b') as your default value, " + "you MUST specify a valid c_default." + ) + block = """ + fn + follow_symlinks: int = a + b + """ + self.expect_failure(block, err, lineno=1) + + def test_param_no_docstring(self): + function = self.parse_function(""" + module os + os.access + follow_symlinks: bool = True + something_else: str = '' + """) + self.assertEqual(3, len(function.parameters)) + conv = function.parameters['something_else'].converter + self.assertIsInstance(conv, str_converter) + + def test_param_default_parameters_out_of_order(self): + err = ( + "Can't have a parameter without a default ('something_else') " + "after a parameter with a default!" + ) + block = """ + module os + os.access + follow_symlinks: bool = True + something_else: str + """ + self.expect_failure(block, err, lineno=3) + + def disabled_test_converter_arguments(self): + function = self.parse_function(""" + module os + os.access + path: path_t(allow_fd=1) + """) + p = function.parameters['path'] + self.assertEqual(1, p.converter.args['allow_fd']) + + def test_function_docstring(self): + function = self.parse_function(""" + module os + os.stat as os_stat_fn + + path: str + Path to be examined + Ensure that multiple lines are indented correctly. + + Perform a stat system call on the given path. + + Ensure that multiple lines are indented correctly. + Ensure that multiple lines are indented correctly. + """) + self.checkDocstring(function, """ + stat($module, /, path) + -- + + Perform a stat system call on the given path. + + path + Path to be examined + Ensure that multiple lines are indented correctly. + + Ensure that multiple lines are indented correctly. + Ensure that multiple lines are indented correctly. + """) + + def test_docstring_trailing_whitespace(self): + function = self.parse_function( + "module t\n" + "t.s\n" + " a: object\n" + " Param docstring with trailing whitespace \n" + "Func docstring summary with trailing whitespace \n" + " \n" + "Func docstring body with trailing whitespace \n" + ) + self.checkDocstring(function, """ + s($module, /, a) + -- + + Func docstring summary with trailing whitespace + + a + Param docstring with trailing whitespace + + Func docstring body with trailing whitespace + """) + + def test_explicit_parameters_in_docstring(self): + function = self.parse_function(dedent(""" + module foo + foo.bar + x: int + Documentation for x. + y: int + + This is the documentation for foo. + + Okay, we're done here. + """)) + self.checkDocstring(function, """ + bar($module, /, x, y) + -- + + This is the documentation for foo. + + x + Documentation for x. + + Okay, we're done here. + """) + + def test_docstring_with_comments(self): + function = self.parse_function(dedent(""" + module foo + foo.bar + x: int + # We're about to have + # the documentation for x. + Documentation for x. + # We've just had + # the documentation for x. + y: int + + # We're about to have + # the documentation for foo. + This is the documentation for foo. + # We've just had + # the documentation for foo. + + Okay, we're done here. + """)) + self.checkDocstring(function, """ + bar($module, /, x, y) + -- + + This is the documentation for foo. + + x + Documentation for x. + + Okay, we're done here. + """) + + def test_parser_regression_special_character_in_parameter_column_of_docstring_first_line(self): + function = self.parse_function(dedent(""" + module os + os.stat + path: str + This/used to break Clinic! + """)) + self.checkDocstring(function, """ + stat($module, /, path) + -- + + This/used to break Clinic! + """) + + def test_c_name(self): + function = self.parse_function(""" + module os + os.stat as os_stat_fn + """) + self.assertEqual("os_stat_fn", function.c_basename) + + def test_base_invalid_syntax(self): + block = """ + module os + os.stat + invalid syntax: int = 42 + """ + err = "Function 'stat' has an invalid parameter declaration: 'invalid syntax: int = 42'" + self.expect_failure(block, err, lineno=2) + + def test_param_default_invalid_syntax(self): + block = """ + module os + os.stat + x: int = invalid syntax + """ + err = "Function 'stat' has an invalid parameter declaration:" + self.expect_failure(block, err, lineno=2) + + def test_cloning_nonexistent_function_correctly_fails(self): + block = """ + cloned = fooooooooooooooooo + This is trying to clone a nonexistent function!! + """ + err = "Couldn't find existing function 'fooooooooooooooooo'!" + with support.captured_stderr() as stderr: + self.expect_failure(block, err, lineno=0) + expected_debug_print = dedent("""\ + cls=None, module=, existing='fooooooooooooooooo' + (cls or module).functions=[] + """) + stderr = stderr.getvalue() + self.assertIn(expected_debug_print, stderr) + + def test_return_converter(self): + function = self.parse_function(""" + module os + os.stat -> int + """) + self.assertIsInstance(function.return_converter, int_return_converter) + + def test_return_converter_invalid_syntax(self): + block = """ + module os + os.stat -> invalid syntax + """ + err = "Badly formed annotation for 'os.stat': 'invalid syntax'" + self.expect_failure(block, err) + + def test_legacy_converter_disallowed_in_return_annotation(self): + block = """ + module os + os.stat -> "s" + """ + err = "Legacy converter 's' not allowed as a return converter" + self.expect_failure(block, err) + + def test_unknown_return_converter(self): + block = """ + module os + os.stat -> fooooooooooooooooooooooo + """ + err = "No available return converter called 'fooooooooooooooooooooooo'" + self.expect_failure(block, err) + + def test_star(self): + function = self.parse_function(""" + module os + os.access + * + follow_symlinks: bool = True + """) + p = function.parameters['follow_symlinks'] + self.assertEqual(inspect.Parameter.KEYWORD_ONLY, p.kind) + self.assertEqual(0, p.group) + + def test_group(self): + function = self.parse_function(""" + module window + window.border + [ + ls: int + ] + / + """) + p = function.parameters['ls'] + self.assertEqual(1, p.group) + + def test_left_group(self): + function = self.parse_function(""" + module curses + curses.addch + [ + y: int + Y-coordinate. + x: int + X-coordinate. + ] + ch: char + Character to add. + [ + attr: long + Attributes for the character. + ] + / + """) + dataset = ( + ('y', -1), ('x', -1), + ('ch', 0), + ('attr', 1), + ) + for name, group in dataset: + with self.subTest(name=name, group=group): + p = function.parameters[name] + self.assertEqual(p.group, group) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + self.checkDocstring(function, """ + addch([y, x,] ch, [attr]) + + + y + Y-coordinate. + x + X-coordinate. + ch + Character to add. + attr + Attributes for the character. + """) + + def test_nested_groups(self): + function = self.parse_function(""" + module curses + curses.imaginary + [ + [ + y1: int + Y-coordinate. + y2: int + Y-coordinate. + ] + x1: int + X-coordinate. + x2: int + X-coordinate. + ] + ch: char + Character to add. + [ + attr1: long + Attributes for the character. + attr2: long + Attributes for the character. + attr3: long + Attributes for the character. + [ + attr4: long + Attributes for the character. + attr5: long + Attributes for the character. + attr6: long + Attributes for the character. + ] + ] + / + """) + dataset = ( + ('y1', -2), ('y2', -2), + ('x1', -1), ('x2', -1), + ('ch', 0), + ('attr1', 1), ('attr2', 1), ('attr3', 1), + ('attr4', 2), ('attr5', 2), ('attr6', 2), + ) + for name, group in dataset: + with self.subTest(name=name, group=group): + p = function.parameters[name] + self.assertEqual(p.group, group) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + self.checkDocstring(function, """ + imaginary([[y1, y2,] x1, x2,] ch, [attr1, attr2, attr3, [attr4, attr5, + attr6]]) + + + y1 + Y-coordinate. + y2 + Y-coordinate. + x1 + X-coordinate. + x2 + X-coordinate. + ch + Character to add. + attr1 + Attributes for the character. + attr2 + Attributes for the character. + attr3 + Attributes for the character. + attr4 + Attributes for the character. + attr5 + Attributes for the character. + attr6 + Attributes for the character. + """) + + def test_disallowed_grouping__two_top_groups_on_left(self): + err = ( + "Function 'two_top_groups_on_left' has an unsupported group " + "configuration. (Unexpected state 2.b)" + ) + block = """ + module foo + foo.two_top_groups_on_left + [ + group1 : int + ] + [ + group2 : int + ] + param: int + """ + self.expect_failure(block, err, lineno=5) + + def test_disallowed_grouping__two_top_groups_on_right(self): + block = """ + module foo + foo.two_top_groups_on_right + param: int + [ + group1 : int + ] + [ + group2 : int + ] + """ + err = ( + "Function 'two_top_groups_on_right' has an unsupported group " + "configuration. (Unexpected state 6.b)" + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__parameter_after_group_on_right(self): + block = """ + module foo + foo.parameter_after_group_on_right + param: int + [ + [ + group1 : int + ] + group2 : int + ] + """ + err = ( + "Function parameter_after_group_on_right has an unsupported group " + "configuration. (Unexpected state 6.a)" + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__group_after_parameter_on_left(self): + block = """ + module foo + foo.group_after_parameter_on_left + [ + group2 : int + [ + group1 : int + ] + ] + param: int + """ + err = ( + "Function 'group_after_parameter_on_left' has an unsupported group " + "configuration. (Unexpected state 2.b)" + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__empty_group_on_left(self): + block = """ + module foo + foo.empty_group + [ + [ + ] + group2 : int + ] + param: int + """ + err = ( + "Function 'empty_group' has an empty group. " + "All groups must contain at least one parameter." + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__empty_group_on_right(self): + block = """ + module foo + foo.empty_group + param: int + [ + [ + ] + group2 : int + ] + """ + err = ( + "Function 'empty_group' has an empty group. " + "All groups must contain at least one parameter." + ) + self.expect_failure(block, err) + + def test_disallowed_grouping__no_matching_bracket(self): + block = """ + module foo + foo.empty_group + param: int + ] + group2: int + ] + """ + err = "Function 'empty_group' has a ']' without a matching '['" + self.expect_failure(block, err) + + def test_disallowed_grouping__must_be_position_only(self): + dataset = (""" + with_kwds + [ + * + a: object + ] + """, """ + with_kwds + [ + a: object + ] + """) + err = ( + "You cannot use optional groups ('[' and ']') unless all " + "parameters are positional-only ('/')" + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + + def test_no_parameters(self): + function = self.parse_function(""" + module foo + foo.bar + + Docstring + + """) + self.assertEqual("bar($module, /)\n--\n\nDocstring", function.docstring) + self.assertEqual(1, len(function.parameters)) # self! + + def test_init_with_no_parameters(self): + function = self.parse_function(""" + module foo + class foo.Bar "unused" "notneeded" + foo.Bar.__init__ + + Docstring + + """, signatures_in_block=3, function_index=2) + + # self is not in the signature + self.assertEqual("Bar()\n--\n\nDocstring", function.docstring) + # but it *is* a parameter + self.assertEqual(1, len(function.parameters)) + + def test_illegal_module_line(self): + block = """ + module foo + foo.bar => int + / + """ + err = "Illegal function name: 'foo.bar => int'" + self.expect_failure(block, err) + + def test_illegal_c_basename(self): + block = """ + module foo + foo.bar as 935 + / + """ + err = "Illegal C basename: '935'" + self.expect_failure(block, err) + + def test_no_c_basename(self): + block = "foo as " + err = "No C basename provided after 'as' keyword" + self.expect_failure(block, err, strip=False) + + def test_single_star(self): + block = """ + module foo + foo.bar + * + * + """ + err = "Function 'bar' uses '*' more than once." + self.expect_failure(block, err) + + def test_parameters_required_after_star(self): + dataset = ( + "module foo\nfoo.bar\n *", + "module foo\nfoo.bar\n *\nDocstring here.", + "module foo\nfoo.bar\n this: int\n *", + "module foo\nfoo.bar\n this: int\n *\nDocstring.", + ) + err = "Function 'bar' specifies '*' without following parameters." + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + + def test_fulldisplayname_class(self): + dataset = ( + ("T", """ + class T "void *" "" + T.__init__ + """), + ("m.T", """ + module m + class m.T "void *" "" + @classmethod + m.T.__new__ + """), + ("m.T.C", """ + module m + class m.T "void *" "" + class m.T.C "void *" "" + m.T.C.__init__ + """), + ) + for name, code in dataset: + with self.subTest(name=name, code=code): + block = self.parse(code) + func = block.signatures[-1] + self.assertEqual(func.fulldisplayname, name) + + def test_fulldisplayname_meth(self): + dataset = ( + ("func", "func"), + ("m.func", """ + module m + m.func + """), + ("T.meth", """ + class T "void *" "" + T.meth + """), + ("m.T.meth", """ + module m + class m.T "void *" "" + m.T.meth + """), + ("m.T.C.meth", """ + module m + class m.T "void *" "" + class m.T.C "void *" "" + m.T.C.meth + """), + ) + for name, code in dataset: + with self.subTest(name=name, code=code): + block = self.parse(code) + func = block.signatures[-1] + self.assertEqual(func.fulldisplayname, name) + + def test_depr_star_invalid_format_1(self): + block = """ + module foo + foo.bar + this: int + * [from 3] + Docstring. + """ + err = ( + "Function 'bar': expected format '[from major.minor]' " + "where 'major' and 'minor' are integers; got '3'" + ) + self.expect_failure(block, err, lineno=3) + + def test_depr_star_invalid_format_2(self): + block = """ + module foo + foo.bar + this: int + * [from a.b] + Docstring. + """ + err = ( + "Function 'bar': expected format '[from major.minor]' " + "where 'major' and 'minor' are integers; got 'a.b'" + ) + self.expect_failure(block, err, lineno=3) + + def test_depr_star_invalid_format_3(self): + block = """ + module foo + foo.bar + this: int + * [from 1.2.3] + Docstring. + """ + err = ( + "Function 'bar': expected format '[from major.minor]' " + "where 'major' and 'minor' are integers; got '1.2.3'" + ) + self.expect_failure(block, err, lineno=3) + + def test_parameters_required_after_depr_star(self): + block = """ + module foo + foo.bar + this: int + * [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_parameters_required_after_depr_star2(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + * + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_parameters_required_after_depr_star3(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + *args: tuple + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_star(self): + block = """ + module foo + foo.bar + a: int + * + * [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '* [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_vararg(self): + block = """ + module foo + foo.bar + a: int + *args: tuple + * [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '* [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_duplicate(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + b: int + * [from 3.14] + c: int + Docstring. + """ + err = "Function 'bar' uses '* [from 3.14]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_star_duplicate2(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + b: int + * [from 3.15] + c: int + Docstring. + """ + err = "Function 'bar': '* [from 3.15]' must precede '* [from 3.14]'" + self.expect_failure(block, err, lineno=5) + + def test_depr_slash_duplicate(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + b: int + / [from 3.14] + c: int + Docstring. + """ + err = "Function 'bar' uses '/ [from 3.14]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_slash_duplicate2(self): + block = """ + module foo + foo.bar + a: int + / [from 3.15] + b: int + / [from 3.14] + c: int + Docstring. + """ + err = "Function 'bar': '/ [from 3.14]' must precede '/ [from 3.15]'" + self.expect_failure(block, err, lineno=5) + + def test_single_slash(self): + block = """ + module foo + foo.bar + / + / + """ + err = ( + "Function 'bar' has an unsupported group configuration. " + "(Unexpected state 0.d)" + ) + self.expect_failure(block, err) + + def test_parameters_required_before_depr_slash(self): + block = """ + module foo + foo.bar + / [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '/ [from ...]' without " + "preceding parameters." + ) + self.expect_failure(block, err, lineno=2) + + def test_parameters_required_before_depr_slash2(self): + block = """ + module foo + foo.bar + a: int + / + / [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '/ [from ...]' without " + "preceding parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_double_slash(self): + block = """ + module foo + foo.bar + a: int + / + b: int + / + """ + err = "Function 'bar' uses '/' more than once." + self.expect_failure(block, err) + + def test_slash_after_star(self): + block = """ + module foo + foo.bar + x: int + y: int + * + z: int + / + """ + err = "Function 'bar': '/' must precede '*'" + self.expect_failure(block, err) + + def test_slash_after_vararg(self): + block = """ + module foo + foo.bar + x: int + y: int + *args: tuple + z: int + / + """ + err = "Function 'bar': '/' must precede '*'" + self.expect_failure(block, err) + + def test_depr_star_must_come_after_slash(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + / + b: int + Docstring. + """ + err = "Function 'bar': '/' must precede '* [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '* [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_star_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + * + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_vararg_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + *args: tuple + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_slash_must_come_after_slash(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + / + b: int + Docstring. + """ + err = "Function 'bar': '/' must precede '/ [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_parameters_not_permitted_after_slash_for_now(self): + block = """ + module foo + foo.bar + / + x: int + """ + err = ( + "Function 'bar' has an unsupported group configuration. " + "(Unexpected state 0.d)" + ) + self.expect_failure(block, err) + + def test_parameters_no_more_than_one_vararg(self): + err = "Function 'bar' uses '*' more than once." + block = """ + module foo + foo.bar + *vararg1: tuple + *vararg2: tuple + """ + self.expect_failure(block, err, lineno=3) + + def test_function_not_at_column_0(self): + function = self.parse_function(""" + module foo + foo.bar + x: int + Nested docstring here, goeth. + * + y: str + Not at column 0! + """) + self.checkDocstring(function, """ + bar($module, /, x, *, y) + -- + + Not at column 0! + + x + Nested docstring here, goeth. + """) + + def test_docstring_only_summary(self): + function = self.parse_function(""" + module m + m.f + summary + """) + self.checkDocstring(function, """ + f($module, /) + -- + + summary + """) + + def test_docstring_empty_lines(self): + function = self.parse_function(""" + module m + m.f + + + """) + self.checkDocstring(function, """ + f($module, /) + -- + """) + + def test_docstring_explicit_params_placement(self): + function = self.parse_function(""" + module m + m.f + a: int + Param docstring for 'a' will be included + b: int + c: int + Param docstring for 'c' will be included + This is the summary line. + + We'll now place the params section here: + {parameters} + And now for something completely different! + (Note the added newline) + """) + self.checkDocstring(function, """ + f($module, /, a, b, c) + -- + + This is the summary line. + + We'll now place the params section here: + a + Param docstring for 'a' will be included + c + Param docstring for 'c' will be included + + And now for something completely different! + (Note the added newline) + """) + + def test_indent_stack_no_tabs(self): + block = """ + module foo + foo.bar + *vararg1: tuple + \t*vararg2: tuple + """ + err = ("Tab characters are illegal in the Clinic DSL: " + r"'\t*vararg2: tuple'") + self.expect_failure(block, err) + + def test_indent_stack_illegal_outdent(self): + block = """ + module foo + foo.bar + a: object + b: object + """ + err = "Illegal outdent" + self.expect_failure(block, err) + + def test_directive(self): + parser = DSLParser(_make_clinic()) + parser.flag = False + parser.directives['setflag'] = lambda : setattr(parser, 'flag', True) + block = Block("setflag") + parser.parse(block) + self.assertTrue(parser.flag) + + def test_legacy_converters(self): + block = self.parse('module os\nos.access\n path: "s"') + module, function = block.signatures + conv = (function.parameters['path']).converter + self.assertIsInstance(conv, str_converter) + + def test_legacy_converters_non_string_constant_annotation(self): + err = "Annotations must be either a name, a function call, or a string" + dataset = ( + 'module os\nos.access\n path: 42', + 'module os\nos.access\n path: 42.42', + 'module os\nos.access\n path: 42j', + 'module os\nos.access\n path: b"42"', + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err, lineno=2) + + def test_other_bizarre_things_in_annotations_fail(self): + err = "Annotations must be either a name, a function call, or a string" + dataset = ( + 'module os\nos.access\n path: {"some": "dictionary"}', + 'module os\nos.access\n path: ["list", "of", "strings"]', + 'module os\nos.access\n path: (x for x in range(42))', + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err, lineno=2) + + def test_kwarg_splats_disallowed_in_function_call_annotations(self): + err = "Cannot use a kwarg splat in a function-call annotation" + dataset = ( + 'module fo\nfo.barbaz\n o: bool(**{None: "bang!"})', + 'module fo\nfo.barbaz -> bool(**{None: "bang!"})', + 'module fo\nfo.barbaz -> bool(**{"bang": 42})', + 'module fo\nfo.barbaz\n o: bool(**{"bang": None})', + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + + def test_self_param_placement(self): + err = ( + "A 'self' parameter, if specified, must be the very first thing " + "in the parameter block." + ) + block = """ + module foo + foo.func + a: int + self: self(type="PyObject *") + """ + self.expect_failure(block, err, lineno=3) + + def test_self_param_cannot_be_optional(self): + err = "A 'self' parameter cannot be marked optional." + block = """ + module foo + foo.func + self: self(type="PyObject *") = None + """ + self.expect_failure(block, err, lineno=2) + + def test_defining_class_param_placement(self): + err = ( + "A 'defining_class' parameter, if specified, must either be the " + "first thing in the parameter block, or come just after 'self'." + ) + block = """ + module foo + foo.func + self: self(type="PyObject *") + a: int + cls: defining_class + """ + self.expect_failure(block, err, lineno=4) + + def test_defining_class_param_cannot_be_optional(self): + err = "A 'defining_class' parameter cannot be marked optional." + block = """ + module foo + foo.func + cls: defining_class(type="PyObject *") = None + """ + self.expect_failure(block, err, lineno=2) + + def test_slot_methods_cannot_access_defining_class(self): + block = """ + module foo + class Foo "" "" + Foo.__init__ + cls: defining_class + a: object + """ + err = "Slot methods cannot access their defining class." + with self.assertRaisesRegex(ValueError, err): + self.parse_function(block) + + def test_new_must_be_a_class_method(self): + err = "'__new__' must be a class method!" + block = """ + module foo + class Foo "" "" + Foo.__new__ + """ + self.expect_failure(block, err, lineno=2) + + def test_init_must_be_a_normal_method(self): + err_template = "'__init__' must be a normal method; got 'FunctionKind.{}'!" + annotations = { + "@classmethod": "CLASS_METHOD", + "@staticmethod": "STATIC_METHOD", + "@getter": "GETTER", + } + for annotation, invalid_kind in annotations.items(): + with self.subTest(annotation=annotation, invalid_kind=invalid_kind): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.__init__ + """ + expected_error = err_template.format(invalid_kind) + self.expect_failure(block, expected_error, lineno=3) + + def test_init_cannot_define_a_return_type(self): + block = """ + class Foo "" "" + Foo.__init__ -> long + """ + expected_error = "__init__ methods cannot define a return type" + self.expect_failure(block, expected_error, lineno=1) + + def test_invalid_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property -> int + """ + expected_error = f"{annotation} method cannot define a return type" + self.expect_failure(block, expected_error, lineno=3) + + block = f""" + module foo + class Foo "" "" + {annotation} + Foo.property + obj: int + / + """ + expected_error = f"{annotation} methods cannot define parameters" + self.expect_failure(block, expected_error) + + def test_setter_docstring(self): + block = """ + module foo + class Foo "" "" + @setter + Foo.property + + foo + + bar + [clinic start generated code]*/ + """ + expected_error = "docstrings are only supported for @getter, not @setter" + self.expect_failure(block, expected_error) + + def test_duplicate_getset(self): + annotations = ["@getter", "@setter"] + for annotation in annotations: + with self.subTest(annotation=annotation): + block = f""" + module foo + class Foo "" "" + {annotation} + {annotation} + Foo.property -> int + """ + expected_error = f"Cannot apply {annotation} twice to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getter_and_setter_disallowed_on_same_function(self): + dup_annotations = [("@getter", "@setter"), ("@setter", "@getter")] + for dup in dup_annotations: + with self.subTest(dup=dup): + block = f""" + module foo + class Foo "" "" + {dup[0]} + {dup[1]} + Foo.property -> int + """ + expected_error = "Cannot apply both @getter and @setter to the same function!" + self.expect_failure(block, expected_error, lineno=3) + + def test_getset_no_class(self): + for annotation in "@getter", "@setter": + with self.subTest(annotation=annotation): + block = f""" + module m + {annotation} + m.func + """ + expected_error = "@getter and @setter must be methods" + self.expect_failure(block, expected_error, lineno=2) + + def test_duplicate_coexist(self): + err = "Called @coexist twice" + block = """ + module m + @coexist + @coexist + m.fn + """ + self.expect_failure(block, err, lineno=2) + + def test_unused_param(self): + block = self.parse(""" + module foo + foo.func + fn: object + k: float + i: float(unused=True) + / + * + flag: bool(unused=True) = False + """) + sig = block.signatures[1] # Function index == 1 + params = sig.parameters + conv = lambda fn: params[fn].converter + dataset = ( + {"name": "fn", "unused": False}, + {"name": "k", "unused": False}, + {"name": "i", "unused": True}, + {"name": "flag", "unused": True}, + ) + for param in dataset: + name, unused = param.values() + with self.subTest(name=name, unused=unused): + p = conv(name) + # Verify that the unused flag is parsed correctly. + self.assertEqual(unused, p.unused) + + # Now, check that we'll produce correct code. + decl = p.simple_declaration(in_parser=False) + if unused: + self.assertIn("Py_UNUSED", decl) + else: + self.assertNotIn("Py_UNUSED", decl) + + # Make sure the Py_UNUSED macro is not used in the parser body. + parser_decl = p.simple_declaration(in_parser=True) + self.assertNotIn("Py_UNUSED", parser_decl) + + def test_scaffolding(self): + # test repr on special values + self.assertEqual(repr(unspecified), '') + self.assertEqual(repr(NULL), '') + + # test that fail fails + with support.captured_stdout() as stdout: + errmsg = 'The igloos are melting' + with self.assertRaisesRegex(ClinicError, errmsg) as cm: + fail(errmsg, filename='clown.txt', line_number=69) + exc = cm.exception + self.assertEqual(exc.filename, 'clown.txt') + self.assertEqual(exc.lineno, 69) + self.assertEqual(stdout.getvalue(), "") + + def test_non_ascii_character_in_docstring(self): + block = """ + module test + test.fn + a: int + á param docstring + docstring fü bár baß + """ + with support.captured_stdout() as stdout: + self.parse(block) + # The line numbers are off; this is a known limitation. + expected = dedent("""\ + Warning: + Non-ascii characters are not allowed in docstrings: 'á' + + Warning: + Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' + + """) + self.assertEqual(stdout.getvalue(), expected) + + def test_illegal_c_identifier(self): + err = "Illegal C identifier: 17a" + block = """ + module test + test.fn + a as 17a: int + """ + self.expect_failure(block, err, lineno=2) + + def test_cannot_convert_special_method(self): + err = "'__len__' is a special method and cannot be converted" + block = """ + class T "" "" + T.__len__ + """ + self.expect_failure(block, err, lineno=1) + + def test_cannot_specify_pydefault_without_default(self): + err = "You can't specify py_default without specifying a default value!" + block = """ + fn + a: object(py_default='NULL') + """ + self.expect_failure(block, err, lineno=1) + + def test_vararg_cannot_take_default_value(self): + err = "Function 'fn' has an invalid parameter declaration:" + block = """ + fn + *args: tuple = None + """ + self.expect_failure(block, err, lineno=1) + + def test_default_is_not_of_correct_type(self): + err = ("int_converter: default value 2.5 for field 'a' " + "is not of type 'int'") + block = """ + fn + a: int = 2.5 + """ + self.expect_failure(block, err, lineno=1) + + def test_invalid_legacy_converter(self): + err = "'fhi' is not a valid legacy converter" + block = """ + fn + a: 'fhi' + """ + self.expect_failure(block, err, lineno=1) + + def test_parent_class_or_module_does_not_exist(self): + err = "Parent class or module 'baz' does not exist" + block = """ + module m + baz.func + """ + self.expect_failure(block, err, lineno=1) + + def test_duplicate_param_name(self): + err = "You can't have two parameters named 'a'" + block = """ + module m + m.func + a: int + a: float + """ + self.expect_failure(block, err, lineno=3) + + def test_param_requires_custom_c_name(self): + err = "Parameter 'module' requires a custom C name" + block = """ + module m + m.func + module: int + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_assert_no_group(self): + err = "Function 'func' has a ']' without a matching '['" + block = """ + module m + m.func + ] + docstring + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_no_summary(self): + err = "Docstring for 'm.func' does not have a summary line!" + block = """ + module m + m.func + docstring1 + docstring2 + """ + self.expect_failure(block, err, lineno=3) + + def test_state_func_docstring_only_one_param_template(self): + err = "You may not specify {parameters} more than once in a docstring!" + block = """ + module m + m.func + docstring summary + + these are the params: + {parameters} + these are the params again: + {parameters} + """ + self.expect_failure(block, err, lineno=7) + + def test_kind_defining_class(self): + function = self.parse_function(""" + module m + class m.C "PyObject *" "" + m.C.meth + cls: defining_class + """, signatures_in_block=3, function_index=2) + p = function.parameters['cls'] + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + def test_disallow_defining_class_at_module_level(self): + err = "A 'defining_class' parameter cannot be defined at module level." + block = """ + module m + m.func + cls: defining_class + """ + self.expect_failure(block, err, lineno=2) + + +class ClinicExternalTest(TestCase): + maxDiff = None + + def setUp(self): + save_restore_converters(self) + + def run_clinic(self, *args): + with ( + support.captured_stdout() as out, + support.captured_stderr() as err, + self.assertRaises(SystemExit) as cm + ): + clinic.main(args) + return out.getvalue(), err.getvalue(), cm.exception.code + + def expect_success(self, *args): + out, err, code = self.run_clinic(*args) + if code != 0: + self.fail("\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(err, "") + return out + + def expect_failure(self, *args): + out, err, code = self.run_clinic(*args) + self.assertNotEqual(code, 0, f"Unexpected success: {args=}") + return out, err + + def test_external(self): + CLINIC_TEST = 'clinic.test.c' + source = support.findfile(CLINIC_TEST) + with open(source, encoding='utf-8') as f: + orig_contents = f.read() + + # Run clinic CLI and verify that it does not complain. + self.addCleanup(unlink, TESTFN) + out = self.expect_success("-f", "-o", TESTFN, source) + self.assertEqual(out, "") + + with open(TESTFN, encoding='utf-8') as f: + new_contents = f.read() + + self.assertEqual(new_contents, orig_contents) + + def test_no_change(self): + # bpo-42398: Test that the destination file is left unchanged if the + # content does not change. Moreover, check also that the file + # modification time does not change in this case. + code = dedent(""" + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=da39a3ee5e6b4b0d]*/ + """) + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(code) + pre_mtime = os.stat(fn).st_mtime_ns + self.expect_success(fn) + post_mtime = os.stat(fn).st_mtime_ns + # Don't change the file modification time + # if the content does not change + self.assertEqual(pre_mtime, post_mtime) + + def test_cli_force(self): + invalid_input = dedent(""" + /*[clinic input] + output preset block + module test + test.fn + a: int + [clinic start generated code]*/ + + const char *hand_edited = "output block is overwritten"; + /*[clinic end generated code: output=bogus input=bogus]*/ + """) + fail_msg = ( + "Checksum mismatch! Expected 'bogus', computed '2ed19'. " + "Suggested fix: remove all generated code including the end marker, " + "or use the '-f' option.\n" + ) + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(invalid_input) + # First, run the CLI without -f and expect failure. + # Note, we cannot check the entire fail msg, because the path to + # the tmp file will change for every run. + _, err = self.expect_failure(fn) + self.assertEndsWith(err, fail_msg) + # Then, force regeneration; success expected. + out = self.expect_success("-f", fn) + self.assertEqual(out, "") + # Verify by checking the checksum. + checksum = ( + "/*[clinic end generated code: " + "output=a2957bc4d43a3c2f input=9543a8d2da235301]*/\n" + ) + with open(fn, encoding='utf-8') as f: + generated = f.read() + self.assertEndsWith(generated, checksum) + + def test_cli_make(self): + c_code = dedent(""" + /*[clinic input] + [clinic start generated code]*/ + """) + py_code = "pass" + c_files = "file1.c", "file2.c" + py_files = "file1.py", "file2.py" + + def create_files(files, srcdir, code): + for fn in files: + path = os.path.join(srcdir, fn) + with open(path, "w", encoding="utf-8") as f: + f.write(code) + + with os_helper.temp_dir() as tmp_dir: + # add some folders, some C files and a Python file + create_files(c_files, tmp_dir, c_code) + create_files(py_files, tmp_dir, py_code) + + # create C files in externals/ dir + ext_path = os.path.join(tmp_dir, "externals") + with os_helper.temp_dir(path=ext_path) as externals: + create_files(c_files, externals, c_code) + + # run clinic in verbose mode with --make on tmpdir + out = self.expect_success("-v", "--make", "--srcdir", tmp_dir) + + # expect verbose mode to only mention the C files in tmp_dir + for filename in c_files: + with self.subTest(filename=filename): + path = os.path.join(tmp_dir, filename) + self.assertIn(path, out) + for filename in py_files: + with self.subTest(filename=filename): + path = os.path.join(tmp_dir, filename) + self.assertNotIn(path, out) + # don't expect C files from the externals dir + for filename in c_files: + with self.subTest(filename=filename): + path = os.path.join(ext_path, filename) + self.assertNotIn(path, out) + + def test_cli_make_exclude(self): + code = dedent(""" + /*[clinic input] + [clinic start generated code]*/ + """) + with os_helper.temp_dir(quiet=False) as tmp_dir: + # add some folders, some C files and a Python file + for fn in "file1.c", "file2.c", "file3.c", "file4.c": + path = os.path.join(tmp_dir, fn) + with open(path, "w", encoding="utf-8") as f: + f.write(code) + + # Run clinic in verbose mode with --make on tmpdir. + # Exclude file2.c and file3.c. + out = self.expect_success( + "-v", "--make", "--srcdir", tmp_dir, + "--exclude", os.path.join(tmp_dir, "file2.c"), + # The added ./ should be normalised away. + "--exclude", os.path.join(tmp_dir, "./file3.c"), + # Relative paths should also work. + "--exclude", "file4.c" + ) + + # expect verbose mode to only mention the C files in tmp_dir + self.assertIn("file1.c", out) + self.assertNotIn("file2.c", out) + self.assertNotIn("file3.c", out) + self.assertNotIn("file4.c", out) + + def test_cli_verbose(self): + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write("") + out = self.expect_success("-v", fn) + self.assertEqual(out.strip(), fn) + + @support.force_not_colorized + def test_cli_help(self): + out = self.expect_success("-h") + self.assertIn("usage: clinic.py", out) + + def test_cli_converters(self): + prelude = dedent(""" + Legacy converters: + B C D L O S U Y Z Z# + b c d f h i l p s s# s* u u# w* y y# y* z z# z* + + Converters: + """) + expected_converters = ( + "bool", + "byte", + "char", + "defining_class", + "double", + "fildes", + "float", + "int", + "long", + "long_long", + "object", + "Py_buffer", + "Py_complex", + "Py_ssize_t", + "Py_UNICODE", + "PyByteArrayObject", + "PyBytesObject", + "self", + "short", + "size_t", + "slice_index", + "str", + "uint16", + "uint32", + "uint64", + "uint8", + "unicode", + "unicode_fs_decoded", + "unicode_fs_encoded", + "unsigned_char", + "unsigned_int", + "unsigned_long", + "unsigned_long_long", + "unsigned_short", + ) + finale = dedent(""" + Return converters: + bool() + double() + float() + int() + long() + object() + Py_ssize_t() + size_t() + unsigned_int() + unsigned_long() + + All converters also accept (c_default=None, py_default=None, annotation=None). + All return converters also accept (py_default=None). + """) + out = self.expect_success("--converters") + # We cannot simply compare the output, because the repr of the *accept* + # param may change (it's a set, thus unordered). So, let's compare the + # start and end of the expected output, and then assert that the + # converters appear lined up in alphabetical order. + self.assertStartsWith(out, prelude) + self.assertEndsWith(out, finale) + + out = out.removeprefix(prelude) + out = out.removesuffix(finale) + lines = out.split("\n") + for converter, line in zip(expected_converters, lines): + line = line.lstrip() + with self.subTest(converter=converter): + self.assertStartsWith(line, converter) + + def test_cli_fail_converters_and_filename(self): + _, err = self.expect_failure("--converters", "test.c") + msg = "can't specify --converters and a filename at the same time" + self.assertIn(msg, err) + + def test_cli_fail_no_filename(self): + _, err = self.expect_failure() + self.assertIn("no input files", err) + + def test_cli_fail_output_and_multiple_files(self): + _, err = self.expect_failure("-o", "out.c", "input.c", "moreinput.c") + msg = "error: can't use -o with multiple filenames" + self.assertIn(msg, err) + + def test_cli_fail_filename_or_output_and_make(self): + msg = "can't use -o or filenames with --make" + for opts in ("-o", "out.c"), ("filename.c",): + with self.subTest(opts=opts): + _, err = self.expect_failure("--make", *opts) + self.assertIn(msg, err) + + def test_cli_fail_make_without_srcdir(self): + _, err = self.expect_failure("--make", "--srcdir", "") + msg = "error: --srcdir must not be empty with --make" + self.assertIn(msg, err) + + def test_file_dest(self): + block = dedent(""" + /*[clinic input] + destination test new file {path}.h + output everything test + func + a: object + / + [clinic start generated code]*/ + """) + expected_checksum_line = ( + "/*[clinic end generated code: " + "output=da39a3ee5e6b4b0d input=b602ab8e173ac3bd]*/\n" + ) + expected_output = dedent("""\ + /*[clinic input] + preserve + [clinic start generated code]*/ + + PyDoc_VAR(func__doc__); + + PyDoc_STRVAR(func__doc__, + "func($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FUNC_METHODDEF \\ + {"func", (PyCFunction)func, METH_O, func__doc__}, + + static PyObject * + func(PyObject *module, PyObject *a) + /*[clinic end generated code: output=3dde2d13002165b9 input=a9049054013a1b77]*/ + """) + with os_helper.temp_dir() as tmp_dir: + in_fn = os.path.join(tmp_dir, "test.c") + out_fn = os.path.join(tmp_dir, "test.c.h") + with open(in_fn, "w", encoding="utf-8") as f: + f.write(block) + with open(out_fn, "w", encoding="utf-8") as f: + f.write("") # Write an empty output file! + # Clinic should complain about the empty output file. + _, err = self.expect_failure(in_fn) + expected_err = (f"Modified destination file {out_fn!r}; " + "not overwriting!") + self.assertIn(expected_err, err) + # Run clinic again, this time with the -f option. + _ = self.expect_success("-f", in_fn) + # Read back the generated output. + with open(in_fn, encoding="utf-8") as f: + data = f.read() + expected_block = f"{block}{expected_checksum_line}" + self.assertEqual(data, expected_block) + with open(out_fn, encoding="utf-8") as f: + data = f.read() + self.assertEqual(data, expected_output) + +try: + import _testclinic as ac_tester +except ImportError: + ac_tester = None + +@unittest.skipIf(ac_tester is None, "_testclinic is missing") +class ClinicFunctionalTest(unittest.TestCase): + locals().update((name, getattr(ac_tester, name)) + for name in dir(ac_tester) if name.startswith('test_')) + + def check_depr(self, regex, fn, /, *args, **kwds): + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + # Record the line number, so we're sure we've got the correct stack + # level on the deprecation warning. + _, lineno = fn(*args, **kwds), sys._getframe().f_lineno + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + + def check_depr_star(self, pnames, fn, /, *args, name=None, **kwds): + if name is None: + name = fn.__qualname__ + if isinstance(fn, type): + name = f'{fn.__module__}.{name}' + regex = ( + fr"Passing( more than)?( [0-9]+)? positional argument(s)? to " + fr"{re.escape(name)}\(\) is deprecated. Parameters? {pnames} will " + fr"become( a)? keyword-only parameters? in Python 3\.14" + ) + self.check_depr(regex, fn, *args, **kwds) + + def check_depr_kwd(self, pnames, fn, *args, name=None, **kwds): + if name is None: + name = fn.__qualname__ + if isinstance(fn, type): + name = f'{fn.__module__}.{name}' + pl = 's' if ' ' in pnames else '' + regex = ( + fr"Passing keyword argument{pl} {pnames} to " + fr"{re.escape(name)}\(\) is deprecated. Parameter{pl} {pnames} " + fr"will become positional-only in Python 3\.14." + ) + self.check_depr(regex, fn, *args, **kwds) + + def test_objects_converter(self): + with self.assertRaises(TypeError): + ac_tester.objects_converter() + self.assertEqual(ac_tester.objects_converter(1, 2), (1, 2)) + self.assertEqual(ac_tester.objects_converter([], 'whatever class'), ([], 'whatever class')) + self.assertEqual(ac_tester.objects_converter(1), (1, None)) + + def test_bytes_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.bytes_object_converter(1) + self.assertEqual(ac_tester.bytes_object_converter(b'BytesObject'), (b'BytesObject',)) + + def test_byte_array_object_converter(self): + with self.assertRaises(TypeError): + ac_tester.byte_array_object_converter(1) + byte_arr = bytearray(b'ByteArrayObject') + self.assertEqual(ac_tester.byte_array_object_converter(byte_arr), (byte_arr,)) + + def test_unicode_converter(self): + with self.assertRaises(TypeError): + ac_tester.unicode_converter(1) + self.assertEqual(ac_tester.unicode_converter('unicode'), ('unicode',)) + + def test_bool_converter(self): + with self.assertRaises(TypeError): + ac_tester.bool_converter(False, False, 'not a int') + self.assertEqual(ac_tester.bool_converter(), (True, True, True)) + self.assertEqual(ac_tester.bool_converter('', [], 5), (False, False, True)) + self.assertEqual(ac_tester.bool_converter(('not empty',), {1: 2}, 0), (True, True, False)) + + def test_bool_converter_c_default(self): + self.assertEqual(ac_tester.bool_converter_c_default(), (1, 0, -2, -3)) + self.assertEqual(ac_tester.bool_converter_c_default(False, True, False, True), + (0, 1, 0, 1)) + + def test_char_converter(self): + with self.assertRaises(TypeError): + ac_tester.char_converter(1) + with self.assertRaises(TypeError): + ac_tester.char_converter(b'ab') + chars = [b'A', b'\a', b'\b', b'\t', b'\n', b'\v', b'\f', b'\r', b'"', b"'", b'?', b'\\', b'\000', b'\377'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(), expected) + chars = [b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'a', b'b', b'c', b'd'] + expected = tuple(ord(c) for c in chars) + self.assertEqual(ac_tester.char_converter(*chars), expected) + + def test_unsigned_char_converter(self): + from _testcapi import UCHAR_MAX + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(UCHAR_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_char_converter(0, UCHAR_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_char_converter([]) + self.assertEqual(ac_tester.unsigned_char_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, UCHAR_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, (UCHAR_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_short_converter(self): + from _testcapi import SHRT_MIN, SHRT_MAX + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.short_converter(SHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.short_converter([]) + self.assertEqual(ac_tester.short_converter(-1234), (-1234,)) + self.assertEqual(ac_tester.short_converter(4321), (4321,)) + + def test_unsigned_short_converter(self): + from _testcapi import USHRT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_short_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(USHRT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_short_converter(0, USHRT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_short_converter([]) + self.assertEqual(ac_tester.unsigned_short_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, USHRT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, (USHRT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_int_converter(self): + from _testcapi import INT_MIN, INT_MAX + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.int_converter(INT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.int_converter(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.int_converter([]) + self.assertEqual(ac_tester.int_converter(), (12, 34, 45)) + self.assertEqual(ac_tester.int_converter(1, 2, '3'), (1, 2, ord('3'))) + + def test_unsigned_int_converter(self): + from _testcapi import UINT_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_int_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(UINT_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_int_converter(0, UINT_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_int_converter([]) + self.assertEqual(ac_tester.unsigned_int_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, UINT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, (UINT_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_converter(self): + from _testcapi import LONG_MIN, LONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_converter(LONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_converter([]) + self.assertEqual(ac_tester.long_converter(), (12,)) + self.assertEqual(ac_tester.long_converter(-1234), (-1234,)) + + def test_unsigned_long_converter(self): + from _testcapi import ULONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(ULONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_converter(0, ULONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, ULONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, (ULONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_long_long_converter(self): + from _testcapi import LLONG_MIN, LLONG_MAX + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.long_long_converter(LLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.long_long_converter([]) + self.assertEqual(ac_tester.long_long_converter(), (12,)) + self.assertEqual(ac_tester.long_long_converter(-1234), (-1234,)) + + def test_unsigned_long_long_converter(self): + from _testcapi import ULLONG_MAX + with self.assertRaises(ValueError): + ac_tester.unsigned_long_long_converter(-1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(ULLONG_MAX + 1) + with self.assertRaises(OverflowError): + ac_tester.unsigned_long_long_converter(0, ULLONG_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.unsigned_long_long_converter([]) + self.assertEqual(ac_tester.unsigned_long_long_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, ULLONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, (ULLONG_MAX + 1) * 3 + 123), (0, 0, 123)) + + def test_py_ssize_t_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MIN - 1) + with self.assertRaises(OverflowError): + ac_tester.py_ssize_t_converter(PY_SSIZE_T_MAX + 1) + with self.assertRaises(TypeError): + ac_tester.py_ssize_t_converter([]) + self.assertEqual(ac_tester.py_ssize_t_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.py_ssize_t_converter(1, 2, None), (1, 2, 56)) + + def test_slice_index_converter(self): + from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX + with self.assertRaises(TypeError): + ac_tester.slice_index_converter([]) + self.assertEqual(ac_tester.slice_index_converter(), (12, 34, 56)) + self.assertEqual(ac_tester.slice_index_converter(1, 2, None), (1, 2, 56)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MAX, PY_SSIZE_T_MAX + 1, PY_SSIZE_T_MAX + 1234), + (PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX)) + self.assertEqual(ac_tester.slice_index_converter(PY_SSIZE_T_MIN, PY_SSIZE_T_MIN - 1, PY_SSIZE_T_MIN - 1234), + (PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, PY_SSIZE_T_MIN)) + + def test_size_t_converter(self): + with self.assertRaises(ValueError): + ac_tester.size_t_converter(-1) + with self.assertRaises(TypeError): + ac_tester.size_t_converter([]) + self.assertEqual(ac_tester.size_t_converter(), (12,)) + + def test_float_converter(self): + with self.assertRaises(TypeError): + ac_tester.float_converter([]) + self.assertEqual(ac_tester.float_converter(), (12.5,)) + self.assertEqual(ac_tester.float_converter(-0.5), (-0.5,)) + + def test_double_converter(self): + with self.assertRaises(TypeError): + ac_tester.double_converter([]) + self.assertEqual(ac_tester.double_converter(), (12.5,)) + self.assertEqual(ac_tester.double_converter(-0.5), (-0.5,)) + + def test_py_complex_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_complex_converter([]) + self.assertEqual(ac_tester.py_complex_converter(complex(1, 2)), (complex(1, 2),)) + self.assertEqual(ac_tester.py_complex_converter(complex('-1-2j')), (complex('-1-2j'),)) + self.assertEqual(ac_tester.py_complex_converter(-0.5), (-0.5,)) + self.assertEqual(ac_tester.py_complex_converter(10), (10,)) + + def test_str_converter(self): + with self.assertRaises(TypeError): + ac_tester.str_converter(1) + with self.assertRaises(TypeError): + ac_tester.str_converter('a', 'b', 'c') + with self.assertRaises(ValueError): + ac_tester.str_converter('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter('a', b'b', 'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', b'c'), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter('a', b'b', 'c\0c'), ('a', 'b', 'c\0c')) + + def test_str_converter_encoding(self): + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding(1) + self.assertEqual(ac_tester.str_converter_encoding('a', 'b', 'c'), ('a', 'b', 'c')) + with self.assertRaises(TypeError): + ac_tester.str_converter_encoding('a', b'b\0b', 'c') + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c')])), ('a', 'b', 'c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', bytearray([ord('c'), 0, ord('c')])), + ('a', 'b', 'c\x00c')) + self.assertEqual(ac_tester.str_converter_encoding('a', b'b', b'c\x00c'), ('a', 'b', 'c\x00c')) + + def test_py_buffer_converter(self): + with self.assertRaises(TypeError): + ac_tester.py_buffer_converter('a', 'b') + self.assertEqual(ac_tester.py_buffer_converter('abc', bytearray([1, 2, 3])), (b'abc', b'\x01\x02\x03')) + + def test_keywords(self): + self.assertEqual(ac_tester.keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.keywords(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords(a=1, b=2), (1, 2)) + + def test_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly(1, 2) + self.assertEqual(ac_tester.keywords_kwonly(1, b=2), (1, 2)) + self.assertEqual(ac_tester.keywords_kwonly(a=1, b=2), (1, 2)) + + def test_keywords_opt(self): + self.assertEqual(ac_tester.keywords_opt(1), (1, None, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, 3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_opt(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_keywords_opt_kwonly(self): + self.assertEqual(ac_tester.keywords_opt_kwonly(1), (1, None, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_opt_kwonly(1, 2, 3) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, c=3), (1, None, 3, None)) + self.assertEqual(ac_tester.keywords_opt_kwonly(a=1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_keywords_kwonly_opt(self): + self.assertEqual(ac_tester.keywords_kwonly_opt(1), (1, None, None)) + with self.assertRaises(TypeError): + ac_tester.keywords_kwonly_opt(1, 2) + self.assertEqual(ac_tester.keywords_kwonly_opt(1, b=2), (1, 2, None)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, c=3), (1, None, 3)) + self.assertEqual(ac_tester.keywords_kwonly_opt(a=1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords(1, 2), (1, 2)) + self.assertEqual(ac_tester.posonly_keywords(1, b=2), (1, 2)) + + def test_posonly_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly(a=1, b=2) + self.assertEqual(ac_tester.posonly_kwonly(1, b=2), (1, 2)) + + def test_posonly_keywords_kwonly(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, 2, c=3), (1, 2, 3)) + self.assertEqual(ac_tester.posonly_keywords_kwonly(1, b=2, c=3), (1, 2, 3)) + + def test_posonly_keywords_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt(a=1, b=2, c=3, d=4) + self.assertEqual(ac_tester.posonly_keywords_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_opt_keywords_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, 3, 4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt(a=1, b=2, c=3, d=4) + + def test_posonly_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(1, 2) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2), (1, 2, None, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_kwonly_opt(1, b=2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_kwonly_opt(a=1, b=2, c=3, d=4) + + def test_posonly_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + + def test_posonly_keywords_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, b=2) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(1, 2, 3) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_kwonly_opt(a=1, b=2, c=3) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_keywords_opt_kwonly_opt(self): + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2), (1, 2, None, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, 4) + with self.assertRaises(TypeError): + ac_tester.posonly_keywords_opt_kwonly_opt(a=1, b=2) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, b=2, c=3), (1, 2, 3, None, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4, None)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, 3, d=4, e=5), (1, 2, 3, 4, 5)) + self.assertEqual(ac_tester.posonly_keywords_opt_kwonly_opt(1, 2, c=3, d=4, e=5), (1, 2, 3, 4, 5)) + + def test_posonly_opt_keywords_opt_kwonly_opt(self): + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1), (1, None, None, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2), (1, 2, None, None)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, b=2) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3), (1, 2, 3, None)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, d=4), (1, 2, 3, 4)) + self.assertEqual(ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaises(TypeError): + ac_tester.posonly_opt_keywords_opt_kwonly_opt(1, 2, 3, 4) + + def test_keyword_only_parameter(self): + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter() + with self.assertRaises(TypeError): + ac_tester.keyword_only_parameter(1) + self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + + if ac_tester is not None: + @repeat_fn(ac_tester.varpos, + ac_tester.varpos_array, + ac_tester.TestClass.varpos_no_fastcall, + ac_tester.TestClass.varpos_array_no_fastcall) + def test_varpos(self, fn): + # fn(*args) + self.assertEqual(fn(), ()) + self.assertEqual(fn(1, 2), (1, 2)) + + @repeat_fn(ac_tester.posonly_varpos, + ac_tester.posonly_varpos_array, + ac_tester.TestClass.posonly_varpos_no_fastcall, + ac_tester.TestClass.posonly_varpos_array_no_fastcall) + def test_posonly_varpos(self, fn): + # fn(a, b, /, *args) + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + @repeat_fn(ac_tester.posonly_req_opt_varpos, + ac_tester.posonly_req_opt_varpos_array, + ac_tester.TestClass.posonly_req_opt_varpos_no_fastcall, + ac_tester.TestClass.posonly_req_opt_varpos_array_no_fastcall) + def test_posonly_req_opt_varpos(self, fn): + # fn(a, b=False, /, *args) + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, a=1) + self.assertEqual(fn(1), (1, False, ())) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + @repeat_fn(ac_tester.posonly_poskw_varpos, + ac_tester.posonly_poskw_varpos_array, + ac_tester.TestClass.posonly_poskw_varpos_no_fastcall, + ac_tester.TestClass.posonly_poskw_varpos_array_no_fastcall) + def test_posonly_poskw_varpos(self, fn): + # fn(a, /, b, *args) + self.assertRaises(TypeError, fn) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, b=2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + self.assertRaises(TypeError, fn, b=4) + errmsg = re.escape("given by name ('b') and position (2)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, b=4) + + def test_poskw_varpos(self): + # fn(a, *args) + fn = ac_tester.poskw_varpos + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(a=1), (1, ())) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1), (1, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_poskw_varpos_kwonly_opt(self): + # fn(a, *args, b=False) + fn = ac_tester.poskw_varpos_kwonly_opt + self.assertRaises(TypeError, fn) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), True)) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), (1, (2, 3, 4), True)) + self.assertEqual(fn(a=1), (1, (), False)) + self.assertEqual(fn(a=1, b=2), (1, (), True)) + + def test_poskw_varpos_kwonly_opt2(self): + # fn(a, *args, b=False, c=False) + fn = ac_tester.poskw_varpos_kwonly_opt2 + self.assertRaises(TypeError, fn) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), 2, False)) + self.assertEqual(fn(1, b=2, c=3), (1, (), 2, 3)) + self.assertEqual(fn(1, 2, 3), (1, (2, 3), False, False)) + self.assertEqual(fn(1, 2, 3, b=4), (1, (2, 3), 4, False)) + self.assertEqual(fn(1, 2, 3, b=4, c=5), (1, (2, 3), 4, 5)) + self.assertEqual(fn(a=1), (1, (), False, False)) + self.assertEqual(fn(a=1, b=2), (1, (), 2, False)) + self.assertEqual(fn(a=1, b=2, c=3), (1, (), 2, 3)) + + def test_varpos_kwonly_opt(self): + # fn(*args, b=False) + fn = ac_tester.varpos_kwonly_opt + self.assertEqual(fn(), ((), False)) + self.assertEqual(fn(b=2), ((), 2)) + self.assertEqual(fn(1, b=2), ((1, ), 2)) + self.assertEqual(fn(1, 2, 3, 4), ((1, 2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + + def test_varpos_kwonly_req_opt(self): + fn = ac_tester.varpos_kwonly_req_opt + self.assertRaises(TypeError, fn) + self.assertEqual(fn(a=1), ((), 1, False, False)) + self.assertEqual(fn(a=1, b=2), ((), 1, 2, False)) + self.assertEqual(fn(a=1, b=2, c=3), ((), 1, 2, 3)) + self.assertRaises(TypeError, fn, 1) + self.assertEqual(fn(1, a=2), ((1,), 2, False, False)) + self.assertEqual(fn(1, a=2, b=3), ((1,), 2, 3, False)) + self.assertEqual(fn(1, a=2, b=3, c=4), ((1,), 2, 3, 4)) + + def test_gh_32092_oob(self): + ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) + + def test_gh_32092_kw_pass(self): + ac_tester.gh_32092_kw_pass(1, 2, 3) + + def test_gh_99233_refcount(self): + arg = '*A unique string is not referenced by anywhere else.*' + arg_refcount_origin = sys.getrefcount(arg) + ac_tester.gh_99233_refcount(arg) + arg_refcount_after = sys.getrefcount(arg) + self.assertEqual(arg_refcount_origin, arg_refcount_after) + + def test_gh_99240_double_free(self): + err = re.escape( + "gh_99240_double_free() argument 2 must be encoded string " + "without null bytes, not str" + ) + with self.assertRaisesRegex(TypeError, err): + ac_tester.gh_99240_double_free('a', '\0b') + + def test_null_or_tuple_for_varargs(self): + # fn(name, *constraints, covariant=False) + fn = ac_tester.null_or_tuple_for_varargs + # All of these should not crash: + self.assertEqual(fn('a'), ('a', (), False)) + self.assertEqual(fn('a', 1, 2, 3, covariant=True), ('a', (1, 2, 3), True)) + self.assertEqual(fn(name='a'), ('a', (), False)) + self.assertEqual(fn(name='a', covariant=True), ('a', (), True)) + self.assertEqual(fn(covariant=True, name='a'), ('a', (), True)) + + self.assertRaises(TypeError, fn, covariant=True) + errmsg = re.escape("given by name ('name') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, name='a') + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, name='a', covariant=True) + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, covariant=True, name='a') + + def test_cloned_func_exception_message(self): + incorrect_arg = -1 # f1() and f2() accept a single str + with self.assertRaisesRegex(TypeError, "clone_f1"): + ac_tester.clone_f1(incorrect_arg) + with self.assertRaisesRegex(TypeError, "clone_f2"): + ac_tester.clone_f2(incorrect_arg) + + def test_cloned_func_with_converter_exception_message(self): + for name in "clone_with_conv_f1", "clone_with_conv_f2": + with self.subTest(name=name): + func = getattr(ac_tester, name) + self.assertEqual(func(), name) + + def test_get_defining_class(self): + obj = ac_tester.TestClass() + meth = obj.get_defining_class + self.assertIs(obj.get_defining_class(), ac_tester.TestClass) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(meth, 1) + check(meth, a=1) + + def test_get_defining_class_capi(self): + from _testcapi import pyobject_vectorcall + obj = ac_tester.TestClass() + meth = obj.get_defining_class + pyobject_vectorcall(meth, None, None) + pyobject_vectorcall(meth, (), None) + pyobject_vectorcall(meth, (), ()) + pyobject_vectorcall(meth, None, ()) + self.assertIs(pyobject_vectorcall(meth, (), ()), ac_tester.TestClass) + + check = partial(self.assertRaisesRegex, TypeError, "no arguments") + check(pyobject_vectorcall, meth, (1,), None) + check(pyobject_vectorcall, meth, (1,), ("a",)) + + def test_get_defining_class_arg(self): + obj = ac_tester.TestClass() + self.assertEqual(obj.get_defining_class_arg("arg"), + (ac_tester.TestClass, "arg")) + self.assertEqual(obj.get_defining_class_arg(arg=123), + (ac_tester.TestClass, 123)) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass, arg="arg") + + # wrong number of arguments + with self.assertRaises(TypeError): + obj.get_defining_class_arg() + with self.assertRaises(TypeError): + obj.get_defining_class_arg("arg1", "arg2") + + def test_defclass_varpos(self): + # fn(*args) + cls = ac_tester.TestClass + obj = cls() + fn = obj.defclass_varpos + self.assertEqual(fn(), (cls, ())) + self.assertEqual(fn(1, 2), (cls, (1, 2))) + fn = cls.defclass_varpos + self.assertRaises(TypeError, fn) + self.assertEqual(fn(obj), (cls, ())) + self.assertEqual(fn(obj, 1, 2), (cls, (1, 2))) + + def test_defclass_posonly_varpos(self): + # fn(a, b, /, *args) + cls = ac_tester.TestClass + obj = cls() + fn = obj.defclass_posonly_varpos + errmsg = 'takes at least 2 positional arguments' + self.assertRaisesRegex(TypeError, errmsg, fn) + self.assertRaisesRegex(TypeError, errmsg, fn, 1) + self.assertEqual(fn(1, 2), (cls, 1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (cls, 1, 2, (3, 4))) + fn = cls.defclass_posonly_varpos + self.assertRaises(TypeError, fn) + self.assertRaisesRegex(TypeError, errmsg, fn, obj) + self.assertRaisesRegex(TypeError, errmsg, fn, obj, 1) + self.assertEqual(fn(obj, 1, 2), (cls, 1, 2, ())) + self.assertEqual(fn(obj, 1, 2, 3, 4), (cls, 1, 2, (3, 4))) + + def test_depr_star_new(self): + cls = ac_tester.DeprStarNew + cls() + cls(a=None) + self.check_depr_star("'a'", cls, None) + + def test_depr_star_new_cloned(self): + fn = ac_tester.DeprStarNew().cloned + fn() + fn(a=None) + self.check_depr_star("'a'", fn, None, name='_testclinic.DeprStarNew.cloned') + + def test_depr_star_init(self): + cls = ac_tester.DeprStarInit + cls() + cls(a=None) + self.check_depr_star("'a'", cls, None) + + def test_depr_star_init_cloned(self): + fn = ac_tester.DeprStarInit().cloned + fn() + fn(a=None) + self.check_depr_star("'a'", fn, None, name='_testclinic.DeprStarInit.cloned') + + def test_depr_star_init_noinline(self): + cls = ac_tester.DeprStarInitNoInline + self.assertRaises(TypeError, cls, "a") + cls(a="a", b="b") + cls(a="a", b="b", c="c") + cls("a", b="b") + cls("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", cls) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, cls, "a", "b", "c", "d") + + def test_depr_kwd_new(self): + cls = ac_tester.DeprKwdNew + cls() + cls(None) + self.check_depr_kwd("'a'", cls, a=None) + + def test_depr_kwd_init(self): + cls = ac_tester.DeprKwdInit + cls() + cls(None) + self.check_depr_kwd("'a'", cls, a=None) + + def test_depr_kwd_init_noinline(self): + cls = ac_tester.DeprKwdInitNoInline + cls = ac_tester.depr_star_noinline + self.assertRaises(TypeError, cls, "a") + cls(a="a", b="b") + cls(a="a", b="b", c="c") + cls("a", b="b") + cls("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", cls) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, cls, "a", "b", "c", "d") + + def test_depr_star_pos0_len1(self): + fn = ac_tester.depr_star_pos0_len1 + fn(a=None) + self.check_depr_star("'a'", fn, "a") + + def test_depr_star_pos0_len2(self): + fn = ac_tester.depr_star_pos0_len2 + fn(a=0, b=0) + check = partial(self.check_depr_star, "'a' and 'b'", fn) + check("a", b=0) + check("a", "b") + + def test_depr_star_pos0_len3_with_kwd(self): + fn = ac_tester.depr_star_pos0_len3_with_kwd + fn(a=0, b=0, c=0, d=0) + check = partial(self.check_depr_star, "'a', 'b' and 'c'", fn) + check("a", b=0, c=0, d=0) + check("a", "b", c=0, d=0) + check("a", "b", "c", d=0) + + def test_depr_star_pos1_len1_opt(self): + fn = ac_tester.depr_star_pos1_len1_opt + fn(a=0, b=0) + fn("a", b=0) + fn(a=0) # b is optional + check = partial(self.check_depr_star, "'b'", fn) + check("a", "b") + + def test_depr_star_pos1_len1(self): + fn = ac_tester.depr_star_pos1_len1 + fn(a=0, b=0) + fn("a", b=0) + check = partial(self.check_depr_star, "'b'", fn) + check("a", "b") + + def test_depr_star_pos1_len2_with_kwd(self): + fn = ac_tester.depr_star_pos1_len2_with_kwd + fn(a=0, b=0, c=0, d=0), + fn("a", b=0, c=0, d=0), + check = partial(self.check_depr_star, "'b' and 'c'", fn) + check("a", "b", c=0, d=0), + check("a", "b", "c", d=0), + + def test_depr_star_pos2_len1(self): + fn = ac_tester.depr_star_pos2_len1 + fn(a=0, b=0, c=0) + fn("a", b=0, c=0) + fn("a", "b", c=0) + check = partial(self.check_depr_star, "'c'", fn) + check("a", "b", "c") + + def test_depr_star_pos2_len2(self): + fn = ac_tester.depr_star_pos2_len2 + fn(a=0, b=0, c=0, d=0) + fn("a", b=0, c=0, d=0) + fn("a", "b", c=0, d=0) + check = partial(self.check_depr_star, "'c' and 'd'", fn) + check("a", "b", "c", d=0) + check("a", "b", "c", "d") + + def test_depr_star_pos2_len2_with_kwd(self): + fn = ac_tester.depr_star_pos2_len2_with_kwd + fn(a=0, b=0, c=0, d=0, e=0) + fn("a", b=0, c=0, d=0, e=0) + fn("a", "b", c=0, d=0, e=0) + check = partial(self.check_depr_star, "'c' and 'd'", fn) + check("a", "b", "c", d=0, e=0) + check("a", "b", "c", "d", e=0) + + def test_depr_star_noinline(self): + fn = ac_tester.depr_star_noinline + self.assertRaises(TypeError, fn, "a") + fn(a="a", b="b") + fn(a="a", b="b", c="c") + fn("a", b="b") + fn("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", fn) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + + def test_depr_star_multi(self): + fn = ac_tester.depr_star_multi + self.assertRaises(TypeError, fn, "a") + fn("a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + errmsg = ( + "Passing more than 1 positional argument to depr_star_multi() is deprecated. " + "Parameter 'b' will become a keyword-only parameter in Python 3.16. " + "Parameters 'c' and 'd' will become keyword-only parameters in Python 3.15. " + "Parameters 'e', 'f' and 'g' will become keyword-only parameters in Python 3.14.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", c="c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", "c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", "c", "d", e="e", f="f", g="g", h="h") + check("a", "b", "c", "d", "e", f="f", g="g", h="h") + check("a", "b", "c", "d", "e", "f", g="g", h="h") + check("a", "b", "c", "d", "e", "f", "g", h="h") + self.assertRaises(TypeError, fn, "a", "b", "c", "d", "e", "f", "g", "h") + + def test_depr_kwd_required_1(self): + fn = ac_tester.depr_kwd_required_1 + fn("a", "b") + self.assertRaises(TypeError, fn, "a") + self.assertRaises(TypeError, fn, "a", "b", "c") + check = partial(self.check_depr_kwd, "'b'", fn) + check("a", b="b") + self.assertRaises(TypeError, fn, a="a", b="b") + + def test_depr_kwd_required_2(self): + fn = ac_tester.depr_kwd_required_2 + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a", "b") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", "b", c="c") + check("a", b="b", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_optional_1(self): + fn = ac_tester.depr_kwd_optional_1 + fn("a") + fn("a", "b") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a", "b", "c") + check = partial(self.check_depr_kwd, "'b'", fn) + check("a", b="b") + self.assertRaises(TypeError, fn, a="a", b="b") + + def test_depr_kwd_optional_2(self): + fn = ac_tester.depr_kwd_optional_2 + fn("a") + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", c="c") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_optional_3(self): + fn = ac_tester.depr_kwd_optional_3 + fn() + fn("a") + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'a', 'b' and 'c'", fn) + check("a", "b", c="c") + check("a", b="b") + check(a="a") + + def test_depr_kwd_required_optional(self): + fn = ac_tester.depr_kwd_required_optional + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_noinline(self): + fn = ac_tester.depr_kwd_noinline + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_multi(self): + fn = ac_tester.depr_kwd_multi + fn("a", "b", "c", "d", "e", "f", "g", h="h") + errmsg = ( + "Passing keyword arguments 'b', 'c', 'd', 'e', 'f' and 'g' to depr_kwd_multi() is deprecated. " + "Parameter 'b' will become positional-only in Python 3.14. " + "Parameters 'c' and 'd' will become positional-only in Python 3.15. " + "Parameters 'e', 'f' and 'g' will become positional-only in Python 3.16.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", "c", "d", "e", "f", g="g", h="h") + check("a", "b", "c", "d", "e", f="f", g="g", h="h") + check("a", "b", "c", "d", e="e", f="f", g="g", h="h") + check("a", "b", "c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", c="c", d="d", e="e", f="f", g="g", h="h") + check("a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + + def test_depr_multi(self): + fn = ac_tester.depr_multi + self.assertRaises(TypeError, fn, "a", "b", "c", "d", "e", "f", "g") + errmsg = ( + "Passing more than 4 positional arguments to depr_multi() is deprecated. " + "Parameter 'e' will become a keyword-only parameter in Python 3.15. " + "Parameter 'f' will become a keyword-only parameter in Python 3.14.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", "c", "d", "e", "f", g="g") + check("a", "b", "c", "d", "e", f="f", g="g") + fn("a", "b", "c", "d", e="e", f="f", g="g") + fn("a", "b", "c", d="d", e="e", f="f", g="g") + errmsg = ( + "Passing keyword arguments 'b' and 'c' to depr_multi() is deprecated. " + "Parameter 'b' will become positional-only in Python 3.14. " + "Parameter 'c' will become positional-only in Python 3.15.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", c="c", d="d", e="e", f="f", g="g") + check("a", b="b", c="c", d="d", e="e", f="f", g="g") + self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") + + +class LimitedCAPIOutputTests(unittest.TestCase): + + def setUp(self): + self.clinic = _make_clinic(limited_capi=True) + + @staticmethod + def wrap_clinic_input(block): + return dedent(f""" + /*[clinic input] + output everything buffer + {block} + [clinic start generated code]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + + def test_limited_capi_float(self): + block = self.wrap_clinic_input(""" + func + f: float + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("float f;", generated) + self.assertIn("f = (float) PyFloat_AsDouble", generated) + + def test_limited_capi_double(self): + block = self.wrap_clinic_input(""" + func + f: double + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("double f;", generated) + self.assertIn("f = PyFloat_AsDouble", generated) + + +try: + import _testclinic_limited +except ImportError: + _testclinic_limited = None + +@unittest.skipIf(_testclinic_limited is None, "_testclinic_limited is missing") +class LimitedCAPIFunctionalTest(unittest.TestCase): + locals().update((name, getattr(_testclinic_limited, name)) + for name in dir(_testclinic_limited) if name.startswith('test_')) + + def test_my_int_func(self): + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func() + self.assertEqual(_testclinic_limited.my_int_func(3), 3) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func(1.0) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_func("xyz") + + def test_my_int_sum(self): + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum() + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1) + self.assertEqual(_testclinic_limited.my_int_sum(1, 2), 3) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1.0, 2) + with self.assertRaises(TypeError): + _testclinic_limited.my_int_sum(1, "str") + + def test_my_double_sum(self): + for func in ( + _testclinic_limited.my_float_sum, + _testclinic_limited.my_double_sum, + ): + with self.subTest(func=func.__name__): + self.assertEqual(func(1.0, 2.5), 3.5) + with self.assertRaises(TypeError): + func() + with self.assertRaises(TypeError): + func(1) + with self.assertRaises(TypeError): + func(1., "2") + + def test_get_file_descriptor(self): + # test 'file descriptor' converter: call PyObject_AsFileDescriptor() + get_fd = _testclinic_limited.get_file_descriptor + + class MyInt(int): + pass + + class MyFile: + def __init__(self, fd): + self._fd = fd + def fileno(self): + return self._fd + + for fd in (0, 1, 2, 5, 123_456): + self.assertEqual(get_fd(fd), fd) + + myint = MyInt(fd) + self.assertEqual(get_fd(myint), fd) + + myfile = MyFile(fd) + self.assertEqual(get_fd(myfile), fd) + + with self.assertRaises(OverflowError): + get_fd(2**256) + with self.assertWarnsRegex(RuntimeWarning, + "bool is used as a file descriptor"): + get_fd(True) + with self.assertRaises(TypeError): + get_fd(1.0) + with self.assertRaises(TypeError): + get_fd("abc") + with self.assertRaises(TypeError): + get_fd(None) + + +class PermutationTests(unittest.TestCase): + """Test permutation support functions.""" + + def test_permute_left_option_groups(self): + expected = ( + (), + (3,), + (2, 3), + (1, 2, 3), + ) + data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. + actual = tuple(permute_left_option_groups(data)) + self.assertEqual(actual, expected) + + def test_permute_right_option_groups(self): + expected = ( + (), + (1,), + (1, 2), + (1, 2, 3), + ) + data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. + actual = tuple(permute_right_option_groups(data)) + self.assertEqual(actual, expected) + + def test_permute_optional_groups(self): + empty = { + "left": (), "required": (), "right": (), + "expected": ((),), + } + noleft1 = { + "left": (), "required": ("b",), "right": ("c",), + "expected": ( + ("b",), + ("b", "c"), + ), + } + noleft2 = { + "left": (), "required": ("b", "c",), "right": ("d",), + "expected": ( + ("b", "c"), + ("b", "c", "d"), + ), + } + noleft3 = { + "left": (), "required": ("b", "c",), "right": ("d", "e"), + "expected": ( + ("b", "c"), + ("b", "c", "d"), + ("b", "c", "d", "e"), + ), + } + noright1 = { + "left": ("a",), "required": ("b",), "right": (), + "expected": ( + ("b",), + ("a", "b"), + ), + } + noright2 = { + "left": ("a",), "required": ("b", "c"), "right": (), + "expected": ( + ("b", "c"), + ("a", "b", "c"), + ), + } + noright3 = { + "left": ("a", "b"), "required": ("c",), "right": (), + "expected": ( + ("c",), + ("b", "c"), + ("a", "b", "c"), + ), + } + leftandright1 = { + "left": ("a",), "required": ("b",), "right": ("c",), + "expected": ( + ("b",), + ("a", "b"), # Prefer left. + ("a", "b", "c"), + ), + } + leftandright2 = { + "left": ("a", "b"), "required": ("c", "d"), "right": ("e", "f"), + "expected": ( + ("c", "d"), + ("b", "c", "d"), # Prefer left. + ("a", "b", "c", "d"), # Prefer left. + ("a", "b", "c", "d", "e"), + ("a", "b", "c", "d", "e", "f"), + ), + } + dataset = ( + empty, + noleft1, noleft2, noleft3, + noright1, noright2, noright3, + leftandright1, leftandright2, + ) + for params in dataset: + with self.subTest(**params): + left, required, right, expected = params.values() + permutations = permute_optional_groups(left, required, right) + actual = tuple(permutations) + self.assertEqual(actual, expected) + + +class FormatHelperTests(unittest.TestCase): + + def test_strip_leading_and_trailing_blank_lines(self): + dataset = ( + # Input lines, expected output. + ("a\nb", "a\nb"), + ("a\nb\n", "a\nb"), + ("a\nb ", "a\nb"), + ("\na\nb\n\n", "a\nb"), + ("\n\na\nb\n\n", "a\nb"), + ("\n\na\n\nb\n\n", "a\n\nb"), + # Note, leading whitespace is preserved: + (" a\nb", " a\nb"), + (" a\nb ", " a\nb"), + (" \n \n a\nb \n \n ", " a\nb"), + ) + for lines, expected in dataset: + with self.subTest(lines=lines, expected=expected): + out = libclinic.normalize_snippet(lines) + self.assertEqual(out, expected) + + def test_normalize_snippet(self): + snippet = """ + one + two + three + """ + + # Expected outputs: + zero_indent = ( + "one\n" + "two\n" + "three" + ) + four_indent = ( + " one\n" + " two\n" + " three" + ) + eight_indent = ( + " one\n" + " two\n" + " three" + ) + expected_outputs = {0: zero_indent, 4: four_indent, 8: eight_indent} + for indent, expected in expected_outputs.items(): + with self.subTest(indent=indent): + actual = libclinic.normalize_snippet(snippet, indent=indent) + self.assertEqual(actual, expected) + + def test_escaped_docstring(self): + dataset = ( + # input, expected + (r"abc", r'"abc"'), + (r"\abc", r'"\\abc"'), + (r"\a\bc", r'"\\a\\bc"'), + (r"\a\\bc", r'"\\a\\\\bc"'), + (r'"abc"', r'"\"abc\""'), + (r"'a'", r'"\'a\'"'), + ) + for line, expected in dataset: + with self.subTest(line=line, expected=expected): + out = libclinic.docstring_for_c_string(line) + self.assertEqual(out, expected) + + def test_format_escape(self): + line = "{}, {a}" + expected = "{{}}, {{a}}" + out = libclinic.format_escape(line) + self.assertEqual(out, expected) + + def test_c_bytes_repr(self): + c_bytes_repr = libclinic.c_bytes_repr + self.assertEqual(c_bytes_repr(b''), '""') + self.assertEqual(c_bytes_repr(b'abc'), '"abc"') + self.assertEqual(c_bytes_repr(b'\a\b\f\n\r\t\v'), r'"\a\b\f\n\r\t\v"') + self.assertEqual(c_bytes_repr(b' \0\x7f'), r'" \000\177"') + self.assertEqual(c_bytes_repr(b'"'), r'"\""') + self.assertEqual(c_bytes_repr(b"'"), r'''"'"''') + self.assertEqual(c_bytes_repr(b'\\'), r'"\\"') + self.assertEqual(c_bytes_repr(b'??/'), r'"?\?/"') + self.assertEqual(c_bytes_repr(b'???/'), r'"?\?\?/"') + self.assertEqual(c_bytes_repr(b'/*****/ /*/ */*'), r'"/\*****\/ /\*\/ *\/\*"') + self.assertEqual(c_bytes_repr(b'\xa0'), r'"\240"') + self.assertEqual(c_bytes_repr(b'\xff'), r'"\377"') + + def test_c_str_repr(self): + c_str_repr = libclinic.c_str_repr + self.assertEqual(c_str_repr(''), '""') + self.assertEqual(c_str_repr('abc'), '"abc"') + self.assertEqual(c_str_repr('\a\b\f\n\r\t\v'), r'"\a\b\f\n\r\t\v"') + self.assertEqual(c_str_repr(' \0\x7f'), r'" \000\177"') + self.assertEqual(c_str_repr('"'), r'"\""') + self.assertEqual(c_str_repr("'"), r'''"'"''') + self.assertEqual(c_str_repr('\\'), r'"\\"') + self.assertEqual(c_str_repr('??/'), r'"?\?/"') + self.assertEqual(c_str_repr('???/'), r'"?\?\?/"') + self.assertEqual(c_str_repr('/*****/ /*/ */*'), r'"/\*****\/ /\*\/ *\/\*"') + self.assertEqual(c_str_repr('\xa0'), r'"\u00a0"') + self.assertEqual(c_str_repr('\xff'), r'"\u00ff"') + self.assertEqual(c_str_repr('\u20ac'), r'"\u20ac"') + self.assertEqual(c_str_repr('\U0001f40d'), r'"\U0001f40d"') + + def test_c_unichar_repr(self): + c_unichar_repr = libclinic.c_unichar_repr + self.assertEqual(c_unichar_repr('a'), "'a'") + self.assertEqual(c_unichar_repr('\n'), r"'\n'") + self.assertEqual(c_unichar_repr('\b'), r"'\b'") + self.assertEqual(c_unichar_repr('\0'), '0') + self.assertEqual(c_unichar_repr('\1'), '0x01') + self.assertEqual(c_unichar_repr('\x7f'), '0x7f') + self.assertEqual(c_unichar_repr(' '), "' '") + self.assertEqual(c_unichar_repr('"'), """'"'""") + self.assertEqual(c_unichar_repr("'"), r"'\''") + self.assertEqual(c_unichar_repr('\\'), r"'\\'") + self.assertEqual(c_unichar_repr('?'), "'?'") + self.assertEqual(c_unichar_repr('\xa0'), '0xa0') + self.assertEqual(c_unichar_repr('\xff'), '0xff') + self.assertEqual(c_unichar_repr('\u20ac'), '0x20ac') + self.assertEqual(c_unichar_repr('\U0001f40d'), '0x1f40d') + + def test_indent_all_lines(self): + # Blank lines are expected to be unchanged. + self.assertEqual(libclinic.indent_all_lines("", prefix="bar"), "") + + lines = ( + "one\n" + "two" # The missing newline is deliberate. + ) + expected = ( + "barone\n" + "bartwo" + ) + out = libclinic.indent_all_lines(lines, prefix="bar") + self.assertEqual(out, expected) + + # If last line is empty, expect it to be unchanged. + lines = ( + "\n" + "one\n" + "two\n" + "" + ) + expected = ( + "bar\n" + "barone\n" + "bartwo\n" + "" + ) + out = libclinic.indent_all_lines(lines, prefix="bar") + self.assertEqual(out, expected) + + def test_suffix_all_lines(self): + # Blank lines are expected to be unchanged. + self.assertEqual(libclinic.suffix_all_lines("", suffix="foo"), "") + + lines = ( + "one\n" + "two" # The missing newline is deliberate. + ) + expected = ( + "onefoo\n" + "twofoo" + ) + out = libclinic.suffix_all_lines(lines, suffix="foo") + self.assertEqual(out, expected) + + # If last line is empty, expect it to be unchanged. + lines = ( + "\n" + "one\n" + "two\n" + "" + ) + expected = ( + "foo\n" + "onefoo\n" + "twofoo\n" + "" + ) + out = libclinic.suffix_all_lines(lines, suffix="foo") + self.assertEqual(out, expected) + + +class ClinicReprTests(unittest.TestCase): + def test_Block_repr(self): + block = Block("foo") + expected_repr = "" + self.assertEqual(repr(block), expected_repr) + + block2 = Block("bar", "baz", [], "eggs", "spam") + expected_repr_2 = "" + self.assertEqual(repr(block2), expected_repr_2) + + block3 = Block( + input="longboi_" * 100, + dsl_name="wow_so_long", + signatures=[], + output="very_long_" * 100, + indent="" + ) + expected_repr_3 = ( + "" + ) + self.assertEqual(repr(block3), expected_repr_3) + + def test_Destination_repr(self): + c = _make_clinic() + + destination = Destination( + "foo", type="file", clinic=c, args=("eggs",) + ) + self.assertEqual( + repr(destination), "" + ) + + destination2 = Destination("bar", type="buffer", clinic=c) + self.assertEqual(repr(destination2), "") + + def test_Module_repr(self): + module = Module("foo", _make_clinic()) + self.assertRegex(repr(module), r"") + + def test_Class_repr(self): + cls = Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') + self.assertRegex(repr(cls), r"") + + def test_FunctionKind_repr(self): + self.assertEqual( + repr(FunctionKind.CLASS_METHOD), "" + ) + + def test_Function_and_Parameter_reprs(self): + function = Function( + name='foo', + module=_make_clinic(), + cls=None, + c_basename=None, + full_name='foofoo', + return_converter=int_return_converter(), + kind=FunctionKind.METHOD_INIT, + coexist=False + ) + self.assertEqual(repr(function), "") + + converter = self_converter('bar', 'bar', function) + parameter = Parameter( + "bar", + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, + function=function, + converter=converter + ) + self.assertEqual(repr(parameter), "") + + def test_Monitor_repr(self): + monitor = libclinic.cpp.Monitor("test.c") + self.assertRegex(repr(monitor), r"") + + monitor.line_number = 42 + monitor.stack.append(("token1", "condition1")) + self.assertRegex( + repr(monitor), r"" + ) + + monitor.stack.append(("token2", "condition2")) + self.assertRegex( + repr(monitor), + r"" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py new file mode 100644 index 00000000000..fc34ac2fdc9 --- /dev/null +++ b/Lib/test/test_generated_cases.py @@ -0,0 +1,2074 @@ +import contextlib +import os +import re +import sys +import tempfile +import unittest + +from io import StringIO +from test import support +from test import test_tools + + +def skip_if_different_mount_drives(): + if sys.platform != "win32": + return + ROOT = os.path.dirname(os.path.dirname(__file__)) + root_drive = os.path.splitroot(ROOT)[0] + cwd_drive = os.path.splitroot(os.getcwd())[0] + if root_drive != cwd_drive: + # May raise ValueError if ROOT and the current working + # different have different mount drives (on Windows). + raise unittest.SkipTest( + f"the current working directory and the Python source code " + f"directory have different mount drives " + f"({cwd_drive} and {root_drive})" + ) + + +skip_if_different_mount_drives() + + +test_tools.skip_if_missing("cases_generator") +with test_tools.imports_under_tool("cases_generator"): + from analyzer import analyze_forest, StackItem + from cwriter import CWriter + import parser + from stack import Local, Stack + import tier1_generator + import opcode_metadata_generator + import optimizer_generator + + +def handle_stderr(): + if support.verbose > 1: + return contextlib.nullcontext() + else: + return support.captured_stderr() + + +def parse_src(src): + p = parser.Parser(src, "test.c") + nodes = [] + while node := p.definition(): + nodes.append(node) + return nodes + + +class TestEffects(unittest.TestCase): + def test_effect_sizes(self): + stack = Stack() + inputs = [ + x := StackItem("x", None, "1"), + y := StackItem("y", None, "oparg"), + z := StackItem("z", None, "oparg*2"), + ] + outputs = [ + StackItem("x", None, "1"), + StackItem("b", None, "oparg*4"), + StackItem("c", None, "1"), + ] + null = CWriter.null() + stack.pop(z, null) + stack.pop(y, null) + stack.pop(x, null) + for out in outputs: + stack.push(Local.undefined(out)) + self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2") + self.assertEqual(stack.physical_sp.to_c(), "0") + self.assertEqual(stack.logical_sp.to_c(), "1 - oparg - oparg*2 + oparg*4") + + +class TestGeneratedCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") + self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") + self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_output_filename, + self.temp_metadata_filename, + self.temp_pymetadata_filename, + self.temp_executor_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + tier1_generator.generate_tier1_from_files( + [self.temp_input_filename], self.temp_output_filename, False + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.read() + _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) + instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) + _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) + labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) + actual = instructions.strip() + "\n\n " + labels.strip() + + self.assertEqual(actual.strip(), expected.strip()) + + def test_inst_no_args(self): + input = """ + inst(OP, (--)) { + SPAM(); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + SPAM(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_pop(self): + input = """ + inst(OP, (value --)) { + SPAM(value); + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef value; + value = stack_pointer[-1]; + SPAM(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_push(self): + input = """ + inst(OP, (-- res)) { + res = SPAM(); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef res; + res = SPAM(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_inst_one_push_one_pop(self): + input = """ + inst(OP, (value -- res)) { + res = SPAM(value); + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + res = SPAM(value); + stack_pointer[-1] = res; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_binary_op(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); + + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = SPAM(left, right); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_overlap(self): + input = """ + inst(OP, (left, right -- left, result)) { + result = SPAM(left, right); + INPUTS_DEAD(); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef result; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + result = SPAM(left, right); + stack_pointer[-1] = result; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_predictions(self): + input = """ + inst(OP1, (arg -- res)) { + DEAD(arg); + res = Py_None; + } + inst(OP3, (arg -- res)) { + DEAD(arg); + DEOPT_IF(xxx); + res = Py_None; + } + family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + PREDICTED_OP1:; + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + res = Py_None; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(OP3) { + #if Py_TAIL_CALL_INTERP + int opcode = OP3; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + if (xxx) { + UPDATE_MISS_STATS(OP1); + assert(_PyOpcode_Deopt[opcode] == (OP1)); + JUMP_TO_PREDICTED(OP1); + } + res = Py_None; + stack_pointer[-1] = res; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_sync_sp(self): + input = """ + inst(A, (arg -- res)) { + DEAD(arg); + SYNC_SP(); + escaping_call(); + res = Py_None; + } + inst(B, (arg -- res)) { + DEAD(arg); + res = Py_None; + SYNC_SP(); + escaping_call(); + } + """ + output = """ + TARGET(A) { + #if Py_TAIL_CALL_INTERP + int opcode = A; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(A); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = Py_None; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(B) { + #if Py_TAIL_CALL_INTERP + int opcode = B; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(B); + _PyStackRef arg; + _PyStackRef res; + arg = stack_pointer[-1]; + res = Py_None; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + + def test_pep7_condition(self): + input = """ + inst(OP, (arg1 -- out)) { + if (arg1) + out = 0; + else { + out = 1; + } + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_error_if_plain(self): + input = """ + inst(OP, (--)) { + ERROR_IF(cond); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + if (cond) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_plain_with_comment(self): + input = """ + inst(OP, (--)) { + ERROR_IF(cond); // Comment is ok + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + if (cond) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop(self): + input = """ + inst(OP, (left, right -- res)) { + SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond); + res = 0; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + SPAM(left, right); + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + res = 0; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop_with_result(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + res = SPAM(left, right); + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_cache_effect(self): + input = """ + inst(OP, (counter/1, extra/2, value --)) { + DEAD(value); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + _PyStackRef value; + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t extra = read_u32(&this_instr[2].cache); + (void)extra; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_suppress_dispatch(self): + input = """ + label(somewhere) { + } + + inst(OP, (--)) { + goto somewhere; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + JUMP_TO_LABEL(somewhere); + } + + LABEL(somewhere) + { + } + """ + self.run_cases_test(input, output) + + def test_macro_instruction(self): + input = """ + inst(OP1, (counter/1, left, right -- left, right)) { + op1(left, right); + } + op(OP2, (extra/2, arg2, left, right -- res)) { + res = op2(arg2, left, right); + INPUTS_DEAD(); + } + macro(OP) = OP1 + cache/2 + OP2; + inst(OP3, (unused/5, arg2, left, right -- res)) { + res = op3(arg2, left, right); + INPUTS_DEAD(); + } + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP); + PREDICTED_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef arg2; + _PyStackRef res; + // _OP1 + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 2 cache entries */ + // OP2 + { + arg2 = stack_pointer[-3]; + uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; + _PyFrame_SetStackPointer(frame, stack_pointer); + res = op2(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(OP1); + _PyStackRef left; + _PyStackRef right; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(OP3) { + #if Py_TAIL_CALL_INTERP + int opcode = OP3; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP3); + static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); + _PyStackRef arg2; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 5 cache entries */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + arg2 = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + res = op3(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_caches(self): + input = """ + inst(OP, (unused/1, unused/2 --)) { + body; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + body; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_no_flags(self): + input = """ + pseudo(OP, (in -- out1, out2)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_with_flags(self): + input = """ + pseudo(OP, (in1, in2 --), (HAS_ARG, HAS_JUMP)) = { + OP1, + }; + + inst(OP1, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pseudo_instruction_as_sequence(self): + input = """ + pseudo(OP, (in -- out1, out2)) = [ + OP1, OP2 + ]; + + inst(OP1, (--)) { + } + + inst(OP2, (--)) { + } + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + + TARGET(OP2) { + #if Py_TAIL_CALL_INTERP + int opcode = OP2; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + + def test_array_input(self): + input = """ + inst(OP, (below, values[oparg*2], above --)) { + SPAM(values, oparg); + DEAD(below); + DEAD(values); + DEAD(above); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; + above = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg*2]; + below = stack_pointer[-2 - oparg*2]; + SPAM(values, oparg); + stack_pointer += -2 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_output(self): + input = """ + inst(OP, (unused, unused -- below, values[oparg*3], above)) { + SPAM(values, oparg); + below = 0; + above = 0; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; + values = &stack_pointer[-1]; + SPAM(values, oparg); + below = 0; + above = 0; + stack_pointer[-2] = below; + stack_pointer[-1 + oparg*3] = above; + stack_pointer += oparg*3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_input_output(self): + input = """ + inst(OP, (values[oparg] -- values[oparg], above)) { + SPAM(values, oparg); + above = 0; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *values; + _PyStackRef above; + values = &stack_pointer[-oparg]; + SPAM(values, oparg); + above = 0; + stack_pointer[0] = above; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_error_if(self): + input = """ + inst(OP, (extra, values[oparg] --)) { + DEAD(extra); + DEAD(values); + ERROR_IF(oparg == 0); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef extra; + _PyStackRef *values; + values = &stack_pointer[-oparg]; + extra = stack_pointer[-1 - oparg]; + if (oparg == 0) { + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_macro_push_push(self): + input = """ + op(A, (-- val1)) { + val1 = SPAM(); + } + op(B, (-- val2)) { + val2 = SPAM(); + } + macro(M) = A + B; + """ + output = """ + TARGET(M) { + #if Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + _PyStackRef val1; + _PyStackRef val2; + // A + { + val1 = SPAM(); + } + // B + { + val2 = SPAM(); + } + stack_pointer[0] = val1; + stack_pointer[1] = val2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_override_inst(self): + input = """ + inst(OP, (--)) { + spam; + } + override inst(OP, (--)) { + ham; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_override_op(self): + input = """ + op(OP, (--)) { + spam; + } + macro(M) = OP; + override op(OP, (--)) { + ham; + } + """ + output = """ + TARGET(M) { + #if Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_annotated_inst(self): + input = """ + pure inst(OP, (--)) { + ham; + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + ham; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_annotated_op(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + macro(M) = OP; + """ + output = """ + TARGET(M) { + #if Py_TAIL_CALL_INTERP + int opcode = M; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + SPAM(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + input = """ + pure register specializing op(OP, (--)) { + SPAM(); + } + macro(M) = OP; + """ + self.run_cases_test(input, output) + + def test_deopt_and_exit(self): + input = """ + pure op(OP, (arg1 -- out)) { + DEOPT_IF(1); + EXIT_IF(1); + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_of_one(self): + input = """ + inst(OP, (arg[1] -- out[1])) { + out[0] = arg[0]; + DEAD(arg); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef *out; + arg = &stack_pointer[-1]; + out = &stack_pointer[-1]; + out[0] = arg[0]; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pointer_to_stackref(self): + input = """ + inst(OP, (arg: _PyStackRef * -- out)) { + out = *arg; + DEAD(arg); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef out; + arg = (_PyStackRef *)stack_pointer[-1].bits; + out = *arg; + stack_pointer[-1] = out; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_cached_value(self): + input = """ + op(FIRST, (arg1 -- out)) { + out = arg1; + } + + op(SECOND, (unused -- unused)) { + } + + macro(BOTH) = FIRST + SECOND; + """ + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_unused_named_values(self): + input = """ + op(OP, (named -- named)) { + } + + macro(INST) = OP; + """ + output = """ + TARGET(INST) { + #if Py_TAIL_CALL_INTERP + int opcode = INST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INST); + DISPATCH(); + } + + """ + self.run_cases_test(input, output) + + def test_used_unused_used(self): + input = """ + op(FIRST, (w -- w)) { + USE(w); + } + + op(SECOND, (x -- x)) { + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef w; + _PyStackRef y; + // FIRST + { + w = stack_pointer[-1]; + USE(w); + } + // SECOND + { + } + // THIRD + { + y = w; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_used_used(self): + input = """ + op(FIRST, (w -- w)) { + } + + op(SECOND, (x -- x)) { + USE(x); + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + // FIRST + { + } + // SECOND + { + x = stack_pointer[-1]; + USE(x); + } + // THIRD + { + y = x; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_flush(self): + input = """ + op(FIRST, ( -- a, b)) { + a = 0; + b = 1; + } + + op(SECOND, (a, b -- )) { + USE(a, b); + INPUTS_DEAD(); + } + + macro(TEST) = FIRST + flush + SECOND; + """ + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 0; + b = 1; + } + // flush + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + // SECOND + { + USE(a, b); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_on_error_peeks(self): + + input = """ + op(FIRST, (x, y -- a, b)) { + a = x; + DEAD(x); + b = y; + DEAD(y); + } + + op(SECOND, (a, b -- a, b)) { + } + + op(THIRD, (j, k --)) { + INPUTS_DEAD(); // Mark j and k as used + ERROR_IF(cond); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + _PyStackRef a; + _PyStackRef b; + // FIRST + { + y = stack_pointer[-1]; + x = stack_pointer[-2]; + a = x; + b = y; + } + // SECOND + { + } + // THIRD + { + if (cond) { + JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_push_then_error(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a -- a, b)) { + b = 1; + ERROR_IF(cond); + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + TARGET(TEST) { + #if Py_TAIL_CALL_INTERP + int opcode = TEST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 1; + } + // SECOND + { + b = 1; + if (cond) { + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + } + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_true(self): + + input = """ + inst(OP1, ( --)) { + ERROR_IF(true); + } + inst(OP2, ( --)) { + ERROR_IF(1); + } + """ + output = """ + TARGET(OP1) { + #if Py_TAIL_CALL_INTERP + int opcode = OP1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + JUMP_TO_LABEL(error); + } + + TARGET(OP2) { + #if Py_TAIL_CALL_INTERP + int opcode = OP2; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + JUMP_TO_LABEL(error); + } + """ + self.run_cases_test(input, output) + + def test_scalar_array_inconsistency(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_size_inconsistency(self): + + input = """ + op(FIRST, ( -- a[2])) { + a[0] = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_reload(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + code(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + code(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_save_reload_paired(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_reload_only(self): + + input = """ + inst(BALANCED, ( -- )) { + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_only(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if Py_TAIL_CALL_INTERP + int opcode = BALANCED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_instruction_size_macro(self): + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + """ + + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + frame->return_offset = 1u ; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + # Two instructions of different sizes referencing the same + # uop containing the `INSTRUCTION_SIZE` macro is not allowed. + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + macro(OP2) = unused/1 + OP; + """ + + output = "" # No output needed as this should raise an error. + with self.assertRaisesRegex(SyntaxError, "All instructions containing a uop"): + self.run_cases_test(input, output) + + def test_escaping_call_next_to_cmacro(self): + input = """ + inst(OP, (--)) { + #ifdef Py_GIL_DISABLED + escaping_call(); + #else + another_escaping_call(); + #endif + yet_another_escaping_call(); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #else + _PyFrame_SetStackPointer(frame, stack_pointer); + another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + yet_another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pystackref_frompyobject_new_next_to_cmacro(self): + input = """ + inst(OP, (-- out1, out2)) { + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef out1; + _PyStackRef out2; + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = out1; + stack_pointer[1] = out2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_no_escaping_calls_in_branching_macros(self): + + input = """ + inst(OP, ( -- )) { + DEOPT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + EXIT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + ERROR_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_kill_in_wrong_order(self): + input = """ + inst(OP, (a, b -- c)) { + c = b; + PyStackRef_CLOSE(a); + PyStackRef_CLOSE(b); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_complex_label(self): + input = """ + label(other_label) { + } + + label(other_label2) { + } + + label(my_label) { + // Comment + do_thing(); + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + + output = """ + LABEL(other_label) + { + } + + LABEL(other_label2) + { + } + + LABEL(my_label) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (complex) { + JUMP_TO_LABEL(other_label); + } + JUMP_TO_LABEL(other_label2); + } + """ + self.run_cases_test(input, output) + + def test_spilled_label(self): + input = """ + spilled label(one) { + RELOAD_STACK(); + goto two; + } + + label(two) { + SAVE_STACK(); + goto one; + } + """ + + output = """ + LABEL(one) + { + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(two); + } + + LABEL(two) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMP_TO_LABEL(one); + } + """ + self.run_cases_test(input, output) + + + def test_incorrect_spills(self): + input1 = """ + spilled label(one) { + goto two; + } + + label(two) { + } + """ + + input2 = """ + spilled label(one) { + } + + label(two) { + goto one; + } + """ + with self.assertRaisesRegex(SyntaxError, ".*reload.*"): + self.run_cases_test(input1, "") + with self.assertRaisesRegex(SyntaxError, ".*spill.*"): + self.run_cases_test(input2, "") + + + def test_multiple_labels(self): + input = """ + label(my_label_1) { + // Comment + do_thing1(); + goto my_label_2; + } + + label(my_label_2) { + // Comment + do_thing2(); + goto my_label_1; + } + """ + + output = """ + LABEL(my_label_1) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing1(); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(my_label_2); + } + + LABEL(my_label_2) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + do_thing2(); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(my_label_1); + } + """ + self.run_cases_test(input, output) + + def test_reassigning_live_inputs(self): + input = """ + inst(OP, (in -- in)) { + in = 0; + } + """ + + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef in; + in = stack_pointer[-1]; + in = 0; + stack_pointer[-1] = in; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_reassigning_dead_inputs(self): + input = """ + inst(OP, (in -- )) { + temp = use(in); + DEAD(in); + in = temp; + PyStackRef_CLOSE(in); + } + """ + output = """ + TARGET(OP) { + #if Py_TAIL_CALL_INTERP + int opcode = OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef in; + in = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + temp = use(in); + stack_pointer = _PyFrame_GetStackPointer(frame); + in = temp; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(in); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + +class TestGeneratedAbstractCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_input2_filename, + self.temp_output_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, input2: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with open(self.temp_input2_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input2) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + optimizer_generator.generate_tier2_abstract_from_files( + [self.temp_input_filename, self.temp_input2_filename], + self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + self.assertEqual(actual.strip(), expected.strip()) + + def test_overridden_abstract(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + """ + input2 = """ + pure op(OP, (--)) { + eggs(); + } + """ + output = """ + case OP: { + eggs(); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_overridden_abstract_args(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + op(OP2, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + input2 = """ + op(OP, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + output = """ + case OP: { + JitOptSymbol *arg1; + JitOptSymbol *out; + arg1 = stack_pointer[-1]; + out = EGGS(arg1); + stack_pointer[-1] = out; + break; + } + + case OP2: { + JitOptSymbol *out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_no_overridden_case(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + + pure op(OP2, (arg1 -- out)) { + } + + """ + input2 = """ + pure op(OP2, (arg1 -- out)) { + out = NULL; + } + """ + output = """ + case OP: { + JitOptSymbol *out; + out = sym_new_not_null(ctx); + stack_pointer[-1] = out; + break; + } + + case OP2: { + JitOptSymbol *out; + out = NULL; + stack_pointer[-1] = out; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_missing_override_failure(self): + input = """ + pure op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + pure op(OTHER, (arg1 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(AssertionError, "All abstract uops"): + self.run_cases_test(input, input2, output) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py new file mode 100644 index 00000000000..b32db4426f2 --- /dev/null +++ b/Lib/test/test_peg_generator/__init__.py @@ -0,0 +1,12 @@ +import os.path +from test import support +from test.support import load_package_tests + + +# Creating a virtual environment and building C extensions is slow +support.requires('cpu') + + +# Load all tests in package +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_peg_generator/__main__.py b/Lib/test/test_peg_generator/__main__.py new file mode 100644 index 00000000000..1fab1fddb57 --- /dev/null +++ b/Lib/test/test_peg_generator/__main__.py @@ -0,0 +1,4 @@ +import unittest +from . import load_tests + +unittest.main() diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py new file mode 100644 index 00000000000..aa01a9b8f7e --- /dev/null +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -0,0 +1,523 @@ +import contextlib +import subprocess +import sysconfig +import textwrap +import unittest +import os +import shutil +import tempfile +from pathlib import Path + +from test import test_tools +from test import support +from test.support import os_helper, import_helper +from test.support.script_helper import assert_python_ok + +if support.check_cflags_pgo(): + raise unittest.SkipTest("peg_generator test disabled under PGO build") + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import ( + parse_string, + generate_parser_c_extension, + generate_c_parser_source, + ) + + +TEST_TEMPLATE = """ +tmp_dir = {extension_path!r} + +import ast +import traceback +import sys +import unittest + +from test import test_tools +with test_tools.imports_under_tool("peg_generator"): + from pegen.ast_dump import ast_dump + +sys.path.insert(0, tmp_dir) +import parse + +class Tests(unittest.TestCase): + + def check_input_strings_for_grammar( + self, + valid_cases = (), + invalid_cases = (), + ): + if valid_cases: + for case in valid_cases: + parse.parse_string(case, mode=0) + + if invalid_cases: + for case in invalid_cases: + with self.assertRaises(SyntaxError): + parse.parse_string(case, mode=0) + + def verify_ast_generation(self, stmt): + expected_ast = ast.parse(stmt) + actual_ast = parse.parse_string(stmt, mode=1) + self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast)) + + def test_parse(self): + {test_source} + +unittest.main() +""" + + +@support.requires_subprocess() +class TestCParser(unittest.TestCase): + + _has_run = False + + @classmethod + def setUpClass(cls): + if cls._has_run: + # Since gh-104798 (Use setuptools in peg-generator and reenable + # tests), this test case has been producing ref leaks. Initial + # debugging points to bug(s) in setuptools and/or importlib. + # See gh-105063 for more info. + raise unittest.SkipTest("gh-105063: can not rerun because of ref. leaks") + cls._has_run = True + + # When running under regtest, a separate tempdir is used + # as the current directory and watched for left-overs. + # Reusing that as the base for temporary directories + # ensures everything is cleaned up properly and + # cleans up afterwards if not (with warnings). + cls.tmp_base = os.getcwd() + if os.path.samefile(cls.tmp_base, os_helper.SAVEDCWD): + cls.tmp_base = None + # Create a directory for the reuseable static library part of + # the pegen extension build process. This greatly reduces the + # runtime overhead of spawning compiler processes. + cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base) + cls.addClassCleanup(shutil.rmtree, cls.library_dir) + + with contextlib.ExitStack() as stack: + python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools("venv")) + sitepackages = subprocess.check_output( + [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], + text=True, + ).strip() + stack.enter_context(import_helper.DirsOnSysPath(sitepackages)) + cls.addClassCleanup(stack.pop_all().close) + + @support.requires_venv_with_pip() + def setUp(self): + self._backup_config_vars = dict(sysconfig._CONFIG_VARS) + cmd = support.missing_compiler_executable() + if cmd is not None: + self.skipTest("The %r command is not found" % cmd) + self.old_cwd = os.getcwd() + self.tmp_path = tempfile.mkdtemp(dir=self.tmp_base) + self.enterContext(os_helper.change_cwd(self.tmp_path)) + + def tearDown(self): + os.chdir(self.old_cwd) + shutil.rmtree(self.tmp_path) + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(self._backup_config_vars) + + def build_extension(self, grammar_source): + grammar = parse_string(grammar_source, GrammarParser) + # Because setUp() already changes the current directory to the + # temporary path, use a relative path here to prevent excessive + # path lengths when compiling. + generate_parser_c_extension(grammar, Path('.'), library_dir=self.library_dir) + + def run_test(self, grammar_source, test_source): + self.build_extension(grammar_source) + test_source = textwrap.indent(textwrap.dedent(test_source), 8 * " ") + assert_python_ok( + "-c", + TEST_TEMPLATE.format(extension_path=self.tmp_path, test_source=test_source), + ) + + def test_c_parser(self) -> None: + grammar_source = """ + start[mod_ty]: a[asdl_stmt_seq*]=stmt* $ { _PyAST_Module(a, NULL, p->arena) } + stmt[stmt_ty]: a=expr_stmt { a } + expr_stmt[stmt_ty]: a=expression NEWLINE { _PyAST_Expr(a, EXTRA) } + expression[expr_ty]: ( l=expression '+' r=term { _PyAST_BinOp(l, Add, r, EXTRA) } + | l=expression '-' r=term { _PyAST_BinOp(l, Sub, r, EXTRA) } + | t=term { t } + ) + term[expr_ty]: ( l=term '*' r=factor { _PyAST_BinOp(l, Mult, r, EXTRA) } + | l=term '/' r=factor { _PyAST_BinOp(l, Div, r, EXTRA) } + | f=factor { f } + ) + factor[expr_ty]: ('(' e=expression ')' { e } + | a=atom { a } + ) + atom[expr_ty]: ( n=NAME { n } + | n=NUMBER { n } + | s=STRING { s } + ) + """ + test_source = """ + expressions = [ + "4+5", + "4-5", + "4*5", + "1+4*5", + "1+4/5", + "(1+1) + (1+1)", + "(1+1) - (1+1)", + "(1+1) * (1+1)", + "(1+1) / (1+1)", + ] + + for expr in expressions: + the_ast = parse.parse_string(expr, mode=1) + expected_ast = ast.parse(expr) + self.assertEqual(ast_dump(the_ast), ast_dump(expected_ast)) + """ + self.run_test(grammar_source, test_source) + + def test_lookahead(self) -> None: + grammar_source = """ + start: NAME &NAME expr NEWLINE? ENDMARKER + expr: NAME | NUMBER + """ + test_source = """ + valid_cases = ["foo bar"] + invalid_cases = ["foo 34"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_negative_lookahead(self) -> None: + grammar_source = """ + start: NAME !NAME expr NEWLINE? ENDMARKER + expr: NAME | NUMBER + """ + test_source = """ + valid_cases = ["foo 34"] + invalid_cases = ["foo bar"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_cut(self) -> None: + grammar_source = """ + start: X ~ Y Z | X Q S + X: 'x' + Y: 'y' + Z: 'z' + Q: 'q' + S: 's' + """ + test_source = """ + valid_cases = ["x y z"] + invalid_cases = ["x q s"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_gather(self) -> None: + grammar_source = """ + start: ';'.pass_stmt+ NEWLINE + pass_stmt: 'pass' + """ + test_source = """ + valid_cases = ["pass", "pass; pass"] + invalid_cases = ["pass;", "pass; pass;"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_left_recursion(self) -> None: + grammar_source = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + """ + test_source = """ + valid_cases = ["-34", "34", "34 + 12", "1 + 1 + 2 + 3"] + self.check_input_strings_for_grammar(valid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_advanced_left_recursive(self) -> None: + grammar_source = """ + start: NUMBER | sign start + sign: ['-'] + """ + test_source = """ + valid_cases = ["23", "-34"] + self.check_input_strings_for_grammar(valid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_mutually_left_recursive(self) -> None: + grammar_source = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + test_source = """ + valid_cases = ["B E", "D A C A E"] + self.check_input_strings_for_grammar(valid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_nasty_mutually_left_recursive(self) -> None: + grammar_source = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + test_source = """ + valid_cases = ["x ="] + invalid_cases = ["x - + ="] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_return_stmt_noexpr_action(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a } + statement[stmt_ty]: simple_stmt + simple_stmt[stmt_ty]: small_stmt + small_stmt[stmt_ty]: return_stmt + return_stmt[stmt_ty]: a='return' NEWLINE { _PyAST_Return(NULL, EXTRA) } + """ + test_source = """ + stmt = "return" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_gather_action_ast(self) -> None: + grammar_source = """ + start[mod_ty]: a[asdl_stmt_seq*]=';'.pass_stmt+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA)} + """ + test_source = """ + stmt = "pass; pass" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_pass_stmt_action(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a } + statement[stmt_ty]: simple_stmt + simple_stmt[stmt_ty]: small_stmt + small_stmt[stmt_ty]: pass_stmt + pass_stmt[stmt_ty]: a='pass' NEWLINE { _PyAST_Pass(EXTRA) } + """ + test_source = """ + stmt = "pass" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_if_stmt_action(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } + statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | simple_stmt + + simple_stmt[asdl_stmt_seq*]: a=small_stmt b=further_small_stmt* [';'] NEWLINE { + (asdl_stmt_seq*)_PyPegen_seq_insert_in_front(p, a, b) } + further_small_stmt[stmt_ty]: ';' a=small_stmt { a } + + block: simple_stmt | NEWLINE INDENT a=statements DEDENT { a } + + compound_stmt: if_stmt + + if_stmt: 'if' a=full_expression ':' b=block { _PyAST_If(a, b, NULL, EXTRA) } + + small_stmt[stmt_ty]: pass_stmt + + pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) } + + full_expression: NAME + """ + test_source = """ + stmt = "pass" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_same_name_different_types(self) -> None: + grammar_source = """ + start[mod_ty]: a[asdl_stmt_seq*]=import_from+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena)} + import_from[stmt_ty]: ( a='from' !'import' c=simple_name 'import' d=import_as_names_from { + _PyAST_ImportFrom(c->v.Name.id, d, 0, EXTRA) } + | a='from' '.' 'import' c=import_as_names_from { + _PyAST_ImportFrom(NULL, c, 1, EXTRA) } + ) + simple_name[expr_ty]: NAME + import_as_names_from[asdl_alias_seq*]: a[asdl_alias_seq*]=','.import_as_name_from+ { a } + import_as_name_from[alias_ty]: a=NAME 'as' b=NAME { _PyAST_alias(((expr_ty) a)->v.Name.id, ((expr_ty) b)->v.Name.id, EXTRA) } + """ + test_source = """ + for stmt in ("from a import b as c", "from . import a as b"): + expected_ast = ast.parse(stmt) + actual_ast = parse.parse_string(stmt, mode=1) + self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast)) + """ + self.run_test(grammar_source, test_source) + + def test_with_stmt_with_paren(self) -> None: + grammar_source = """ + start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } + statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } + compound_stmt[stmt_ty]: with_stmt + with_stmt[stmt_ty]: ( + a='with' '(' b[asdl_withitem_seq*]=','.with_item+ ')' ':' c=block { + _PyAST_With(b, (asdl_stmt_seq*) _PyPegen_singleton_seq(p, c), NULL, EXTRA) } + ) + with_item[withitem_ty]: ( + e=NAME o=['as' t=NAME { t }] { _PyAST_withitem(e, _PyPegen_set_expr_context(p, o, Store), p->arena) } + ) + block[stmt_ty]: a=pass_stmt NEWLINE { a } | NEWLINE INDENT a=pass_stmt DEDENT { a } + pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) } + """ + test_source = """ + stmt = "with (\\n a as b,\\n c as d\\n): pass" + the_ast = parse.parse_string(stmt, mode=1) + self.assertStartsWith(ast_dump(the_ast), + "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), " + "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]" + ) + """ + self.run_test(grammar_source, test_source) + + def test_ternary_operator(self) -> None: + grammar_source = """ + start[mod_ty]: a=expr ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + expr[asdl_stmt_seq*]: a=listcomp NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, _PyAST_Expr(a, EXTRA)) } + listcomp[expr_ty]: ( + a='[' b=NAME c=for_if_clauses d=']' { _PyAST_ListComp(b, c, EXTRA) } + ) + for_if_clauses[asdl_comprehension_seq*]: ( + a[asdl_comprehension_seq*]=(y=['async'] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })* + { _PyAST_comprehension(_PyAST_Name(((expr_ty) a)->v.Name.id, Store, EXTRA), b, c, (y == NULL) ? 0 : 1, p->arena) })+ { a } + ) + """ + test_source = """ + stmt = "[i for i in a if b]" + self.verify_ast_generation(stmt) + """ + self.run_test(grammar_source, test_source) + + def test_syntax_error_for_string(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: NAME + """ + test_source = r""" + for text in ("a b 42 b a", "\u540d \u540d 42 \u540d \u540d"): + try: + parse.parse_string(text, mode=0) + except SyntaxError as e: + tb = traceback.format_exc() + self.assertTrue('File "", line 1' in tb) + self.assertTrue(f"SyntaxError: invalid syntax" in tb) + """ + self.run_test(grammar_source, test_source) + + def test_headers_and_trailer(self) -> None: + grammar_source = """ + @header 'SOME HEADER' + @subheader 'SOME SUBHEADER' + @trailer 'SOME TRAILER' + start: expr+ NEWLINE? ENDMARKER + expr: x=NAME + """ + grammar = parse_string(grammar_source, GrammarParser) + parser_source = generate_c_parser_source(grammar) + + self.assertTrue("SOME HEADER" in parser_source) + self.assertTrue("SOME SUBHEADER" in parser_source) + self.assertTrue("SOME TRAILER" in parser_source) + + def test_error_in_rules(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: NAME {PyTuple_New(-1)} + """ + # PyTuple_New raises SystemError if an invalid argument was passed. + test_source = """ + with self.assertRaises(SystemError): + parse.parse_string("a", mode=0) + """ + self.run_test(grammar_source, test_source) + + def test_no_soft_keywords(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: 'foo' + """ + grammar = parse_string(grammar_source, GrammarParser) + parser_source = generate_c_parser_source(grammar) + assert "expect_soft_keyword" not in parser_source + + def test_soft_keywords(self) -> None: + grammar_source = """ + start: expr+ NEWLINE? ENDMARKER + expr: "foo" + """ + grammar = parse_string(grammar_source, GrammarParser) + parser_source = generate_c_parser_source(grammar) + assert "expect_soft_keyword" in parser_source + + def test_soft_keywords_parse(self) -> None: + grammar_source = """ + start: "if" expr '+' expr NEWLINE + expr: NAME + """ + test_source = """ + valid_cases = ["if if + if"] + invalid_cases = ["if if"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_soft_keywords_lookahead(self) -> None: + grammar_source = """ + start: &"if" "if" expr '+' expr NEWLINE + expr: NAME + """ + test_source = """ + valid_cases = ["if if + if"] + invalid_cases = ["if if"] + self.check_input_strings_for_grammar(valid_cases, invalid_cases) + """ + self.run_test(grammar_source, test_source) + + def test_forced(self) -> None: + grammar_source = """ + start: NAME &&':' | NAME + """ + test_source = """ + self.assertEqual(parse.parse_string("number :", mode=0), None) + with self.assertRaises(SyntaxError) as e: + parse.parse_string("a", mode=0) + self.assertIn("expected ':'", str(e.exception)) + """ + self.run_test(grammar_source, test_source) + + def test_forced_with_group(self) -> None: + grammar_source = """ + start: NAME &&(':' | ';') | NAME + """ + test_source = """ + self.assertEqual(parse.parse_string("number :", mode=0), None) + self.assertEqual(parse.parse_string("number ;", mode=0), None) + with self.assertRaises(SyntaxError) as e: + parse.parse_string("a", mode=0) + self.assertIn("expected (':' | ';')", e.exception.args[0]) + """ + self.run_test(grammar_source, test_source) diff --git a/Lib/test/test_peg_generator/test_first_sets.py b/Lib/test/test_peg_generator/test_first_sets.py new file mode 100644 index 00000000000..d6f8322f034 --- /dev/null +++ b/Lib/test/test_peg_generator/test_first_sets.py @@ -0,0 +1,286 @@ +import unittest + +from test import test_tools +from typing import Dict, Set + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import parse_string + from pegen.first_sets import FirstSetCalculator + from pegen.grammar import Grammar + + +class TestFirstSets(unittest.TestCase): + def calculate_first_sets(self, grammar_source: str) -> Dict[str, Set[str]]: + grammar: Grammar = parse_string(grammar_source, GrammarParser) + return FirstSetCalculator(grammar.rules).calculate() + + def test_alternatives(self) -> None: + grammar = """ + start: expr NEWLINE? ENDMARKER + expr: A | B + A: 'a' | '-' + B: 'b' | '+' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "A": {"'a'", "'-'"}, + "B": {"'+'", "'b'"}, + "expr": {"'+'", "'a'", "'b'", "'-'"}, + "start": {"'+'", "'a'", "'b'", "'-'"}, + }, + ) + + def test_optionals(self) -> None: + grammar = """ + start: expr NEWLINE + expr: ['a'] ['b'] 'c' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "expr": {"'c'", "'a'", "'b'"}, + "start": {"'c'", "'a'", "'b'"}, + }, + ) + + def test_repeat_with_separator(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"thing": {"NUMBER"}, "start": {"NUMBER"}}, + ) + + def test_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: (term)? 'b' + term: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "term": {"NUMBER"}, + "sum": {"NUMBER", "'b'"}, + "start": {"'b'", "NUMBER"}, + }, + ) + + def test_optional_literal(self) -> None: + grammar = """ + start: sum NEWLINE + sum: '+' ? term + term: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "term": {"NUMBER"}, + "sum": {"'+'", "NUMBER"}, + "start": {"'+'", "NUMBER"}, + }, + ) + + def test_optional_after(self) -> None: + grammar = """ + start: term NEWLINE + term: NUMBER ['+'] + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"term": {"NUMBER"}, "start": {"NUMBER"}}, + ) + + def test_optional_before(self) -> None: + grammar = """ + start: term NEWLINE + term: ['+'] NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}}, + ) + + def test_repeat_0(self) -> None: + grammar = """ + start: thing* "+" NEWLINE + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}}, + ) + + def test_repeat_0_with_group(self) -> None: + grammar = """ + start: ('+' '-')* term NEWLINE + term: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}}, + ) + + def test_repeat_1(self) -> None: + grammar = """ + start: thing+ '-' NEWLINE + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"thing": {"NUMBER"}, "start": {"NUMBER"}}, + ) + + def test_repeat_1_with_group(self) -> None: + grammar = """ + start: ('+' term)+ term NEWLINE + term: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}} + ) + + def test_gather(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"thing": {"NUMBER"}, "start": {"NUMBER"}}, + ) + + def test_positive_lookahead(self) -> None: + grammar = """ + start: expr NEWLINE + expr: &'a' opt + opt: 'a' | 'b' | 'c' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "expr": {"'a'"}, + "start": {"'a'"}, + "opt": {"'b'", "'c'", "'a'"}, + }, + ) + + def test_negative_lookahead(self) -> None: + grammar = """ + start: expr NEWLINE + expr: !'a' opt + opt: 'a' | 'b' | 'c' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "opt": {"'b'", "'a'", "'c'"}, + "expr": {"'b'", "'c'"}, + "start": {"'b'", "'c'"}, + }, + ) + + def test_left_recursion(self) -> None: + grammar = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + foo: 'foo' + bar: 'bar' + baz: 'baz' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "expr": {"NUMBER", "'-'"}, + "term": {"NUMBER"}, + "start": {"NUMBER", "'-'"}, + "foo": {"'foo'"}, + "bar": {"'bar'"}, + "baz": {"'baz'"}, + }, + ) + + def test_advance_left_recursion(self) -> None: + grammar = """ + start: NUMBER | sign start + sign: ['-'] + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}}, + ) + + def test_mutual_left_recursion(self) -> None: + grammar = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "foo": {"'D'", "'B'"}, + "bar": {"'D'"}, + "start": {"'D'", "'B'"}, + }, + ) + + def test_nasty_left_recursion(self) -> None: + # TODO: Validate this + grammar = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + self.assertEqual( + self.calculate_first_sets(grammar), + {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}}, + ) + + def test_nullable_rule(self) -> None: + grammar = """ + start: sign thing $ + sign: ['-'] + thing: NUMBER + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "sign": {"", "'-'"}, + "thing": {"NUMBER"}, + "start": {"NUMBER", "'-'"}, + }, + ) + + def test_epsilon_production_in_start_rule(self) -> None: + grammar = """ + start: ['-'] $ + """ + self.assertEqual( + self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}} + ) + + def test_multiple_nullable_rules(self) -> None: + grammar = """ + start: sign thing other another $ + sign: ['-'] + thing: ['+'] + other: '*' + another: '/' + """ + self.assertEqual( + self.calculate_first_sets(grammar), + { + "sign": {"", "'-'"}, + "thing": {"'+'", ""}, + "start": {"'+'", "'-'", "'*'"}, + "other": {"'*'"}, + "another": {"'/'"}, + }, + ) diff --git a/Lib/test/test_peg_generator/test_grammar_validator.py b/Lib/test/test_peg_generator/test_grammar_validator.py new file mode 100644 index 00000000000..857aced8ae5 --- /dev/null +++ b/Lib/test/test_peg_generator/test_grammar_validator.py @@ -0,0 +1,77 @@ +import unittest +from test import test_tools + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.validator import SubRuleValidator, ValidationError + from pegen.validator import RaiseRuleValidator, CutValidator + from pegen.testutil import parse_string + from pegen.grammar import Grammar + + +class TestPegen(unittest.TestCase): + def test_rule_with_no_collision(self) -> None: + grammar_source = """ + start: bad_rule + sum: + | NAME '-' NAME + | NAME '+' NAME + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = SubRuleValidator(grammar) + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) + + def test_rule_with_simple_collision(self) -> None: + grammar_source = """ + start: bad_rule + sum: + | NAME '+' NAME + | NAME '+' NAME ';' + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = SubRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) + + def test_rule_with_collision_after_some_other_rules(self) -> None: + grammar_source = """ + start: bad_rule + sum: + | NAME '+' NAME + | NAME '*' NAME ';' + | NAME '-' NAME + | NAME '+' NAME ';' + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = SubRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) + + def test_raising_valid_rule(self) -> None: + grammar_source = """ + start: NAME { RAISE_SYNTAX_ERROR("this is not allowed") } + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = RaiseRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) + + def test_cut_validator(self) -> None: + grammar_source = """ + star: (OP ~ OP)* + plus: (OP ~ OP)+ + bracket: [OP ~ OP] + gather: OP.(OP ~ OP)+ + nested: [OP | NAME ~ OP] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = CutValidator(grammar) + for rule_name, rule in grammar.rules.items(): + with self.subTest(rule_name): + with self.assertRaises(ValidationError): + validator.validate_rule(rule_name, rule) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py new file mode 100644 index 00000000000..58ce558c548 --- /dev/null +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -0,0 +1,1132 @@ +import ast +import difflib +import io +import textwrap +import unittest + +from test import test_tools +from typing import Dict, Any +from tokenize import TokenInfo, NAME, NEWLINE, NUMBER, OP + +test_tools.skip_if_missing("peg_generator") +with test_tools.imports_under_tool("peg_generator"): + from pegen.grammar_parser import GeneratedParser as GrammarParser + from pegen.testutil import parse_string, generate_parser, make_parser + from pegen.grammar import GrammarVisitor, GrammarError, Grammar + from pegen.grammar_visualizer import ASTGrammarPrinter + from pegen.parser import Parser + from pegen.parser_generator import compute_nullables, compute_left_recursives + from pegen.python_generator import PythonParserGenerator + + +class TestPegen(unittest.TestCase): + def test_parse_grammar(self) -> None: + grammar_source = """ + start: sum NEWLINE + sum: t1=term '+' t2=term { action } | term + term: NUMBER + """ + expected = """ + start: sum NEWLINE + sum: term '+' term | term + term: NUMBER + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + rules = grammar.rules + self.assertEqual(str(grammar), textwrap.dedent(expected).strip()) + # Check the str() and repr() of a few rules; AST nodes don't support ==. + self.assertEqual(str(rules["start"]), "start: sum NEWLINE") + self.assertEqual(str(rules["sum"]), "sum: term '+' term | term") + expected_repr = ( + "Rule('term', None, Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))" + ) + self.assertEqual(repr(rules["term"]), expected_repr) + + def test_repeated_rules(self) -> None: + grammar_source = """ + start: the_rule NEWLINE + the_rule: 'b' NEWLINE + the_rule: 'a' NEWLINE + """ + with self.assertRaisesRegex(GrammarError, "Repeated rule 'the_rule'"): + parse_string(grammar_source, GrammarParser) + + def test_long_rule_str(self) -> None: + grammar_source = """ + start: zero | one | one zero | one one | one zero zero | one zero one | one one zero | one one one + """ + expected = """ + start: + | zero + | one + | one zero + | one one + | one zero zero + | one zero one + | one one zero + | one one one + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + self.assertEqual(str(grammar.rules["start"]), textwrap.dedent(expected).strip()) + + def test_typed_rules(self) -> None: + grammar = """ + start[int]: sum NEWLINE + sum[int]: t1=term '+' t2=term { action } | term + term[int]: NUMBER + """ + rules = parse_string(grammar, GrammarParser).rules + # Check the str() and repr() of a few rules; AST nodes don't support ==. + self.assertEqual(str(rules["start"]), "start: sum NEWLINE") + self.assertEqual(str(rules["sum"]), "sum: term '+' term | term") + self.assertEqual( + repr(rules["term"]), + "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))", + ) + + def test_gather(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + rules = parse_string(grammar, GrammarParser).rules + self.assertEqual(str(rules["start"]), "start: ','.thing+ NEWLINE") + self.assertStartsWith(repr(rules["start"]), + "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" + ) + self.assertEqual(str(rules["thing"]), "thing: NUMBER") + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + node = parse_string("1, 2\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n" + ), + TokenInfo( + NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n" + ), + ], + ) + + def test_expr_grammar(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term '+' term | term + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + ], + ) + + def test_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term ('+' term)? + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n" + ), + [ + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n" + ), + TokenInfo( + NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n" + ), + ], + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n" + ), + ], + ) + node = parse_string("1\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + None, + ], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ], + ) + + def test_optional_literal(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term '+' ? + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1+\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+\n" + ), + TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+\n"), + ], + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+\n"), + ], + ) + node = parse_string("1\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + None, + ], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ], + ) + + def test_alt_optional_operator(self) -> None: + grammar = """ + start: sum NEWLINE + sum: term ['+' term] + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n" + ), + [ + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n" + ), + TokenInfo( + NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n" + ), + ], + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n" + ), + ], + ) + node = parse_string("1\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + None, + ], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ], + ) + + def test_repeat_0_simple(self) -> None: + grammar = """ + start: thing thing* NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 2 3\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"), + [ + TokenInfo( + NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n" + ), + TokenInfo( + NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n" + ), + ], + ) + node = parse_string("1\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + [], + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + ], + ) + + def test_repeat_0_complex(self) -> None: + grammar = """ + start: term ('+' term)* NEWLINE + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n" + ), + [ + [ + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="2", + start=(1, 4), + end=(1, 5), + line="1 + 2 + 3\n", + ), + ], + [ + TokenInfo( + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="3", + start=(1, 8), + end=(1, 9), + line="1 + 2 + 3\n", + ), + ], + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + ), + ], + ) + + def test_repeat_1_simple(self) -> None: + grammar = """ + start: thing thing+ NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 2 3\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"), + [ + TokenInfo( + NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n" + ), + TokenInfo( + NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n" + ), + ], + ) + with self.assertRaises(SyntaxError): + parse_string("1\n", parser_class) + + def test_repeat_1_complex(self) -> None: + grammar = """ + start: term ('+' term)+ NEWLINE + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual( + node, + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n" + ), + [ + [ + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="2", + start=(1, 4), + end=(1, 5), + line="1 + 2 + 3\n", + ), + ], + [ + TokenInfo( + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="3", + start=(1, 8), + end=(1, 9), + line="1 + 2 + 3\n", + ), + ], + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + ), + ], + ) + with self.assertRaises(SyntaxError): + parse_string("1\n", parser_class) + + def test_repeat_with_sep_simple(self) -> None: + grammar = """ + start: ','.thing+ NEWLINE + thing: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("1, 2, 3\n", parser_class) + self.assertEqual( + node, + [ + [ + TokenInfo( + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3\n" + ), + TokenInfo( + NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3\n" + ), + TokenInfo( + NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3\n" + ), + ], + ) + + def test_left_recursive(self) -> None: + grammar_source = """ + start: expr NEWLINE + expr: ('-' term | expr '+' term | term) + term: NUMBER + foo: NAME+ + bar: NAME* + baz: NAME? + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + parser_class = generate_parser(grammar) + rules = grammar.rules + self.assertFalse(rules["start"].left_recursive) + self.assertTrue(rules["expr"].left_recursive) + self.assertFalse(rules["term"].left_recursive) + self.assertFalse(rules["foo"].left_recursive) + self.assertFalse(rules["bar"].left_recursive) + self.assertFalse(rules["baz"].left_recursive) + node = parse_string("1 + 2 + 3\n", parser_class) + self.assertEqual( + node, + [ + [ + [ + TokenInfo( + NUMBER, + string="1", + start=(1, 0), + end=(1, 1), + line="1 + 2 + 3\n", + ), + TokenInfo( + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, + string="2", + start=(1, 4), + end=(1, 5), + line="1 + 2 + 3\n", + ), + ], + TokenInfo( + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + ), + TokenInfo( + NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n" + ), + ], + TokenInfo( + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + ), + ], + ) + + def test_python_expr(self) -> None: + grammar = """ + start: expr NEWLINE? $ { ast.Expression(expr) } + expr: ( expr '+' term { ast.BinOp(expr, ast.Add(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } + | expr '-' term { ast.BinOp(expr, ast.Sub(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } + | term { term } + ) + term: ( l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, lineno=l.lineno, col_offset=l.col_offset, end_lineno=r.end_lineno, end_col_offset=r.end_col_offset) } + | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, lineno=l.lineno, col_offset=l.col_offset, end_lineno=r.end_lineno, end_col_offset=r.end_col_offset) } + | factor { factor } + ) + factor: ( '(' expr ')' { expr } + | atom { atom } + ) + atom: ( n=NAME { ast.Name(id=n.string, ctx=ast.Load(), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) } + | n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) } + ) + """ + parser_class = make_parser(grammar) + node = parse_string("(1 + 2*3 + 5)/(6 - 2)\n", parser_class) + code = compile(node, "", "eval") + val = eval(code) + self.assertEqual(val, 3.0) + + def test_f_string_in_action(self) -> None: + grammar = """ + start: n=NAME NEWLINE? $ { f"name -> {n.string}" } + """ + parser_class = make_parser(grammar) + node = parse_string("a", parser_class) + self.assertEqual(node.strip(), "name -> a") + + def test_nullable(self) -> None: + grammar_source = """ + start: sign NUMBER + sign: ['-' | '+'] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + rules = grammar.rules + nullables = compute_nullables(rules) + self.assertNotIn(rules["start"], nullables) # Not None! + self.assertIn(rules["sign"], nullables) + + def test_advanced_left_recursive(self) -> None: + grammar_source = """ + start: NUMBER | sign start + sign: ['-'] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + rules = grammar.rules + nullables = compute_nullables(rules) + compute_left_recursives(rules) + self.assertNotIn(rules["start"], nullables) # Not None! + self.assertIn(rules["sign"], nullables) + self.assertTrue(rules["start"].left_recursive) + self.assertFalse(rules["sign"].left_recursive) + + def test_mutually_left_recursive(self) -> None: + grammar_source = """ + start: foo 'E' + foo: bar 'A' | 'B' + bar: foo 'C' | 'D' + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + rules = grammar.rules + self.assertFalse(rules["start"].left_recursive) + self.assertTrue(rules["foo"].left_recursive) + self.assertTrue(rules["bar"].left_recursive) + genr.generate("") + ns: Dict[str, Any] = {} + exec(out.getvalue(), ns) + parser_class: Type[Parser] = ns["GeneratedParser"] + node = parse_string("D A C A E", parser_class) + + self.assertEqual( + node, + [ + [ + [ + [ + TokenInfo( + type=NAME, + string="D", + start=(1, 0), + end=(1, 1), + line="D A C A E", + ), + TokenInfo( + type=NAME, + string="A", + start=(1, 2), + end=(1, 3), + line="D A C A E", + ), + ], + TokenInfo( + type=NAME, + string="C", + start=(1, 4), + end=(1, 5), + line="D A C A E", + ), + ], + TokenInfo( + type=NAME, + string="A", + start=(1, 6), + end=(1, 7), + line="D A C A E", + ), + ], + TokenInfo( + type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E" + ), + ], + ) + node = parse_string("B C A E", parser_class) + self.assertEqual( + node, + [ + [ + [ + TokenInfo( + type=NAME, + string="B", + start=(1, 0), + end=(1, 1), + line="B C A E", + ), + TokenInfo( + type=NAME, + string="C", + start=(1, 2), + end=(1, 3), + line="B C A E", + ), + ], + TokenInfo( + type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E" + ), + ], + TokenInfo( + type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E" + ), + ], + ) + + def test_nasty_mutually_left_recursive(self) -> None: + # This grammar does not recognize 'x - + =', much to my chagrin. + # But that's the way PEG works. + # [Breathlessly] + # The problem is that the toplevel target call + # recurses into maybe, which recognizes 'x - +', + # and then the toplevel target looks for another '+', + # which fails, so it retreats to NAME, + # which succeeds, so we end up just recognizing 'x', + # and then start fails because there's no '=' after that. + grammar_source = """ + start: target '=' + target: maybe '+' | NAME + maybe: maybe '-' | target + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator(grammar, out) + genr.generate("") + ns: Dict[str, Any] = {} + exec(out.getvalue(), ns) + parser_class = ns["GeneratedParser"] + with self.assertRaises(SyntaxError): + parse_string("x - + =", parser_class) + + def test_lookahead(self) -> None: + grammar = """ + start: (expr_stmt | assign_stmt) &'.' + expr_stmt: !(target '=') expr + assign_stmt: target '=' expr + expr: term ('+' term)* + target: NAME + term: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("foo = 12 + 12 .", parser_class) + self.maxDiff = None + self.assertEqual( + node, + [ + TokenInfo( + NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 ." + ), + TokenInfo( + OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 ." + ), + [ + TokenInfo( + NUMBER, + string="12", + start=(1, 6), + end=(1, 8), + line="foo = 12 + 12 .", + ), + [ + [ + TokenInfo( + OP, + string="+", + start=(1, 9), + end=(1, 10), + line="foo = 12 + 12 .", + ), + TokenInfo( + NUMBER, + string="12", + start=(1, 11), + end=(1, 13), + line="foo = 12 + 12 .", + ), + ] + ], + ], + ], + ) + + def test_named_lookahead_error(self) -> None: + grammar = """ + start: foo=!'x' NAME + """ + with self.assertRaises(SyntaxError): + make_parser(grammar) + + def test_start_leader(self) -> None: + grammar = """ + start: attr | NAME + attr: start '.' NAME + """ + # Would assert False without a special case in compute_left_recursives(). + make_parser(grammar) + + def test_opt_sequence(self) -> None: + grammar = """ + start: [NAME*] + """ + # This case was failing because of a double trailing comma at the end + # of a line in the generated source. See bpo-41044 + make_parser(grammar) + + def test_left_recursion_too_complex(self) -> None: + grammar = """ + start: foo + foo: bar '+' | baz '+' | '+' + bar: baz '-' | foo '-' | '-' + baz: foo '*' | bar '*' | '*' + """ + with self.assertRaises(ValueError) as errinfo: + make_parser(grammar) + self.assertTrue("no leader" in str(errinfo.exception.value)) + + def test_cut(self) -> None: + grammar = """ + start: '(' ~ expr ')' + expr: NUMBER + """ + parser_class = make_parser(grammar) + node = parse_string("(1)", parser_class) + self.assertEqual( + node, + [ + TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"), + TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)"), + TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"), + ], + ) + + def test_cut_is_local_in_rule(self) -> None: + grammar = """ + start: + | inner + | 'x' { "ok" } + inner: + | 'x' ~ 'y' + | 'x' + """ + parser_class = make_parser(grammar) + node = parse_string("x", parser_class) + self.assertEqual(node, 'ok') + + def test_cut_is_local_in_parens(self) -> None: + # we currently don't guarantee this behavior, see gh-143054 + grammar = """ + start: + | ('x' ~ 'y' | 'x') + | 'x' { "ok" } + """ + parser_class = make_parser(grammar) + node = parse_string("x", parser_class) + self.assertEqual(node, 'ok') + + def test_dangling_reference(self) -> None: + grammar = """ + start: foo ENDMARKER + foo: bar NAME + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + def test_bad_token_reference(self) -> None: + grammar = """ + start: foo + foo: NAMEE + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + def test_missing_start(self) -> None: + grammar = """ + foo: NAME + """ + with self.assertRaises(GrammarError): + parser_class = make_parser(grammar) + + def test_invalid_rule_name(self) -> None: + grammar = """ + start: _a b + _a: 'a' + b: 'b' + """ + with self.assertRaisesRegex(GrammarError, "cannot start with underscore: '_a'"): + parser_class = make_parser(grammar) + + def test_invalid_variable_name(self) -> None: + grammar = """ + start: a b + a: _x='a' + b: 'b' + """ + with self.assertRaisesRegex(GrammarError, "cannot start with underscore: '_x'"): + parser_class = make_parser(grammar) + + def test_invalid_variable_name_in_temporal_rule(self) -> None: + grammar = """ + start: a b + a: (_x='a' | 'b') | 'c' + b: 'b' + """ + with self.assertRaisesRegex(GrammarError, "cannot start with underscore: '_x'"): + parser_class = make_parser(grammar) + + def test_soft_keyword(self) -> None: + grammar = """ + start: + | "number" n=NUMBER { eval(n.string) } + | "string" n=STRING { n.string } + | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string } + """ + parser_class = make_parser(grammar) + self.assertEqual(parse_string("number 1", parser_class), 1) + self.assertEqual(parse_string("string 'b'", parser_class), "'b'") + self.assertEqual( + parse_string("number test 1", parser_class), "test = 1" + ) + assert ( + parse_string("string test 'b'", parser_class) == "test = 'b'" + ) + with self.assertRaises(SyntaxError): + parse_string("test 1", parser_class) + + def test_forced(self) -> None: + grammar = """ + start: NAME &&':' | NAME + """ + parser_class = make_parser(grammar) + self.assertTrue(parse_string("number :", parser_class)) + with self.assertRaises(SyntaxError) as e: + parse_string("a", parser_class) + + self.assertIn("expected ':'", str(e.exception)) + + def test_forced_with_group(self) -> None: + grammar = """ + start: NAME &&(':' | ';') | NAME + """ + parser_class = make_parser(grammar) + self.assertTrue(parse_string("number :", parser_class)) + self.assertTrue(parse_string("number ;", parser_class)) + with self.assertRaises(SyntaxError) as e: + parse_string("a", parser_class) + self.assertIn("expected (':' | ';')", e.exception.args[0]) + + def test_unreachable_explicit(self) -> None: + source = """ + start: NAME { UNREACHABLE } + """ + grammar = parse_string(source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator( + grammar, out, unreachable_formatting="This is a test" + ) + genr.generate("") + self.assertIn("This is a test", out.getvalue()) + + def test_unreachable_implicit1(self) -> None: + source = """ + start: NAME | invalid_input + invalid_input: NUMBER { None } + """ + grammar = parse_string(source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator( + grammar, out, unreachable_formatting="This is a test" + ) + genr.generate("") + self.assertIn("This is a test", out.getvalue()) + + def test_unreachable_implicit2(self) -> None: + source = """ + start: NAME | '(' invalid_input ')' + invalid_input: NUMBER { None } + """ + grammar = parse_string(source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator( + grammar, out, unreachable_formatting="This is a test" + ) + genr.generate("") + self.assertIn("This is a test", out.getvalue()) + + def test_unreachable_implicit3(self) -> None: + source = """ + start: NAME | invalid_input { None } + invalid_input: NUMBER + """ + grammar = parse_string(source, GrammarParser) + out = io.StringIO() + genr = PythonParserGenerator( + grammar, out, unreachable_formatting="This is a test" + ) + genr.generate("") + self.assertNotIn("This is a test", out.getvalue()) + + def test_locations_in_alt_action_and_group(self) -> None: + grammar = """ + start: t=term NEWLINE? $ { ast.Expression(t) } + term: + | l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, LOCATIONS) } + | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, LOCATIONS) } + | factor + factor: + | ( + n=NAME { ast.Name(id=n.string, ctx=ast.Load(), LOCATIONS) } | + n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), LOCATIONS) } + ) + """ + parser_class = make_parser(grammar) + source = "2*3\n" + o = ast.dump(parse_string(source, parser_class).body, include_attributes=True) + p = ast.dump(ast.parse(source).body[0].value, include_attributes=True).replace( + " kind=None,", "" + ) + diff = "\n".join( + difflib.unified_diff( + o.split("\n"), p.split("\n"), "cpython", "python-pegen" + ) + ) + self.assertFalse(diff) + + +class TestGrammarVisitor: + class Visitor(GrammarVisitor): + def __init__(self) -> None: + self.n_nodes = 0 + + def visit(self, node: Any, *args: Any, **kwargs: Any) -> None: + self.n_nodes += 1 + super().visit(node, *args, **kwargs) + + def test_parse_trivial_grammar(self) -> None: + grammar = """ + start: 'a' + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + self.assertEqual(visitor.n_nodes, 6) + + def test_parse_or_grammar(self) -> None: + grammar = """ + start: rule + rule: 'a' | 'b' + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/NameLeaf -> 6 + # Rule/Rhs/ -> 2 + # Alt/NamedItem/StringLeaf -> 3 + # Alt/NamedItem/StringLeaf -> 3 + + self.assertEqual(visitor.n_nodes, 14) + + def test_parse_repeat1_grammar(self) -> None: + grammar = """ + start: 'a'+ + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/Repeat1/StringLeaf -> 6 + self.assertEqual(visitor.n_nodes, 7) + + def test_parse_repeat0_grammar(self) -> None: + grammar = """ + start: 'a'* + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/Repeat0/StringLeaf -> 6 + + self.assertEqual(visitor.n_nodes, 7) + + def test_parse_optional_grammar(self) -> None: + grammar = """ + start: 'a' ['b'] + """ + rules = parse_string(grammar, GrammarParser) + visitor = self.Visitor() + + visitor.visit(rules) + + # Grammar/Rule/Rhs/Alt/NamedItem/StringLeaf -> 6 + # NamedItem/Opt/Rhs/Alt/NamedItem/Stringleaf -> 6 + + self.assertEqual(visitor.n_nodes, 12) + + +class TestGrammarVisualizer(unittest.TestCase): + def test_simple_rule(self) -> None: + grammar = """ + start: 'a' 'b' + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'a'") + └──NamedItem + └──StringLeaf("'b'") + """ + ) + + self.assertEqual(output, expected_output) + + def test_multiple_rules(self) -> None: + grammar = """ + start: a b + a: 'a' + b: 'b' + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──NameLeaf('a') + └──NamedItem + └──NameLeaf('b') + + └──Rule + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'a'") + + └──Rule + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'b'") + """ + ) + + self.assertEqual(output, expected_output) + + def test_deep_nested_rule(self) -> None: + grammar = """ + start: 'a' ['b'['c'['d']]] + """ + rules = parse_string(grammar, GrammarParser) + + printer = ASTGrammarPrinter() + lines: List[str] = [] + printer.print_grammar_ast(rules, printer=lines.append) + + output = "\n".join(lines) + expected_output = textwrap.dedent( + """\ + └──Rule + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'a'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'b'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + ├──NamedItem + │ └──StringLeaf("'c'") + └──NamedItem + └──Opt + └──Rhs + └──Alt + └──NamedItem + └──StringLeaf("'d'") + """ + ) + + self.assertEqual(output, expected_output) diff --git a/Lib/test/test_tools/i18n_data/ascii-escapes.pot b/Lib/test/test_tools/i18n_data/ascii-escapes.pot index 18d868b6a20..cc5a9f6ba61 100644 --- a/Lib/test/test_tools/i18n_data/ascii-escapes.pot +++ b/Lib/test/test_tools/i18n_data/ascii-escapes.pot @@ -15,30 +15,36 @@ msgstr "" "Generated-By: pygettext.py 1.5\n" +#. Special characters that are always escaped in the POT file #: escapes.py:5 msgid "" "\"\t\n" "\r\\" msgstr "" +#. All ascii characters 0-31 #: escapes.py:8 msgid "" "\000\001\002\003\004\005\006\007\010\t\n" "\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" msgstr "" +#. All ascii characters 32-126 #: escapes.py:13 msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" msgstr "" +#. ascii char 127 #: escapes.py:17 msgid "\177" msgstr "" +#. some characters in the 128-255 range #: escapes.py:20 -msgid "€   ÿ" +msgid "\302\200 \302\240 ÿ" msgstr "" +#. some characters >= 256 encoded as 2, 3 and 4 bytes, respectively #: escapes.py:23 msgid "α ㄱ 𓂀" msgstr "" diff --git a/Lib/test/test_tools/i18n_data/comments.pot b/Lib/test/test_tools/i18n_data/comments.pot new file mode 100644 index 00000000000..a1df46d453c --- /dev/null +++ b/Lib/test/test_tools/i18n_data/comments.pot @@ -0,0 +1,110 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: comments.py:4 +msgid "foo" +msgstr "" + +#. i18n: This is a translator comment +#: comments.py:7 +msgid "bar" +msgstr "" + +#. i18n: This is a translator comment +#. i18n: This is another translator comment +#: comments.py:11 +msgid "baz" +msgstr "" + +#. i18n: This is a translator comment +#. with multiple +#. lines +#: comments.py:16 +msgid "qux" +msgstr "" + +#. i18n: This is a translator comment +#: comments.py:21 +msgid "quux" +msgstr "" + +#. i18n: This is a translator comment +#. with multiple lines +#. i18n: This is another translator comment +#. with multiple lines +#: comments.py:27 +msgid "corge" +msgstr "" + +#: comments.py:31 +msgid "grault" +msgstr "" + +#. i18n: This is another translator comment +#: comments.py:36 +msgid "garply" +msgstr "" + +#: comments.py:40 +msgid "george" +msgstr "" + +#. i18n: This is another translator comment +#: comments.py:45 +msgid "waldo" +msgstr "" + +#. i18n: This is a translator comment +#. i18n: This is also a translator comment +#. i18n: This is another translator comment +#: comments.py:50 +msgid "waldo2" +msgstr "" + +#. i18n: This is a translator comment +#. i18n: This is another translator comment +#. i18n: This is yet another translator comment +#. i18n: This is a translator comment +#. with multiple lines +#: comments.py:53 comments.py:56 comments.py:59 comments.py:63 +msgid "fred" +msgstr "" + +#: comments.py:65 +msgid "plugh" +msgstr "" + +#: comments.py:67 +msgid "foobar" +msgstr "" + +#. i18n: This is a translator comment +#: comments.py:71 +msgid "xyzzy" +msgstr "" + +#: comments.py:72 +msgid "thud" +msgstr "" + +#. i18n: This is a translator comment +#. i18n: This is another translator comment +#. i18n: This is yet another translator comment +#: comments.py:78 +msgid "foos" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/comments.py b/Lib/test/test_tools/i18n_data/comments.py new file mode 100644 index 00000000000..dca4dfa57b1 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/comments.py @@ -0,0 +1,78 @@ +from gettext import gettext as _ + +# Not a translator comment +_('foo') + +# i18n: This is a translator comment +_('bar') + +# i18n: This is a translator comment +# i18n: This is another translator comment +_('baz') + +# i18n: This is a translator comment +# with multiple +# lines +_('qux') + +# This comment should not be included because +# it does not start with the prefix +# i18n: This is a translator comment +_('quux') + +# i18n: This is a translator comment +# with multiple lines +# i18n: This is another translator comment +# with multiple lines +_('corge') + +# i18n: This comment should be ignored + +_('grault') + +# i18n: This comment should be ignored + +# i18n: This is another translator comment +_('garply') + +# i18n: comment should be ignored +x = 1 +_('george') + +# i18n: This comment should be ignored +x = 1 +# i18n: This is another translator comment +_('waldo') + +# i18n: This is a translator comment +x = 1 # i18n: This is also a translator comment +# i18n: This is another translator comment +_('waldo2') + +# i18n: This is a translator comment +_('fred') + +# i18n: This is another translator comment +_('fred') + +# i18n: This is yet another translator comment +_('fred') + +# i18n: This is a translator comment +# with multiple lines +_('fred') + +_('plugh') # i18n: This comment should be ignored + +_('foo' # i18n: This comment should be ignored + 'bar') # i18n: This comment should be ignored + +# i18n: This is a translator comment +_('xyzzy') +_('thud') + + +## i18n: This is a translator comment +# # i18n: This is another translator comment +### ### i18n: This is yet another translator comment +_('foos') diff --git a/Lib/test/test_tools/i18n_data/custom_keywords.pot b/Lib/test/test_tools/i18n_data/custom_keywords.pot new file mode 100644 index 00000000000..03a9cba3a20 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/custom_keywords.pot @@ -0,0 +1,51 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: custom_keywords.py:10 custom_keywords.py:11 +msgid "bar" +msgstr "" + +#: custom_keywords.py:13 +msgid "cat" +msgid_plural "cats" +msgstr[0] "" +msgstr[1] "" + +#: custom_keywords.py:14 +msgid "dog" +msgid_plural "dogs" +msgstr[0] "" +msgstr[1] "" + +#: custom_keywords.py:16 +msgctxt "context" +msgid "bar" +msgstr "" + +#: custom_keywords.py:18 +msgctxt "context" +msgid "cat" +msgid_plural "cats" +msgstr[0] "" +msgstr[1] "" + +#: custom_keywords.py:34 +msgid "overridden" +msgid_plural "default" +msgstr[0] "" +msgstr[1] "" + diff --git a/Lib/test/test_tools/i18n_data/custom_keywords.py b/Lib/test/test_tools/i18n_data/custom_keywords.py new file mode 100644 index 00000000000..ba0ffe77180 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/custom_keywords.py @@ -0,0 +1,34 @@ +from gettext import ( + gettext as foo, + ngettext as nfoo, + pgettext as pfoo, + npgettext as npfoo, + gettext as bar, + gettext as _, +) + +foo('bar') +foo('bar', 'baz') + +nfoo('cat', 'cats', 1) +nfoo('dog', 'dogs') + +pfoo('context', 'bar') + +npfoo('context', 'cat', 'cats', 1) + +# This is an unknown keyword and should be ignored +bar('baz') + +# 'nfoo' requires at least 2 arguments +nfoo('dog') + +# 'pfoo' requires at least 2 arguments +pfoo('context') + +# 'npfoo' requires at least 3 arguments +npfoo('context') +npfoo('context', 'cat') + +# --keyword should override the default keyword +_('overridden', 'default') diff --git a/Lib/test/test_tools/i18n_data/docstrings.pot b/Lib/test/test_tools/i18n_data/docstrings.pot index 5af1d41422f..387db2413a5 100644 --- a/Lib/test/test_tools/i18n_data/docstrings.pot +++ b/Lib/test/test_tools/i18n_data/docstrings.pot @@ -15,26 +15,40 @@ msgstr "" "Generated-By: pygettext.py 1.5\n" -#: docstrings.py:7 +#: docstrings.py:1 +#, docstring +msgid "Module docstring" +msgstr "" + +#: docstrings.py:9 #, docstring msgid "" msgstr "" -#: docstrings.py:18 +#: docstrings.py:15 +#, docstring +msgid "docstring" +msgstr "" + +#: docstrings.py:20 #, docstring msgid "" "multiline\n" -" docstring\n" -" " +"docstring" msgstr "" -#: docstrings.py:25 +#: docstrings.py:27 #, docstring msgid "docstring1" msgstr "" -#: docstrings.py:30 +#: docstrings.py:38 +#, docstring +msgid "nested docstring" +msgstr "" + +#: docstrings.py:43 #, docstring -msgid "Hello, {}!" +msgid "nested class docstring" msgstr "" diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py index 85d7f159d37..151a55a4b56 100644 --- a/Lib/test/test_tools/i18n_data/docstrings.py +++ b/Lib/test/test_tools/i18n_data/docstrings.py @@ -1,3 +1,5 @@ +"""Module docstring""" + # Test docstring extraction from gettext import gettext as _ @@ -10,10 +12,10 @@ def test(x): # Leading empty line def test2(x): - """docstring""" # XXX This should be extracted but isn't. + """docstring""" -# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`. +# Multiline docstrings are cleaned with `inspect.cleandoc`. def test3(x): """multiline docstring @@ -27,15 +29,15 @@ def test4(x): def test5(x): - """Hello, {}!""".format("world!") # XXX This should not be extracted. + """Hello, {}!""".format("world!") # This should not be extracted. # Nested docstrings def test6(x): def inner(y): - """nested docstring""" # XXX This should be extracted but isn't. + """nested docstring""" class Outer: class Inner: - "nested class docstring" # XXX This should be extracted but isn't. + "nested class docstring" diff --git a/Lib/test/test_tools/i18n_data/escapes.pot b/Lib/test/test_tools/i18n_data/escapes.pot index 2c7899d59da..4dfac0f451d 100644 --- a/Lib/test/test_tools/i18n_data/escapes.pot +++ b/Lib/test/test_tools/i18n_data/escapes.pot @@ -15,30 +15,36 @@ msgstr "" "Generated-By: pygettext.py 1.5\n" +#. Special characters that are always escaped in the POT file #: escapes.py:5 msgid "" "\"\t\n" "\r\\" msgstr "" +#. All ascii characters 0-31 #: escapes.py:8 msgid "" "\000\001\002\003\004\005\006\007\010\t\n" "\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" msgstr "" +#. All ascii characters 32-126 #: escapes.py:13 msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" msgstr "" +#. ascii char 127 #: escapes.py:17 msgid "\177" msgstr "" +#. some characters in the 128-255 range #: escapes.py:20 msgid "\302\200 \302\240 \303\277" msgstr "" +#. some characters >= 256 encoded as 2, 3 and 4 bytes, respectively #: escapes.py:23 msgid "\316\261 \343\204\261 \360\223\202\200" msgstr "" diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot index ddfbd18349e..e8167acfc07 100644 --- a/Lib/test/test_tools/i18n_data/messages.pot +++ b/Lib/test/test_tools/i18n_data/messages.pot @@ -15,53 +15,85 @@ msgstr "" "Generated-By: pygettext.py 1.5\n" -#: messages.py:5 +#: messages.py:16 msgid "" msgstr "" -#: messages.py:8 messages.py:9 +#: messages.py:19 messages.py:20 messages.py:21 msgid "parentheses" msgstr "" -#: messages.py:12 +#: messages.py:24 msgid "Hello, world!" msgstr "" -#: messages.py:15 +#: messages.py:27 msgid "" "Hello,\n" " multiline!\n" msgstr "" -#: messages.py:29 +#: messages.py:46 messages.py:89 messages.py:90 messages.py:93 messages.py:94 +#: messages.py:99 messages.py:100 messages.py:101 +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + +#: messages.py:47 +msgid "something" +msgstr "" + +#: messages.py:50 msgid "Hello, {}!" msgstr "" -#: messages.py:33 +#: messages.py:54 msgid "1" msgstr "" -#: messages.py:33 +#: messages.py:54 msgid "2" msgstr "" -#: messages.py:34 messages.py:35 +#: messages.py:55 messages.py:56 msgid "A" msgstr "" -#: messages.py:34 messages.py:35 +#: messages.py:55 messages.py:56 msgid "B" msgstr "" -#: messages.py:36 +#: messages.py:57 msgid "set" msgstr "" -#: messages.py:42 +#: messages.py:62 messages.py:63 msgid "nested string" msgstr "" -#: messages.py:47 +#: messages.py:68 msgid "baz" msgstr "" +#: messages.py:71 messages.py:75 +msgid "default value" +msgstr "" + +#: messages.py:91 messages.py:92 messages.py:95 messages.py:96 +msgctxt "context" +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + +#: messages.py:102 +msgid "domain foo" +msgstr "" + +#: messages.py:118 messages.py:119 +msgid "world" +msgid_plural "worlds" +msgstr[0] "" +msgstr[1] "" + diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py index f220294b8d5..9457bcb8611 100644 --- a/Lib/test/test_tools/i18n_data/messages.py +++ b/Lib/test/test_tools/i18n_data/messages.py @@ -1,5 +1,16 @@ # Test message extraction -from gettext import gettext as _ +from gettext import ( + gettext, + ngettext, + pgettext, + npgettext, + dgettext, + dngettext, + dpgettext, + dnpgettext +) + +_ = gettext # Empty string _("") @@ -7,6 +18,7 @@ # Extra parentheses (_("parentheses")) ((_("parentheses"))) +_(("parentheses")) # Multiline strings _("Hello, " @@ -21,13 +33,22 @@ _(None) _(1) _(False) -_(x="kwargs are not allowed") +_(["invalid"]) +_({"invalid"}) +_("string"[3]) +_("string"[:3]) +_({"string": "foo"}) + +# pygettext does not allow keyword arguments, but both xgettext and pybabel do +_(x="kwargs are not allowed!") + +# Unusual, but valid arguments _("foo", "bar") _("something", x="something else") # .format() _("Hello, {}!").format("world") # valid -_("Hello, {}!".format("world")) # invalid +_("Hello, {}!".format("world")) # invalid, but xgettext extracts the first string # Nested structures _("1"), _("2") @@ -38,7 +59,7 @@ # Nested functions and classes def test(): - _("nested string") # XXX This should be extracted but isn't. + _("nested string") [_("nested string")] @@ -47,11 +68,11 @@ def bar(self): return _("baz") -def bar(x=_('default value')): # XXX This should be extracted but isn't. +def bar(x=_('default value')): pass -def baz(x=[_('default value')]): # XXX This should be extracted but isn't. +def baz(x=[_('default value')]): pass @@ -62,3 +83,37 @@ def _(x): def _(x="don't extract me"): pass + + +# Other gettext functions +gettext("foo") +ngettext("foo", "foos", 1) +pgettext("context", "foo") +npgettext("context", "foo", "foos", 1) +dgettext("domain", "foo") +dngettext("domain", "foo", "foos", 1) +dpgettext("domain", "context", "foo") +dnpgettext("domain", "context", "foo", "foos", 1) + +# Complex arguments +ngettext("foo", "foos", 42 + (10 - 20)) +ngettext("foo", "foos", *args) +ngettext("foo", "foos", **kwargs) +dgettext(["some", {"complex"}, ("argument",)], "domain foo") + +# Invalid calls which are not extracted +gettext() +ngettext('foo') +pgettext('context') +npgettext('context', 'foo') +dgettext('domain') +dngettext('domain', 'foo') +dpgettext('domain', 'context') +dnpgettext('domain', 'context', 'foo') +dgettext(*args, 'foo') +dpgettext(*args, 'context', 'foo') +dnpgettext(*args, 'context', 'foo', 'foos') + +# f-strings +f"Hello, {_('world')}!" +f"Hello, {ngettext('world', 'worlds', 3)}!" diff --git a/Lib/test/test_tools/i18n_data/multiple_keywords.pot b/Lib/test/test_tools/i18n_data/multiple_keywords.pot new file mode 100644 index 00000000000..954cb8e9948 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/multiple_keywords.pot @@ -0,0 +1,38 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: multiple_keywords.py:3 +msgid "bar" +msgstr "" + +#: multiple_keywords.py:5 +msgctxt "baz" +msgid "qux" +msgstr "" + +#: multiple_keywords.py:9 +msgctxt "corge" +msgid "grault" +msgstr "" + +#: multiple_keywords.py:11 +msgctxt "xyzzy" +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + diff --git a/Lib/test/test_tools/i18n_data/multiple_keywords.py b/Lib/test/test_tools/i18n_data/multiple_keywords.py new file mode 100644 index 00000000000..7bde349505b --- /dev/null +++ b/Lib/test/test_tools/i18n_data/multiple_keywords.py @@ -0,0 +1,11 @@ +from gettext import gettext as foo + +foo('bar') + +foo('baz', 'qux') + +# The 't' specifier is not supported, so the following +# call is extracted as pgettext instead of ngettext. +foo('corge', 'grault', 1) + +foo('xyzzy', 'foo', 'foos', 1) diff --git a/Lib/test/test_tools/test_compute_changes.py b/Lib/test/test_tools/test_compute_changes.py new file mode 100644 index 00000000000..b20ff975fc2 --- /dev/null +++ b/Lib/test/test_tools/test_compute_changes.py @@ -0,0 +1,144 @@ +"""Tests to cover the Tools/build/compute-changes.py script.""" + +import importlib +import os +import unittest +from pathlib import Path +from unittest.mock import patch + +from test.test_tools import skip_if_missing, imports_under_tool + +skip_if_missing("build") + +with patch.dict(os.environ, {"GITHUB_DEFAULT_BRANCH": "main"}): + with imports_under_tool("build"): + compute_changes = importlib.import_module("compute-changes") + +process_changed_files = compute_changes.process_changed_files +Outputs = compute_changes.Outputs +ANDROID_DIRS = compute_changes.ANDROID_DIRS +IOS_DIRS = compute_changes.IOS_DIRS +MACOS_DIRS = compute_changes.MACOS_DIRS +WASI_DIRS = compute_changes.WASI_DIRS +RUN_TESTS_IGNORE = compute_changes.RUN_TESTS_IGNORE +UNIX_BUILD_SYSTEM_FILE_NAMES = compute_changes.UNIX_BUILD_SYSTEM_FILE_NAMES +LIBRARY_FUZZER_PATHS = compute_changes.LIBRARY_FUZZER_PATHS + + +class TestProcessChangedFiles(unittest.TestCase): + + def test_windows(self): + f = {Path(".github/workflows/reusable-windows.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_windows_tests) + + def test_docs(self): + for f in ( + ".github/workflows/reusable-docs.yml", + "Doc/library/datetime.rst", + "Doc/Makefile", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_docs) + self.assertFalse(result.run_tests) + + def test_ci_fuzz_stdlib(self): + for p in LIBRARY_FUZZER_PATHS: + with self.subTest(p=p): + if p.is_dir(): + f = p / "file" + elif p.is_file(): + f = p + else: + continue + result = process_changed_files({f}) + self.assertTrue(result.run_ci_fuzz_stdlib) + + def test_android(self): + for d in ANDROID_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_android) + self.assertFalse(result.run_windows_tests) + + def test_ios(self): + for d in IOS_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_ios) + self.assertFalse(result.run_windows_tests) + + def test_macos(self): + f = {Path(".github/workflows/reusable-macos.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_macos) + + for d in MACOS_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_macos) + self.assertFalse(result.run_windows_tests) + + def test_wasi(self): + f = {Path(".github/workflows/reusable-wasi.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + + for d in WASI_DIRS: + with self.subTest(d=d): + result = process_changed_files({d / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + self.assertFalse(result.run_windows_tests) + + def test_unix(self): + for f in UNIX_BUILD_SYSTEM_FILE_NAMES: + with self.subTest(f=f): + result = process_changed_files({f}) + self.assertTrue(result.run_tests) + self.assertFalse(result.run_windows_tests) + + def test_msi(self): + for f in ( + ".github/workflows/reusable-windows-msi.yml", + "Tools/msi/build.bat", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_windows_msi) + + def test_all_run(self): + for f in ( + ".github/workflows/some-new-workflow.yml", + ".github/workflows/build.yml", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_android) + self.assertTrue(result.run_ios) + self.assertTrue(result.run_macos) + self.assertTrue(result.run_ubuntu) + self.assertTrue(result.run_wasi) + + def test_all_ignored(self): + for f in RUN_TESTS_IGNORE: + with self.subTest(f=f): + self.assertEqual(process_changed_files({Path(f)}), Outputs()) + + def test_wasi_and_android(self): + f = {Path(".github/workflows/reusable-wasi.yml"), Path("Android/file")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index ffa1b1178ed..d1831d68f02 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -8,7 +8,7 @@ from pathlib import Path from test.support.script_helper import assert_python_ok -from test.test_tools import skip_if_missing, toolsdir +from test.test_tools import imports_under_tool, skip_if_missing, toolsdir from test.support.os_helper import temp_cwd, temp_dir @@ -17,6 +17,11 @@ DATA_DIR = Path(__file__).resolve().parent / 'i18n_data' +with imports_under_tool("i18n"): + from pygettext import (parse_spec, process_keywords, DEFAULTKEYWORDS, + unparse_spec) + + def normalize_POT_file(pot): """Normalize the POT creation timestamp, charset and file locations to make the POT file easier to compare. @@ -87,7 +92,8 @@ def assert_POT_equal(self, expected, actual): self.maxDiff = None self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual)) - def extract_from_str(self, module_content, *, args=(), strict=True): + def extract_from_str(self, module_content, *, args=(), strict=True, + with_stderr=False, raw=False): """Return all msgids extracted from module_content.""" filename = 'test.py' with temp_cwd(None): @@ -98,12 +104,19 @@ def extract_from_str(self, module_content, *, args=(), strict=True): self.assertEqual(res.err, b'') with open('messages.pot', encoding='utf-8') as fp: data = fp.read() - return self.get_msgids(data) + if not raw: + data = self.get_msgids(data) + if not with_stderr: + return data + return data, res.err def extract_docstrings_from_str(self, module_content): """Return all docstrings extracted from module_content.""" return self.extract_from_str(module_content, args=('--docstrings',), strict=False) + def get_stderr(self, module_content): + return self.extract_from_str(module_content, strict=False, with_stderr=True)[1] + def test_header(self): """Make sure the required fields are in the header, according to: http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry @@ -149,6 +162,14 @@ def test_POT_Creation_Date(self): # This will raise if the date format does not exactly match. datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z') + def test_output_option(self): + for opt in ('-o', '--output='): + with temp_cwd(): + assert_python_ok(self.script, f'{opt}test') + self.assertTrue(os.path.exists('test')) + res = assert_python_ok(self.script, f'{opt}-') + self.assertIn(b'Project-Id-Version: PACKAGE VERSION', res.out) + def test_funcdocstring(self): for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): with self.subTest(doc): @@ -332,14 +353,14 @@ def test_calls_in_fstring_with_multiple_args(self): msgids = self.extract_docstrings_from_str(dedent('''\ f"{_('foo', 'bar')}" ''')) - self.assertNotIn('foo', msgids) + self.assertIn('foo', msgids) self.assertNotIn('bar', msgids) def test_calls_in_fstring_with_keyword_args(self): msgids = self.extract_docstrings_from_str(dedent('''\ f"{_('foo', bar='baz')}" ''')) - self.assertNotIn('foo', msgids) + self.assertIn('foo', msgids) self.assertNotIn('bar', msgids) self.assertNotIn('baz', msgids) @@ -400,17 +421,195 @@ def test_files_list(self): self.assertIn(f'msgid "{text2}"', data) self.assertNotIn(text3, data) + def test_help_text(self): + """Test that the help text is displayed.""" + res = assert_python_ok(self.script, '--help') + self.assertEqual(res.out, b'') + self.assertIn(b'pygettext -- Python equivalent of xgettext(1)', res.err) + + def test_error_messages(self): + """Test that pygettext outputs error messages to stderr.""" + stderr = self.get_stderr(dedent('''\ + _(1+2) + ngettext('foo') + dgettext(*args, 'foo') + ''')) + + # Normalize line endings on Windows + stderr = stderr.decode('utf-8').replace('\r', '') + + self.assertEqual( + stderr, + "*** test.py:1: Expected a string constant for argument 1, got 1 + 2\n" + "*** test.py:2: Expected at least 2 positional argument(s) in gettext call, got 1\n" + "*** test.py:3: Variable positional arguments are not allowed in gettext calls\n" + ) + + def test_extract_all_comments(self): + """ + Test that the --add-comments option without an + explicit tag extracts all translator comments. + """ + for arg in ('--add-comments', '-c'): + with self.subTest(arg=arg): + data = self.extract_from_str(dedent('''\ + # Translator comment + _("foo") + '''), args=(arg,), raw=True) + self.assertIn('#. Translator comment', data) + + def test_comments_with_multiple_tags(self): + """ + Test that multiple --add-comments tags can be specified. + """ + for arg in ('--add-comments={}', '-c{}'): + with self.subTest(arg=arg): + args = (arg.format('foo:'), arg.format('bar:')) + data = self.extract_from_str(dedent('''\ + # foo: comment + _("foo") + + # bar: comment + _("bar") + + # baz: comment + _("baz") + '''), args=args, raw=True) + self.assertIn('#. foo: comment', data) + self.assertIn('#. bar: comment', data) + self.assertNotIn('#. baz: comment', data) + + def test_comments_not_extracted_without_tags(self): + """ + Test that translator comments are not extracted without + specifying --add-comments. + """ + data = self.extract_from_str(dedent('''\ + # Translator comment + _("foo") + '''), raw=True) + self.assertNotIn('#.', data) + + def test_parse_keyword_spec(self): + valid = ( + ('foo', ('foo', {'msgid': 0})), + ('foo:1', ('foo', {'msgid': 0})), + ('foo:1,2', ('foo', {'msgid': 0, 'msgid_plural': 1})), + ('foo:1, 2', ('foo', {'msgid': 0, 'msgid_plural': 1})), + ('foo:1,2c', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:2c,1', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:2c ,1', ('foo', {'msgid': 0, 'msgctxt': 1})), + ('foo:1,2,3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), + ('foo:1, 2, 3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), + ('foo:3c,1,2', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})), + ) + for spec, expected in valid: + with self.subTest(spec=spec): + self.assertEqual(parse_spec(spec), expected) + # test unparse-parse round-trip + self.assertEqual(parse_spec(unparse_spec(*expected)), expected) + + invalid = ( + ('foo:', "Invalid keyword spec 'foo:': missing argument positions"), + ('foo:bar', "Invalid keyword spec 'foo:bar': position is not an integer"), + ('foo:0', "Invalid keyword spec 'foo:0': argument positions must be strictly positive"), + ('foo:-2', "Invalid keyword spec 'foo:-2': argument positions must be strictly positive"), + ('foo:1,1', "Invalid keyword spec 'foo:1,1': duplicate positions"), + ('foo:1,2,1', "Invalid keyword spec 'foo:1,2,1': duplicate positions"), + ('foo:1c,2,1c', "Invalid keyword spec 'foo:1c,2,1c': duplicate positions"), + ('foo:1c,2,3c', "Invalid keyword spec 'foo:1c,2,3c': msgctxt can only appear once"), + ('foo:1,2,3', "Invalid keyword spec 'foo:1,2,3': too many positions"), + ('foo:1c', "Invalid keyword spec 'foo:1c': msgctxt cannot appear without msgid"), + ) + for spec, message in invalid: + with self.subTest(spec=spec): + with self.assertRaises(ValueError) as cm: + parse_spec(spec) + self.assertEqual(str(cm.exception), message) + + def test_process_keywords(self): + default_keywords = {name: [spec] for name, spec + in DEFAULTKEYWORDS.items()} + inputs = ( + (['foo'], True), + (['_:1,2'], True), + (['foo', 'foo:1,2'], True), + (['foo'], False), + (['_:1,2', '_:1c,2,3', 'pgettext'], False), + # Duplicate entries + (['foo', 'foo'], True), + (['_'], False) + ) + expected = ( + {'foo': [{'msgid': 0}]}, + {'_': [{'msgid': 0, 'msgid_plural': 1}]}, + {'foo': [{'msgid': 0}, {'msgid': 0, 'msgid_plural': 1}]}, + default_keywords | {'foo': [{'msgid': 0}]}, + default_keywords | {'_': [{'msgid': 0, 'msgid_plural': 1}, + {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2}, + {'msgid': 0}], + 'pgettext': [{'msgid': 0}, + {'msgctxt': 0, 'msgid': 1}]}, + {'foo': [{'msgid': 0}]}, + default_keywords, + ) + for (keywords, no_default_keywords), expected in zip(inputs, expected): + with self.subTest(keywords=keywords, + no_default_keywords=no_default_keywords): + processed = process_keywords( + keywords, + no_default_keywords=no_default_keywords) + self.assertEqual(processed, expected) + + def test_multiple_keywords_same_funcname_errors(self): + # If at least one keyword spec for a given funcname matches, + # no error should be printed. + msgids, stderr = self.extract_from_str(dedent('''\ + _("foo", 42) + _(42, "bar") + '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True) + self.assertIn('foo', msgids) + self.assertIn('bar', msgids) + self.assertEqual(stderr, b'') + + # If no keyword spec for a given funcname matches, + # all errors are printed. + msgids, stderr = self.extract_from_str(dedent('''\ + _(x, 42) + _(42, y) + '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True, + strict=False) + self.assertEqual(msgids, ['']) + # Normalize line endings on Windows + stderr = stderr.decode('utf-8').replace('\r', '') + self.assertEqual( + stderr, + '*** test.py:1: No keywords matched gettext call "_":\n' + '\tkeyword="_": Expected a string constant for argument 1, got x\n' + '\tkeyword="_:2": Expected a string constant for argument 2, got 42\n' + '*** test.py:2: No keywords matched gettext call "_":\n' + '\tkeyword="_": Expected a string constant for argument 1, got 42\n' + '\tkeyword="_:2": Expected a string constant for argument 2, got y\n') + def extract_from_snapshots(): snapshots = { - 'messages.py': ('--docstrings',), + 'messages.py': (), 'fileloc.py': ('--docstrings',), 'docstrings.py': ('--docstrings',), + 'comments.py': ('--add-comments=i18n:',), + 'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2', + '--keyword=pfoo:1c,2', + '--keyword=npfoo:1c,2,3', '--keyword=_:1,2'), + 'multiple_keywords.py': ('--keyword=foo:1c,2,3', '--keyword=foo:1c,2', + '--keyword=foo:1,2', + # repeat a keyword to make sure it is extracted only once + '--keyword=foo', '--keyword=foo'), # == Test character escaping # Escape ascii and unicode: - 'escapes.py': ('--escape',), + 'escapes.py': ('--escape', '--add-comments='), # Escape only ascii and let unicode pass through: - ('escapes.py', 'ascii-escapes.pot'): (), + ('escapes.py', 'ascii-escapes.pot'): ('--add-comments=',), } for filename, args in snapshots.items(): diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 4c7588d4d93..31a51606739 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -48,15 +48,18 @@ def test_makefile_test_folders(self): if dirname == '__pycache__' or dirname.startswith('.'): dirs.clear() # do not process subfolders continue - # Skip empty dirs: + + # Skip empty dirs (ignoring hidden files and __pycache__): + files = [ + filename for filename in files + if not filename.startswith('.') + ] + dirs = [ + dirname for dirname in dirs + if not dirname.startswith('.') and dirname != "__pycache__" + ] if not dirs and not files: continue - # Skip dirs with hidden-only files: - if files and all( - filename.startswith('.') or filename == '__pycache__' - for filename in files - ): - continue relpath = os.path.relpath(dirpath, support.STDLIB_DIR) with self.subTest(relpath=relpath): diff --git a/Lib/test/test_tools/test_msgfmt.py b/Lib/test/test_tools/test_msgfmt.py index 8cd31680f76..7be606bbff6 100644 --- a/Lib/test/test_tools/test_msgfmt.py +++ b/Lib/test/test_tools/test_msgfmt.py @@ -1,6 +1,7 @@ """Tests for the Tools/i18n/msgfmt.py tool.""" import json +import struct import sys import unittest from gettext import GNUTranslations @@ -8,18 +9,21 @@ from test.support.os_helper import temp_cwd from test.support.script_helper import assert_python_failure, assert_python_ok -from test.test_tools import skip_if_missing, toolsdir +from test.test_tools import imports_under_tool, skip_if_missing, toolsdir skip_if_missing('i18n') data_dir = (Path(__file__).parent / 'msgfmt_data').resolve() script_dir = Path(toolsdir) / 'i18n' -msgfmt = script_dir / 'msgfmt.py' +msgfmt_py = script_dir / 'msgfmt.py' + +with imports_under_tool("i18n"): + import msgfmt def compile_messages(po_file, mo_file): - assert_python_ok(msgfmt, '-o', mo_file, po_file) + assert_python_ok(msgfmt_py, '-o', mo_file, po_file) class CompilationTest(unittest.TestCase): @@ -40,6 +44,31 @@ def test_compilation(self): self.assertDictEqual(actual._catalog, expected._catalog) + def test_binary_header(self): + with temp_cwd(): + tmp_mo_file = 'messages.mo' + compile_messages(data_dir / "general.po", tmp_mo_file) + with open(tmp_mo_file, 'rb') as f: + mo_data = f.read() + + ( + magic, + version, + num_strings, + orig_table_offset, + trans_table_offset, + hash_table_size, + hash_table_offset, + ) = struct.unpack("=7I", mo_data[:28]) + + self.assertEqual(magic, 0x950412de) + self.assertEqual(version, 0) + self.assertEqual(num_strings, 9) + self.assertEqual(orig_table_offset, 28) + self.assertEqual(trans_table_offset, 100) + self.assertEqual(hash_table_size, 0) + self.assertEqual(hash_table_offset, 0) + def test_translations(self): with open(data_dir / 'general.mo', 'rb') as f: t = GNUTranslations(f) @@ -62,6 +91,14 @@ def test_translations(self): '%d emails sent.', 2), '%d emails sent.') + def test_po_with_bom(self): + with temp_cwd(): + Path('bom.po').write_bytes(b'\xef\xbb\xbfmsgid "Python"\nmsgstr "Pioton"\n') + + res = assert_python_failure(msgfmt_py, 'bom.po') + err = res.err.decode('utf-8') + self.assertIn('The file bom.po starts with a UTF-8 BOM', err) + def test_invalid_msgid_plural(self): with temp_cwd(): Path('invalid.po').write_text('''\ @@ -69,7 +106,7 @@ def test_invalid_msgid_plural(self): msgstr[0] "singular" ''') - res = assert_python_failure(msgfmt, 'invalid.po') + res = assert_python_failure(msgfmt_py, 'invalid.po') err = res.err.decode('utf-8') self.assertIn('msgid_plural not preceded by msgid', err) @@ -80,7 +117,7 @@ def test_plural_without_msgid_plural(self): msgstr[0] "bar" ''') - res = assert_python_failure(msgfmt, 'invalid.po') + res = assert_python_failure(msgfmt_py, 'invalid.po') err = res.err.decode('utf-8') self.assertIn('plural without msgid_plural', err) @@ -92,7 +129,7 @@ def test_indexed_msgstr_without_msgid_plural(self): msgstr "bar" ''') - res = assert_python_failure(msgfmt, 'invalid.po') + res = assert_python_failure(msgfmt_py, 'invalid.po') err = res.err.decode('utf-8') self.assertIn('indexed msgstr required for plural', err) @@ -102,38 +139,136 @@ def test_generic_syntax_error(self): "foo" ''') - res = assert_python_failure(msgfmt, 'invalid.po') + res = assert_python_failure(msgfmt_py, 'invalid.po') err = res.err.decode('utf-8') self.assertIn('Syntax error', err) + +class POParserTest(unittest.TestCase): + @classmethod + def tearDownClass(cls): + # msgfmt uses a global variable to store messages, + # clear it after the tests. + msgfmt.MESSAGES.clear() + + def test_strings(self): + # Test that the PO parser correctly handles and unescape + # strings in the PO file. + # The PO file format allows for a variety of escape sequences, + # octal and hex escapes. + valid_strings = ( + # empty strings + ('""', ''), + ('"" "" ""', ''), + # allowed escape sequences + (r'"\\"', '\\'), + (r'"\""', '"'), + (r'"\t"', '\t'), + (r'"\n"', '\n'), + (r'"\r"', '\r'), + (r'"\f"', '\f'), + (r'"\a"', '\a'), + (r'"\b"', '\b'), + (r'"\v"', '\v'), + # non-empty strings + ('"foo"', 'foo'), + ('"foo" "bar"', 'foobar'), + ('"foo""bar"', 'foobar'), + ('"" "foo" ""', 'foo'), + # newlines and tabs + (r'"foo\nbar"', 'foo\nbar'), + (r'"foo\n" "bar"', 'foo\nbar'), + (r'"foo\tbar"', 'foo\tbar'), + (r'"foo\t" "bar"', 'foo\tbar'), + # escaped quotes + (r'"foo\"bar"', 'foo"bar'), + (r'"foo\"" "bar"', 'foo"bar'), + (r'"foo\\" "bar"', 'foo\\bar'), + # octal escapes + (r'"\120\171\164\150\157\156"', 'Python'), + (r'"\120\171\164" "\150\157\156"', 'Python'), + (r'"\"\120\171\164" "\150\157\156\""', '"Python"'), + # hex escapes + (r'"\x50\x79\x74\x68\x6f\x6e"', 'Python'), + (r'"\x50\x79\x74" "\x68\x6f\x6e"', 'Python'), + (r'"\"\x50\x79\x74" "\x68\x6f\x6e\""', '"Python"'), + ) + + with temp_cwd(): + for po_string, expected in valid_strings: + with self.subTest(po_string=po_string): + # Construct a PO file with a single entry, + # compile it, read it into a catalog and + # check the result. + po = f'msgid {po_string}\nmsgstr "translation"' + Path('messages.po').write_text(po) + # Reset the global MESSAGES dictionary + msgfmt.MESSAGES.clear() + msgfmt.make('messages.po', 'messages.mo') + + with open('messages.mo', 'rb') as f: + actual = GNUTranslations(f) + + self.assertDictEqual(actual._catalog, {expected: 'translation'}) + + invalid_strings = ( + # "''", # invalid but currently accepted + '"', + '"""', + '"" "', + 'foo', + '"" "foo', + '"foo" foo', + '42', + '"" 42 ""', + # disallowed escape sequences + # r'"\'"', # invalid but currently accepted + # r'"\e"', # invalid but currently accepted + # r'"\8"', # invalid but currently accepted + # r'"\9"', # invalid but currently accepted + r'"\x"', + r'"\u1234"', + r'"\N{ROMAN NUMERAL NINE}"' + ) + with temp_cwd(): + for invalid_string in invalid_strings: + with self.subTest(string=invalid_string): + po = f'msgid {invalid_string}\nmsgstr "translation"' + Path('messages.po').write_text(po) + # Reset the global MESSAGES dictionary + msgfmt.MESSAGES.clear() + with self.assertRaises(Exception): + msgfmt.make('messages.po', 'messages.mo') + + class CLITest(unittest.TestCase): def test_help(self): for option in ('--help', '-h'): - res = assert_python_ok(msgfmt, option) + res = assert_python_ok(msgfmt_py, option) err = res.err.decode('utf-8') self.assertIn('Generate binary message catalog from textual translation description.', err) def test_version(self): for option in ('--version', '-V'): - res = assert_python_ok(msgfmt, option) + res = assert_python_ok(msgfmt_py, option) out = res.out.decode('utf-8').strip() self.assertEqual('msgfmt.py 1.2', out) def test_invalid_option(self): - res = assert_python_failure(msgfmt, '--invalid-option') + res = assert_python_failure(msgfmt_py, '--invalid-option') err = res.err.decode('utf-8') self.assertIn('Generate binary message catalog from textual translation description.', err) self.assertIn('option --invalid-option not recognized', err) def test_no_input_file(self): - res = assert_python_ok(msgfmt) + res = assert_python_ok(msgfmt_py) err = res.err.decode('utf-8').replace('\r\n', '\n') self.assertIn('No input file given\n' "Try `msgfmt --help' for more information.", err) def test_nonexistent_file(self): - assert_python_failure(msgfmt, 'nonexistent.po') + assert_python_failure(msgfmt_py, 'nonexistent.po') def update_catalog_snapshots(): diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py new file mode 100644 index 00000000000..7469a1047f8 --- /dev/null +++ b/Lib/test/test_type_cache.py @@ -0,0 +1,265 @@ +""" Tests for the internal type cache in CPython. """ +import dis +import unittest +import warnings +from test import support +from test.support import import_helper, requires_specialization, requires_specialization_ft +try: + from sys import _clear_type_cache +except ImportError: + _clear_type_cache = None + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") +type_get_version = _testcapi.type_get_version +type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe +type_assign_version = _testcapi.type_assign_version +type_modified = _testcapi.type_modified + +def clear_type_cache(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + _clear_type_cache() + +@support.cpython_only +@unittest.skipIf(_clear_type_cache is None, "requires sys._clear_type_cache") +class TypeCacheTests(unittest.TestCase): + def test_tp_version_tag_unique(self): + """tp_version_tag should be unique assuming no overflow, even after + clearing type cache. + """ + # Check if global version tag has already overflowed. + Y = type('Y', (), {}) + Y.x = 1 + Y.x # Force a _PyType_Lookup, populating version tag + y_ver = type_get_version(Y) + # Overflow, or not enough left to conduct the test. + if y_ver == 0 or y_ver > 0xFFFFF000: + self.skipTest("Out of type version tags") + # Note: try to avoid any method lookups within this loop, + # It will affect global version tag. + all_version_tags = [] + append_result = all_version_tags.append + assertNotEqual = self.assertNotEqual + for _ in range(30): + clear_type_cache() + X = type('Y', (), {}) + X.x = 1 + X.x + tp_version_tag_after = type_get_version(X) + assertNotEqual(tp_version_tag_after, 0, msg="Version overflowed") + append_result(tp_version_tag_after) + self.assertEqual(len(set(all_version_tags)), 30, + msg=f"{all_version_tags} contains non-unique versions") + + def test_type_assign_version(self): + class C: + x = 5 + + self.assertEqual(type_assign_version(C), 1) + c_ver = type_get_version(C) + + C.x = 6 + self.assertEqual(type_get_version(C), 0) + self.assertEqual(type_assign_version(C), 1) + self.assertNotEqual(type_get_version(C), 0) + self.assertNotEqual(type_get_version(C), c_ver) + + def test_type_assign_specific_version(self): + """meta-test for type_assign_specific_version_unsafe""" + class C: + pass + + type_assign_version(C) + orig_version = type_get_version(C) + if orig_version == 0: + self.skipTest("Could not assign a valid type version") + + type_modified(C) + type_assign_specific_version_unsafe(C, orig_version + 5) + type_assign_version(C) # this should do nothing + + new_version = type_get_version(C) + self.assertEqual(new_version, orig_version + 5) + + clear_type_cache() + + def test_per_class_limit(self): + class C: + x = 0 + + type_assign_version(C) + orig_version = type_get_version(C) + for i in range(1001): + C.x = i + type_assign_version(C) + + new_version = type_get_version(C) + self.assertEqual(new_version, 0) + + def test_119462(self): + + class Holder: + value = None + + @classmethod + def set_value(cls): + cls.value = object() + + class HolderSub(Holder): + pass + + for _ in range(1050): + Holder.set_value() + HolderSub.value + +@support.cpython_only +class TypeCacheWithSpecializationTests(unittest.TestCase): + def tearDown(self): + clear_type_cache() + + def _assign_valid_version_or_skip(self, type_): + type_modified(type_) + type_assign_version(type_) + if type_get_version(type_) == 0: + self.skipTest("Could not assign valid type version") + + def _no_more_versions(self, user_type): + type_modified(user_type) + for _ in range(1001): + type_assign_specific_version_unsafe(user_type, 1000_000_000) + type_assign_specific_version_unsafe(user_type, 0) + self.assertEqual(type_get_version(user_type), 0) + + def _all_opnames(self, func): + return set(instr.opname for instr in dis.Bytecode(func, adaptive=True)) + + def _check_specialization(self, func, arg, opname, *, should_specialize): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + func(arg) + + if should_specialize: + self.assertNotIn(opname, self._all_opnames(func)) + else: + self.assertIn(opname, self._all_opnames(func)) + + @requires_specialization + def test_class_load_attr_specialization_user_type(self): + class A: + def foo(self): + pass + + self._assign_valid_version_or_skip(A) + + def load_foo_1(type_): + type_.foo + + self._check_specialization(load_foo_1, A, "LOAD_ATTR", should_specialize=True) + del load_foo_1 + + self._no_more_versions(A) + + def load_foo_2(type_): + return type_.foo + + self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) + + @requires_specialization + def test_class_load_attr_specialization_static_type(self): + self.assertNotEqual(type_get_version(str), 0) + self.assertNotEqual(type_get_version(bytes), 0) + + def get_capitalize_1(type_): + return type_.capitalize + + self._check_specialization(get_capitalize_1, str, "LOAD_ATTR", should_specialize=True) + self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') + self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') + + @requires_specialization + def test_property_load_attr_specialization_user_type(self): + class G: + @property + def x(self): + return 9 + + self._assign_valid_version_or_skip(G) + + def load_x_1(instance): + instance.x + + self._check_specialization(load_x_1, G(), "LOAD_ATTR", should_specialize=True) + del load_x_1 + + self._no_more_versions(G) + + def load_x_2(instance): + instance.x + + self._check_specialization(load_x_2, G(), "LOAD_ATTR", should_specialize=False) + + @requires_specialization + def test_store_attr_specialization_user_type(self): + class B: + __slots__ = ("bar",) + + self._assign_valid_version_or_skip(B) + + def store_bar_1(type_): + type_.bar = 10 + + self._check_specialization(store_bar_1, B(), "STORE_ATTR", should_specialize=True) + del store_bar_1 + + self._no_more_versions(B) + + def store_bar_2(type_): + type_.bar = 10 + + self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False) + + @requires_specialization_ft + def test_class_call_specialization_user_type(self): + class F: + def __init__(self): + pass + + self._assign_valid_version_or_skip(F) + + def call_class_1(type_): + type_() + + self._check_specialization(call_class_1, F, "CALL", should_specialize=True) + del call_class_1 + + self._no_more_versions(F) + + def call_class_2(type_): + type_() + + self._check_specialization(call_class_2, F, "CALL", should_specialize=False) + + @requires_specialization + def test_to_bool_specialization_user_type(self): + class H: + pass + + self._assign_valid_version_or_skip(H) + + def to_bool_1(instance): + not instance + + self._check_specialization(to_bool_1, H(), "TO_BOOL", should_specialize=True) + del to_bool_1 + + self._no_more_versions(H) + + def to_bool_2(instance): + not instance + + self._check_specialization(to_bool_2, H(), "TO_BOOL", should_specialize=False) + + +if __name__ == "__main__": + unittest.main() From 41c98b17805162c3a0d29fdc306cfaff8ebe0ee3 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 17 May 2026 15:50:51 +0300 Subject: [PATCH 3/4] Remove `test_capi` --- Lib/test/test_capi/__init__.py | 12 - Lib/test/test_capi/__main__.py | 3 - Lib/test/test_capi/check_config.py | 77 - Lib/test/test_capi/test_abstract.py | 1082 -------- Lib/test/test_capi/test_bytearray.py | 174 -- Lib/test/test_capi/test_bytes.py | 303 --- Lib/test/test_capi/test_codecs.py | 879 ------- Lib/test/test_capi/test_complex.py | 299 --- Lib/test/test_capi/test_config.py | 404 --- Lib/test/test_capi/test_dict.py | 550 ---- Lib/test/test_capi/test_emscripten.py | 25 - Lib/test/test_capi/test_eval.py | 103 - Lib/test/test_capi/test_eval_code_ex.py | 126 - Lib/test/test_capi/test_exceptions.py | 680 ----- Lib/test/test_capi/test_file.py | 325 --- Lib/test/test_capi/test_float.py | 259 -- Lib/test/test_capi/test_frame.py | 56 - Lib/test/test_capi/test_function.py | 340 --- Lib/test/test_capi/test_getargs.py | 1421 ----------- Lib/test/test_capi/test_hash.py | 93 - Lib/test/test_capi/test_immortal.py | 42 - Lib/test/test_capi/test_import.py | 381 --- Lib/test/test_capi/test_list.py | 359 --- Lib/test/test_capi/test_long.py | 818 ------ Lib/test/test_capi/test_mem.py | 183 -- Lib/test/test_capi/test_misc.py | 2922 ---------------------- Lib/test/test_capi/test_number.py | 352 --- Lib/test/test_capi/test_object.py | 258 -- Lib/test/test_capi/test_opt.py | 2021 --------------- Lib/test/test_capi/test_pyatomic.py | 15 - Lib/test/test_capi/test_run.py | 118 - Lib/test/test_capi/test_set.py | 271 -- Lib/test/test_capi/test_structmembers.py | 186 -- Lib/test/test_capi/test_sys.py | 154 -- Lib/test/test_capi/test_time.py | 78 - Lib/test/test_capi/test_tuple.py | 301 --- Lib/test/test_capi/test_type.py | 276 -- Lib/test/test_capi/test_unicode.py | 2039 --------------- Lib/test/test_capi/test_watchers.py | 678 ----- 39 files changed, 18663 deletions(-) delete mode 100644 Lib/test/test_capi/__init__.py delete mode 100644 Lib/test/test_capi/__main__.py delete mode 100644 Lib/test/test_capi/check_config.py delete mode 100644 Lib/test/test_capi/test_abstract.py delete mode 100644 Lib/test/test_capi/test_bytearray.py delete mode 100644 Lib/test/test_capi/test_bytes.py delete mode 100644 Lib/test/test_capi/test_codecs.py delete mode 100644 Lib/test/test_capi/test_complex.py delete mode 100644 Lib/test/test_capi/test_config.py delete mode 100644 Lib/test/test_capi/test_dict.py delete mode 100644 Lib/test/test_capi/test_emscripten.py delete mode 100644 Lib/test/test_capi/test_eval.py delete mode 100644 Lib/test/test_capi/test_eval_code_ex.py delete mode 100644 Lib/test/test_capi/test_exceptions.py delete mode 100644 Lib/test/test_capi/test_file.py delete mode 100644 Lib/test/test_capi/test_float.py delete mode 100644 Lib/test/test_capi/test_frame.py delete mode 100644 Lib/test/test_capi/test_function.py delete mode 100644 Lib/test/test_capi/test_getargs.py delete mode 100644 Lib/test/test_capi/test_hash.py delete mode 100644 Lib/test/test_capi/test_immortal.py delete mode 100644 Lib/test/test_capi/test_import.py delete mode 100644 Lib/test/test_capi/test_list.py delete mode 100644 Lib/test/test_capi/test_long.py delete mode 100644 Lib/test/test_capi/test_mem.py delete mode 100644 Lib/test/test_capi/test_misc.py delete mode 100644 Lib/test/test_capi/test_number.py delete mode 100644 Lib/test/test_capi/test_object.py delete mode 100644 Lib/test/test_capi/test_opt.py delete mode 100644 Lib/test/test_capi/test_pyatomic.py delete mode 100644 Lib/test/test_capi/test_run.py delete mode 100644 Lib/test/test_capi/test_set.py delete mode 100644 Lib/test/test_capi/test_structmembers.py delete mode 100644 Lib/test/test_capi/test_sys.py delete mode 100644 Lib/test/test_capi/test_time.py delete mode 100644 Lib/test/test_capi/test_tuple.py delete mode 100644 Lib/test/test_capi/test_type.py delete mode 100644 Lib/test/test_capi/test_unicode.py delete mode 100644 Lib/test/test_capi/test_watchers.py diff --git a/Lib/test/test_capi/__init__.py b/Lib/test/test_capi/__init__.py deleted file mode 100644 index 5a8ba684539..00000000000 --- a/Lib/test/test_capi/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -import unittest -from test.support import load_package_tests -from test.support import TEST_MODULES_ENABLED - - -if not TEST_MODULES_ENABLED: - raise unittest.SkipTest("requires test modules") - - -def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_capi/__main__.py b/Lib/test/test_capi/__main__.py deleted file mode 100644 index 05d01775ddf..00000000000 --- a/Lib/test/test_capi/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -import unittest - -unittest.main('test.test_capi') diff --git a/Lib/test/test_capi/check_config.py b/Lib/test/test_capi/check_config.py deleted file mode 100644 index bc4f3a0beb9..00000000000 --- a/Lib/test/test_capi/check_config.py +++ /dev/null @@ -1,77 +0,0 @@ -# This script is used by test_misc. - -import _imp -import _testinternalcapi -import json -import os -import sys - - -def import_singlephase(): - assert '_testsinglephase' not in sys.modules - try: - import _testsinglephase # noqa: F401 - except ImportError: - sys.modules.pop('_testsinglephase', None) - return False - else: - del sys.modules['_testsinglephase'] - return True - - -def check_singlephase(override): - # Check using the default setting. - settings_initial = _testinternalcapi.get_interp_settings() - allowed_initial = import_singlephase() - assert(_testinternalcapi.get_interp_settings() == settings_initial) - - # Apply the override and check. - override_initial = _imp._override_multi_interp_extensions_check(override) - settings_after = _testinternalcapi.get_interp_settings() - allowed_after = import_singlephase() - - # Apply the override again and check. - noop = {} - override_after = _imp._override_multi_interp_extensions_check(override) - settings_noop = _testinternalcapi.get_interp_settings() - if settings_noop != settings_after: - noop['settings_noop'] = settings_noop - allowed_noop = import_singlephase() - if allowed_noop != allowed_after: - noop['allowed_noop'] = allowed_noop - - # Restore the original setting and check. - override_noop = _imp._override_multi_interp_extensions_check(override_initial) - if override_noop != override_after: - noop['override_noop'] = override_noop - settings_restored = _testinternalcapi.get_interp_settings() - allowed_restored = import_singlephase() - - # Restore the original setting again. - override_restored = _imp._override_multi_interp_extensions_check(override_initial) - assert(_testinternalcapi.get_interp_settings() == settings_restored) - - return dict({ - 'requested': override, - 'override__initial': override_initial, - 'override_after': override_after, - 'override_restored': override_restored, - 'settings__initial': settings_initial, - 'settings_after': settings_after, - 'settings_restored': settings_restored, - 'allowed__initial': allowed_initial, - 'allowed_after': allowed_after, - 'allowed_restored': allowed_restored, - }, **noop) - - -def run_singlephase_check(override, outfd): - with os.fdopen(outfd, 'w') as outfile: - sys.stdout = outfile - sys.stderr = outfile - try: - results = check_singlephase(override) - json.dump(results, outfile) - finally: - sys.stdout = sys.__stdout__ - sys.stderr = sys.__stderr__ diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py deleted file mode 100644 index 7d548ae87c0..00000000000 --- a/Lib/test/test_capi/test_abstract.py +++ /dev/null @@ -1,1082 +0,0 @@ -import unittest -from collections import OrderedDict -from test import support -from test.support import import_helper - -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') -from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX - -NULL = None - -class StrSubclass(str): - pass - -class BytesSubclass(bytes): - pass - -class WithStr: - def __init__(self, value): - self.value = value - def __str__(self): - return self.value - -class WithRepr: - def __init__(self, value): - self.value = value - def __repr__(self): - return self.value - -class WithBytes: - def __init__(self, value): - self.value = value - def __bytes__(self): - return self.value - -class TestObject: - @property - def evil(self): - raise RuntimeError('do not get evil') - @evil.setter - def evil(self, value): - raise RuntimeError('do not set evil') - @evil.deleter - def evil(self): - raise RuntimeError('do not del evil') - -class ProxyGetItem: - def __init__(self, obj): - self.obj = obj - def __getitem__(self, key): - return self.obj[key] - -class ProxySetItem: - def __init__(self, obj): - self.obj = obj - def __setitem__(self, key, value): - self.obj[key] = value - -class ProxyDelItem: - def __init__(self, obj): - self.obj = obj - def __delitem__(self, key): - del self.obj[key] - -def gen(): - yield 'a' - yield 'b' - yield 'c' - - -class CAPITest(unittest.TestCase): - def assertTypedEqual(self, actual, expected): - self.assertIs(type(actual), type(expected)) - self.assertEqual(actual, expected) - - def test_object_str(self): - # Test PyObject_Str() - object_str = _testlimitedcapi.object_str - self.assertTypedEqual(object_str(''), '') - self.assertTypedEqual(object_str('abc'), 'abc') - self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') - self.assertTypedEqual(object_str(StrSubclass('abc')), 'abc') - self.assertTypedEqual(object_str(WithStr('abc')), 'abc') - self.assertTypedEqual(object_str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) - self.assertTypedEqual(object_str(WithRepr('')), '') - self.assertTypedEqual(object_str(WithRepr(StrSubclass(''))), StrSubclass('')) - self.assertTypedEqual(object_str(NULL), '') - - def test_object_repr(self): - # Test PyObject_Repr() - object_repr = _testlimitedcapi.object_repr - self.assertTypedEqual(object_repr(''), "''") - self.assertTypedEqual(object_repr('abc'), "'abc'") - self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") - self.assertTypedEqual(object_repr(StrSubclass('abc')), "'abc'") - self.assertTypedEqual(object_repr(WithRepr('')), '') - self.assertTypedEqual(object_repr(WithRepr(StrSubclass(''))), StrSubclass('')) - self.assertTypedEqual(object_repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') - self.assertTypedEqual(object_repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) - self.assertTypedEqual(object_repr(NULL), '') - - def test_object_ascii(self): - # Test PyObject_ASCII() - object_ascii = _testlimitedcapi.object_ascii - self.assertTypedEqual(object_ascii(''), "''") - self.assertTypedEqual(object_ascii('abc'), "'abc'") - self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") - self.assertTypedEqual(object_ascii(StrSubclass('abc')), "'abc'") - self.assertTypedEqual(object_ascii(WithRepr('')), '') - self.assertTypedEqual(object_ascii(WithRepr(StrSubclass(''))), StrSubclass('')) - self.assertTypedEqual(object_ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') - self.assertTypedEqual(object_ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') - self.assertTypedEqual(object_ascii(NULL), '') - - def test_object_bytes(self): - # Test PyObject_Bytes() - object_bytes = _testlimitedcapi.object_bytes - self.assertTypedEqual(object_bytes(b''), b'') - self.assertTypedEqual(object_bytes(b'abc'), b'abc') - self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') - self.assertTypedEqual(object_bytes(WithBytes(b'abc')), b'abc') - self.assertTypedEqual(object_bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) - self.assertTypedEqual(object_bytes(bytearray(b'abc')), b'abc') - self.assertTypedEqual(object_bytes(memoryview(b'abc')), b'abc') - self.assertTypedEqual(object_bytes([97, 98, 99]), b'abc') - self.assertTypedEqual(object_bytes((97, 98, 99)), b'abc') - self.assertTypedEqual(object_bytes(iter([97, 98, 99])), b'abc') - self.assertRaises(TypeError, object_bytes, WithBytes(bytearray(b'abc'))) - self.assertRaises(TypeError, object_bytes, WithBytes([97, 98, 99])) - self.assertRaises(TypeError, object_bytes, 3) - self.assertRaises(TypeError, object_bytes, 'abc') - self.assertRaises(TypeError, object_bytes, object()) - self.assertTypedEqual(object_bytes(NULL), b'') - - def test_object_getattr(self): - xgetattr = _testlimitedcapi.object_getattr - obj = TestObject() - obj.a = 11 - setattr(obj, '\U0001f40d', 22) - self.assertEqual(xgetattr(obj, 'a'), 11) - self.assertRaises(AttributeError, xgetattr, obj, 'b') - self.assertEqual(xgetattr(obj, '\U0001f40d'), 22) - - self.assertRaises(RuntimeError, xgetattr, obj, 'evil') - self.assertRaises(TypeError, xgetattr, obj, 1) - # CRASHES xgetattr(obj, NULL) - # CRASHES xgetattr(NULL, 'a') - - def test_object_getattrstring(self): - getattrstring = _testlimitedcapi.object_getattrstring - obj = TestObject() - obj.a = 11 - setattr(obj, '\U0001f40d', 22) - self.assertEqual(getattrstring(obj, b'a'), 11) - self.assertRaises(AttributeError, getattrstring, obj, b'b') - self.assertEqual(getattrstring(obj, '\U0001f40d'.encode()), 22) - - self.assertRaises(RuntimeError, getattrstring, obj, b'evil') - self.assertRaises(UnicodeDecodeError, getattrstring, obj, b'\xff') - # CRASHES getattrstring(obj, NULL) - # CRASHES getattrstring(NULL, b'a') - - def test_object_getoptionalattr(self): - getoptionalattr = _testcapi.object_getoptionalattr - obj = TestObject() - obj.a = 11 - setattr(obj, '\U0001f40d', 22) - self.assertEqual(getoptionalattr(obj, 'a'), 11) - self.assertIs(getoptionalattr(obj, 'b'), AttributeError) - self.assertEqual(getoptionalattr(obj, '\U0001f40d'), 22) - - self.assertRaises(RuntimeError, getoptionalattr, obj, 'evil') - self.assertRaises(TypeError, getoptionalattr, obj, 1) - # CRASHES getoptionalattr(obj, NULL) - # CRASHES getoptionalattr(NULL, 'a') - - def test_object_getoptionalattrstring(self): - getoptionalattrstring = _testcapi.object_getoptionalattrstring - obj = TestObject() - obj.a = 11 - setattr(obj, '\U0001f40d', 22) - self.assertEqual(getoptionalattrstring(obj, b'a'), 11) - self.assertIs(getoptionalattrstring(obj, b'b'), AttributeError) - self.assertEqual(getoptionalattrstring(obj, '\U0001f40d'.encode()), 22) - - self.assertRaises(RuntimeError, getoptionalattrstring, obj, b'evil') - self.assertRaises(UnicodeDecodeError, getoptionalattrstring, obj, b'\xff') - # CRASHES getoptionalattrstring(obj, NULL) - # CRASHES getoptionalattrstring(NULL, b'a') - - def test_object_hasattr(self): - xhasattr = _testlimitedcapi.object_hasattr - obj = TestObject() - obj.a = 1 - setattr(obj, '\U0001f40d', 2) - self.assertTrue(xhasattr(obj, 'a')) - self.assertFalse(xhasattr(obj, 'b')) - self.assertTrue(xhasattr(obj, '\U0001f40d')) - - with support.catch_unraisable_exception() as cm: - self.assertFalse(xhasattr(obj, 'evil')) - self.assertEqual(cm.unraisable.exc_type, RuntimeError) - self.assertEqual(str(cm.unraisable.exc_value), - 'do not get evil') - - with support.catch_unraisable_exception() as cm: - self.assertFalse(xhasattr(obj, 1)) - self.assertEqual(cm.unraisable.exc_type, TypeError) - self.assertEqual(str(cm.unraisable.exc_value), - "attribute name must be string, not 'int'") - - # CRASHES xhasattr(obj, NULL) - # CRASHES xhasattr(NULL, 'a') - - def test_object_hasattrstring(self): - hasattrstring = _testlimitedcapi.object_hasattrstring - obj = TestObject() - obj.a = 1 - setattr(obj, '\U0001f40d', 2) - self.assertTrue(hasattrstring(obj, b'a')) - self.assertFalse(hasattrstring(obj, b'b')) - self.assertTrue(hasattrstring(obj, '\U0001f40d'.encode())) - - with support.catch_unraisable_exception() as cm: - self.assertFalse(hasattrstring(obj, b'evil')) - self.assertEqual(cm.unraisable.exc_type, RuntimeError) - self.assertEqual(str(cm.unraisable.exc_value), - 'do not get evil') - - with support.catch_unraisable_exception() as cm: - self.assertFalse(hasattrstring(obj, b'\xff')) - self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) - self.assertRegex(str(cm.unraisable.exc_value), - "'utf-8' codec can't decode") - - # CRASHES hasattrstring(obj, NULL) - # CRASHES hasattrstring(NULL, b'a') - - def test_object_hasattrwitherror(self): - xhasattr = _testcapi.object_hasattrwitherror - obj = TestObject() - obj.a = 1 - setattr(obj, '\U0001f40d', 2) - self.assertTrue(xhasattr(obj, 'a')) - self.assertFalse(xhasattr(obj, 'b')) - self.assertTrue(xhasattr(obj, '\U0001f40d')) - - self.assertRaises(RuntimeError, xhasattr, obj, 'evil') - self.assertRaises(TypeError, xhasattr, obj, 1) - # CRASHES xhasattr(obj, NULL) - # CRASHES xhasattr(NULL, 'a') - - def test_object_hasattrstringwitherror(self): - hasattrstring = _testcapi.object_hasattrstringwitherror - obj = TestObject() - obj.a = 1 - setattr(obj, '\U0001f40d', 2) - self.assertTrue(hasattrstring(obj, b'a')) - self.assertFalse(hasattrstring(obj, b'b')) - self.assertTrue(hasattrstring(obj, '\U0001f40d'.encode())) - - self.assertRaises(RuntimeError, hasattrstring, obj, b'evil') - self.assertRaises(UnicodeDecodeError, hasattrstring, obj, b'\xff') - # CRASHES hasattrstring(obj, NULL) - # CRASHES hasattrstring(NULL, b'a') - - def test_object_setattr(self): - xsetattr = _testlimitedcapi.object_setattr - obj = TestObject() - xsetattr(obj, 'a', 5) - self.assertEqual(obj.a, 5) - xsetattr(obj, '\U0001f40d', 8) - self.assertEqual(getattr(obj, '\U0001f40d'), 8) - - # PyObject_SetAttr(obj, attr_name, NULL) removes the attribute - xsetattr(obj, 'a', NULL) - self.assertNotHasAttr(obj, 'a') - self.assertRaises(AttributeError, xsetattr, obj, 'b', NULL) - self.assertRaises(RuntimeError, xsetattr, obj, 'evil', NULL) - - self.assertRaises(RuntimeError, xsetattr, obj, 'evil', 'good') - self.assertRaises(AttributeError, xsetattr, 42, 'a', 5) - self.assertRaises(TypeError, xsetattr, obj, 1, 5) - # CRASHES xsetattr(obj, NULL, 5) - # CRASHES xsetattr(NULL, 'a', 5) - - def test_object_setattrstring(self): - setattrstring = _testlimitedcapi.object_setattrstring - obj = TestObject() - setattrstring(obj, b'a', 5) - self.assertEqual(obj.a, 5) - setattrstring(obj, '\U0001f40d'.encode(), 8) - self.assertEqual(getattr(obj, '\U0001f40d'), 8) - - # PyObject_SetAttrString(obj, attr_name, NULL) removes the attribute - setattrstring(obj, b'a', NULL) - self.assertNotHasAttr(obj, 'a') - self.assertRaises(AttributeError, setattrstring, obj, b'b', NULL) - self.assertRaises(RuntimeError, setattrstring, obj, b'evil', NULL) - - self.assertRaises(RuntimeError, setattrstring, obj, b'evil', 'good') - self.assertRaises(AttributeError, setattrstring, 42, b'a', 5) - self.assertRaises(TypeError, setattrstring, obj, 1, 5) - self.assertRaises(UnicodeDecodeError, setattrstring, obj, b'\xff', 5) - # CRASHES setattrstring(obj, NULL, 5) - # CRASHES setattrstring(NULL, b'a', 5) - - def test_object_delattr(self): - xdelattr = _testlimitedcapi.object_delattr - obj = TestObject() - obj.a = 1 - setattr(obj, '\U0001f40d', 2) - xdelattr(obj, 'a') - self.assertNotHasAttr(obj, 'a') - self.assertRaises(AttributeError, xdelattr, obj, 'b') - xdelattr(obj, '\U0001f40d') - self.assertNotHasAttr(obj, '\U0001f40d') - - self.assertRaises(AttributeError, xdelattr, 42, 'numerator') - self.assertRaises(RuntimeError, xdelattr, obj, 'evil') - self.assertRaises(TypeError, xdelattr, obj, 1) - # CRASHES xdelattr(obj, NULL) - # CRASHES xdelattr(NULL, 'a') - - def test_object_delattrstring(self): - delattrstring = _testlimitedcapi.object_delattrstring - obj = TestObject() - obj.a = 1 - setattr(obj, '\U0001f40d', 2) - delattrstring(obj, b'a') - self.assertNotHasAttr(obj, 'a') - self.assertRaises(AttributeError, delattrstring, obj, b'b') - delattrstring(obj, '\U0001f40d'.encode()) - self.assertNotHasAttr(obj, '\U0001f40d') - - self.assertRaises(AttributeError, delattrstring, 42, b'numerator') - self.assertRaises(RuntimeError, delattrstring, obj, b'evil') - self.assertRaises(UnicodeDecodeError, delattrstring, obj, b'\xff') - # CRASHES delattrstring(obj, NULL) - # CRASHES delattrstring(NULL, b'a') - - - def test_mapping_check(self): - check = _testlimitedcapi.mapping_check - self.assertTrue(check({1: 2})) - self.assertTrue(check([1, 2])) - self.assertTrue(check((1, 2))) - self.assertTrue(check('abc')) - self.assertTrue(check(b'abc')) - self.assertFalse(check(42)) - self.assertFalse(check(object())) - self.assertFalse(check(NULL)) - - def test_mapping_size(self): - for size in _testlimitedcapi.mapping_size, _testlimitedcapi.mapping_length: - self.assertEqual(size({1: 2}), 1) - self.assertEqual(size([1, 2]), 2) - self.assertEqual(size((1, 2)), 2) - self.assertEqual(size('abc'), 3) - self.assertEqual(size(b'abc'), 3) - - self.assertRaises(TypeError, size, 42) - self.assertRaises(TypeError, size, object()) - self.assertRaises(SystemError, size, NULL) - - def test_object_getitem(self): - getitem = _testlimitedcapi.object_getitem - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitem(dct, 'a'), 1) - self.assertRaises(KeyError, getitem, dct, 'b') - self.assertEqual(getitem(dct, '\U0001f40d'), 2) - - dct2 = ProxyGetItem(dct) - self.assertEqual(getitem(dct2, 'a'), 1) - self.assertRaises(KeyError, getitem, dct2, 'b') - - self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') - - self.assertRaises(TypeError, getitem, 42, 'a') - self.assertRaises(TypeError, getitem, {}, []) # unhashable - self.assertRaises(SystemError, getitem, {}, NULL) - self.assertRaises(IndexError, getitem, [], 1) - self.assertRaises(TypeError, getitem, [], 'a') - self.assertRaises(SystemError, getitem, NULL, 'a') - - def test_mapping_getitemstring(self): - getitemstring = _testlimitedcapi.mapping_getitemstring - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitemstring(dct, b'a'), 1) - self.assertRaises(KeyError, getitemstring, dct, b'b') - self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) - - dct2 = ProxyGetItem(dct) - self.assertEqual(getitemstring(dct2, b'a'), 1) - self.assertRaises(KeyError, getitemstring, dct2, b'b') - - self.assertRaises(TypeError, getitemstring, 42, b'a') - self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') - self.assertRaises(SystemError, getitemstring, {}, NULL) - self.assertRaises(TypeError, getitemstring, [], b'a') - self.assertRaises(SystemError, getitemstring, NULL, b'a') - - def test_mapping_getoptionalitem(self): - getitem = _testcapi.mapping_getoptionalitem - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitem(dct, 'a'), 1) - self.assertEqual(getitem(dct, 'b'), KeyError) - self.assertEqual(getitem(dct, '\U0001f40d'), 2) - - dct2 = ProxyGetItem(dct) - self.assertEqual(getitem(dct2, 'a'), 1) - self.assertEqual(getitem(dct2, 'b'), KeyError) - - self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') - - self.assertRaises(TypeError, getitem, 42, 'a') - self.assertRaises(TypeError, getitem, {}, []) # unhashable - self.assertRaises(IndexError, getitem, [], 1) - self.assertRaises(TypeError, getitem, [], 'a') - # CRASHES getitem({}, NULL) - # CRASHES getitem(NULL, 'a') - - def test_mapping_getoptionalitemstring(self): - getitemstring = _testcapi.mapping_getoptionalitemstring - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitemstring(dct, b'a'), 1) - self.assertEqual(getitemstring(dct, b'b'), KeyError) - self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) - - dct2 = ProxyGetItem(dct) - self.assertEqual(getitemstring(dct2, b'a'), 1) - self.assertEqual(getitemstring(dct2, b'b'), KeyError) - - self.assertRaises(TypeError, getitemstring, 42, b'a') - self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') - self.assertRaises(SystemError, getitemstring, {}, NULL) - self.assertRaises(TypeError, getitemstring, [], b'a') - # CRASHES getitemstring(NULL, b'a') - - def test_mapping_haskey(self): - haskey = _testlimitedcapi.mapping_haskey - dct = {'a': 1, '\U0001f40d': 2} - self.assertTrue(haskey(dct, 'a')) - self.assertFalse(haskey(dct, 'b')) - self.assertTrue(haskey(dct, '\U0001f40d')) - - dct2 = ProxyGetItem(dct) - self.assertTrue(haskey(dct2, 'a')) - self.assertFalse(haskey(dct2, 'b')) - - self.assertTrue(haskey(['a', 'b', 'c'], 1)) - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskey(42, 'a')) - self.assertEqual(cm.unraisable.exc_type, TypeError) - self.assertEqual(str(cm.unraisable.exc_value), - "'int' object is not subscriptable") - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskey({}, [])) - self.assertEqual(cm.unraisable.exc_type, TypeError) - self.assertEqual(str(cm.unraisable.exc_value), - "cannot use 'list' as a dict key " - "(unhashable type: 'list')") - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskey([], 1)) - self.assertEqual(cm.unraisable.exc_type, IndexError) - self.assertEqual(str(cm.unraisable.exc_value), - 'list index out of range') - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskey([], 'a')) - self.assertEqual(cm.unraisable.exc_type, TypeError) - self.assertEqual(str(cm.unraisable.exc_value), - 'list indices must be integers or slices, not str') - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskey({}, NULL)) - self.assertEqual(cm.unraisable.exc_type, SystemError) - self.assertEqual(str(cm.unraisable.exc_value), - 'null argument to internal routine') - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskey(NULL, 'a')) - self.assertEqual(cm.unraisable.exc_type, SystemError) - self.assertEqual(str(cm.unraisable.exc_value), - 'null argument to internal routine') - - def test_mapping_haskeystring(self): - haskeystring = _testlimitedcapi.mapping_haskeystring - dct = {'a': 1, '\U0001f40d': 2} - self.assertTrue(haskeystring(dct, b'a')) - self.assertFalse(haskeystring(dct, b'b')) - self.assertTrue(haskeystring(dct, '\U0001f40d'.encode())) - - dct2 = ProxyGetItem(dct) - self.assertTrue(haskeystring(dct2, b'a')) - self.assertFalse(haskeystring(dct2, b'b')) - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskeystring(42, b'a')) - self.assertEqual(cm.unraisable.exc_type, TypeError) - self.assertEqual(str(cm.unraisable.exc_value), - "'int' object is not subscriptable") - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskeystring({}, b'\xff')) - self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) - self.assertRegex(str(cm.unraisable.exc_value), - "'utf-8' codec can't decode") - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskeystring({}, NULL)) - self.assertEqual(cm.unraisable.exc_type, SystemError) - self.assertEqual(str(cm.unraisable.exc_value), - "null argument to internal routine") - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskeystring([], b'a')) - self.assertEqual(cm.unraisable.exc_type, TypeError) - self.assertEqual(str(cm.unraisable.exc_value), - 'list indices must be integers or slices, not str') - - with support.catch_unraisable_exception() as cm: - self.assertFalse(haskeystring(NULL, b'a')) - self.assertEqual(cm.unraisable.exc_type, SystemError) - self.assertEqual(str(cm.unraisable.exc_value), - "null argument to internal routine") - - def test_mapping_haskeywitherror(self): - haskey = _testlimitedcapi.mapping_haskeywitherror - dct = {'a': 1, '\U0001f40d': 2} - self.assertTrue(haskey(dct, 'a')) - self.assertFalse(haskey(dct, 'b')) - self.assertTrue(haskey(dct, '\U0001f40d')) - - dct2 = ProxyGetItem(dct) - self.assertTrue(haskey(dct2, 'a')) - self.assertFalse(haskey(dct2, 'b')) - - self.assertTrue(haskey(['a', 'b', 'c'], 1)) - - self.assertRaises(TypeError, haskey, 42, 'a') - self.assertRaises(TypeError, haskey, {}, []) # unhashable - self.assertRaises(IndexError, haskey, [], 1) - self.assertRaises(TypeError, haskey, [], 'a') - - # CRASHES haskey({}, NULL)) - # CRASHES haskey(NULL, 'a')) - - def test_mapping_haskeystringwitherror(self): - haskeystring = _testlimitedcapi.mapping_haskeystringwitherror - dct = {'a': 1, '\U0001f40d': 2} - self.assertTrue(haskeystring(dct, b'a')) - self.assertFalse(haskeystring(dct, b'b')) - self.assertTrue(haskeystring(dct, '\U0001f40d'.encode())) - - dct2 = ProxyGetItem(dct) - self.assertTrue(haskeystring(dct2, b'a')) - self.assertFalse(haskeystring(dct2, b'b')) - - self.assertRaises(TypeError, haskeystring, 42, b'a') - self.assertRaises(UnicodeDecodeError, haskeystring, {}, b'\xff') - self.assertRaises(SystemError, haskeystring, {}, NULL) - self.assertRaises(TypeError, haskeystring, [], b'a') - # CRASHES haskeystring(NULL, b'a') - - def test_object_setitem(self): - setitem = _testlimitedcapi.object_setitem - dct = {} - setitem(dct, 'a', 5) - self.assertEqual(dct, {'a': 5}) - setitem(dct, '\U0001f40d', 8) - self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) - - dct = {} - dct2 = ProxySetItem(dct) - setitem(dct2, 'a', 5) - self.assertEqual(dct, {'a': 5}) - - lst = ['a', 'b', 'c'] - setitem(lst, 1, 'x') - self.assertEqual(lst, ['a', 'x', 'c']) - - self.assertRaises(TypeError, setitem, 42, 'a', 5) - self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable - self.assertRaises(SystemError, setitem, {}, NULL, 5) - self.assertRaises(SystemError, setitem, {}, 'a', NULL) - self.assertRaises(IndexError, setitem, [], 1, 5) - self.assertRaises(TypeError, setitem, [], 'a', 5) - self.assertRaises(TypeError, setitem, (), 1, 5) - self.assertRaises(SystemError, setitem, NULL, 'a', 5) - - def test_mapping_setitemstring(self): - setitemstring = _testlimitedcapi.mapping_setitemstring - dct = {} - setitemstring(dct, b'a', 5) - self.assertEqual(dct, {'a': 5}) - setitemstring(dct, '\U0001f40d'.encode(), 8) - self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) - - dct = {} - dct2 = ProxySetItem(dct) - setitemstring(dct2, b'a', 5) - self.assertEqual(dct, {'a': 5}) - - self.assertRaises(TypeError, setitemstring, 42, b'a', 5) - self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) - self.assertRaises(SystemError, setitemstring, {}, NULL, 5) - self.assertRaises(SystemError, setitemstring, {}, b'a', NULL) - self.assertRaises(TypeError, setitemstring, [], b'a', 5) - self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) - - def test_object_delitem(self): - for delitem in _testlimitedcapi.object_delitem, _testlimitedcapi.mapping_delitem: - dct = {'a': 1, 'c': 2, '\U0001f40d': 3} - delitem(dct, 'a') - self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) - self.assertRaises(KeyError, delitem, dct, 'b') - delitem(dct, '\U0001f40d') - self.assertEqual(dct, {'c': 2}) - - dct = {'a': 1, 'c': 2} - dct2 = ProxyDelItem(dct) - delitem(dct2, 'a') - self.assertEqual(dct, {'c': 2}) - self.assertRaises(KeyError, delitem, dct2, 'b') - - lst = ['a', 'b', 'c'] - delitem(lst, 1) - self.assertEqual(lst, ['a', 'c']) - - self.assertRaises(TypeError, delitem, 42, 'a') - self.assertRaises(TypeError, delitem, {}, []) # unhashable - self.assertRaises(SystemError, delitem, {}, NULL) - self.assertRaises(IndexError, delitem, [], 1) - self.assertRaises(TypeError, delitem, [], 'a') - self.assertRaises(SystemError, delitem, NULL, 'a') - - def test_mapping_delitemstring(self): - delitemstring = _testlimitedcapi.mapping_delitemstring - dct = {'a': 1, 'c': 2, '\U0001f40d': 3} - delitemstring(dct, b'a') - self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) - self.assertRaises(KeyError, delitemstring, dct, b'b') - delitemstring(dct, '\U0001f40d'.encode()) - self.assertEqual(dct, {'c': 2}) - - dct = {'a': 1, 'c': 2} - dct2 = ProxyDelItem(dct) - delitemstring(dct2, b'a') - self.assertEqual(dct, {'c': 2}) - self.assertRaises(KeyError, delitemstring, dct2, b'b') - - self.assertRaises(TypeError, delitemstring, 42, b'a') - self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') - self.assertRaises(SystemError, delitemstring, {}, NULL) - self.assertRaises(TypeError, delitemstring, [], b'a') - self.assertRaises(SystemError, delitemstring, NULL, b'a') - - def test_mapping_keys_valuesitems(self): - class Mapping1(dict): - def keys(self): - return list(super().keys()) - def values(self): - return list(super().values()) - def items(self): - return list(super().items()) - class Mapping2(dict): - def keys(self): - return tuple(super().keys()) - def values(self): - return tuple(super().values()) - def items(self): - return tuple(super().items()) - dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} - - for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), - dict_obj, OrderedDict(dict_obj), - Mapping1(dict_obj), Mapping2(dict_obj)]: - self.assertListEqual(_testlimitedcapi.mapping_keys(mapping), - list(mapping.keys())) - self.assertListEqual(_testlimitedcapi.mapping_values(mapping), - list(mapping.values())) - self.assertListEqual(_testlimitedcapi.mapping_items(mapping), - list(mapping.items())) - - def test_mapping_keys_valuesitems_bad_arg(self): - self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, object()) - self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, object()) - self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, object()) - self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, []) - self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, []) - self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, []) - self.assertRaises(SystemError, _testlimitedcapi.mapping_keys, NULL) - self.assertRaises(SystemError, _testlimitedcapi.mapping_values, NULL) - self.assertRaises(SystemError, _testlimitedcapi.mapping_items, NULL) - - class BadMapping: - def keys(self): - return None - def values(self): - return None - def items(self): - return None - bad_mapping = BadMapping() - self.assertRaises(TypeError, _testlimitedcapi.mapping_keys, bad_mapping) - self.assertRaises(TypeError, _testlimitedcapi.mapping_values, bad_mapping) - self.assertRaises(TypeError, _testlimitedcapi.mapping_items, bad_mapping) - - def test_sequence_check(self): - check = _testlimitedcapi.sequence_check - self.assertFalse(check({1: 2})) - self.assertTrue(check([1, 2])) - self.assertTrue(check((1, 2))) - self.assertTrue(check('abc')) - self.assertTrue(check(b'abc')) - self.assertFalse(check(42)) - self.assertFalse(check(object())) - # CRASHES check(NULL) - - def test_sequence_size(self): - for size in _testlimitedcapi.sequence_size, _testlimitedcapi.sequence_length: - self.assertEqual(size([1, 2]), 2) - self.assertEqual(size((1, 2)), 2) - self.assertEqual(size('abc'), 3) - self.assertEqual(size(b'abc'), 3) - - self.assertRaises(TypeError, size, {}) - self.assertRaises(TypeError, size, 42) - self.assertRaises(TypeError, size, object()) - self.assertRaises(SystemError, size, NULL) - - def test_sequence_getitem(self): - getitem = _testlimitedcapi.sequence_getitem - lst = ['a', 'b', 'c'] - self.assertEqual(getitem(lst, 1), 'b') - self.assertEqual(getitem(lst, -1), 'c') - self.assertRaises(IndexError, getitem, lst, 3) - self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) - self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) - - self.assertRaises(TypeError, getitem, 42, 1) - self.assertRaises(TypeError, getitem, {}, 1) - self.assertRaises(SystemError, getitem, NULL, 1) - - def test_sequence_concat(self): - concat = _testlimitedcapi.sequence_concat - self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) - self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) - - self.assertRaises(TypeError, concat, [], ()) - self.assertRaises(TypeError, concat, (), []) - self.assertRaises(TypeError, concat, [], 42) - self.assertRaises(TypeError, concat, 42, []) - self.assertRaises(TypeError, concat, 42, 43) - self.assertRaises(SystemError, concat, [], NULL) - self.assertRaises(SystemError, concat, NULL, []) - - def test_sequence_repeat(self): - repeat = _testlimitedcapi.sequence_repeat - self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) - self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) - self.assertEqual(repeat(['a', 'b'], 0), []) - self.assertEqual(repeat(['a', 'b'], -1), []) - self.assertEqual(repeat(['a', 'b'], PY_SSIZE_T_MIN), []) - self.assertEqual(repeat([], PY_SSIZE_T_MAX), []) - self.assertRaises(MemoryError, repeat, ['a', 'b'], PY_SSIZE_T_MAX) - - self.assertRaises(TypeError, repeat, set(), 2) - self.assertRaises(TypeError, repeat, 42, 2) - self.assertRaises(SystemError, repeat, NULL, 2) - - def test_sequence_inplaceconcat(self): - inplaceconcat = _testlimitedcapi.sequence_inplaceconcat - lst = ['a', 'b'] - res = inplaceconcat(lst, [1, 2]) - self.assertEqual(res, ['a', 'b', 1, 2]) - self.assertIs(res, lst) - lst = ['a', 'b'] - res = inplaceconcat(lst, (1, 2)) - self.assertEqual(res, ['a', 'b', 1, 2]) - self.assertIs(res, lst) - self.assertEqual(inplaceconcat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) - - self.assertRaises(TypeError, inplaceconcat, (), []) - self.assertRaises(TypeError, inplaceconcat, [], 42) - self.assertRaises(TypeError, inplaceconcat, 42, []) - self.assertRaises(TypeError, inplaceconcat, 42, 43) - self.assertRaises(SystemError, inplaceconcat, [], NULL) - self.assertRaises(SystemError, inplaceconcat, NULL, []) - - def test_sequence_inplacerepeat(self): - inplacerepeat = _testlimitedcapi.sequence_inplacerepeat - lst = ['a', 'b'] - res = inplacerepeat(lst, 2) - self.assertEqual(res, ['a', 'b', 'a', 'b']) - self.assertIs(res, lst) - self.assertEqual(inplacerepeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) - self.assertEqual(inplacerepeat(['a', 'b'], 0), []) - self.assertEqual(inplacerepeat(['a', 'b'], -1), []) - self.assertEqual(inplacerepeat(['a', 'b'], PY_SSIZE_T_MIN), []) - self.assertEqual(inplacerepeat([], PY_SSIZE_T_MAX), []) - self.assertRaises(MemoryError, inplacerepeat, ['a', 'b'], PY_SSIZE_T_MAX) - - self.assertRaises(TypeError, inplacerepeat, set(), 2) - self.assertRaises(TypeError, inplacerepeat, 42, 2) - self.assertRaises(SystemError, inplacerepeat, NULL, 2) - - def test_sequence_setitem(self): - setitem = _testlimitedcapi.sequence_setitem - lst = ['a', 'b', 'c'] - setitem(lst, 1, 'x') - self.assertEqual(lst, ['a', 'x', 'c']) - setitem(lst, -1, 'y') - self.assertEqual(lst, ['a', 'x', 'y']) - - setitem(lst, 0, NULL) - self.assertEqual(lst, ['x', 'y']) - self.assertRaises(IndexError, setitem, lst, 3, 'x') - self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MAX, 'x') - self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MIN, 'x') - - self.assertRaises(TypeError, setitem, 42, 1, 'x') - self.assertRaises(TypeError, setitem, {}, 1, 'x') - self.assertRaises(SystemError, setitem, NULL, 1, 'x') - - def test_sequence_delitem(self): - delitem = _testlimitedcapi.sequence_delitem - lst = ['a', 'b', 'c'] - delitem(lst, 1) - self.assertEqual(lst, ['a', 'c']) - delitem(lst, -1) - self.assertEqual(lst, ['a']) - self.assertRaises(IndexError, delitem, lst, 3) - self.assertRaises(IndexError, delitem, lst, PY_SSIZE_T_MAX) - self.assertRaises(IndexError, delitem, lst, PY_SSIZE_T_MIN) - - self.assertRaises(TypeError, delitem, 42, 1) - self.assertRaises(TypeError, delitem, {}, 1) - self.assertRaises(SystemError, delitem, NULL, 1) - - def test_sequence_setslice(self): - setslice = _testlimitedcapi.sequence_setslice - - # Correct case: - for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: - for stop in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: - data = [1, 2, 3, 4, 5] - data_copy = [1, 2, 3, 4, 5] - setslice(data, start, stop, [8, 9]) - data_copy[start:stop] = [8, 9] - self.assertEqual(data, data_copy) - - data = [1, 2, 3, 4, 5] - data_copy = [1, 2, 3, 4, 5] - setslice(data, start, stop, NULL) - del data_copy[start:stop] - self.assertEqual(data, data_copy) - - # Custom class: - class Custom: - def __setitem__(self, index, value): - self.index = index - self.value = value - - c = Custom() - setslice(c, 0, 5, 'abc') - self.assertEqual(c.index, slice(0, 5)) - self.assertEqual(c.value, 'abc') - - # Immutable sequences must raise: - bad_seq1 = (1, 2, 3, 4) - self.assertRaises(TypeError, setslice, bad_seq1, 1, 3, (8, 9)) - self.assertEqual(bad_seq1, (1, 2, 3, 4)) - - bad_seq2 = 'abcd' - self.assertRaises(TypeError, setslice, bad_seq2, 1, 3, 'xy') - self.assertEqual(bad_seq2, 'abcd') - - # Not a sequence: - self.assertRaises(TypeError, setslice, object(), 1, 3, 'xy') - self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') - - def test_sequence_delslice(self): - delslice = _testlimitedcapi.sequence_delslice - - # Correct case: - for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: - for stop in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: - data = [1, 2, 3, 4, 5] - data_copy = [1, 2, 3, 4, 5] - delslice(data, start, stop) - del data_copy[start:stop] - self.assertEqual(data, data_copy) - - # Custom class: - class Custom: - def __delitem__(self, index): - self.index = index - - c = Custom() - delslice(c, 0, 5) - self.assertEqual(c.index, slice(0, 5)) - - # Immutable sequences must raise: - bad_seq1 = (1, 2, 3, 4) - self.assertRaises(TypeError, delslice, bad_seq1, 1, 3) - self.assertEqual(bad_seq1, (1, 2, 3, 4)) - - bad_seq2 = 'abcd' - self.assertRaises(TypeError, delslice, bad_seq2, 1, 3) - self.assertEqual(bad_seq2, 'abcd') - - # Not a sequence: - self.assertRaises(TypeError, delslice, object(), 1, 3) - self.assertRaises(SystemError, delslice, NULL, 1, 3) - - mapping = {1: 'a', 2: 'b', 3: 'c'} - self.assertRaises(KeyError, delslice, mapping, 1, 3) - self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) - - def test_sequence_count(self): - count = _testlimitedcapi.sequence_count - - lst = ['a', 'b', 'a'] - self.assertEqual(count(lst, 'a'), 2) - self.assertEqual(count(lst, 'c'), 0) - self.assertEqual(count(iter(lst), 'a'), 2) - self.assertEqual(count(iter(lst), 'c'), 0) - self.assertEqual(count({'a': 2}, 'a'), 1) - - self.assertRaises(TypeError, count, 42, 'a') - self.assertRaises(SystemError, count, [], NULL) - self.assertRaises(SystemError, count, [1], NULL) - self.assertRaises(SystemError, count, NULL, 'a') - - def test_sequence_contains(self): - contains = _testlimitedcapi.sequence_contains - - lst = ['a', 'b', 'a'] - self.assertEqual(contains(lst, 'a'), 1) - self.assertEqual(contains(lst, 'c'), 0) - self.assertEqual(contains(iter(lst), 'a'), 1) - self.assertEqual(contains(iter(lst), 'c'), 0) - self.assertEqual(contains({'a': 2}, 'a'), 1) - - # XXX Only for empty sequences. Should be SystemError? - self.assertEqual(contains([], NULL), 0) - - self.assertRaises(TypeError, contains, 42, 'a') - self.assertRaises(SystemError, contains, [1], NULL) - # CRASHES contains({}, NULL) - # CRASHES contains(set(), NULL) - # CRASHES contains(NULL, 'a') - - def test_sequence_index(self): - index = _testlimitedcapi.sequence_index - - lst = ['a', 'b', 'a'] - self.assertEqual(index(lst, 'a'), 0) - self.assertEqual(index(lst, 'b'), 1) - self.assertRaises(ValueError, index, lst, 'c') - self.assertEqual(index(iter(lst), 'a'), 0) - self.assertEqual(index(iter(lst), 'b'), 1) - self.assertRaises(ValueError, index, iter(lst), 'c') - dct = {'a': 2, 'b': 3} - self.assertEqual(index(dct, 'a'), 0) - self.assertEqual(index(dct, 'b'), 1) - self.assertRaises(ValueError, index, dct, 'c') - - self.assertRaises(TypeError, index, 42, 'a') - self.assertRaises(SystemError, index, [], NULL) - self.assertRaises(SystemError, index, [1], NULL) - self.assertRaises(SystemError, index, NULL, 'a') - - def test_sequence_list(self): - xlist = _testlimitedcapi.sequence_list - self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) - self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) - self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) - self.assertEqual(xlist(gen()), ['a', 'b', 'c']) - - self.assertRaises(TypeError, xlist, 42) - self.assertRaises(SystemError, xlist, NULL) - - def test_sequence_tuple(self): - xtuple = _testlimitedcapi.sequence_tuple - self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) - self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) - self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) - self.assertEqual(xtuple(gen()), ('a', 'b', 'c')) - - self.assertRaises(TypeError, xtuple, 42) - self.assertRaises(SystemError, xtuple, NULL) - - def test_sequence_fast(self): - # Test PySequence_Fast() - sequence_fast = _testlimitedcapi.sequence_fast - sequence_fast_get_size = _testcapi.sequence_fast_get_size - sequence_fast_get_item = _testcapi.sequence_fast_get_item - - tpl = ('a', 'b', 'c') - fast = sequence_fast(tpl, "err_msg") - self.assertIs(fast, tpl) - self.assertEqual(sequence_fast_get_size(fast), 3) - self.assertEqual(sequence_fast_get_item(fast, 2), 'c') - - lst = ['a', 'b', 'c'] - fast = sequence_fast(lst, "err_msg") - self.assertIs(fast, lst) - self.assertEqual(sequence_fast_get_size(fast), 3) - self.assertEqual(sequence_fast_get_item(fast, 2), 'c') - - it = iter(['A', 'B']) - fast = sequence_fast(it, "err_msg") - self.assertEqual(fast, ['A', 'B']) - self.assertEqual(sequence_fast_get_size(fast), 2) - self.assertEqual(sequence_fast_get_item(fast, 1), 'B') - - text = 'fast' - fast = sequence_fast(text, "err_msg") - self.assertEqual(fast, ['f', 'a', 's', 't']) - self.assertEqual(sequence_fast_get_size(fast), 4) - self.assertEqual(sequence_fast_get_item(fast, 0), 'f') - - self.assertRaises(TypeError, sequence_fast, 42, "err_msg") - self.assertRaises(SystemError, sequence_fast, NULL, "err_msg") - - # CRASHES sequence_fast_get_size(NULL) - # CRASHES sequence_fast_get_item(NULL, 0) - - def test_object_generichash(self): - # Test PyObject_GenericHash() - generichash = _testcapi.object_generichash - for obj in object(), 1, 'string', []: - self.assertEqual(generichash(obj), object.__hash__(obj)) - - def run_iter_api_test(self, next_func): - for data in (), [], (1, 2, 3), [1 , 2, 3], "123": - with self.subTest(data=data): - items = [] - it = iter(data) - while (item := next_func(it)) is not None: - items.append(item) - self.assertEqual(items, list(data)) - - class Broken: - def __init__(self): - self.count = 0 - - def __next__(self): - if self.count < 3: - self.count += 1 - return self.count - else: - raise TypeError('bad type') - - it = Broken() - self.assertEqual(next_func(it), 1) - self.assertEqual(next_func(it), 2) - self.assertEqual(next_func(it), 3) - with self.assertRaisesRegex(TypeError, 'bad type'): - next_func(it) - - def test_iter_next(self): - from _testcapi import PyIter_Next - self.run_iter_api_test(PyIter_Next) - # CRASHES PyIter_Next(10) - - def test_iter_nextitem(self): - from _testcapi import PyIter_NextItem - self.run_iter_api_test(PyIter_NextItem) - - regex = "expected.*iterator.*got.*'int'" - with self.assertRaisesRegex(TypeError, regex): - PyIter_NextItem(10) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py deleted file mode 100644 index 52565ea34c6..00000000000 --- a/Lib/test/test_capi/test_bytearray.py +++ /dev/null @@ -1,174 +0,0 @@ -import unittest -from test.support import import_helper - -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') -from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX - -NULL = None - -class ByteArraySubclass(bytearray): - pass - -class BytesLike: - def __init__(self, value): - self.value = value - def __bytes__(self): - return self.value - - -class CAPITest(unittest.TestCase): - def test_check(self): - # Test PyByteArray_Check() - check = _testlimitedcapi.bytearray_check - self.assertTrue(check(bytearray(b''))) - self.assertTrue(check(bytearray(b'abc'))) - self.assertFalse(check(b'abc')) - self.assertTrue(check(ByteArraySubclass(b'abc'))) - self.assertFalse(check(BytesLike(b'abc'))) - self.assertFalse(check(3)) - self.assertFalse(check([])) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_checkexact(self): - # Test PyByteArray_CheckExact() - check = _testlimitedcapi.bytearray_checkexact - self.assertTrue(check(bytearray(b''))) - self.assertTrue(check(bytearray(b'abc'))) - self.assertFalse(check(b'abc')) - self.assertFalse(check(ByteArraySubclass(b'abc'))) - self.assertFalse(check(BytesLike(b'abc'))) - self.assertFalse(check(3)) - self.assertFalse(check([])) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_fromstringandsize(self): - # Test PyByteArray_FromStringAndSize() - fromstringandsize = _testlimitedcapi.bytearray_fromstringandsize - - self.assertEqual(fromstringandsize(b'abc'), bytearray(b'abc')) - self.assertEqual(fromstringandsize(b'abc', 2), bytearray(b'ab')) - self.assertEqual(fromstringandsize(b'abc\0def'), bytearray(b'abc\0def')) - self.assertEqual(fromstringandsize(b'', 0), bytearray()) - self.assertEqual(fromstringandsize(NULL, 0), bytearray()) - self.assertEqual(len(fromstringandsize(NULL, 3)), 3) - self.assertRaises(MemoryError, fromstringandsize, NULL, PY_SSIZE_T_MAX) - - self.assertRaises(SystemError, fromstringandsize, b'abc', -1) - self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) - self.assertRaises(SystemError, fromstringandsize, NULL, -1) - self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) - - def test_fromobject(self): - # Test PyByteArray_FromObject() - fromobject = _testlimitedcapi.bytearray_fromobject - - self.assertEqual(fromobject(b''), bytearray(b'')) - self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) - self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) - self.assertEqual(fromobject(ByteArraySubclass(b'abc')), bytearray(b'abc')) - self.assertEqual(fromobject([97, 98, 99]), bytearray(b'abc')) - self.assertEqual(fromobject(3), bytearray(b'\0\0\0')) - self.assertRaises(TypeError, fromobject, BytesLike(b'abc')) - self.assertRaises(TypeError, fromobject, 'abc') - self.assertRaises(TypeError, fromobject, object()) - - # CRASHES fromobject(NULL) - - def test_size(self): - # Test PyByteArray_Size() - size = _testlimitedcapi.bytearray_size - self.assertEqual(size(bytearray(b'')), 0) - self.assertEqual(size(bytearray(b'abc')), 3) - self.assertEqual(size(ByteArraySubclass(b'abc')), 3) - - # CRASHES size(b'abc') - # CRASHES size(object()) - # CRASHES size(NULL) - - def test_asstring(self): - """Test PyByteArray_AsString()""" - asstring = _testlimitedcapi.bytearray_asstring - self.assertEqual(asstring(bytearray(b''), 1), b'\0') - self.assertEqual(asstring(bytearray(b'abc'), 4), b'abc\0') - self.assertEqual(asstring(ByteArraySubclass(b'abc'), 4), b'abc\0') - self.assertEqual(asstring(bytearray(b'abc\0def'), 8), b'abc\0def\0') - - # CRASHES asstring(b'abc', 0) - # CRASHES asstring(object()', 0) - # CRASHES asstring(NULL, 0) - - def test_concat(self): - """Test PyByteArray_Concat()""" - concat = _testlimitedcapi.bytearray_concat - - ba = bytearray(b'abc') - self.assertEqual(concat(ba, b'def'), bytearray(b'abcdef')) - self.assertEqual(ba, b'abc') - self.assertEqual(concat(ba, ba), bytearray(b'abcabc')) - - self.assertEqual(concat(b'abc', b'def'), bytearray(b'abcdef')) - self.assertEqual(concat(b'a\0b', b'c\0d'), bytearray(b'a\0bc\0d')) - self.assertEqual(concat(bytearray(b'abc'), b'def'), bytearray(b'abcdef')) - self.assertEqual(concat(b'abc', bytearray(b'def')), bytearray(b'abcdef')) - self.assertEqual(concat(bytearray(b'abc'), b''), bytearray(b'abc')) - self.assertEqual(concat(b'', bytearray(b'def')), bytearray(b'def')) - self.assertEqual(concat(bytearray(b''), bytearray(b'')), bytearray(b'')) - self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), - bytearray(b'abcdef')) - self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), - bytearray(b'abcdef')) - - self.assertRaises(TypeError, concat, memoryview(b'axbycz')[::2], b'def') - self.assertRaises(TypeError, concat, b'abc', memoryview(b'dxeyfz')[::2]) - self.assertRaises(TypeError, concat, b'abc', 'def') - self.assertRaises(TypeError, concat, 'abc', b'def') - self.assertRaises(TypeError, concat, 'abc', 'def') - self.assertRaises(TypeError, concat, [], b'def') - self.assertRaises(TypeError, concat, b'abc', []) - self.assertRaises(TypeError, concat, [], []) - - # CRASHES concat(NULL, bytearray(b'def')) - # CRASHES concat(bytearray(b'abc'), NULL) - # CRASHES concat(NULL, object()) - # CRASHES concat(object(), NULL) - - def test_resize(self): - """Test PyByteArray_Resize()""" - resize = _testlimitedcapi.bytearray_resize - - ba = bytearray(b'abcdef') - self.assertEqual(resize(ba, 3), 0) - self.assertEqual(ba, bytearray(b'abc')) - self.assertEqual(resize(ba, 10), 0) - self.assertEqual(len(ba), 10) - self.assertEqual(ba[:3], bytearray(b'abc')) - self.assertEqual(resize(ba, 2**20), 0) - self.assertEqual(len(ba), 2**20) - self.assertEqual(ba[:3], bytearray(b'abc')) - self.assertEqual(resize(ba, 0), 0) - self.assertEqual(ba, bytearray()) - - ba = bytearray(b'') - self.assertEqual(resize(ba, 0), 0) - self.assertEqual(ba, bytearray()) - - ba = ByteArraySubclass(b'abcdef') - self.assertEqual(resize(ba, 3), 0) - self.assertEqual(ba, bytearray(b'abc')) - - self.assertRaises(ValueError, resize, bytearray(), -1) - self.assertRaises(ValueError, resize, bytearray(), -200) - self.assertRaises(MemoryError, resize, bytearray(), PY_SSIZE_T_MAX) - self.assertRaises(MemoryError, resize, bytearray(1000), PY_SSIZE_T_MAX) - - # CRASHES resize(b'abc', 0) - # CRASHES resize(object(), 0) - # CRASHES resize(NULL, 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py deleted file mode 100644 index bc820bd68d9..00000000000 --- a/Lib/test/test_capi/test_bytes.py +++ /dev/null @@ -1,303 +0,0 @@ -import unittest -from test.support import import_helper - -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') -_testcapi = import_helper.import_module('_testcapi') -from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX - -NULL = None - -class BytesSubclass(bytes): - pass - -class BytesLike: - def __init__(self, value): - self.value = value - def __bytes__(self): - return self.value - - -class CAPITest(unittest.TestCase): - def test_check(self): - # Test PyBytes_Check() - check = _testlimitedcapi.bytes_check - self.assertTrue(check(b'abc')) - self.assertTrue(check(b'')) - self.assertFalse(check('abc')) - self.assertFalse(check(bytearray(b'abc'))) - self.assertTrue(check(BytesSubclass(b'abc'))) - self.assertFalse(check(BytesLike(b'abc'))) - self.assertFalse(check(3)) - self.assertFalse(check([])) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_checkexact(self): - # Test PyBytes_CheckExact() - check = _testlimitedcapi.bytes_checkexact - self.assertTrue(check(b'abc')) - self.assertTrue(check(b'')) - self.assertFalse(check('abc')) - self.assertFalse(check(bytearray(b'abc'))) - self.assertFalse(check(BytesSubclass(b'abc'))) - self.assertFalse(check(BytesLike(b'abc'))) - self.assertFalse(check(3)) - self.assertFalse(check([])) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_fromstringandsize(self): - # Test PyBytes_FromStringAndSize() - fromstringandsize = _testlimitedcapi.bytes_fromstringandsize - - self.assertEqual(fromstringandsize(b'abc'), b'abc') - self.assertEqual(fromstringandsize(b'abc', 2), b'ab') - self.assertEqual(fromstringandsize(b'abc\0def'), b'abc\0def') - self.assertEqual(fromstringandsize(b'a'), b'a') - self.assertEqual(fromstringandsize(b'a', 1), b'a') - self.assertEqual(fromstringandsize(b'', 0), b'') - self.assertEqual(fromstringandsize(NULL, 0), b'') - self.assertEqual(len(fromstringandsize(NULL, 3)), 3) - self.assertRaises((MemoryError, OverflowError), - fromstringandsize, NULL, PY_SSIZE_T_MAX) - - self.assertRaises(SystemError, fromstringandsize, b'abc', -1) - self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) - self.assertRaises(SystemError, fromstringandsize, NULL, -1) - self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) - - def test_fromstring(self): - # Test PyBytes_FromString() - fromstring = _testlimitedcapi.bytes_fromstring - - self.assertEqual(fromstring(b'abc\0def'), b'abc') - self.assertEqual(fromstring(b''), b'') - - # CRASHES fromstring(NULL) - - def test_fromobject(self): - # Test PyBytes_FromObject() - fromobject = _testlimitedcapi.bytes_fromobject - - self.assertEqual(fromobject(b''), b'') - self.assertEqual(fromobject(b'abc'), b'abc') - self.assertEqual(fromobject(bytearray(b'abc')), b'abc') - self.assertEqual(fromobject(BytesSubclass(b'abc')), b'abc') - self.assertEqual(fromobject([97, 98, 99]), b'abc') - self.assertRaises(TypeError, fromobject, 3) - self.assertRaises(TypeError, fromobject, BytesLike(b'abc')) - self.assertRaises(TypeError, fromobject, 'abc') - self.assertRaises(TypeError, fromobject, object()) - self.assertRaises(SystemError, fromobject, NULL) - - def test_size(self): - # Test PyBytes_Size() - size = _testlimitedcapi.bytes_size - - self.assertEqual(size(b''), 0) - self.assertEqual(size(b'abc'), 3) - self.assertEqual(size(BytesSubclass(b'abc')), 3) - self.assertRaises(TypeError, size, bytearray(b'abc')) - self.assertRaises(TypeError, size, 'abc') - self.assertRaises(TypeError, size, object()) - - # CRASHES size(NULL) - - def test_asstring(self): - """Test PyBytes_AsString()""" - asstring = _testlimitedcapi.bytes_asstring - - self.assertEqual(asstring(b'abc', 4), b'abc\0') - self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') - self.assertEqual(asstring(b'', 1), b'\0') - self.assertRaises(TypeError, asstring, 'abc', 0) - self.assertRaises(TypeError, asstring, object(), 0) - - # CRASHES asstring(NULL, 0) - - def test_asstringandsize(self): - """Test PyBytes_AsStringAndSize()""" - asstringandsize = _testlimitedcapi.bytes_asstringandsize - asstringandsize_null = _testlimitedcapi.bytes_asstringandsize_null - - self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) - self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) - self.assertEqual(asstringandsize(b'', 1), (b'\0', 0)) - self.assertEqual(asstringandsize_null(b'abc', 4), b'abc\0') - self.assertRaises(ValueError, asstringandsize_null, b'abc\0def', 8) - self.assertRaises(TypeError, asstringandsize, 'abc', 0) - self.assertRaises(TypeError, asstringandsize_null, 'abc', 0) - self.assertRaises(TypeError, asstringandsize, object(), 0) - self.assertRaises(TypeError, asstringandsize_null, object(), 0) - - # CRASHES asstringandsize(NULL, 0) - # CRASHES asstringandsize_null(NULL, 0) - - def test_repr(self): - # Test PyBytes_Repr() - bytes_repr = _testlimitedcapi.bytes_repr - - self.assertEqual(bytes_repr(b'', 0), r"""b''""") - self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") - self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") - self.assertEqual(bytes_repr(b'''a'b"c"d''', 0), r"""b'a\'b"c"d'""") - self.assertEqual(bytes_repr(b'''a'b"c"d''', 1), r"""b'a\'b"c"d'""") - self.assertEqual(bytes_repr(b'''a'b"c''', 0), r"""b'a\'b"c'""") - self.assertEqual(bytes_repr(b'''a'b"c''', 1), r"""b'a\'b"c'""") - self.assertEqual(bytes_repr(b'''a'b'c"d''', 0), r"""b'a\'b\'c"d'""") - self.assertEqual(bytes_repr(b'''a'b'c"d''', 1), r"""b'a\'b\'c"d'""") - self.assertEqual(bytes_repr(b'''a'b'c'd''', 0), r"""b'a\'b\'c\'d'""") - self.assertEqual(bytes_repr(b'''a'b'c'd''', 1), r'''b"a'b'c'd"''') - - self.assertEqual(bytes_repr(BytesSubclass(b'abc'), 0), r"""b'abc'""") - - # UDEFINED bytes_repr(object(), 0) - # CRASHES bytes_repr(NULL, 0) - - def test_concat(self, concat=None): - """Test PyBytes_Concat()""" - if concat is None: - concat = _testlimitedcapi.bytes_concat - - self.assertEqual(concat(b'abc', b'def'), b'abcdef') - self.assertEqual(concat(b'a\0b', b'c\0d'), b'a\0bc\0d') - self.assertEqual(concat(bytearray(b'abc'), b'def'), b'abcdef') - self.assertEqual(concat(b'abc', bytearray(b'def')), b'abcdef') - self.assertEqual(concat(bytearray(b'abc'), b''), b'abc') - self.assertEqual(concat(b'', bytearray(b'def')), b'def') - self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), b'abcdef') - self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), b'abcdef') - self.assertEqual(concat(b'', b''), b'') - - self.assertEqual(concat(b'abc', b'def', True), b'abcdef') - self.assertEqual(concat(b'abc', bytearray(b'def'), True), b'abcdef') - # Check that it does not change the singleton - self.assertEqual(concat(bytes(), b'def', True), b'def') - self.assertEqual(len(bytes()), 0) - - self.assertRaises(TypeError, concat, memoryview(b'axbycz')[::2], b'def') - self.assertRaises(TypeError, concat, b'abc', memoryview(b'dxeyfz')[::2]) - self.assertRaises(TypeError, concat, b'abc', 'def') - self.assertRaises(TypeError, concat, 'abc', b'def') - self.assertRaises(TypeError, concat, 'abc', 'def') - self.assertRaises(TypeError, concat, [], b'def') - self.assertRaises(TypeError, concat, b'abc', []) - self.assertRaises(TypeError, concat, [], []) - - self.assertEqual(concat(NULL, b'def'), NULL) - self.assertEqual(concat(b'abc', NULL), NULL) - self.assertEqual(concat(NULL, object()), NULL) - self.assertEqual(concat(object(), NULL), NULL) - - def test_concatanddel(self): - """Test PyBytes_ConcatAndDel()""" - self.test_concat(_testlimitedcapi.bytes_concatanddel) - - def test_decodeescape(self): - """Test PyBytes_DecodeEscape()""" - decodeescape = _testlimitedcapi.bytes_decodeescape - - self.assertEqual(decodeescape(b''), b'') - self.assertEqual(decodeescape(b'abc'), b'abc') - self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), - b'''\t\n\r\v\f\0\\'"''') - self.assertEqual(decodeescape(b'\t\n\r\x0b\x0c\x00'), b'\t\n\r\v\f\0') - self.assertEqual(decodeescape(br'\xa1\xa2'), b'\xa1\xa2') - self.assertEqual(decodeescape(br'\2\24\241'), b'\x02\x14\xa1') - self.assertEqual(decodeescape(b'\xa1\xa2'), b'\xa1\xa2') - with self.assertWarns(DeprecationWarning): - self.assertEqual(decodeescape(br'\u4f60'), br'\u4f60') - with self.assertWarns(DeprecationWarning): - self.assertEqual(decodeescape(br'\z'), br'\z') - with self.assertWarns(DeprecationWarning): - self.assertEqual(decodeescape(br'\541'), b'a') - - for b in b'\\', br'\x', br'\xa', br'\xz', br'\xaz': - self.assertRaises(ValueError, decodeescape, b) - self.assertRaises(ValueError, decodeescape, b, 'strict') - self.assertEqual(decodeescape(br'x\xa', 'replace'), b'x?') - self.assertEqual(decodeescape(br'x\xay', 'replace'), b'x?y') - self.assertEqual(decodeescape(br'x\xa\xy', 'replace'), b'x??y') - self.assertEqual(decodeescape(br'x\xa\xy', 'ignore'), b'xy') - self.assertRaises(ValueError, decodeescape, b'\\', 'spam') - self.assertEqual(decodeescape(NULL), b'') - self.assertRaises(OverflowError, decodeescape, b'abc', NULL, PY_SSIZE_T_MAX) - self.assertRaises(OverflowError, decodeescape, NULL, NULL, PY_SSIZE_T_MAX) - - # CRASHES decodeescape(b'abc', NULL, -1) - # CRASHES decodeescape(NULL, NULL, 1) - - def test_resize(self): - """Test _PyBytes_Resize()""" - resize = _testcapi.bytes_resize - - for new in True, False: - self.assertEqual(resize(b'abc', 0, new), b'') - self.assertEqual(resize(b'abc', 1, new), b'a') - self.assertEqual(resize(b'abc', 2, new), b'ab') - self.assertEqual(resize(b'abc', 3, new), b'abc') - b = resize(b'abc', 4, new) - self.assertEqual(len(b), 4) - self.assertEqual(b[:3], b'abc') - - self.assertEqual(resize(b'a', 0, new), b'') - self.assertEqual(resize(b'a', 1, new), b'a') - b = resize(b'a', 2, new) - self.assertEqual(len(b), 2) - self.assertEqual(b[:1], b'a') - - self.assertEqual(resize(b'', 0, new), b'') - self.assertEqual(len(resize(b'', 1, new)), 1) - self.assertEqual(len(resize(b'', 2, new)), 2) - - self.assertRaises(SystemError, resize, b'abc', -1, False) - self.assertRaises(SystemError, resize, bytearray(b'abc'), 3, False) - - # CRASHES resize(NULL, 0, False) - # CRASHES resize(NULL, 3, False) - - def test_join(self): - """Test PyBytes_Join()""" - bytes_join = _testcapi.bytes_join - - self.assertEqual(bytes_join(b'', []), b'') - self.assertEqual(bytes_join(b'sep', []), b'') - - self.assertEqual(bytes_join(b'', [b'a', b'b', b'c']), b'abc') - self.assertEqual(bytes_join(b'-', [b'a', b'b', b'c']), b'a-b-c') - self.assertEqual(bytes_join(b' - ', [b'a', b'b', b'c']), b'a - b - c') - self.assertEqual(bytes_join(b'-', [bytearray(b'abc'), - memoryview(b'def')]), - b'abc-def') - - self.assertEqual(bytes_join(b'-', iter([b'a', b'b', b'c'])), b'a-b-c') - - # invalid 'sep' argument - with self.assertRaises(TypeError): - bytes_join(bytearray(b'sep'), []) - with self.assertRaises(TypeError): - bytes_join(memoryview(b'sep'), []) - with self.assertRaises(TypeError): - bytes_join('', []) # empty Unicode string - with self.assertRaises(TypeError): - bytes_join('unicode', []) - with self.assertRaises(TypeError): - bytes_join(123, []) - with self.assertRaises(SystemError): - self.assertEqual(bytes_join(NULL, [b'a', b'b', b'c']), b'abc') - - # invalid 'iterable' argument - with self.assertRaises(TypeError): - bytes_join(b'', [b'bytes', 'unicode']) - with self.assertRaises(TypeError): - bytes_join(b'', [b'bytes', 123]) - with self.assertRaises(TypeError): - bytes_join(b'', 123) - with self.assertRaises(SystemError): - bytes_join(b'', NULL) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py deleted file mode 100644 index a0355c7a388..00000000000 --- a/Lib/test/test_capi/test_codecs.py +++ /dev/null @@ -1,879 +0,0 @@ -import codecs -import contextlib -import io -import re -import sys -import unittest -import unittest.mock as mock -import _testcapi -from test.support import import_helper - -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') - -NULL = None -BAD_ARGUMENT = re.escape('bad argument type for built-in operation') - - -class CAPIUnicodeTest(unittest.TestCase): - # TODO: Test the following functions: - # - # PyUnicode_BuildEncodingMap - # PyUnicode_FSConverter - # PyUnicode_FSDecoder - # PyUnicode_DecodeMBCS - # PyUnicode_DecodeMBCSStateful - # PyUnicode_DecodeCodePageStateful - # PyUnicode_AsMBCSString - # PyUnicode_EncodeCodePage - # PyUnicode_DecodeLocaleAndSize - # PyUnicode_DecodeLocale - # PyUnicode_EncodeLocale - # PyUnicode_DecodeFSDefault - # PyUnicode_DecodeFSDefaultAndSize - # PyUnicode_EncodeFSDefault - - def test_fromencodedobject(self): - """Test PyUnicode_FromEncodedObject()""" - fromencodedobject = _testlimitedcapi.unicode_fromencodedobject - - self.assertEqual(fromencodedobject(b'abc', NULL), 'abc') - self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc') - b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80' - s = 'a\xa1\u4f60\U0001f600' - self.assertEqual(fromencodedobject(b, NULL), s) - self.assertEqual(fromencodedobject(b, 'utf-8'), s) - self.assertEqual(fromencodedobject(b, 'latin1'), b.decode('latin1')) - self.assertRaises(UnicodeDecodeError, fromencodedobject, b, 'ascii') - self.assertEqual(fromencodedobject(b, 'ascii', 'replace'), - 'a' + '\ufffd'*9) - self.assertEqual(fromencodedobject(bytearray(b), NULL), s) - self.assertEqual(fromencodedobject(bytearray(b), 'utf-8'), s) - self.assertRaises(LookupError, fromencodedobject, b'abc', 'foo') - self.assertRaises(LookupError, fromencodedobject, b, 'ascii', 'foo') - self.assertRaises(TypeError, fromencodedobject, 'abc', NULL) - self.assertRaises(TypeError, fromencodedobject, 'abc', 'ascii') - self.assertRaises(TypeError, fromencodedobject, [], NULL) - self.assertRaises(TypeError, fromencodedobject, [], 'ascii') - self.assertRaises(SystemError, fromencodedobject, NULL, NULL) - self.assertRaises(SystemError, fromencodedobject, NULL, 'ascii') - - def test_decode(self): - """Test PyUnicode_Decode()""" - decode = _testlimitedcapi.unicode_decode - - self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]') - self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]') - self.assertEqual(decode(b'[\xa4]', 'iso8859-15', 'strict'), '[\u20ac]') - self.assertRaises(UnicodeDecodeError, decode, b'[\xa4]', 'utf-8') - self.assertEqual(decode(b'[\xa4]', 'utf-8', 'replace'), '[\ufffd]') - - self.assertEqual(decode(b'[\xe2\x82\xac]', NULL), '[\u20ac]') - self.assertEqual(decode(b'[\xa4]', NULL, 'replace'), '[\ufffd]') - - self.assertRaises(LookupError, decode, b'\xa4', 'foo') - self.assertRaises(LookupError, decode, b'\xa4', 'utf-8', 'foo') - # TODO: Test PyUnicode_Decode() with NULL as data and - # negative size. - - def test_asencodedstring(self): - """Test PyUnicode_AsEncodedString()""" - asencodedstring = _testlimitedcapi.unicode_asencodedstring - - self.assertEqual(asencodedstring('abc', NULL), b'abc') - self.assertEqual(asencodedstring('abc', 'ascii'), b'abc') - s = 'a\xa1\u4f60\U0001f600' - b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80' - self.assertEqual(asencodedstring(s, NULL), b) - self.assertEqual(asencodedstring(s, 'utf-8'), b) - self.assertEqual(asencodedstring('\xa1\xa2', 'latin1'), b'\xa1\xa2') - self.assertRaises(UnicodeEncodeError, asencodedstring, '\xa1\xa2', 'ascii') - self.assertEqual(asencodedstring(s, 'ascii', 'replace'), b'a???') - - self.assertRaises(LookupError, asencodedstring, 'abc', 'foo') - self.assertRaises(LookupError, asencodedstring, s, 'ascii', 'foo') - self.assertRaises(TypeError, asencodedstring, b'abc', NULL) - self.assertRaises(TypeError, asencodedstring, b'abc', 'ascii') - self.assertRaises(TypeError, asencodedstring, [], NULL) - self.assertRaises(TypeError, asencodedstring, [], 'ascii') - # CRASHES asencodedstring(NULL, NULL) - # CRASHES asencodedstring(NULL, 'ascii') - - def test_decodeutf8(self): - """Test PyUnicode_DecodeUTF8()""" - decodeutf8 = _testlimitedcapi.unicode_decodeutf8 - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - b = s.encode('utf-8') - self.assertEqual(decodeutf8(b), s) - self.assertEqual(decodeutf8(b, 'strict'), s) - - self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') - self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') - self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') - self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') - self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') - self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') - - self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') - # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and - # negative size. - - def test_decodeutf8stateful(self): - """Test PyUnicode_DecodeUTF8Stateful()""" - decodeutf8stateful = _testlimitedcapi.unicode_decodeutf8stateful - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - b = s.encode('utf-8') - self.assertEqual(decodeutf8stateful(b), (s, len(b))) - self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) - - self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') - self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') - self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') - self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) - self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) - self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') - self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) - - self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') - # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and - # negative size. - # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of - # "consumed". - - def test_asutf8string(self): - """Test PyUnicode_AsUTF8String()""" - asutf8string = _testlimitedcapi.unicode_asutf8string - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - self.assertEqual(asutf8string(s), s.encode('utf-8')) - - self.assertRaises(UnicodeEncodeError, asutf8string, '\ud8ff') - self.assertRaises(TypeError, asutf8string, b'abc') - self.assertRaises(TypeError, asutf8string, []) - # CRASHES asutf8string(NULL) - - def test_decodeutf16(self): - """Test PyUnicode_DecodeUTF16()""" - decodeutf16 = _testlimitedcapi.unicode_decodeutf16 - - naturalbyteorder = -1 if sys.byteorder == 'little' else 1 - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - b = s.encode('utf-16') - self.assertEqual(decodeutf16(0, b), (naturalbyteorder, s)) - b = s.encode('utf-16le') - self.assertEqual(decodeutf16(-1, b), (-1, s)) - self.assertEqual(decodeutf16(0, b'\xff\xfe'+b), (-1, s)) - b = s.encode('utf-16be') - self.assertEqual(decodeutf16(1, b), (1, s)) - self.assertEqual(decodeutf16(0, b'\xfe\xff'+b), (1, s)) - - self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'a') - self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'a') - self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xff\xfea') - self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xfe\xffa') - - self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x00\xde') - self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xde\x00') - self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xde\xde') - self.assertEqual(decodeutf16(-1, b'\x00\xde', 'replace'), (-1, '\ufffd')) - self.assertEqual(decodeutf16(1, b'\xde\x00', 'replace'), (1, '\ufffd')) - self.assertEqual(decodeutf16(0, b'\xde\xde', 'replace'), (0, '\ufffd')) - self.assertEqual(decodeutf16(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd')) - self.assertEqual(decodeutf16(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd')) - - self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x3d\xd8') - self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xd8\x3d') - self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xd8\xd8') - self.assertEqual(decodeutf16(-1, b'\x3d\xd8', 'replace'), (-1, '\ufffd')) - self.assertEqual(decodeutf16(1, b'\xd8\x3d', 'replace'), (1, '\ufffd')) - self.assertEqual(decodeutf16(0, b'\xd8\xd8', 'replace'), (0, '\ufffd')) - self.assertEqual(decodeutf16(0, b'\xff\xfe\x3d\xd8', 'replace'), (-1, '\ufffd')) - self.assertEqual(decodeutf16(0, b'\xfe\xff\xd8\x3d', 'replace'), (1, '\ufffd')) - - self.assertRaises(LookupError, decodeutf16, -1, b'\x00\xde', 'foo') - self.assertRaises(LookupError, decodeutf16, 1, b'\xde\x00', 'foo') - self.assertRaises(LookupError, decodeutf16, 0, b'\xde\xde', 'foo') - # TODO: Test PyUnicode_DecodeUTF16() with NULL as data and - # negative size. - - def test_decodeutf16stateful(self): - """Test PyUnicode_DecodeUTF16Stateful()""" - decodeutf16stateful = _testlimitedcapi.unicode_decodeutf16stateful - - naturalbyteorder = -1 if sys.byteorder == 'little' else 1 - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - b = s.encode('utf-16') - self.assertEqual(decodeutf16stateful(0, b), (naturalbyteorder, s, len(b))) - b = s.encode('utf-16le') - self.assertEqual(decodeutf16stateful(-1, b), (-1, s, len(b))) - self.assertEqual(decodeutf16stateful(0, b'\xff\xfe'+b), (-1, s, len(b)+2)) - b = s.encode('utf-16be') - self.assertEqual(decodeutf16stateful(1, b), (1, s, len(b))) - self.assertEqual(decodeutf16stateful(0, b'\xfe\xff'+b), (1, s, len(b)+2)) - - self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d'), (-1, 'a', 2)) - self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8'), (-1, 'a', 2)) - self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8\x00'), (-1, 'a', 2)) - self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8'), (1, 'a', 2)) - self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d'), (1, 'a', 2)) - self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d\xde'), (1, 'a', 2)) - self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x61\x00\x3d\xd8\x00'), (-1, 'a', 4)) - self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\x00\x61\xd8\x3d\xde'), (1, 'a', 4)) - - self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x00\xde') - self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xde\x00') - self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 0, b'\xde\xde') - self.assertEqual(decodeutf16stateful(-1, b'\x00\xde', 'replace'), (-1, '\ufffd', 2)) - self.assertEqual(decodeutf16stateful(1, b'\xde\x00', 'replace'), (1, '\ufffd', 2)) - self.assertEqual(decodeutf16stateful(0, b'\xde\xde', 'replace'), (0, '\ufffd', 2)) - self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd', 4)) - self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd', 4)) - - self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x3d\xd8\x61\x00') - self.assertEqual(decodeutf16stateful(-1, b'\x3d\xd8\x61\x00', 'replace'), (-1, '\ufffda', 4)) - self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xd8\x3d\x00\x61') - self.assertEqual(decodeutf16stateful(1, b'\xd8\x3d\x00\x61', 'replace'), (1, '\ufffda', 4)) - - self.assertRaises(LookupError, decodeutf16stateful, -1, b'\x00\xde', 'foo') - self.assertRaises(LookupError, decodeutf16stateful, 1, b'\xde\x00', 'foo') - self.assertRaises(LookupError, decodeutf16stateful, 0, b'\xde\xde', 'foo') - # TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as data and - # negative size. - # TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as the address of - # "consumed". - - def test_asutf16string(self): - """Test PyUnicode_AsUTF16String()""" - asutf16string = _testlimitedcapi.unicode_asutf16string - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - self.assertEqual(asutf16string(s), s.encode('utf-16')) - - self.assertRaises(UnicodeEncodeError, asutf16string, '\ud8ff') - self.assertRaises(TypeError, asutf16string, b'abc') - self.assertRaises(TypeError, asutf16string, []) - # CRASHES asutf16string(NULL) - - def test_decodeutf32(self): - """Test PyUnicode_DecodeUTF8()""" - decodeutf32 = _testlimitedcapi.unicode_decodeutf32 - - naturalbyteorder = -1 if sys.byteorder == 'little' else 1 - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - b = s.encode('utf-32') - self.assertEqual(decodeutf32(0, b), (naturalbyteorder, s)) - b = s.encode('utf-32le') - self.assertEqual(decodeutf32(-1, b), (-1, s)) - self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00'+b), (-1, s)) - b = s.encode('utf-32be') - self.assertEqual(decodeutf32(1, b), (1, s)) - self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff'+b), (1, s)) - - self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x61\x00\x00\x00\x00') - self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\x00\x61\x00') - self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00') - self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00') - - self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\xff\xff\xff\xff') - self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\xff\xff\xff\xff') - self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xff\xff\xff') - self.assertEqual(decodeutf32(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd')) - self.assertEqual(decodeutf32(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd')) - self.assertEqual(decodeutf32(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd')) - self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd')) - self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd')) - - self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x3d\xd8\x00\x00') - self.assertEqual(decodeutf32(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd')) - self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\xd8\x3d') - self.assertEqual(decodeutf32(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd')) - - self.assertRaises(LookupError, decodeutf32, -1, b'\xff\xff\xff\xff', 'foo') - self.assertRaises(LookupError, decodeutf32, 1, b'\xff\xff\xff\xff', 'foo') - self.assertRaises(LookupError, decodeutf32, 0, b'\xff\xff\xff\xff', 'foo') - # TODO: Test PyUnicode_DecodeUTF32() with NULL as data and - # negative size. - - def test_decodeutf32stateful(self): - """Test PyUnicode_DecodeUTF32Stateful()""" - decodeutf32stateful = _testlimitedcapi.unicode_decodeutf32stateful - - naturalbyteorder = -1 if sys.byteorder == 'little' else 1 - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - b = s.encode('utf-32') - self.assertEqual(decodeutf32stateful(0, b), (naturalbyteorder, s, len(b))) - b = s.encode('utf-32le') - self.assertEqual(decodeutf32stateful(-1, b), (-1, s, len(b))) - self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, s, len(b)+4)) - b = s.encode('utf-32be') - self.assertEqual(decodeutf32stateful(1, b), (1, s, len(b))) - self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, s, len(b)+4)) - - self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00'), (-1, 'a', 4)) - self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6'), (-1, 'a', 4)) - self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 4)) - self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00'), (1, 'a', 4)) - self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01'), (1, 'a', 4)) - self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 4)) - self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 8)) - self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 8)) - - for b in b'\xff', b'\xff\xff', b'\xff\xff\xff': - self.assertEqual(decodeutf32stateful(-1, b), (-1, '', 0)) - self.assertEqual(decodeutf32stateful(1, b), (1, '', 0)) - self.assertEqual(decodeutf32stateful(0, b), (0, '', 0)) - self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, '', 4)) - self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, '', 4)) - self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\xff\xff\xff\xff') - self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\xff\xff\xff\xff') - self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 0, b'\xff\xff\xff\xff') - self.assertEqual(decodeutf32stateful(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 4)) - self.assertEqual(decodeutf32stateful(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 4)) - self.assertEqual(decodeutf32stateful(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd', 4)) - self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 8)) - self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 8)) - - self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\x3d\xd8\x00\x00') - self.assertEqual(decodeutf32stateful(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd', 4)) - self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\x00\x00\xd8\x3d') - self.assertEqual(decodeutf32stateful(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd', 4)) - - self.assertRaises(LookupError, decodeutf32stateful, -1, b'\xff\xff\xff\xff', 'foo') - self.assertRaises(LookupError, decodeutf32stateful, 1, b'\xff\xff\xff\xff', 'foo') - self.assertRaises(LookupError, decodeutf32stateful, 0, b'\xff\xff\xff\xff', 'foo') - # TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as data and - # negative size. - # TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as the address of - # "consumed". - - def test_asutf32string(self): - """Test PyUnicode_AsUTF32String()""" - asutf32string = _testlimitedcapi.unicode_asutf32string - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: - self.assertEqual(asutf32string(s), s.encode('utf-32')) - - self.assertRaises(UnicodeEncodeError, asutf32string, '\ud8ff') - self.assertRaises(TypeError, asutf32string, b'abc') - self.assertRaises(TypeError, asutf32string, []) - # CRASHES asutf32string(NULL) - - def test_decodelatin1(self): - """Test PyUnicode_DecodeLatin1()""" - decodelatin1 = _testlimitedcapi.unicode_decodelatin1 - - self.assertEqual(decodelatin1(b'abc'), 'abc') - self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc') - self.assertEqual(decodelatin1(b'\xa1\xa2'), '\xa1\xa2') - self.assertEqual(decodelatin1(b'\xa1\xa2', 'strict'), '\xa1\xa2') - # TODO: Test PyUnicode_DecodeLatin1() with NULL as data and - # negative size. - - def test_aslatin1string(self): - """Test PyUnicode_AsLatin1String()""" - aslatin1string = _testlimitedcapi.unicode_aslatin1string - - self.assertEqual(aslatin1string('abc'), b'abc') - self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2') - - self.assertRaises(UnicodeEncodeError, aslatin1string, '\u4f60') - self.assertRaises(TypeError, aslatin1string, b'abc') - self.assertRaises(TypeError, aslatin1string, []) - # CRASHES aslatin1string(NULL) - - def test_decodeascii(self): - """Test PyUnicode_DecodeASCII()""" - decodeascii = _testlimitedcapi.unicode_decodeascii - - self.assertEqual(decodeascii(b'abc'), 'abc') - self.assertEqual(decodeascii(b'abc', 'strict'), 'abc') - - self.assertRaises(UnicodeDecodeError, decodeascii, b'\xff') - self.assertEqual(decodeascii(b'a\xff', 'replace'), 'a\ufffd') - self.assertEqual(decodeascii(b'a\xffb', 'replace'), 'a\ufffdb') - - self.assertRaises(LookupError, decodeascii, b'a\xff', 'foo') - # TODO: Test PyUnicode_DecodeASCII() with NULL as data and - # negative size. - - def test_asasciistring(self): - """Test PyUnicode_AsASCIIString()""" - asasciistring = _testlimitedcapi.unicode_asasciistring - - self.assertEqual(asasciistring('abc'), b'abc') - - self.assertRaises(UnicodeEncodeError, asasciistring, '\x80') - self.assertRaises(TypeError, asasciistring, b'abc') - self.assertRaises(TypeError, asasciistring, []) - # CRASHES asasciistring(NULL) - - def test_decodecharmap(self): - """Test PyUnicode_DecodeCharmap()""" - decodecharmap = _testlimitedcapi.unicode_decodecharmap - - self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac') - self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac') - self.assertEqual(decodecharmap(b'\1\0\2', 'abc'), 'bac') - self.assertEqual(decodecharmap(b'\1\0\2', ['\xa1', '\xa2', '\xa3']), '\xa2\xa1\xa3') - self.assertEqual(decodecharmap(b'\1\0\2', ['\u4f60', '\u597d', '\u4e16']), '\u597d\u4f60\u4e16') - self.assertEqual(decodecharmap(b'\1\0\2', ['\U0001f600', '\U0001f601', '\U0001f602']), '\U0001f601\U0001f600\U0001f602') - - self.assertEqual(decodecharmap(b'\1\0\2', [97, 98, 99]), 'bac') - self.assertEqual(decodecharmap(b'\1\0\2', ['', 'b', 'cd']), 'bcd') - - self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {}) - self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {0: None}) - self.assertEqual(decodecharmap(b'\1\0\2', [None, 'b', 'c'], 'replace'), 'b\ufffdc') - self.assertEqual(decodecharmap(b'\1\0\2\xff', NULL), '\1\0\2\xff') - self.assertRaises(TypeError, decodecharmap, b'\0', 42) - - # TODO: Test PyUnicode_DecodeCharmap() with NULL as data and - # negative size. - - def test_ascharmapstring(self): - """Test PyUnicode_AsCharmapString()""" - ascharmapstring = _testlimitedcapi.unicode_ascharmapstring - - self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7') - self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7') - self.assertEqual(ascharmapstring('\u4f60\u597d\u4e16', {0x4f60: 3, 0x597d: 0, 0x4e16: 7}), b'\3\0\7') - self.assertEqual(ascharmapstring('\U0001f600\U0001f601\U0001f602', {0x1f600: 3, 0x1f601: 0, 0x1f602: 7}), b'\3\0\7') - self.assertEqual(ascharmapstring('abc', {97: 3, 98: b'', 99: b'spam'}), b'\3spam') - - self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {}) - self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {97: None}) - self.assertRaises(TypeError, ascharmapstring, b'a', {}) - self.assertRaises(TypeError, ascharmapstring, [], {}) - self.assertRaises(TypeError, ascharmapstring, 'a', NULL) - # CRASHES ascharmapstring(NULL, {}) - - def test_decodeunicodeescape(self): - """Test PyUnicode_DecodeUnicodeEscape()""" - decodeunicodeescape = _testlimitedcapi.unicode_decodeunicodeescape - - self.assertEqual(decodeunicodeescape(b'abc'), 'abc') - self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\') - self.assertEqual(decodeunicodeescape(b'\t\n\r\x0b\x0c\x00'), '\t\n\r\v\f\0') - self.assertEqual(decodeunicodeescape(br'\xa1\xa2'), '\xa1\xa2') - self.assertEqual(decodeunicodeescape(b'\xa1\xa2'), '\xa1\xa2') - self.assertEqual(decodeunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d') - self.assertEqual(decodeunicodeescape(br'\U0001f600'), '\U0001f600') - with self.assertWarns(DeprecationWarning): - self.assertEqual(decodeunicodeescape(br'\z'), r'\z') - - for b in b'\\', br'\xa', br'\u4f6', br'\U0001f60': - self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b) - self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b, 'strict') - self.assertEqual(decodeunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd') - self.assertEqual(decodeunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy') - - self.assertRaises(LookupError, decodeunicodeescape, b'\\', 'foo') - # TODO: Test PyUnicode_DecodeUnicodeEscape() with NULL as data and - # negative size. - - def test_asunicodeescapestring(self): - """Test PyUnicode_AsUnicodeEscapeString()""" - asunicodeescapestring = _testlimitedcapi.unicode_asunicodeescapestring - - self.assertEqual(asunicodeescapestring('abc'), b'abc') - self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\') - self.assertEqual(asunicodeescapestring('\xa1\xa2'), br'\xa1\xa2') - self.assertEqual(asunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d') - self.assertEqual(asunicodeescapestring('\U0001f600'), br'\U0001f600') - - self.assertRaises(TypeError, asunicodeescapestring, b'abc') - self.assertRaises(TypeError, asunicodeescapestring, []) - # CRASHES asunicodeescapestring(NULL) - - def test_decoderawunicodeescape(self): - """Test PyUnicode_DecodeRawUnicodeEscape()""" - decoderawunicodeescape = _testlimitedcapi.unicode_decoderawunicodeescape - - self.assertEqual(decoderawunicodeescape(b'abc'), 'abc') - self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\') - self.assertEqual(decoderawunicodeescape(b'\xa1\xa2'), '\xa1\xa2') - self.assertEqual(decoderawunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d') - self.assertEqual(decoderawunicodeescape(br'\U0001f600'), '\U0001f600') - self.assertEqual(decoderawunicodeescape(br'\xa1\xa2'), r'\xa1\xa2') - self.assertEqual(decoderawunicodeescape(br'\z'), r'\z') - - for b in br'\u4f6', br'\U0001f60': - self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b) - self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b, 'strict') - self.assertEqual(decoderawunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd') - self.assertEqual(decoderawunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy') - - self.assertRaises(LookupError, decoderawunicodeescape, br'\U0001f60', 'foo') - # TODO: Test PyUnicode_DecodeRawUnicodeEscape() with NULL as data and - # negative size. - - def test_asrawunicodeescapestring(self): - """Test PyUnicode_AsRawUnicodeEscapeString()""" - asrawunicodeescapestring = _testlimitedcapi.unicode_asrawunicodeescapestring - - self.assertEqual(asrawunicodeescapestring('abc'), b'abc') - self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\') - self.assertEqual(asrawunicodeescapestring('\xa1\xa2'), b'\xa1\xa2') - self.assertEqual(asrawunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d') - self.assertEqual(asrawunicodeescapestring('\U0001f600'), br'\U0001f600') - - self.assertRaises(TypeError, asrawunicodeescapestring, b'abc') - self.assertRaises(TypeError, asrawunicodeescapestring, []) - # CRASHES asrawunicodeescapestring(NULL) - - -class CAPICodecs(unittest.TestCase): - - def setUp(self): - # Encoding names are normalized internally by converting them - # to lowercase and their hyphens are replaced by underscores. - self.encoding_name = 'test.test_capi.test_codecs.codec_reversed' - # Make sure that our custom codec is not already registered (that - # way we know whether we correctly unregistered the custom codec - # after a test or not). - self.assertRaises(LookupError, codecs.lookup, self.encoding_name) - # create the search function without registering yet - self._create_custom_codec() - - def _create_custom_codec(self): - def codec_encoder(m, errors='strict'): - return (type(m)().join(reversed(m)), len(m)) - - def codec_decoder(c, errors='strict'): - return (type(c)().join(reversed(c)), len(c)) - - class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return codec_encoder(input) - - class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codec_decoder(input) - - class StreamReader(codecs.StreamReader): - def encode(self, input, errors='strict'): - return codec_encoder(input, errors=errors) - - def decode(self, input, errors='strict'): - return codec_decoder(input, errors=errors) - - class StreamWriter(codecs.StreamWriter): - def encode(self, input, errors='strict'): - return codec_encoder(input, errors=errors) - - def decode(self, input, errors='strict'): - return codec_decoder(input, errors=errors) - - info = codecs.CodecInfo( - encode=codec_encoder, - decode=codec_decoder, - streamreader=StreamReader, - streamwriter=StreamWriter, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - name=self.encoding_name - ) - - def search_function(encoding): - if encoding == self.encoding_name: - return info - return None - - self.codec_info = info - self.search_function = search_function - - @contextlib.contextmanager - def use_custom_encoder(self): - self.assertRaises(LookupError, codecs.lookup, self.encoding_name) - codecs.register(self.search_function) - yield - codecs.unregister(self.search_function) - self.assertRaises(LookupError, codecs.lookup, self.encoding_name) - - def test_codec_register(self): - search_function, encoding = self.search_function, self.encoding_name - # register the search function using the C API - self.assertIsNone(_testcapi.codec_register(search_function)) - # in case the test failed before cleaning up - self.addCleanup(codecs.unregister, self.search_function) - self.assertIs(codecs.lookup(encoding), search_function(encoding)) - self.assertEqual(codecs.encode('123', encoding=encoding), '321') - # unregister the search function using the regular API - codecs.unregister(search_function) - self.assertRaises(LookupError, codecs.lookup, encoding) - - def test_codec_unregister(self): - search_function, encoding = self.search_function, self.encoding_name - self.assertRaises(LookupError, codecs.lookup, encoding) - # register the search function using the regular API - codecs.register(search_function) - # in case the test failed before cleaning up - self.addCleanup(codecs.unregister, self.search_function) - self.assertIsNotNone(codecs.lookup(encoding)) - # unregister the search function using the C API - self.assertIsNone(_testcapi.codec_unregister(search_function)) - self.assertRaises(LookupError, codecs.lookup, encoding) - - def test_codec_known_encoding(self): - self.assertRaises(LookupError, codecs.lookup, 'unknown-codec') - self.assertFalse(_testcapi.codec_known_encoding('unknown-codec')) - self.assertFalse(_testcapi.codec_known_encoding('unknown_codec')) - self.assertFalse(_testcapi.codec_known_encoding('UNKNOWN-codec')) - - encoding_name = self.encoding_name - self.assertRaises(LookupError, codecs.lookup, encoding_name) - - codecs.register(self.search_function) - self.addCleanup(codecs.unregister, self.search_function) - - for name in [ - encoding_name, - encoding_name.upper(), - encoding_name.replace('_', '-'), - ]: - with self.subTest(name): - self.assertTrue(_testcapi.codec_known_encoding(name)) - - def test_codec_encode(self): - encode = _testcapi.codec_encode - self.assertEqual(encode('a', 'utf-8', NULL), b'a') - self.assertEqual(encode('a', 'utf-8', 'strict'), b'a') - self.assertEqual(encode('[é]', 'ascii', 'ignore'), b'[]') - - self.assertRaises(TypeError, encode, NULL, 'ascii', 'strict') - with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): - encode('a', NULL, 'strict') - - def test_codec_decode(self): - decode = _testcapi.codec_decode - - s = 'a\xa1\u4f60\U0001f600' - b = s.encode() - - self.assertEqual(decode(b, 'utf-8', 'strict'), s) - self.assertEqual(decode(b, 'utf-8', NULL), s) - self.assertEqual(decode(b, 'latin1', 'strict'), b.decode('latin1')) - self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', 'strict') - self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', NULL) - self.assertEqual(decode(b, 'ascii', 'replace'), 'a' + '\ufffd'*9) - - # _codecs.decode() only reports an unknown error handling name when - # the corresponding error handling function is used; this difers - # from PyUnicode_Decode() which checks that both the encoding and - # the error handling name are recognized before even attempting to - # call the decoder. - self.assertEqual(decode(b'', 'utf-8', 'unknown-error-handler'), '') - self.assertEqual(decode(b'a', 'utf-8', 'unknown-error-handler'), 'a') - - self.assertRaises(TypeError, decode, NULL, 'ascii', 'strict') - with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): - decode(b, NULL, 'strict') - - def test_codec_encoder(self): - codec_encoder = _testcapi.codec_encoder - - with self.use_custom_encoder(): - encoder = codec_encoder(self.encoding_name) - self.assertIs(encoder, self.codec_info.encode) - - with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): - codec_encoder(NULL) - - def test_codec_decoder(self): - codec_decoder = _testcapi.codec_decoder - - with self.use_custom_encoder(): - decoder = codec_decoder(self.encoding_name) - self.assertIs(decoder, self.codec_info.decode) - - with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): - codec_decoder(NULL) - - def test_codec_incremental_encoder(self): - codec_incremental_encoder = _testcapi.codec_incremental_encoder - - with self.use_custom_encoder(): - encoding = self.encoding_name - - for errors in ['strict', NULL]: - with self.subTest(errors): - encoder = codec_incremental_encoder(encoding, errors) - self.assertIsInstance(encoder, self.codec_info.incrementalencoder) - - with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): - codec_incremental_encoder(NULL, 'strict') - - def test_codec_incremental_decoder(self): - codec_incremental_decoder = _testcapi.codec_incremental_decoder - - with self.use_custom_encoder(): - encoding = self.encoding_name - - for errors in ['strict', NULL]: - with self.subTest(errors): - decoder = codec_incremental_decoder(encoding, errors) - self.assertIsInstance(decoder, self.codec_info.incrementaldecoder) - - with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): - codec_incremental_decoder(NULL, 'strict') - - def test_codec_stream_reader(self): - codec_stream_reader = _testcapi.codec_stream_reader - - with self.use_custom_encoder(): - encoding, stream = self.encoding_name, io.StringIO() - for errors in ['strict', NULL]: - with self.subTest(errors): - writer = codec_stream_reader(encoding, stream, errors) - self.assertIsInstance(writer, self.codec_info.streamreader) - - with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): - codec_stream_reader(NULL, stream, 'strict') - - def test_codec_stream_writer(self): - codec_stream_writer = _testcapi.codec_stream_writer - - with self.use_custom_encoder(): - encoding, stream = self.encoding_name, io.StringIO() - for errors in ['strict', NULL]: - with self.subTest(errors): - writer = codec_stream_writer(encoding, stream, errors) - self.assertIsInstance(writer, self.codec_info.streamwriter) - - with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): - codec_stream_writer(NULL, stream, 'strict') - - -class CAPICodecErrors(unittest.TestCase): - - @classmethod - def _generate_exception_args(cls): - for objlen in range(5): - maxind = 2 * max(2, objlen) - for start in range(-maxind, maxind + 1): - for end in range(-maxind, maxind + 1): - yield objlen, start, end - - @classmethod - def generate_encode_errors(cls): - return tuple( - UnicodeEncodeError('utf-8', '0' * objlen, start, end, 'why') - for objlen, start, end in cls._generate_exception_args() - ) - - @classmethod - def generate_decode_errors(cls): - return tuple( - UnicodeDecodeError('utf-8', b'0' * objlen, start, end, 'why') - for objlen, start, end in cls._generate_exception_args() - ) - - @classmethod - def generate_translate_errors(cls): - return tuple( - UnicodeTranslateError('0' * objlen, start, end, 'why') - for objlen, start, end in cls._generate_exception_args() - ) - - @classmethod - def setUpClass(cls): - cls.unicode_encode_errors = cls.generate_encode_errors() - cls.unicode_decode_errors = cls.generate_decode_errors() - cls.unicode_translate_errors = cls.generate_translate_errors() - cls.all_unicode_errors = ( - cls.unicode_encode_errors - + cls.unicode_decode_errors - + cls.unicode_translate_errors - ) - cls.bad_unicode_errors = ( - ValueError(), - ) - - def test_codec_register_error(self): - # for cleaning up between tests - from _codecs import _unregister_error as _codecs_unregister_error - - self.assertRaises(LookupError, _testcapi.codec_lookup_error, 'custom') - - def custom_error_handler(exc): - raise exc - - error_handler = mock.Mock(wraps=custom_error_handler) - _testcapi.codec_register_error('custom', error_handler) - self.addCleanup(_codecs_unregister_error, 'custom') - - self.assertRaises(UnicodeEncodeError, codecs.encode, - '\xff', 'ascii', errors='custom') - error_handler.assert_called_once() - error_handler.reset_mock() - - self.assertRaises(UnicodeDecodeError, codecs.decode, - b'\xff', 'ascii', errors='custom') - error_handler.assert_called_once() - - # _codecs._unregister_error directly delegates to the internal C - # function so a Python-level function test is sufficient (it is - # tested in test_codeccallbacks). - - def test_codec_lookup_error(self): - codec_lookup_error = _testcapi.codec_lookup_error - self.assertIs(codec_lookup_error(NULL), codecs.strict_errors) - self.assertIs(codec_lookup_error('strict'), codecs.strict_errors) - self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) - self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) - self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) - self.assertIs(codec_lookup_error('backslashreplace'), codecs.backslashreplace_errors) - self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) - self.assertRaises(LookupError, codec_lookup_error, 'unknown') - - def test_codec_strict_errors_handler(self): - handler = _testcapi.codec_strict_errors - for exc in self.all_unicode_errors + self.bad_unicode_errors: - with self.subTest(handler=handler, exc=exc): - self.assertRaises(type(exc), handler, exc) - - def test_codec_ignore_errors_handler(self): - handler = _testcapi.codec_ignore_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors) - - def test_codec_replace_errors_handler(self): - handler = _testcapi.codec_replace_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors) - - def test_codec_xmlcharrefreplace_errors_handler(self): - handler = _testcapi.codec_xmlcharrefreplace_errors - self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) - - def test_codec_backslashreplace_errors_handler(self): - handler = _testcapi.codec_backslashreplace_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors) - - def test_codec_namereplace_errors_handler(self): - handler = _testlimitedcapi.codec_namereplace_errors - self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) - - def do_test_codec_errors_handler(self, handler, exceptions): - self.assertNotEqual(len(exceptions), 0) - for exc in exceptions: - with self.subTest(handler=handler, exc=exc): - # test that the handler does not crash - res = handler(exc) - self.assertIsInstance(res, tuple) - self.assertEqual(len(res), 2) - replacement, continue_from = res - self.assertIsInstance(replacement, str) - self.assertIsInstance(continue_from, int) - self.assertGreaterEqual(continue_from, 0) - self.assertLessEqual(continue_from, len(exc.object)) - - for bad_exc in ( - self.bad_unicode_errors - + tuple(e for e in self.all_unicode_errors if e not in exceptions) - ): - with self.subTest('bad type', handler=handler, exc=bad_exc): - self.assertRaises(TypeError, handler, bad_exc) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py deleted file mode 100644 index c3189a67cc7..00000000000 --- a/Lib/test/test_capi/test_complex.py +++ /dev/null @@ -1,299 +0,0 @@ -from math import isnan -import errno -import unittest -import warnings - -from test.test_capi.test_getargs import (BadComplex, BadComplex2, Complex, - FloatSubclass, Float, BadFloat, - BadFloat2, ComplexSubclass) -from test.support import import_helper -from test.support.testcase import ComplexesAreIdenticalMixin - - -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') -_testinternalcapi = import_helper.import_module('_testinternalcapi') - -NULL = None -INF = float("inf") -NAN = float("nan") -DBL_MAX = _testcapi.DBL_MAX - - -class BadComplex3: - def __complex__(self): - raise RuntimeError - - -class CAPIComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase): - def test_check(self): - # Test PyComplex_Check() - check = _testlimitedcapi.complex_check - - self.assertTrue(check(1+2j)) - self.assertTrue(check(ComplexSubclass(1+2j))) - self.assertFalse(check(Complex())) - self.assertFalse(check(3)) - self.assertFalse(check(3.0)) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_checkexact(self): - # PyComplex_CheckExact() - checkexact = _testlimitedcapi.complex_checkexact - - self.assertTrue(checkexact(1+2j)) - self.assertFalse(checkexact(ComplexSubclass(1+2j))) - self.assertFalse(checkexact(Complex())) - self.assertFalse(checkexact(3)) - self.assertFalse(checkexact(3.0)) - self.assertFalse(checkexact(object())) - - # CRASHES checkexact(NULL) - - def test_fromccomplex(self): - # Test PyComplex_FromCComplex() - fromccomplex = _testcapi.complex_fromccomplex - - self.assertEqual(fromccomplex(1+2j), 1.0+2.0j) - - def test_fromdoubles(self): - # Test PyComplex_FromDoubles() - fromdoubles = _testlimitedcapi.complex_fromdoubles - - self.assertEqual(fromdoubles(1.0, 2.0), 1.0+2.0j) - - def test_realasdouble(self): - # Test PyComplex_RealAsDouble() - realasdouble = _testlimitedcapi.complex_realasdouble - - self.assertEqual(realasdouble(1+2j), 1.0) - self.assertEqual(realasdouble(-1+0j), -1.0) - self.assertEqual(realasdouble(4.25), 4.25) - self.assertEqual(realasdouble(-1.0), -1.0) - self.assertEqual(realasdouble(42), 42.) - self.assertEqual(realasdouble(-1), -1.0) - - # Test subclasses of complex/float - self.assertEqual(realasdouble(ComplexSubclass(1+2j)), 1.0) - self.assertEqual(realasdouble(FloatSubclass(4.25)), 4.25) - - # Test types with __complex__ dunder method - self.assertEqual(realasdouble(Complex()), 4.25) - self.assertRaises(TypeError, realasdouble, BadComplex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(realasdouble(BadComplex2()), 4.25) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, realasdouble, BadComplex2()) - self.assertRaises(RuntimeError, realasdouble, BadComplex3()) - - # Test types with __float__ dunder method - self.assertEqual(realasdouble(Float()), 4.25) - self.assertRaises(TypeError, realasdouble, BadFloat()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(realasdouble(BadFloat2()), 4.25) - - self.assertRaises(TypeError, realasdouble, object()) - - # CRASHES realasdouble(NULL) - - def test_imagasdouble(self): - # Test PyComplex_ImagAsDouble() - imagasdouble = _testlimitedcapi.complex_imagasdouble - - self.assertEqual(imagasdouble(1+2j), 2.0) - self.assertEqual(imagasdouble(1-1j), -1.0) - self.assertEqual(imagasdouble(4.25), 0.0) - self.assertEqual(imagasdouble(42), 0.0) - - # Test subclasses of complex/float - self.assertEqual(imagasdouble(ComplexSubclass(1+2j)), 2.0) - self.assertEqual(imagasdouble(FloatSubclass(4.25)), 0.0) - - # Test types with __complex__ dunder method - self.assertEqual(imagasdouble(Complex()), 0.5) - self.assertRaises(TypeError, imagasdouble, BadComplex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(imagasdouble(BadComplex2()), 0.5) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, imagasdouble, BadComplex2()) - self.assertRaises(RuntimeError, imagasdouble, BadComplex3()) - - # Test types with __float__ dunder method - self.assertEqual(imagasdouble(Float()), 0.0) - self.assertRaises(TypeError, imagasdouble, BadFloat()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(imagasdouble(BadFloat2()), 0.0) - - self.assertRaises(TypeError, imagasdouble, object()) - - # CRASHES imagasdouble(NULL) - - def test_asccomplex(self): - # Test PyComplex_AsCComplex() - asccomplex = _testcapi.complex_asccomplex - - self.assertEqual(asccomplex(1+2j), 1.0+2.0j) - self.assertEqual(asccomplex(-1+2j), -1.0+2.0j) - self.assertEqual(asccomplex(4.25), 4.25+0.0j) - self.assertEqual(asccomplex(-1.0), -1.0+0.0j) - self.assertEqual(asccomplex(42), 42+0j) - self.assertEqual(asccomplex(-1), -1.0+0.0j) - - # Test subclasses of complex/float - self.assertEqual(asccomplex(ComplexSubclass(1+2j)), 1.0+2.0j) - self.assertEqual(asccomplex(FloatSubclass(4.25)), 4.25+0.0j) - - # Test types with __complex__ dunder method - self.assertEqual(asccomplex(Complex()), 4.25+0.5j) - self.assertRaises(TypeError, asccomplex, BadComplex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(asccomplex(BadComplex2()), 4.25+0.5j) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, asccomplex, BadComplex2()) - self.assertRaises(RuntimeError, asccomplex, BadComplex3()) - - # Test types with __float__ dunder method - self.assertEqual(asccomplex(Float()), 4.25+0.0j) - self.assertRaises(TypeError, asccomplex, BadFloat()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(asccomplex(BadFloat2()), 4.25+0.0j) - - self.assertRaises(TypeError, asccomplex, object()) - - # CRASHES asccomplex(NULL) - - def test_py_c_sum(self): - # Test _Py_c_sum() - _py_c_sum = _testcapi._py_c_sum - - self.assertEqual(_py_c_sum(1, 1j), (1+1j, 0)) - - def test_py_cr_sum(self): - # Test _Py_cr_sum() - _py_cr_sum = _testinternalcapi._py_cr_sum - - self.assertComplexesAreIdentical(_py_cr_sum(-0j, -0.0)[0], - complex(-0.0, -0.0)) - - def test_py_c_diff(self): - # Test _Py_c_diff() - _py_c_diff = _testcapi._py_c_diff - - self.assertEqual(_py_c_diff(1, 1j), (1-1j, 0)) - - def test_py_cr_diff(self): - # Test _Py_cr_diff() - _py_cr_diff = _testinternalcapi._py_cr_diff - - self.assertComplexesAreIdentical(_py_cr_diff(-0j, 0.0)[0], - complex(-0.0, -0.0)) - - def test_py_rc_diff(self): - # Test _Py_rc_diff() - _py_rc_diff = _testinternalcapi._py_rc_diff - - self.assertComplexesAreIdentical(_py_rc_diff(-0.0, 0j)[0], - complex(-0.0, -0.0)) - - def test_py_c_neg(self): - # Test _Py_c_neg() - _py_c_neg = _testcapi._py_c_neg - - self.assertEqual(_py_c_neg(1+1j), -1-1j) - - def test_py_c_prod(self): - # Test _Py_c_prod() - _py_c_prod = _testcapi._py_c_prod - - self.assertEqual(_py_c_prod(2, 1j), (2j, 0)) - - def test_py_cr_prod(self): - # Test _Py_cr_prod() - _py_cr_prod = _testinternalcapi._py_cr_prod - - self.assertComplexesAreIdentical(_py_cr_prod(complex('inf+1j'), INF)[0], - complex('inf+infj')) - - def test_py_c_quot(self): - # Test _Py_c_quot() - _py_c_quot = _testcapi._py_c_quot - - self.assertEqual(_py_c_quot(1, 1j), (-1j, 0)) - self.assertEqual(_py_c_quot(1, -1j), (1j, 0)) - self.assertEqual(_py_c_quot(1j, 2), (0.5j, 0)) - self.assertEqual(_py_c_quot(1j, -2), (-0.5j, 0)) - self.assertEqual(_py_c_quot(1, 2j), (-0.5j, 0)) - - z, e = _py_c_quot(NAN, 1j) - self.assertTrue(isnan(z.real)) - self.assertTrue(isnan(z.imag)) - self.assertEqual(e, 0) - - z, e = _py_c_quot(1j, NAN) - self.assertTrue(isnan(z.real)) - self.assertTrue(isnan(z.imag)) - self.assertEqual(e, 0) - - self.assertEqual(_py_c_quot(1, 0j)[1], errno.EDOM) - - def test_py_cr_quot(self): - # Test _Py_cr_quot() - _py_cr_quot = _testinternalcapi._py_cr_quot - - self.assertComplexesAreIdentical(_py_cr_quot(complex('inf+1j'), 2**1000)[0], - INF + 2**-1000*1j) - - def test_py_rc_quot(self): - # Test _Py_rc_quot() - _py_rc_quot = _testinternalcapi._py_rc_quot - - self.assertComplexesAreIdentical(_py_rc_quot(1.0, complex('nan-infj'))[0], - 0j) - - def test_py_c_pow(self): - # Test _Py_c_pow() - _py_c_pow = _testcapi._py_c_pow - - self.assertEqual(_py_c_pow(1j, 0j), (1+0j, 0)) - self.assertEqual(_py_c_pow(1, 1j), (1+0j, 0)) - self.assertEqual(_py_c_pow(0j, 1), (0j, 0)) - self.assertAlmostEqual(_py_c_pow(1j, 2)[0], -1.0+0j) - - r, e = _py_c_pow(1+1j, -1) - self.assertAlmostEqual(r, 0.5-0.5j) - self.assertEqual(e, 0) - - self.assertEqual(_py_c_pow(0j, -1)[1], errno.EDOM) - self.assertEqual(_py_c_pow(0j, 1j)[1], errno.EDOM) - max_num = DBL_MAX+1j - self.assertEqual(_py_c_pow(max_num, max_num), - (complex(INF, INF), errno.ERANGE)) - self.assertEqual(_py_c_pow(max_num, 2), - (complex(INF, INF), errno.ERANGE)) - - - def test_py_c_abs(self): - # Test _Py_c_abs() - _py_c_abs = _testcapi._py_c_abs - - self.assertEqual(_py_c_abs(-1), (1.0, 0)) - self.assertEqual(_py_c_abs(1j), (1.0, 0)) - - self.assertEqual(_py_c_abs(complex('+inf+1j')), (INF, 0)) - self.assertEqual(_py_c_abs(complex('-inf+1j')), (INF, 0)) - self.assertEqual(_py_c_abs(complex('1.25+infj')), (INF, 0)) - self.assertEqual(_py_c_abs(complex('1.25-infj')), (INF, 0)) - - self.assertTrue(isnan(_py_c_abs(complex('1.25+nanj'))[0])) - self.assertTrue(isnan(_py_c_abs(complex('nan-1j'))[0])) - - self.assertEqual(_py_c_abs(complex(*[DBL_MAX]*2))[1], errno.ERANGE) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py deleted file mode 100644 index a2d70dd3af4..00000000000 --- a/Lib/test/test_capi/test_config.py +++ /dev/null @@ -1,404 +0,0 @@ -""" -Tests PyConfig_Get() and PyConfig_Set() C API (PEP 741). -""" -import os -import sys -import sysconfig -import types -import unittest -from test import support -from test.support import import_helper - -_testcapi = import_helper.import_module('_testcapi') - - -# Is the Py_STATS macro defined? -Py_STATS = hasattr(sys, '_stats_on') - - -class CAPITests(unittest.TestCase): - def test_config_get(self): - # Test PyConfig_Get() - config_get = _testcapi.config_get - config_names = _testcapi.config_names - - TEST_VALUE = { - str: "TEST_MARKER_STR", - str | None: "TEST_MARKER_OPT_STR", - list[str]: ("TEST_MARKER_STR_TUPLE",), - dict[str, str | bool]: {"x": "value", "y": True}, - } - - # read config options and check their type - options = [ - ("allocator", int, None), - ("argv", list[str], "argv"), - ("base_exec_prefix", str | None, "base_exec_prefix"), - ("base_executable", str | None, "_base_executable"), - ("base_prefix", str | None, "base_prefix"), - ("buffered_stdio", bool, None), - ("bytes_warning", int, None), - ("check_hash_pycs_mode", str, None), - ("code_debug_ranges", bool, None), - ("configure_c_stdio", bool, None), - ("coerce_c_locale", bool, None), - ("coerce_c_locale_warn", bool, None), - ("configure_locale", bool, None), - ("cpu_count", int, None), - ("dev_mode", bool, None), - ("dump_refs", bool, None), - ("dump_refs_file", str | None, None), - ("exec_prefix", str | None, "exec_prefix"), - ("executable", str | None, "executable"), - ("faulthandler", bool, None), - ("filesystem_encoding", str, None), - ("filesystem_errors", str, None), - ("hash_seed", int, None), - ("home", str | None, None), - ("thread_inherit_context", int, None), - ("context_aware_warnings", int, None), - ("import_time", int, None), - ("inspect", bool, None), - ("install_signal_handlers", bool, None), - ("int_max_str_digits", int, None), - ("interactive", bool, None), - ("isolated", bool, None), - ("malloc_stats", bool, None), - ("module_search_paths", list[str], "path"), - ("optimization_level", int, None), - ("orig_argv", list[str], "orig_argv"), - ("parser_debug", bool, None), - ("parse_argv", bool, None), - ("pathconfig_warnings", bool, None), - ("perf_profiling", int, None), - ("platlibdir", str, "platlibdir"), - ("prefix", str | None, "prefix"), - ("program_name", str, None), - ("pycache_prefix", str | None, "pycache_prefix"), - ("quiet", bool, None), - ("remote_debug", int, None), - ("run_command", str | None, None), - ("run_filename", str | None, None), - ("run_module", str | None, None), - ("safe_path", bool, None), - ("show_ref_count", bool, None), - ("site_import", bool, None), - ("skip_source_first_line", bool, None), - ("stdio_encoding", str, None), - ("stdio_errors", str, None), - ("stdlib_dir", str | None, "_stdlib_dir"), - ("tracemalloc", int, None), - ("use_environment", bool, None), - ("use_frozen_modules", bool, None), - ("use_hash_seed", bool, None), - ("user_site_directory", bool, None), - ("utf8_mode", bool, None), - ("verbose", int, None), - ("warn_default_encoding", bool, None), - ("warnoptions", list[str], "warnoptions"), - ("write_bytecode", bool, None), - ("xoptions", dict[str, str | bool], "_xoptions"), - ] - if support.Py_DEBUG: - options.append(("run_presite", str | None, None)) - if support.Py_GIL_DISABLED: - options.append(("enable_gil", int, None)) - options.append(("tlbc_enabled", int, None)) - if support.MS_WINDOWS: - options.extend(( - ("legacy_windows_stdio", bool, None), - ("legacy_windows_fs_encoding", bool, None), - )) - if Py_STATS: - options.extend(( - ("_pystats", bool, None), - )) - if support.is_apple: - options.extend(( - ("use_system_logger", bool, None), - )) - - for name, option_type, sys_attr in options: - with self.subTest(name=name, option_type=option_type, - sys_attr=sys_attr): - value = config_get(name) - if isinstance(option_type, types.GenericAlias): - self.assertIsInstance(value, option_type.__origin__) - if option_type.__origin__ == dict: - key_type = option_type.__args__[0] - value_type = option_type.__args__[1] - for item in value.items(): - self.assertIsInstance(item[0], key_type) - self.assertIsInstance(item[1], value_type) - else: - item_type = option_type.__args__[0] - for item in value: - self.assertIsInstance(item, item_type) - else: - self.assertIsInstance(value, option_type) - - if sys_attr is not None: - expected = getattr(sys, sys_attr) - self.assertEqual(expected, value) - - override = TEST_VALUE[option_type] - with support.swap_attr(sys, sys_attr, override): - self.assertEqual(config_get(name), override) - - # check that the test checks all options - self.assertEqual(sorted(name for name, option_type, sys_attr in options), - sorted(config_names())) - - def test_config_get_sys_flags(self): - # Test PyConfig_Get() - config_get = _testcapi.config_get - - # compare config options with sys.flags - for flag, name, negate in ( - ("debug", "parser_debug", False), - ("inspect", "inspect", False), - ("interactive", "interactive", False), - ("optimize", "optimization_level", False), - ("dont_write_bytecode", "write_bytecode", True), - ("no_user_site", "user_site_directory", True), - ("no_site", "site_import", True), - ("ignore_environment", "use_environment", True), - ("verbose", "verbose", False), - ("bytes_warning", "bytes_warning", False), - ("quiet", "quiet", False), - # "hash_randomization" is tested below - ("isolated", "isolated", False), - ("dev_mode", "dev_mode", False), - ("utf8_mode", "utf8_mode", False), - ("warn_default_encoding", "warn_default_encoding", False), - ("safe_path", "safe_path", False), - ("int_max_str_digits", "int_max_str_digits", False), - # "gil", "thread_inherit_context" and "context_aware_warnings" are tested below - ): - with self.subTest(flag=flag, name=name, negate=negate): - value = config_get(name) - if negate: - value = not value - self.assertEqual(getattr(sys.flags, flag), value) - - self.assertEqual(sys.flags.hash_randomization, - config_get('use_hash_seed') == 0 - or config_get('hash_seed') != 0) - - if support.Py_GIL_DISABLED: - value = config_get('enable_gil') - expected = (value if value != -1 else None) - self.assertEqual(sys.flags.gil, expected) - - expected_inherit_context = 1 if support.Py_GIL_DISABLED else 0 - self.assertEqual(sys.flags.thread_inherit_context, expected_inherit_context) - - expected_safe_warnings = 1 if support.Py_GIL_DISABLED else 0 - self.assertEqual(sys.flags.context_aware_warnings, expected_safe_warnings) - - def test_config_get_non_existent(self): - # Test PyConfig_Get() on non-existent option name - config_get = _testcapi.config_get - nonexistent_key = 'NONEXISTENT_KEY' - err_msg = f'unknown config option name: {nonexistent_key}' - with self.assertRaisesRegex(ValueError, err_msg): - config_get(nonexistent_key) - - def test_config_get_write_bytecode(self): - # PyConfig_Get("write_bytecode") gets sys.dont_write_bytecode - # as an integer - config_get = _testcapi.config_get - with support.swap_attr(sys, "dont_write_bytecode", 0): - self.assertEqual(config_get('write_bytecode'), 1) - with support.swap_attr(sys, "dont_write_bytecode", "yes"): - self.assertEqual(config_get('write_bytecode'), 0) - with support.swap_attr(sys, "dont_write_bytecode", []): - self.assertEqual(config_get('write_bytecode'), 1) - - def test_config_getint(self): - # Test PyConfig_GetInt() - config_getint = _testcapi.config_getint - - # PyConfig_MEMBER_INT type - self.assertEqual(config_getint('verbose'), sys.flags.verbose) - - # PyConfig_MEMBER_UINT type - self.assertEqual(config_getint('isolated'), sys.flags.isolated) - - # PyConfig_MEMBER_ULONG type - self.assertIsInstance(config_getint('hash_seed'), int) - - # PyPreConfig member - self.assertIsInstance(config_getint('allocator'), int) - - # platlibdir type is str - with self.assertRaises(TypeError): - config_getint('platlibdir') - - def test_get_config_names(self): - names = _testcapi.config_names() - self.assertIsInstance(names, frozenset) - for name in names: - self.assertIsInstance(name, str) - - def test_config_set_sys_attr(self): - # Test PyConfig_Set() with sys attributes - config_get = _testcapi.config_get - config_set = _testcapi.config_set - - # mutable configuration option mapped to sys attributes - for name, sys_attr, option_type in ( - ('argv', 'argv', list[str]), - ('base_exec_prefix', 'base_exec_prefix', str | None), - ('base_executable', '_base_executable', str | None), - ('base_prefix', 'base_prefix', str | None), - ('exec_prefix', 'exec_prefix', str | None), - ('executable', 'executable', str | None), - ('module_search_paths', 'path', list[str]), - ('platlibdir', 'platlibdir', str), - ('prefix', 'prefix', str | None), - ('pycache_prefix', 'pycache_prefix', str | None), - ('stdlib_dir', '_stdlib_dir', str | None), - ('warnoptions', 'warnoptions', list[str]), - ('xoptions', '_xoptions', dict[str, str | bool]), - ): - with self.subTest(name=name): - if option_type == str: - test_values = ('TEST_REPLACE',) - invalid_types = (1, None) - elif option_type == str | None: - test_values = ('TEST_REPLACE', None) - invalid_types = (123,) - elif option_type == list[str]: - test_values = (['TEST_REPLACE'], []) - invalid_types = ('text', 123, [123]) - else: # option_type == dict[str, str | bool]: - test_values = ({"x": "value", "y": True},) - invalid_types = ('text', 123, ['option'], - {123: 'value'}, - {'key': b'bytes'}) - - old_opt_value = config_get(name) - old_sys_value = getattr(sys, sys_attr) - try: - for value in test_values: - config_set(name, value) - self.assertEqual(config_get(name), value) - self.assertEqual(getattr(sys, sys_attr), value) - - for value in invalid_types: - with self.assertRaises(TypeError): - config_set(name, value) - finally: - setattr(sys, sys_attr, old_sys_value) - config_set(name, old_opt_value) - - def test_config_set_sys_flag(self): - # Test PyConfig_Set() with sys.flags - config_get = _testcapi.config_get - config_set = _testcapi.config_set - - # mutable configuration option mapped to sys.flags - class unsigned_int(int): - pass - - def expect_int(value): - value = int(value) - return (value, value) - - def expect_bool(value): - value = int(bool(value)) - return (value, value) - - def expect_bool_not(value): - value = bool(value) - return (int(value), int(not value)) - - for name, sys_flag, option_type, expect_func in ( - # (some flags cannot be set, see comments below.) - ('parser_debug', 'debug', bool, expect_bool), - ('inspect', 'inspect', bool, expect_bool), - ('interactive', 'interactive', bool, expect_bool), - ('optimization_level', 'optimize', unsigned_int, expect_int), - ('write_bytecode', 'dont_write_bytecode', bool, expect_bool_not), - # user_site_directory - # site_import - ('use_environment', 'ignore_environment', bool, expect_bool_not), - ('verbose', 'verbose', unsigned_int, expect_int), - ('bytes_warning', 'bytes_warning', unsigned_int, expect_int), - ('quiet', 'quiet', bool, expect_bool), - # hash_randomization - # isolated - # dev_mode - # utf8_mode - # warn_default_encoding - # safe_path - ('int_max_str_digits', 'int_max_str_digits', unsigned_int, expect_int), - # gil - ): - if name == "int_max_str_digits": - new_values = (0, 5_000, 999_999) - invalid_values = (-1, 40) # value must 0 or >= 4300 - invalid_types = (1.0, "abc") - elif option_type == int: - new_values = (False, True, 0, 1, 5, -5) - invalid_values = () - invalid_types = (1.0, "abc") - else: - new_values = (False, True, 0, 1, 5) - invalid_values = (-5,) - invalid_types = (1.0, "abc") - - with self.subTest(name=name): - old_value = config_get(name) - try: - for value in new_values: - expected, expect_flag = expect_func(value) - - config_set(name, value) - self.assertEqual(config_get(name), expected) - self.assertEqual(getattr(sys.flags, sys_flag), expect_flag) - if name == "write_bytecode": - self.assertEqual(getattr(sys, "dont_write_bytecode"), - expect_flag) - if name == "int_max_str_digits": - self.assertEqual(sys.get_int_max_str_digits(), - expect_flag) - - for value in invalid_values: - with self.assertRaises(ValueError): - config_set(name, value) - - for value in invalid_types: - with self.assertRaises(TypeError): - config_set(name, value) - finally: - config_set(name, old_value) - - def test_config_set_cpu_count(self): - config_get = _testcapi.config_get - config_set = _testcapi.config_set - - old_value = config_get('cpu_count') - try: - config_set('cpu_count', 123) - self.assertEqual(os.cpu_count(), 123) - finally: - config_set('cpu_count', old_value) - - def test_config_set_read_only(self): - # Test PyConfig_Set() on read-only options - config_set = _testcapi.config_set - for name, value in ( - ("allocator", 0), # PyPreConfig member - ("perf_profiling", 8), - ("dev_mode", True), - ("filesystem_encoding", "utf-8"), - ): - with self.subTest(name=name, value=value): - with self.assertRaisesRegex(ValueError, r"read-only"): - config_set(name, value) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py deleted file mode 100644 index e726e3d813d..00000000000 --- a/Lib/test/test_capi/test_dict.py +++ /dev/null @@ -1,550 +0,0 @@ -import unittest -from collections import OrderedDict, UserDict -from types import MappingProxyType -from test import support -from test.support import import_helper - - -_testcapi = import_helper.import_module("_testcapi") -_testlimitedcapi = import_helper.import_module("_testlimitedcapi") - - -NULL = None -INVALID_UTF8 = b'\xff' - -class DictSubclass(dict): - def __getitem__(self, key): - raise RuntimeError('do not get evil') - def __setitem__(self, key, value): - raise RuntimeError('do not set evil') - def __delitem__(self, key): - raise RuntimeError('do not del evil') - -def gen(): - yield 'a' - yield 'b' - yield 'c' - - -class CAPITest(unittest.TestCase): - - def test_dict_check(self): - check = _testlimitedcapi.dict_check - self.assertTrue(check({1: 2})) - self.assertTrue(check(OrderedDict({1: 2}))) - self.assertFalse(check(UserDict({1: 2}))) - self.assertFalse(check([1, 2])) - self.assertFalse(check(object())) - # CRASHES check(NULL) - - def test_dict_checkexact(self): - check = _testlimitedcapi.dict_checkexact - self.assertTrue(check({1: 2})) - self.assertFalse(check(OrderedDict({1: 2}))) - self.assertFalse(check(UserDict({1: 2}))) - self.assertFalse(check([1, 2])) - self.assertFalse(check(object())) - # CRASHES check(NULL) - - def test_dict_new(self): - dict_new = _testlimitedcapi.dict_new - dct = dict_new() - self.assertEqual(dct, {}) - self.assertIs(type(dct), dict) - dct2 = dict_new() - self.assertIsNot(dct2, dct) - - def test_dictproxy_new(self): - dictproxy_new = _testlimitedcapi.dictproxy_new - for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): - proxy = dictproxy_new(dct) - self.assertIs(type(proxy), MappingProxyType) - self.assertEqual(proxy, dct) - with self.assertRaises(TypeError): - proxy[1] = 3 - self.assertEqual(proxy[1], 2) - dct[1] = 4 - self.assertEqual(proxy[1], 4) - - self.assertRaises(TypeError, dictproxy_new, []) - self.assertRaises(TypeError, dictproxy_new, 42) - # CRASHES dictproxy_new(NULL) - - def test_dict_copy(self): - copy = _testlimitedcapi.dict_copy - for dct in {1: 2}, OrderedDict({1: 2}): - dct_copy = copy(dct) - self.assertIs(type(dct_copy), dict) - self.assertEqual(dct_copy, dct) - - self.assertRaises(SystemError, copy, UserDict()) - self.assertRaises(SystemError, copy, []) - self.assertRaises(SystemError, copy, 42) - self.assertRaises(SystemError, copy, NULL) - - def test_dict_clear(self): - clear = _testlimitedcapi.dict_clear - dct = {1: 2} - clear(dct) - self.assertEqual(dct, {}) - - # NOTE: It is not safe to call it with OrderedDict. - - # Has no effect for non-dicts. - dct = UserDict({1: 2}) - clear(dct) - self.assertEqual(dct, {1: 2}) - lst = [1, 2] - clear(lst) - self.assertEqual(lst, [1, 2]) - clear(object()) - - # CRASHES? clear(NULL) - - def test_dict_size(self): - size = _testlimitedcapi.dict_size - self.assertEqual(size({1: 2}), 1) - self.assertEqual(size(OrderedDict({1: 2})), 1) - - self.assertRaises(SystemError, size, UserDict()) - self.assertRaises(SystemError, size, []) - self.assertRaises(SystemError, size, 42) - self.assertRaises(SystemError, size, object()) - self.assertRaises(SystemError, size, NULL) - - def test_dict_getitem(self): - getitem = _testlimitedcapi.dict_getitem - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitem(dct, 'a'), 1) - self.assertIs(getitem(dct, 'b'), KeyError) - self.assertEqual(getitem(dct, '\U0001f40d'), 2) - - dct2 = DictSubclass(dct) - self.assertEqual(getitem(dct2, 'a'), 1) - self.assertIs(getitem(dct2, 'b'), KeyError) - - with support.catch_unraisable_exception() as cm: - self.assertIs(getitem({}, []), KeyError) # unhashable - self.assertEqual(cm.unraisable.exc_type, TypeError) - self.assertEqual(str(cm.unraisable.exc_value), - "unhashable type: 'list'") - - self.assertIs(getitem(42, 'a'), KeyError) - self.assertIs(getitem([1], 0), KeyError) - # CRASHES getitem({}, NULL) - # CRASHES getitem(NULL, 'a') - - def test_dict_getitemstring(self): - getitemstring = _testlimitedcapi.dict_getitemstring - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitemstring(dct, b'a'), 1) - self.assertIs(getitemstring(dct, b'b'), KeyError) - self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) - - dct2 = DictSubclass(dct) - self.assertEqual(getitemstring(dct2, b'a'), 1) - self.assertIs(getitemstring(dct2, b'b'), KeyError) - - with support.catch_unraisable_exception() as cm: - self.assertIs(getitemstring({}, INVALID_UTF8), KeyError) - self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) - self.assertRegex(str(cm.unraisable.exc_value), - "'utf-8' codec can't decode") - - self.assertIs(getitemstring(42, b'a'), KeyError) - self.assertIs(getitemstring([], b'a'), KeyError) - # CRASHES getitemstring({}, NULL) - # CRASHES getitemstring(NULL, b'a') - - def test_dict_getitemref(self): - getitem = _testcapi.dict_getitemref - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitem(dct, 'a'), 1) - self.assertIs(getitem(dct, 'b'), KeyError) - self.assertEqual(getitem(dct, '\U0001f40d'), 2) - - dct2 = DictSubclass(dct) - self.assertEqual(getitem(dct2, 'a'), 1) - self.assertIs(getitem(dct2, 'b'), KeyError) - - self.assertRaises(SystemError, getitem, 42, 'a') - self.assertRaises(TypeError, getitem, {}, []) # unhashable - self.assertRaises(SystemError, getitem, [], 1) - self.assertRaises(SystemError, getitem, [], 'a') - # CRASHES getitem({}, NULL) - # CRASHES getitem(NULL, 'a') - - def test_dict_getitemstringref(self): - getitemstring = _testcapi.dict_getitemstringref - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitemstring(dct, b'a'), 1) - self.assertIs(getitemstring(dct, b'b'), KeyError) - self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) - - dct2 = DictSubclass(dct) - self.assertEqual(getitemstring(dct2, b'a'), 1) - self.assertIs(getitemstring(dct2, b'b'), KeyError) - - self.assertRaises(SystemError, getitemstring, 42, b'a') - self.assertRaises(UnicodeDecodeError, getitemstring, {}, INVALID_UTF8) - self.assertRaises(SystemError, getitemstring, [], b'a') - # CRASHES getitemstring({}, NULL) - # CRASHES getitemstring(NULL, b'a') - - def test_dict_getitemwitherror(self): - getitem = _testlimitedcapi.dict_getitemwitherror - dct = {'a': 1, '\U0001f40d': 2} - self.assertEqual(getitem(dct, 'a'), 1) - self.assertIs(getitem(dct, 'b'), KeyError) - self.assertEqual(getitem(dct, '\U0001f40d'), 2) - - dct2 = DictSubclass(dct) - self.assertEqual(getitem(dct2, 'a'), 1) - self.assertIs(getitem(dct2, 'b'), KeyError) - - self.assertRaises(SystemError, getitem, 42, 'a') - self.assertRaises(TypeError, getitem, {}, []) # unhashable - self.assertRaises(SystemError, getitem, [], 1) - self.assertRaises(SystemError, getitem, [], 'a') - # CRASHES getitem({}, NULL) - # CRASHES getitem(NULL, 'a') - - def test_dict_contains(self): - contains = _testlimitedcapi.dict_contains - dct = {'a': 1, '\U0001f40d': 2} - self.assertTrue(contains(dct, 'a')) - self.assertFalse(contains(dct, 'b')) - self.assertTrue(contains(dct, '\U0001f40d')) - - dct2 = DictSubclass(dct) - self.assertTrue(contains(dct2, 'a')) - self.assertFalse(contains(dct2, 'b')) - - self.assertRaises(TypeError, contains, {}, []) # unhashable - # CRASHES contains({}, NULL) - # CRASHES contains(UserDict(), 'a') - # CRASHES contains(42, 'a') - # CRASHES contains(NULL, 'a') - - def test_dict_contains_string(self): - contains_string = _testcapi.dict_containsstring - dct = {'a': 1, '\U0001f40d': 2} - self.assertTrue(contains_string(dct, b'a')) - self.assertFalse(contains_string(dct, b'b')) - self.assertTrue(contains_string(dct, '\U0001f40d'.encode())) - self.assertRaises(UnicodeDecodeError, contains_string, dct, INVALID_UTF8) - - dct2 = DictSubclass(dct) - self.assertTrue(contains_string(dct2, b'a')) - self.assertFalse(contains_string(dct2, b'b')) - - # CRASHES contains({}, NULL) - # CRASHES contains(NULL, b'a') - - def test_dict_setitem(self): - setitem = _testlimitedcapi.dict_setitem - dct = {} - setitem(dct, 'a', 5) - self.assertEqual(dct, {'a': 5}) - setitem(dct, '\U0001f40d', 8) - self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) - - dct2 = DictSubclass() - setitem(dct2, 'a', 5) - self.assertEqual(dct2, {'a': 5}) - - self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable - self.assertRaises(SystemError, setitem, UserDict(), 'a', 5) - self.assertRaises(SystemError, setitem, [1], 0, 5) - self.assertRaises(SystemError, setitem, 42, 'a', 5) - # CRASHES setitem({}, NULL, 5) - # CRASHES setitem({}, 'a', NULL) - # CRASHES setitem(NULL, 'a', 5) - - def test_dict_setitemstring(self): - setitemstring = _testlimitedcapi.dict_setitemstring - dct = {} - setitemstring(dct, b'a', 5) - self.assertEqual(dct, {'a': 5}) - setitemstring(dct, '\U0001f40d'.encode(), 8) - self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) - - dct2 = DictSubclass() - setitemstring(dct2, b'a', 5) - self.assertEqual(dct2, {'a': 5}) - - self.assertRaises(UnicodeDecodeError, setitemstring, {}, INVALID_UTF8, 5) - self.assertRaises(SystemError, setitemstring, UserDict(), b'a', 5) - self.assertRaises(SystemError, setitemstring, 42, b'a', 5) - # CRASHES setitemstring({}, NULL, 5) - # CRASHES setitemstring({}, b'a', NULL) - # CRASHES setitemstring(NULL, b'a', 5) - - def test_dict_delitem(self): - delitem = _testlimitedcapi.dict_delitem - dct = {'a': 1, 'c': 2, '\U0001f40d': 3} - delitem(dct, 'a') - self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) - self.assertRaises(KeyError, delitem, dct, 'b') - delitem(dct, '\U0001f40d') - self.assertEqual(dct, {'c': 2}) - - dct2 = DictSubclass({'a': 1, 'c': 2}) - delitem(dct2, 'a') - self.assertEqual(dct2, {'c': 2}) - self.assertRaises(KeyError, delitem, dct2, 'b') - - self.assertRaises(TypeError, delitem, {}, []) # unhashable - self.assertRaises(SystemError, delitem, UserDict({'a': 1}), 'a') - self.assertRaises(SystemError, delitem, [1], 0) - self.assertRaises(SystemError, delitem, 42, 'a') - # CRASHES delitem({}, NULL) - # CRASHES delitem(NULL, 'a') - - def test_dict_delitemstring(self): - delitemstring = _testlimitedcapi.dict_delitemstring - dct = {'a': 1, 'c': 2, '\U0001f40d': 3} - delitemstring(dct, b'a') - self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) - self.assertRaises(KeyError, delitemstring, dct, b'b') - delitemstring(dct, '\U0001f40d'.encode()) - self.assertEqual(dct, {'c': 2}) - - dct2 = DictSubclass({'a': 1, 'c': 2}) - delitemstring(dct2, b'a') - self.assertEqual(dct2, {'c': 2}) - self.assertRaises(KeyError, delitemstring, dct2, b'b') - - self.assertRaises(UnicodeDecodeError, delitemstring, {}, INVALID_UTF8) - self.assertRaises(SystemError, delitemstring, UserDict({'a': 1}), b'a') - self.assertRaises(SystemError, delitemstring, 42, b'a') - # CRASHES delitemstring({}, NULL) - # CRASHES delitemstring(NULL, b'a') - - def test_dict_setdefault(self): - setdefault = _testcapi.dict_setdefault - dct = {} - self.assertEqual(setdefault(dct, 'a', 5), 5) - self.assertEqual(dct, {'a': 5}) - self.assertEqual(setdefault(dct, 'a', 8), 5) - self.assertEqual(dct, {'a': 5}) - - dct2 = DictSubclass() - self.assertEqual(setdefault(dct2, 'a', 5), 5) - self.assertEqual(dct2, {'a': 5}) - self.assertEqual(setdefault(dct2, 'a', 8), 5) - self.assertEqual(dct2, {'a': 5}) - - self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable - self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) - self.assertRaises(SystemError, setdefault, [1], 0, 5) - self.assertRaises(SystemError, setdefault, 42, 'a', 5) - # CRASHES setdefault({}, NULL, 5) - # CRASHES setdefault({}, 'a', NULL) - # CRASHES setdefault(NULL, 'a', 5) - - def test_dict_setdefaultref(self): - setdefault = _testcapi.dict_setdefaultref - dct = {} - self.assertEqual(setdefault(dct, 'a', 5), 5) - self.assertEqual(dct, {'a': 5}) - self.assertEqual(setdefault(dct, 'a', 8), 5) - self.assertEqual(dct, {'a': 5}) - - dct2 = DictSubclass() - self.assertEqual(setdefault(dct2, 'a', 5), 5) - self.assertEqual(dct2, {'a': 5}) - self.assertEqual(setdefault(dct2, 'a', 8), 5) - self.assertEqual(dct2, {'a': 5}) - - self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable - self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) - self.assertRaises(SystemError, setdefault, [1], 0, 5) - self.assertRaises(SystemError, setdefault, 42, 'a', 5) - # CRASHES setdefault({}, NULL, 5) - # CRASHES setdefault({}, 'a', NULL) - # CRASHES setdefault(NULL, 'a', 5) - - def test_mapping_keys_valuesitems(self): - class BadMapping(dict): - def keys(self): - return None - def values(self): - return None - def items(self): - return None - dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} - for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: - self.assertListEqual(_testlimitedcapi.dict_keys(mapping), - list(dict_obj.keys())) - self.assertListEqual(_testlimitedcapi.dict_values(mapping), - list(dict_obj.values())) - self.assertListEqual(_testlimitedcapi.dict_items(mapping), - list(dict_obj.items())) - - def test_dict_keys_valuesitems_bad_arg(self): - for mapping in UserDict(), [], object(): - self.assertRaises(SystemError, _testlimitedcapi.dict_keys, mapping) - self.assertRaises(SystemError, _testlimitedcapi.dict_values, mapping) - self.assertRaises(SystemError, _testlimitedcapi.dict_items, mapping) - - def test_dict_next(self): - dict_next = _testlimitedcapi.dict_next - self.assertIsNone(dict_next({}, 0)) - dct = {'a': 1, 'b': 2, 'c': 3} - pos = 0 - pairs = [] - while True: - res = dict_next(dct, pos) - if res is None: - break - rc, pos, key, value = res - self.assertEqual(rc, 1) - pairs.append((key, value)) - self.assertEqual(pairs, list(dct.items())) - - # CRASHES dict_next(NULL, 0) - - def test_dict_update(self): - update = _testlimitedcapi.dict_update - for cls1 in dict, DictSubclass: - for cls2 in dict, DictSubclass, UserDict: - dct = cls1({'a': 1, 'b': 2}) - update(dct, cls2({'b': 3, 'c': 4})) - self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) - - self.assertRaises(AttributeError, update, {}, []) - self.assertRaises(AttributeError, update, {}, 42) - self.assertRaises(SystemError, update, UserDict(), {}) - self.assertRaises(SystemError, update, 42, {}) - self.assertRaises(SystemError, update, {}, NULL) - self.assertRaises(SystemError, update, NULL, {}) - - def test_dict_merge(self): - merge = _testlimitedcapi.dict_merge - for cls1 in dict, DictSubclass: - for cls2 in dict, DictSubclass, UserDict: - dct = cls1({'a': 1, 'b': 2}) - merge(dct, cls2({'b': 3, 'c': 4}), 0) - self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) - dct = cls1({'a': 1, 'b': 2}) - merge(dct, cls2({'b': 3, 'c': 4}), 1) - self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) - - self.assertRaises(AttributeError, merge, {}, [], 0) - self.assertRaises(AttributeError, merge, {}, 42, 0) - self.assertRaises(SystemError, merge, UserDict(), {}, 0) - self.assertRaises(SystemError, merge, 42, {}, 0) - self.assertRaises(SystemError, merge, {}, NULL, 0) - self.assertRaises(SystemError, merge, NULL, {}, 0) - - def test_dict_mergefromseq2(self): - mergefromseq2 = _testlimitedcapi.dict_mergefromseq2 - for cls1 in dict, DictSubclass: - for cls2 in list, iter: - dct = cls1({'a': 1, 'b': 2}) - mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 0) - self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) - dct = cls1({'a': 1, 'b': 2}) - mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 1) - self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) - - self.assertRaises(ValueError, mergefromseq2, {}, [(1,)], 0) - self.assertRaises(ValueError, mergefromseq2, {}, [(1, 2, 3)], 0) - self.assertRaises(TypeError, mergefromseq2, {}, [1], 0) - self.assertRaises(TypeError, mergefromseq2, {}, 42, 0) - # CRASHES mergefromseq2(UserDict(), [], 0) - # CRASHES mergefromseq2(42, [], 0) - # CRASHES mergefromseq2({}, NULL, 0) - # CRASHES mergefromseq2(NULL, {}, 0) - - def test_dict_pop(self): - # Test PyDict_Pop() - dict_pop = _testcapi.dict_pop - dict_pop_null = _testcapi.dict_pop_null - - # key present, get removed value - mydict = {"key": "value", "key2": "value2"} - self.assertEqual(dict_pop(mydict, "key"), (1, "value")) - self.assertEqual(mydict, {"key2": "value2"}) - self.assertEqual(dict_pop(mydict, "key2"), (1, "value2")) - self.assertEqual(mydict, {}) - - # key present, ignore removed value - mydict = {"key": "value", "key2": "value2"} - self.assertEqual(dict_pop_null(mydict, "key"), 1) - self.assertEqual(mydict, {"key2": "value2"}) - self.assertEqual(dict_pop_null(mydict, "key2"), 1) - self.assertEqual(mydict, {}) - - # key missing, expect removed value; empty dict has a fast path - self.assertEqual(dict_pop({}, "key"), (0, NULL)) - self.assertEqual(dict_pop({"a": 1}, "key"), (0, NULL)) - - # key missing, ignored removed value; empty dict has a fast path - self.assertEqual(dict_pop_null({}, "key"), 0) - self.assertEqual(dict_pop_null({"a": 1}, "key"), 0) - - # dict error - not_dict = UserDict({1: 2}) - self.assertRaises(SystemError, dict_pop, not_dict, "key") - self.assertRaises(SystemError, dict_pop_null, not_dict, "key") - - # key error; don't hash key if dict is empty - not_hashable_key = ["list"] - self.assertEqual(dict_pop({}, not_hashable_key), (0, NULL)) - with self.assertRaises(TypeError): - dict_pop({'key': 1}, not_hashable_key) - dict_pop({}, NULL) # key is not checked if dict is empty - - # CRASHES dict_pop(NULL, "key") - # CRASHES dict_pop({"a": 1}, NULL) - - def test_dict_popstring(self): - # Test PyDict_PopString() - dict_popstring = _testcapi.dict_popstring - dict_popstring_null = _testcapi.dict_popstring_null - - # key present, get removed value - mydict = {"key": "value", "key2": "value2"} - self.assertEqual(dict_popstring(mydict, "key"), (1, "value")) - self.assertEqual(mydict, {"key2": "value2"}) - self.assertEqual(dict_popstring(mydict, "key2"), (1, "value2")) - self.assertEqual(mydict, {}) - - # key present, ignore removed value - mydict = {"key": "value", "key2": "value2"} - self.assertEqual(dict_popstring_null(mydict, "key"), 1) - self.assertEqual(mydict, {"key2": "value2"}) - self.assertEqual(dict_popstring_null(mydict, "key2"), 1) - self.assertEqual(mydict, {}) - - # key missing; empty dict has a fast path - self.assertEqual(dict_popstring({}, "key"), (0, NULL)) - self.assertEqual(dict_popstring_null({}, "key"), 0) - self.assertEqual(dict_popstring({"a": 1}, "key"), (0, NULL)) - self.assertEqual(dict_popstring_null({"a": 1}, "key"), 0) - - # non-ASCII key - non_ascii = '\U0001f40d' - dct = {'\U0001f40d': 123} - self.assertEqual(dict_popstring(dct, '\U0001f40d'.encode()), (1, 123)) - dct = {'\U0001f40d': 123} - self.assertEqual(dict_popstring_null(dct, '\U0001f40d'.encode()), 1) - - # dict error - not_dict = UserDict({1: 2}) - self.assertRaises(SystemError, dict_popstring, not_dict, "key") - self.assertRaises(SystemError, dict_popstring_null, not_dict, "key") - - # key error - self.assertRaises(UnicodeDecodeError, dict_popstring, {1: 2}, INVALID_UTF8) - self.assertRaises(UnicodeDecodeError, dict_popstring_null, {1: 2}, INVALID_UTF8) - - # CRASHES dict_popstring(NULL, "key") - # CRASHES dict_popstring({}, NULL) - # CRASHES dict_popstring({"a": 1}, NULL) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_emscripten.py b/Lib/test/test_capi/test_emscripten.py deleted file mode 100644 index 272d9a10ceb..00000000000 --- a/Lib/test/test_capi/test_emscripten.py +++ /dev/null @@ -1,25 +0,0 @@ -import unittest -from test.support import is_emscripten - -if not is_emscripten: - raise unittest.SkipTest("Emscripten-only test") - -from _testinternalcapi import emscripten_set_up_async_input_device -from pathlib import Path - - -class EmscriptenAsyncInputDeviceTest(unittest.TestCase): - def test_emscripten_async_input_device(self): - jspi_supported = emscripten_set_up_async_input_device() - p = Path("/dev/blah") - self.addCleanup(p.unlink) - if not jspi_supported: - with open(p, "r") as f: - self.assertRaises(OSError, f.readline) - return - - with open(p, "r") as f: - for _ in range(10): - self.assertEqual(f.readline().strip(), "ab") - self.assertEqual(f.readline().strip(), "fi") - self.assertEqual(f.readline().strip(), "xy") diff --git a/Lib/test/test_capi/test_eval.py b/Lib/test/test_capi/test_eval.py deleted file mode 100644 index 20ef2695ef3..00000000000 --- a/Lib/test/test_capi/test_eval.py +++ /dev/null @@ -1,103 +0,0 @@ -import sys -import unittest -from test.support import import_helper - -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') - - -class Tests(unittest.TestCase): - def test_eval_get_func_name(self): - eval_get_func_name = _testlimitedcapi.eval_get_func_name - - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(eval_get_func_name(function_example), - "function_example") - self.assertEqual(eval_get_func_name(A.method_example), - "method_example") - self.assertEqual(eval_get_func_name(A().method_example), - "method_example") - self.assertEqual(eval_get_func_name(sum), "sum") # c function - self.assertEqual(eval_get_func_name(A), "type") - - def test_eval_get_func_desc(self): - eval_get_func_desc = _testlimitedcapi.eval_get_func_desc - - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(eval_get_func_desc(function_example), - "()") - self.assertEqual(eval_get_func_desc(A.method_example), - "()") - self.assertEqual(eval_get_func_desc(A().method_example), - "()") - self.assertEqual(eval_get_func_desc(sum), "()") # c function - self.assertEqual(eval_get_func_desc(A), " object") - - def test_eval_getlocals(self): - # Test PyEval_GetLocals() - x = 1 - self.assertEqual(_testlimitedcapi.eval_getlocals(), - {'self': self, - 'x': 1}) - - y = 2 - self.assertEqual(_testlimitedcapi.eval_getlocals(), - {'self': self, - 'x': 1, - 'y': 2}) - - def test_eval_getglobals(self): - # Test PyEval_GetGlobals() - self.assertEqual(_testlimitedcapi.eval_getglobals(), - globals()) - - def test_eval_getbuiltins(self): - # Test PyEval_GetBuiltins() - self.assertEqual(_testlimitedcapi.eval_getbuiltins(), - globals()['__builtins__']) - - def test_eval_getframe(self): - # Test PyEval_GetFrame() - self.assertEqual(_testlimitedcapi.eval_getframe(), - sys._getframe()) - - def test_eval_getframe_builtins(self): - # Test PyEval_GetFrameBuiltins() - self.assertEqual(_testlimitedcapi.eval_getframe_builtins(), - sys._getframe().f_builtins) - - def test_eval_getframe_globals(self): - # Test PyEval_GetFrameGlobals() - self.assertEqual(_testlimitedcapi.eval_getframe_globals(), - sys._getframe().f_globals) - - def test_eval_getframe_locals(self): - # Test PyEval_GetFrameLocals() - self.assertEqual(_testlimitedcapi.eval_getframe_locals(), - sys._getframe().f_locals) - - def test_eval_get_recursion_limit(self): - # Test Py_GetRecursionLimit() - self.assertEqual(_testlimitedcapi.eval_get_recursion_limit(), - sys.getrecursionlimit()) - - def test_eval_set_recursion_limit(self): - # Test Py_SetRecursionLimit() - old_limit = sys.getrecursionlimit() - try: - limit = old_limit + 123 - _testlimitedcapi.eval_set_recursion_limit(limit) - self.assertEqual(sys.getrecursionlimit(), limit) - finally: - sys.setrecursionlimit(old_limit) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_eval_code_ex.py b/Lib/test/test_capi/test_eval_code_ex.py deleted file mode 100644 index b298e5007e5..00000000000 --- a/Lib/test/test_capi/test_eval_code_ex.py +++ /dev/null @@ -1,126 +0,0 @@ -import unittest -import builtins -from collections import UserDict - -from test.support import import_helper -from test.support import swap_attr - - -# Skip this test if the _testcapi module isn't available. -_testcapi = import_helper.import_module('_testcapi') - -NULL = None - - -class PyEval_EvalCodeExTests(unittest.TestCase): - - def test_simple(self): - def f(): - return a - - eval_code_ex = _testcapi.eval_code_ex - code = f.__code__ - self.assertEqual(eval_code_ex(code, dict(a=1)), 1) - - self.assertRaises(NameError, eval_code_ex, code, {}) - self.assertRaises(SystemError, eval_code_ex, code, UserDict(a=1)) - self.assertRaises(SystemError, eval_code_ex, code, []) - self.assertRaises(SystemError, eval_code_ex, code, 1) - # CRASHES eval_code_ex(code, NULL) - # CRASHES eval_code_ex(1, {}) - # CRASHES eval_code_ex(NULL, {}) - - def test_custom_locals(self): - # Monkey-patch __build_class__ to get a class code object. - code = None - def build_class(func, name, /, *bases, **kwds): - nonlocal code - code = func.__code__ - - with swap_attr(builtins, '__build_class__', build_class): - class A: - # Uses LOAD_NAME for a - r[:] = [a] - - eval_code_ex = _testcapi.eval_code_ex - results = [] - g = dict(a=1, r=results) - self.assertIsNone(eval_code_ex(code, g)) - self.assertEqual(results, [1]) - self.assertIsNone(eval_code_ex(code, g, dict(a=2))) - self.assertEqual(results, [2]) - self.assertIsNone(eval_code_ex(code, g, UserDict(a=3))) - self.assertEqual(results, [3]) - self.assertIsNone(eval_code_ex(code, g, {})) - self.assertEqual(results, [1]) - self.assertIsNone(eval_code_ex(code, g, NULL)) - self.assertEqual(results, [1]) - - self.assertRaises(TypeError, eval_code_ex, code, g, []) - self.assertRaises(TypeError, eval_code_ex, code, g, 1) - self.assertRaises(NameError, eval_code_ex, code, dict(r=results), {}) - self.assertRaises(NameError, eval_code_ex, code, dict(r=results), NULL) - self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), []) - self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), 1) - - def test_with_args(self): - def f(a, b, c): - return a - - eval_code_ex = _testcapi.eval_code_ex - code = f.__code__ - self.assertEqual(eval_code_ex(code, {}, {}, (1, 2, 3)), 1) - self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2)) - self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2, 3, 4)) - - def test_with_kwargs(self): - def f(a, b, c): - return a - - eval_code_ex = _testcapi.eval_code_ex - code = f.__code__ - self.assertEqual(eval_code_ex(code, {}, {}, (), dict(a=1, b=2, c=3)), 1) - self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) - self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) - self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2, c=3, d=4)) - - def test_with_default(self): - def f(a): - return a - - eval_code_ex = _testcapi.eval_code_ex - code = f.__code__ - self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (1,)), 1) - self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, ()) - - def test_with_kwarg_default(self): - def f(*, a): - return a - - eval_code_ex = _testcapi.eval_code_ex - code = f.__code__ - self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), dict(a=1)), 1) - self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), {}) - self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), NULL) - self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), UserDict(a=1)) - self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), []) - self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), 1) - - def test_with_closure(self): - a = 1 - b = 2 - def f(): - b - return a - - eval_code_ex = _testcapi.eval_code_ex - code = f.__code__ - self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__), 1) - self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__[::-1]), 2) - - # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, ()), 1) - # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, NULL), 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py deleted file mode 100644 index 6a32201cce5..00000000000 --- a/Lib/test/test_capi/test_exceptions.py +++ /dev/null @@ -1,680 +0,0 @@ -import errno -import os -import re -import sys -import unittest -import textwrap - -from test import support -from test.support import import_helper -from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE -from test.support.script_helper import assert_python_failure, assert_python_ok -from test.support.testcase import ExceptionIsLikeMixin - -from .test_misc import decode_stderr - -# Skip this test if the _testcapi or _testinternalcapi module isn't available. -_testcapi = import_helper.import_module('_testcapi') -_testinternalcapi = import_helper.import_module('_testinternalcapi') - -NULL = None - -class CustomError(Exception): - pass - - -class Test_Exceptions(unittest.TestCase): - - def test_exception(self): - raised_exception = ValueError("5") - new_exc = TypeError("TEST") - try: - raise raised_exception - except ValueError as e: - orig_sys_exception = sys.exception() - orig_exception = _testcapi.set_exception(new_exc) - new_sys_exception = sys.exception() - new_exception = _testcapi.set_exception(orig_exception) - reset_sys_exception = sys.exception() - - self.assertEqual(orig_exception, e) - - self.assertEqual(orig_exception, raised_exception) - self.assertEqual(orig_sys_exception, orig_exception) - self.assertEqual(reset_sys_exception, orig_exception) - self.assertEqual(new_exception, new_exc) - self.assertEqual(new_sys_exception, new_exception) - else: - self.fail("Exception not raised") - - def test_exc_info(self): - raised_exception = ValueError("5") - new_exc = TypeError("TEST") - try: - raise raised_exception - except ValueError as e: - tb = e.__traceback__ - orig_sys_exc_info = sys.exc_info() - orig_exc_info = _testcapi.set_exc_info(new_exc.__class__, new_exc, None) - new_sys_exc_info = sys.exc_info() - new_exc_info = _testcapi.set_exc_info(*orig_exc_info) - reset_sys_exc_info = sys.exc_info() - - self.assertEqual(orig_exc_info[1], e) - - self.assertSequenceEqual(orig_exc_info, (raised_exception.__class__, raised_exception, tb)) - self.assertSequenceEqual(orig_sys_exc_info, orig_exc_info) - self.assertSequenceEqual(reset_sys_exc_info, orig_exc_info) - self.assertSequenceEqual(new_exc_info, (new_exc.__class__, new_exc, None)) - self.assertSequenceEqual(new_sys_exc_info, new_exc_info) - else: - self.assertTrue(False) - - def test_warn_with_stacklevel(self): - code = textwrap.dedent('''\ - import _testcapi - - def foo(): - _testcapi.function_set_warning() - - foo() # line 6 - - - foo() # line 9 - ''') - proc = assert_python_ok("-c", code) - warnings = proc.err.splitlines() - self.assertEqual(warnings, [ - b':6: RuntimeWarning: Testing PyErr_WarnEx', - b':9: RuntimeWarning: Testing PyErr_WarnEx', - ]) - - def test_warn_during_finalization(self): - code = textwrap.dedent('''\ - import _testcapi - - class Foo: - def foo(self): - _testcapi.function_set_warning() - def __del__(self): - self.foo() - - ref = Foo() - ''') - proc = assert_python_ok("-c", code) - warnings = proc.err.splitlines() - # Due to the finalization of the interpreter, the source will be omitted - # because the ``warnings`` module cannot be imported at this time - self.assertEqual(warnings, [ - b':7: RuntimeWarning: Testing PyErr_WarnEx', - ]) - - def test__pyerr_setkeyerror(self): - # Test _PyErr_SetKeyError() - _pyerr_setkeyerror = _testinternalcapi._pyerr_setkeyerror - for arg in ( - "key", - # check that a tuple argument is not unpacked - (1, 2, 3), - # PyErr_SetObject(exc_type, exc_value) uses exc_value if it's - # already an exception, but _PyErr_SetKeyError() always creates a - # new KeyError. - KeyError('arg'), - ): - with self.subTest(arg=arg): - with self.assertRaises(KeyError) as cm: - # Test calling _PyErr_SetKeyError() with an exception set - # to check that the function overrides the current - # exception. - _pyerr_setkeyerror(arg) - self.assertEqual(cm.exception.args, (arg,)) - - -class Test_FatalError(unittest.TestCase): - - def check_fatal_error(self, code, expected, not_expected=()): - with support.SuppressCrashReport(): - rc, out, err = assert_python_failure('-sSI', '-c', code) - - err = decode_stderr(err) - self.assertIn('Fatal Python error: _testcapi_fatal_error_impl: MESSAGE\n', - err) - - match = re.search(r'^Extension modules:(.*) \(total: ([0-9]+)\)$', - err, re.MULTILINE) - if not match: - self.fail(f"Cannot find 'Extension modules:' in {err!r}") - modules = set(match.group(1).strip().split(', ')) - total = int(match.group(2)) - - for name in expected: - self.assertIn(name, modules) - for name in not_expected: - self.assertNotIn(name, modules) - self.assertEqual(len(modules), total) - - @support.requires_subprocess() - def test_fatal_error(self): - # By default, stdlib extension modules are ignored, - # but not test modules. - expected = ('_testcapi',) - not_expected = ('sys',) - code = 'import _testcapi, sys; _testcapi.fatal_error(b"MESSAGE")' - self.check_fatal_error(code, expected, not_expected) - - # Mark _testcapi as stdlib module, but not sys - expected = ('sys',) - not_expected = ('_testcapi',) - code = """if True: - import _testcapi, sys - sys.stdlib_module_names = frozenset({"_testcapi"}) - _testcapi.fatal_error(b"MESSAGE") - """ - self.check_fatal_error(code, expected) - - -class Test_ErrSetAndRestore(unittest.TestCase): - - def test_err_set_raised(self): - with self.assertRaises(ValueError): - _testcapi.err_set_raised(ValueError()) - v = ValueError() - try: - _testcapi.err_set_raised(v) - except ValueError as ex: - self.assertIs(v, ex) - - def test_err_restore(self): - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError) - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError, 1) - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError, 1, None) - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError, ValueError()) - try: - _testcapi.err_restore(KeyError, "hi") - except KeyError as k: - self.assertEqual("hi", k.args[0]) - try: - 1/0 - except Exception as e: - tb = e.__traceback__ - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError, 1, tb) - with self.assertRaises(TypeError): - _testcapi.err_restore(ValueError, 1, 0) - try: - _testcapi.err_restore(ValueError, 1, tb) - except ValueError as v: - self.assertEqual(1, v.args[0]) - self.assertIs(tb, v.__traceback__.tb_next) - - def test_set_object(self): - - # new exception as obj is not an exception - with self.assertRaises(ValueError) as e: - _testcapi.exc_set_object(ValueError, 42) - self.assertEqual(e.exception.args, (42,)) - - # wraps the exception because unrelated types - with self.assertRaises(ValueError) as e: - _testcapi.exc_set_object(ValueError, TypeError(1,2,3)) - wrapped = e.exception.args[0] - self.assertIsInstance(wrapped, TypeError) - self.assertEqual(wrapped.args, (1, 2, 3)) - - # is superclass, so does not wrap - with self.assertRaises(PermissionError) as e: - _testcapi.exc_set_object(OSError, PermissionError(24)) - self.assertEqual(e.exception.args, (24,)) - - class Meta(type): - def __subclasscheck__(cls, sub): - 1/0 - - class Broken(Exception, metaclass=Meta): - pass - - with self.assertRaises(ZeroDivisionError) as e: - _testcapi.exc_set_object(Broken, Broken()) - - def test_set_object_and_fetch(self): - class Broken(Exception): - def __init__(self, *arg): - raise ValueError("Broken __init__") - - exc = _testcapi.exc_set_object_fetch(Broken, 'abcd') - self.assertIsInstance(exc, ValueError) - self.assertEqual(exc.__notes__[0], - "Normalization failed: type=Broken args='abcd'") - - class BadArg: - def __repr__(self): - raise TypeError('Broken arg type') - - exc = _testcapi.exc_set_object_fetch(Broken, BadArg()) - self.assertIsInstance(exc, ValueError) - self.assertEqual(exc.__notes__[0], - 'Normalization failed: type=Broken args=') - - def test_set_string(self): - """Test PyErr_SetString()""" - setstring = _testcapi.err_setstring - with self.assertRaises(ZeroDivisionError) as e: - setstring(ZeroDivisionError, b'error') - self.assertEqual(e.exception.args, ('error',)) - with self.assertRaises(ZeroDivisionError) as e: - setstring(ZeroDivisionError, 'помилка'.encode()) - self.assertEqual(e.exception.args, ('помилка',)) - - with self.assertRaises(UnicodeDecodeError): - setstring(ZeroDivisionError, b'\xff') - self.assertRaises(SystemError, setstring, list, b'error') - # CRASHES setstring(ZeroDivisionError, NULL) - # CRASHES setstring(NULL, b'error') - - def test_format(self): - """Test PyErr_Format()""" - import_helper.import_module('ctypes') - from ctypes import pythonapi, py_object, c_char_p, c_int - name = "PyErr_Format" - PyErr_Format = getattr(pythonapi, name) - PyErr_Format.argtypes = (py_object, c_char_p,) - PyErr_Format.restype = py_object - with self.assertRaises(ZeroDivisionError) as e: - PyErr_Format(ZeroDivisionError, b'%s %d', b'error', c_int(42)) - self.assertEqual(e.exception.args, ('error 42',)) - with self.assertRaises(ZeroDivisionError) as e: - PyErr_Format(ZeroDivisionError, b'%s', 'помилка'.encode()) - self.assertEqual(e.exception.args, ('помилка',)) - - with self.assertRaisesRegex(OverflowError, 'not in range'): - PyErr_Format(ZeroDivisionError, b'%c', c_int(-1)) - with self.assertRaisesRegex(ValueError, 'format string'): - PyErr_Format(ZeroDivisionError, b'\xff') - self.assertRaises(SystemError, PyErr_Format, list, b'error') - # CRASHES PyErr_Format(ZeroDivisionError, NULL) - # CRASHES PyErr_Format(py_object(), b'error') - - def test_setfromerrnowithfilename(self): - """Test PyErr_SetFromErrnoWithFilename()""" - setfromerrnowithfilename = _testcapi.err_setfromerrnowithfilename - ENOENT = errno.ENOENT - with self.assertRaises(FileNotFoundError) as e: - setfromerrnowithfilename(ENOENT, OSError, b'file') - self.assertEqual(e.exception.args, - (ENOENT, 'No such file or directory')) - self.assertEqual(e.exception.errno, ENOENT) - self.assertEqual(e.exception.filename, 'file') - - with self.assertRaises(FileNotFoundError) as e: - setfromerrnowithfilename(ENOENT, OSError, os.fsencode(TESTFN)) - self.assertEqual(e.exception.filename, TESTFN) - - if TESTFN_UNDECODABLE: - with self.assertRaises(FileNotFoundError) as e: - setfromerrnowithfilename(ENOENT, OSError, TESTFN_UNDECODABLE) - self.assertEqual(e.exception.filename, - os.fsdecode(TESTFN_UNDECODABLE)) - - with self.assertRaises(FileNotFoundError) as e: - setfromerrnowithfilename(ENOENT, OSError, NULL) - self.assertIsNone(e.exception.filename) - - with self.assertRaises(OSError) as e: - setfromerrnowithfilename(0, OSError, b'file') - self.assertEqual(e.exception.args, (0, 'Error')) - self.assertEqual(e.exception.errno, 0) - self.assertEqual(e.exception.filename, 'file') - - with self.assertRaises(ZeroDivisionError) as e: - setfromerrnowithfilename(ENOENT, ZeroDivisionError, b'file') - self.assertEqual(e.exception.args, - (ENOENT, 'No such file or directory', 'file')) - # CRASHES setfromerrnowithfilename(ENOENT, NULL, b'error') - - def test_err_writeunraisable(self): - # Test PyErr_WriteUnraisable() - writeunraisable = _testcapi.err_writeunraisable - firstline = self.test_err_writeunraisable.__code__.co_firstlineno - - with support.catch_unraisable_exception() as cm: - writeunraisable(CustomError('oops!'), hex) - self.assertEqual(cm.unraisable.exc_type, CustomError) - self.assertEqual(str(cm.unraisable.exc_value), 'oops!') - self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, - firstline + 6) - self.assertIsNone(cm.unraisable.err_msg) - self.assertEqual(cm.unraisable.object, hex) - - with support.catch_unraisable_exception() as cm: - writeunraisable(CustomError('oops!'), NULL) - self.assertEqual(cm.unraisable.exc_type, CustomError) - self.assertEqual(str(cm.unraisable.exc_value), 'oops!') - self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, - firstline + 15) - self.assertIsNone(cm.unraisable.err_msg) - self.assertIsNone(cm.unraisable.object) - - with (support.swap_attr(sys, 'unraisablehook', None), - support.captured_stderr() as stderr): - writeunraisable(CustomError('oops!'), hex) - lines = stderr.getvalue().splitlines() - self.assertEqual(lines[0], f'Exception ignored in: {hex!r}') - self.assertEqual(lines[1], 'Traceback (most recent call last):') - self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') - - with (support.swap_attr(sys, 'unraisablehook', None), - support.captured_stderr() as stderr): - writeunraisable(CustomError('oops!'), NULL) - lines = stderr.getvalue().splitlines() - self.assertEqual(lines[0], 'Traceback (most recent call last):') - self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') - - # CRASHES writeunraisable(NULL, hex) - # CRASHES writeunraisable(NULL, NULL) - - def test_err_formatunraisable(self): - # Test PyErr_FormatUnraisable() - formatunraisable = _testcapi.err_formatunraisable - firstline = self.test_err_formatunraisable.__code__.co_firstlineno - - with support.catch_unraisable_exception() as cm: - formatunraisable(CustomError('oops!'), b'Error in %R', []) - self.assertEqual(cm.unraisable.exc_type, CustomError) - self.assertEqual(str(cm.unraisable.exc_value), 'oops!') - self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, - firstline + 6) - self.assertEqual(cm.unraisable.err_msg, 'Error in []') - self.assertIsNone(cm.unraisable.object) - - with support.catch_unraisable_exception() as cm: - formatunraisable(CustomError('oops!'), b'undecodable \xff') - self.assertEqual(cm.unraisable.exc_type, CustomError) - self.assertEqual(str(cm.unraisable.exc_value), 'oops!') - self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, - firstline + 15) - self.assertIsNone(cm.unraisable.err_msg) - self.assertIsNone(cm.unraisable.object) - - with support.catch_unraisable_exception() as cm: - formatunraisable(CustomError('oops!'), NULL) - self.assertEqual(cm.unraisable.exc_type, CustomError) - self.assertEqual(str(cm.unraisable.exc_value), 'oops!') - self.assertEqual(cm.unraisable.exc_traceback.tb_lineno, - firstline + 24) - self.assertIsNone(cm.unraisable.err_msg) - self.assertIsNone(cm.unraisable.object) - - with (support.swap_attr(sys, 'unraisablehook', None), - support.captured_stderr() as stderr): - formatunraisable(CustomError('oops!'), b'Error in %R', []) - lines = stderr.getvalue().splitlines() - self.assertEqual(lines[0], f'Error in []:') - self.assertEqual(lines[1], 'Traceback (most recent call last):') - self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') - - with (support.swap_attr(sys, 'unraisablehook', None), - support.captured_stderr() as stderr): - formatunraisable(CustomError('oops!'), b'undecodable \xff') - lines = stderr.getvalue().splitlines() - self.assertEqual(lines[0], 'Traceback (most recent call last):') - self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') - - with (support.swap_attr(sys, 'unraisablehook', None), - support.captured_stderr() as stderr): - formatunraisable(CustomError('oops!'), NULL) - lines = stderr.getvalue().splitlines() - self.assertEqual(lines[0], 'Traceback (most recent call last):') - self.assertEqual(lines[-1], f'{__name__}.CustomError: oops!') - - # CRASHES formatunraisable(NULL, b'Error in %R', []) - # CRASHES formatunraisable(NULL, NULL) - - -class TestUnicodeTranslateError(UnicodeTranslateError): - # UnicodeTranslateError takes 4 arguments instead of 5, - # so we just make a UnicodeTranslateError class that is - # compatible with the UnicodeError.__init__. - def __init__(self, encoding, *args, **kwargs): - super().__init__(*args, **kwargs) - - -class TestUnicodeError(unittest.TestCase): - - def _check_no_crash(self, exc): - # ensure that the __str__() method does not crash - _ = str(exc) - - def test_unicode_encode_error_get_start(self): - get_start = _testcapi.unicode_encode_get_start - self._test_unicode_error_get_start('x', UnicodeEncodeError, get_start) - - def test_unicode_decode_error_get_start(self): - get_start = _testcapi.unicode_decode_get_start - self._test_unicode_error_get_start(b'x', UnicodeDecodeError, get_start) - - def test_unicode_translate_error_get_start(self): - get_start = _testcapi.unicode_translate_get_start - self._test_unicode_error_get_start('x', TestUnicodeTranslateError, get_start) - - def _test_unicode_error_get_start(self, literal, exc_type, get_start): - for obj_len, start, c_start in [ - # normal cases - (5, 0, 0), - (5, 1, 1), - (5, 2, 2), - # out of range start is clamped to max(0, obj_len - 1) - (0, 0, 0), - (0, 1, 0), - (0, 10, 0), - (5, 5, 4), - (5, 10, 4), - # negative values are allowed but clipped in the getter - (0, -1, 0), - (1, -1, 0), - (2, -1, 0), - (2, -2, 0), - ]: - obj = literal * obj_len - with self.subTest(obj, exc_type=exc_type, start=start): - exc = exc_type('utf-8', obj, start, obj_len, 'reason') - self.assertEqual(get_start(exc), c_start) - self._check_no_crash(exc) - - def test_unicode_encode_error_set_start(self): - set_start = _testcapi.unicode_encode_set_start - self._test_unicode_error_set_start('x', UnicodeEncodeError, set_start) - - def test_unicode_decode_error_set_start(self): - set_start = _testcapi.unicode_decode_set_start - self._test_unicode_error_set_start(b'x', UnicodeDecodeError, set_start) - - def test_unicode_translate_error_set_start(self): - set_start = _testcapi.unicode_translate_set_start - self._test_unicode_error_set_start('x', TestUnicodeTranslateError, set_start) - - def _test_unicode_error_set_start(self, literal, exc_type, set_start): - obj_len = 5 - obj = literal * obj_len - for new_start in range(-2 * obj_len, 2 * obj_len): - with self.subTest('C-API', obj=obj, exc_type=exc_type, new_start=new_start): - exc = exc_type('utf-8', obj, 0, obj_len, 'reason') - # arbitrary value is allowed in the C API setter - set_start(exc, new_start) - self.assertEqual(exc.start, new_start) - self._check_no_crash(exc) - - with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_start=new_start): - exc = exc_type('utf-8', obj, 0, obj_len, 'reason') - # arbitrary value is allowed in the attribute setter - exc.start = new_start - self.assertEqual(exc.start, new_start) - self._check_no_crash(exc) - - def test_unicode_encode_error_get_end(self): - get_end = _testcapi.unicode_encode_get_end - self._test_unicode_error_get_end('x', UnicodeEncodeError, get_end) - - def test_unicode_decode_error_get_end(self): - get_end = _testcapi.unicode_decode_get_end - self._test_unicode_error_get_end(b'x', UnicodeDecodeError, get_end) - - def test_unicode_translate_error_get_end(self): - get_end = _testcapi.unicode_translate_get_end - self._test_unicode_error_get_end('x', TestUnicodeTranslateError, get_end) - - def _test_unicode_error_get_end(self, literal, exc_type, get_end): - for obj_len, end, c_end in [ - # normal cases - (5, 0, 1), - (5, 1, 1), - (5, 2, 2), - # out-of-range clipped in [MIN(1, OBJLEN), MAX(MIN(1, OBJLEN), OBJLEN)] - (0, 0, 0), - (0, 1, 0), - (0, 10, 0), - (1, 1, 1), - (1, 2, 1), - (5, 5, 5), - (5, 5, 5), - (5, 10, 5), - # negative values are allowed but clipped in the getter - (0, -1, 0), - (1, -1, 1), - (2, -1, 1), - (2, -2, 1), - ]: - obj = literal * obj_len - with self.subTest(obj, exc_type=exc_type, end=end): - exc = exc_type('utf-8', obj, 0, end, 'reason') - self.assertEqual(get_end(exc), c_end) - self._check_no_crash(exc) - - def test_unicode_encode_error_set_end(self): - set_end = _testcapi.unicode_encode_set_end - self._test_unicode_error_set_end('x', UnicodeEncodeError, set_end) - - def test_unicode_decode_error_set_end(self): - set_end = _testcapi.unicode_decode_set_end - self._test_unicode_error_set_end(b'x', UnicodeDecodeError, set_end) - - def test_unicode_translate_error_set_end(self): - set_end = _testcapi.unicode_translate_set_end - self._test_unicode_error_set_end('x', TestUnicodeTranslateError, set_end) - - def _test_unicode_error_set_end(self, literal, exc_type, set_end): - obj_len = 5 - obj = literal * obj_len - for new_end in range(-2 * obj_len, 2 * obj_len): - with self.subTest('C-API', obj=obj, exc_type=exc_type, new_end=new_end): - exc = exc_type('utf-8', obj, 0, obj_len, 'reason') - # arbitrary value is allowed in the C API setter - set_end(exc, new_end) - self.assertEqual(exc.end, new_end) - self._check_no_crash(exc) - - with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_end=new_end): - exc = exc_type('utf-8', obj, 0, obj_len, 'reason') - # arbitrary value is allowed in the attribute setter - exc.end = new_end - self.assertEqual(exc.end, new_end) - self._check_no_crash(exc) - - -class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): - - def setUp(self): - super().setUp() - try: - raise ExceptionGroup("eg", [TypeError('bad type'), ValueError(42)]) - except ExceptionGroup as e: - self.orig = e - - def test_invalid_args(self): - with self.assertRaisesRegex(TypeError, "orig must be an exception"): - _testcapi.unstable_exc_prep_reraise_star(42, [None]) - - with self.assertRaisesRegex(TypeError, "excs must be a list"): - _testcapi.unstable_exc_prep_reraise_star(self.orig, 42) - - with self.assertRaisesRegex(TypeError, "not an exception"): - _testcapi.unstable_exc_prep_reraise_star(self.orig, [TypeError(42), 42]) - - with self.assertRaisesRegex(ValueError, "orig must be a raised exception"): - _testcapi.unstable_exc_prep_reraise_star(ValueError(42), [TypeError(42)]) - - with self.assertRaisesRegex(ValueError, "orig must be a raised exception"): - _testcapi.unstable_exc_prep_reraise_star(ExceptionGroup("eg", [ValueError(42)]), - [TypeError(42)]) - - - def test_nothing_to_reraise(self): - self.assertEqual( - _testcapi.unstable_exc_prep_reraise_star(self.orig, [None]), None) - - try: - raise ValueError(42) - except ValueError as e: - orig = e - self.assertEqual( - _testcapi.unstable_exc_prep_reraise_star(orig, [None]), None) - - def test_reraise_orig(self): - orig = self.orig - res = _testcapi.unstable_exc_prep_reraise_star(orig, [orig]) - self.assertExceptionIsLike(res, orig) - - def test_raise_orig_parts(self): - orig = self.orig - match, rest = orig.split(TypeError) - - test_cases = [ - ([match, rest], orig), - ([rest, match], orig), - ([match], match), - ([rest], rest), - ([], None), - ] - - for input, expected in test_cases: - with self.subTest(input=input): - res = _testcapi.unstable_exc_prep_reraise_star(orig, input) - self.assertExceptionIsLike(res, expected) - - - def test_raise_with_new_exceptions(self): - orig = self.orig - - match, rest = orig.split(TypeError) - new1 = OSError('bad file') - new2 = RuntimeError('bad runtime') - - test_cases = [ - ([new1, match, rest], ExceptionGroup("", [new1, orig])), - ([match, new1, rest], ExceptionGroup("", [new1, orig])), - ([match, rest, new1], ExceptionGroup("", [new1, orig])), - - ([new1, new2, match, rest], ExceptionGroup("", [new1, new2, orig])), - ([new1, match, new2, rest], ExceptionGroup("", [new1, new2, orig])), - ([new2, rest, match, new1], ExceptionGroup("", [new2, new1, orig])), - ([rest, new2, match, new1], ExceptionGroup("", [new2, new1, orig])), - - - ([new1, new2, rest], ExceptionGroup("", [new1, new2, rest])), - ([new1, match, new2], ExceptionGroup("", [new1, new2, match])), - ([rest, new2, new1], ExceptionGroup("", [new2, new1, rest])), - ([new1, new2], ExceptionGroup("", [new1, new2])), - ([new2, new1], ExceptionGroup("", [new2, new1])), - ] - - for (input, expected) in test_cases: - with self.subTest(input=input): - res = _testcapi.unstable_exc_prep_reraise_star(orig, input) - self.assertExceptionIsLike(res, expected) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_file.py b/Lib/test/test_capi/test_file.py deleted file mode 100644 index 7c1e0026188..00000000000 --- a/Lib/test/test_capi/test_file.py +++ /dev/null @@ -1,325 +0,0 @@ -import io -import os -import unittest -from test import support -from test.support import import_helper, os_helper, warnings_helper - - -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') -_io = import_helper.import_module('_io') -NULL = None -STDOUT_FD = 1 - -with open(__file__, 'rb') as fp: - FIRST_LINE = next(fp).decode() -FIRST_LINE_NORM = FIRST_LINE.rstrip() + '\n' - - -class CAPIFileTest(unittest.TestCase): - def test_pyfile_fromfd(self): - # Test PyFile_FromFd() which is a thin wrapper to _io.open() - pyfile_fromfd = _testlimitedcapi.pyfile_fromfd - filename = __file__ - with open(filename, "rb") as fp: - fd = fp.fileno() - - # FileIO - fp.seek(0) - obj = pyfile_fromfd(fd, filename, "rb", 0, NULL, NULL, NULL, 0) - try: - self.assertIsInstance(obj, _io.FileIO) - self.assertEqual(obj.readline(), FIRST_LINE.encode()) - finally: - obj.close() - - # BufferedReader - fp.seek(0) - obj = pyfile_fromfd(fd, filename, "rb", 1024, NULL, NULL, NULL, 0) - try: - self.assertIsInstance(obj, _io.BufferedReader) - self.assertEqual(obj.readline(), FIRST_LINE.encode()) - finally: - obj.close() - - # TextIOWrapper - fp.seek(0) - obj = pyfile_fromfd(fd, filename, "r", 1, - "utf-8", "replace", NULL, 0) - try: - self.assertIsInstance(obj, _io.TextIOWrapper) - self.assertEqual(obj.encoding, "utf-8") - self.assertEqual(obj.errors, "replace") - self.assertEqual(obj.readline(), FIRST_LINE_NORM) - finally: - obj.close() - - def test_pyfile_getline(self): - # Test PyFile_GetLine(file, n): call file.readline() - # and strip "\n" suffix if n < 0. - pyfile_getline = _testlimitedcapi.pyfile_getline - - # Test Unicode - with open(__file__, "r") as fp: - fp.seek(0) - self.assertEqual(pyfile_getline(fp, -1), - FIRST_LINE_NORM.rstrip('\n')) - fp.seek(0) - self.assertEqual(pyfile_getline(fp, 0), - FIRST_LINE_NORM) - fp.seek(0) - self.assertEqual(pyfile_getline(fp, 6), - FIRST_LINE_NORM[:6]) - - # Test bytes - with open(__file__, "rb") as fp: - fp.seek(0) - self.assertEqual(pyfile_getline(fp, -1), - FIRST_LINE.rstrip('\n').encode()) - fp.seek(0) - self.assertEqual(pyfile_getline(fp, 0), - FIRST_LINE.encode()) - fp.seek(0) - self.assertEqual(pyfile_getline(fp, 6), - FIRST_LINE.encode()[:6]) - - def test_pyfile_writestring(self): - # Test PyFile_WriteString(str, file): call file.write(str) - writestr = _testlimitedcapi.pyfile_writestring - - with io.StringIO() as fp: - self.assertEqual(writestr("a\xe9\u20ac\U0010FFFF".encode(), fp), 0) - with self.assertRaises(UnicodeDecodeError): - writestr(b"\xff", fp) - with self.assertRaises(UnicodeDecodeError): - writestr("\udc80".encode("utf-8", "surrogatepass"), fp) - - text = fp.getvalue() - self.assertEqual(text, "a\xe9\u20ac\U0010FFFF") - - with self.assertRaises(SystemError): - writestr(b"abc", NULL) - - def test_pyfile_writeobject(self): - # Test PyFile_WriteObject(obj, file, flags): - # - Call file.write(str(obj)) if flags equals Py_PRINT_RAW. - # - Call file.write(repr(obj)) otherwise. - writeobject = _testlimitedcapi.pyfile_writeobject - Py_PRINT_RAW = 1 - - with io.StringIO() as fp: - # Test flags=Py_PRINT_RAW - self.assertEqual(writeobject("raw", fp, Py_PRINT_RAW), 0) - writeobject(NULL, fp, Py_PRINT_RAW) - - # Test flags=0 - self.assertEqual(writeobject("repr", fp, 0), 0) - writeobject(NULL, fp, 0) - - text = fp.getvalue() - self.assertEqual(text, "raw'repr'") - - # invalid file type - for invalid_file in (123, "abc", object()): - with self.subTest(file=invalid_file): - with self.assertRaises(AttributeError): - writeobject("abc", invalid_file, Py_PRINT_RAW) - - with self.assertRaises(TypeError): - writeobject("abc", NULL, 0) - - def test_pyobject_asfiledescriptor(self): - # Test PyObject_AsFileDescriptor(obj): - # - Return obj if obj is an integer. - # - Return obj.fileno() otherwise. - # File descriptor must be >= 0. - asfd = _testlimitedcapi.pyobject_asfiledescriptor - - self.assertEqual(asfd(123), 123) - self.assertEqual(asfd(0), 0) - - with open(__file__, "rb") as fp: - self.assertEqual(asfd(fp), fp.fileno()) - - # bool emits RuntimeWarning - msg = r"bool is used as a file descriptor" - with warnings_helper.check_warnings((msg, RuntimeWarning)): - self.assertEqual(asfd(True), 1) - - class FakeFile: - def __init__(self, fd): - self.fd = fd - def fileno(self): - return self.fd - - # file descriptor must be positive - with self.assertRaises(ValueError): - asfd(-1) - with self.assertRaises(ValueError): - asfd(FakeFile(-1)) - - # fileno() result must be an integer - with self.assertRaises(TypeError): - asfd(FakeFile("text")) - - # unsupported types - for obj in ("string", ["list"], object()): - with self.subTest(obj=obj): - with self.assertRaises(TypeError): - asfd(obj) - - # CRASHES asfd(NULL) - - def test_pyfile_newstdprinter(self): - # Test PyFile_NewStdPrinter() - pyfile_newstdprinter = _testcapi.pyfile_newstdprinter - - file = pyfile_newstdprinter(STDOUT_FD) - self.assertEqual(file.closed, False) - self.assertIsNone(file.encoding) - self.assertEqual(file.mode, "w") - - self.assertEqual(file.fileno(), STDOUT_FD) - self.assertEqual(file.isatty(), os.isatty(STDOUT_FD)) - - # flush() is a no-op - self.assertIsNone(file.flush()) - - # close() is a no-op - self.assertIsNone(file.close()) - self.assertEqual(file.closed, False) - - support.check_disallow_instantiation(self, type(file)) - - def test_pyfile_newstdprinter_write(self): - # Test the write() method of PyFile_NewStdPrinter() - pyfile_newstdprinter = _testcapi.pyfile_newstdprinter - - filename = os_helper.TESTFN - self.addCleanup(os_helper.unlink, filename) - - try: - old_stdout = os.dup(STDOUT_FD) - except OSError as exc: - # os.dup(STDOUT_FD) is not supported on WASI - self.skipTest(f"os.dup() failed with {exc!r}") - - try: - with open(filename, "wb") as fp: - # PyFile_NewStdPrinter() only accepts fileno(stdout) - # or fileno(stderr) file descriptor. - fd = fp.fileno() - os.dup2(fd, STDOUT_FD) - - file = pyfile_newstdprinter(STDOUT_FD) - self.assertEqual(file.write("text"), 4) - # The surrogate character is encoded with - # the "surrogateescape" error handler - self.assertEqual(file.write("[\udc80]"), 8) - finally: - os.dup2(old_stdout, STDOUT_FD) - os.close(old_stdout) - - with open(filename, "r") as fp: - self.assertEqual(fp.read(), "text[\\udc80]") - - def test_py_fopen(self): - # Test Py_fopen() and Py_fclose() - py_fopen = _testcapi.py_fopen - - with open(__file__, "rb") as fp: - source = fp.read() - - for filename in (__file__, os.fsencode(__file__)): - with self.subTest(filename=filename): - data = py_fopen(filename, "rb") - self.assertEqual(data, source[:256]) - - data = py_fopen(os_helper.FakePath(filename), "rb") - self.assertEqual(data, source[:256]) - - filenames = [ - os_helper.TESTFN, - os.fsencode(os_helper.TESTFN), - ] - if os_helper.TESTFN_UNDECODABLE is not None: - filenames.append(os_helper.TESTFN_UNDECODABLE) - filenames.append(os.fsdecode(os_helper.TESTFN_UNDECODABLE)) - if os_helper.TESTFN_UNENCODABLE is not None: - filenames.append(os_helper.TESTFN_UNENCODABLE) - for filename in filenames: - with self.subTest(filename=filename): - try: - with open(filename, "wb") as fp: - fp.write(source) - except OSError: - # TESTFN_UNDECODABLE cannot be used to create a file - # on macOS/WASI. - filename = None - continue - try: - data = py_fopen(filename, "rb") - self.assertEqual(data, source[:256]) - finally: - os_helper.unlink(filename) - - # embedded null character/byte in the filename - with self.assertRaises(ValueError): - py_fopen("a\x00b", "rb") - with self.assertRaises(ValueError): - py_fopen(b"a\x00b", "rb") - - # non-ASCII mode failing with "Invalid argument" - with self.assertRaises(OSError): - py_fopen(__file__, b"\xc2\x80") - with self.assertRaises(OSError): - # \x98 is invalid in cp1250, cp1251, cp1257 - # \x9d is invalid in cp1252-cp1255, cp1258 - py_fopen(__file__, b"\xc2\x98\xc2\x9d") - # UnicodeDecodeError can come from the audit hook code - with self.assertRaises((UnicodeDecodeError, OSError)): - py_fopen(__file__, b"\x98\x9d") - - # invalid filename type - for invalid_type in (123, object()): - with self.subTest(filename=invalid_type): - with self.assertRaises(TypeError): - py_fopen(invalid_type, "rb") - - if support.MS_WINDOWS: - with self.assertRaises(OSError): - # On Windows, the file mode is limited to 10 characters - py_fopen(__file__, "rt+, ccs=UTF-8") - - # CRASHES py_fopen(NULL, 'rb') - # CRASHES py_fopen(__file__, NULL) - - def test_py_universalnewlinefgets(self): - py_universalnewlinefgets = _testcapi.py_universalnewlinefgets - filename = os_helper.TESTFN - self.addCleanup(os_helper.unlink, filename) - - with open(filename, "wb") as fp: - fp.write(b"line1\nline2") - - line = py_universalnewlinefgets(filename, 1000) - self.assertEqual(line, b"line1\n") - - with open(filename, "wb") as fp: - fp.write(b"line2\r\nline3") - - line = py_universalnewlinefgets(filename, 1000) - self.assertEqual(line, b"line2\n") - - with open(filename, "wb") as fp: - fp.write(b"line3\rline4") - - line = py_universalnewlinefgets(filename, 1000) - self.assertEqual(line, b"line3\n") - - # PyFile_SetOpenCodeHook() and PyFile_OpenCode() are tested by - # test_embed.test_open_code_hook() - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py deleted file mode 100644 index 8b25607b6d5..00000000000 --- a/Lib/test/test_capi/test_float.py +++ /dev/null @@ -1,259 +0,0 @@ -import math -import random -import sys -import unittest -import warnings - -from test.test_capi.test_getargs import (Float, FloatSubclass, FloatSubclass2, - BadIndex2, BadFloat2, Index, BadIndex, - BadFloat) -from test.support import import_helper - -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') - -NULL = None - -# For PyFloat_Pack/Unpack* -BIG_ENDIAN = 0 -LITTLE_ENDIAN = 1 -EPSILON = { - 2: 2.0 ** -11, # binary16 - 4: 2.0 ** -24, # binary32 - 8: 2.0 ** -53, # binary64 -} - -HAVE_IEEE_754 = float.__getformat__("double").startswith("IEEE") -INF = float("inf") -NAN = float("nan") - - -def make_nan(size, sign, quiet, payload=None): - if size == 8: - payload_mask = 0x7ffffffffffff - i = (sign << 63) + (0x7ff << 52) + (quiet << 51) - elif size == 4: - payload_mask = 0x3fffff - i = (sign << 31) + (0xff << 23) + (quiet << 22) - elif size == 2: - payload_mask = 0x1ff - i = (sign << 15) + (0x1f << 10) + (quiet << 9) - else: - raise ValueError("size must be either 2, 4, or 8") - if payload is None: - payload = random.randint(not quiet, payload_mask) - return i + payload - - -class CAPIFloatTest(unittest.TestCase): - def test_check(self): - # Test PyFloat_Check() - check = _testlimitedcapi.float_check - - self.assertTrue(check(4.25)) - self.assertTrue(check(FloatSubclass(4.25))) - self.assertFalse(check(Float())) - self.assertFalse(check(3)) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_checkexact(self): - # Test PyFloat_CheckExact() - checkexact = _testlimitedcapi.float_checkexact - - self.assertTrue(checkexact(4.25)) - self.assertFalse(checkexact(FloatSubclass(4.25))) - self.assertFalse(checkexact(Float())) - self.assertFalse(checkexact(3)) - self.assertFalse(checkexact(object())) - - # CRASHES checkexact(NULL) - - def test_fromstring(self): - # Test PyFloat_FromString() - fromstring = _testlimitedcapi.float_fromstring - - self.assertEqual(fromstring("4.25"), 4.25) - self.assertEqual(fromstring(b"4.25"), 4.25) - self.assertRaises(ValueError, fromstring, "4.25\0") - self.assertRaises(ValueError, fromstring, b"4.25\0") - - self.assertEqual(fromstring(bytearray(b"4.25")), 4.25) - - self.assertEqual(fromstring(memoryview(b"4.25")), 4.25) - self.assertEqual(fromstring(memoryview(b"4.255")[:-1]), 4.25) - self.assertRaises(TypeError, fromstring, memoryview(b"4.25")[::2]) - - self.assertRaises(TypeError, fromstring, 4.25) - - # CRASHES fromstring(NULL) - - def test_fromdouble(self): - # Test PyFloat_FromDouble() - fromdouble = _testlimitedcapi.float_fromdouble - - self.assertEqual(fromdouble(4.25), 4.25) - - def test_asdouble(self): - # Test PyFloat_AsDouble() - asdouble = _testlimitedcapi.float_asdouble - - class BadFloat3: - def __float__(self): - raise RuntimeError - - self.assertEqual(asdouble(4.25), 4.25) - self.assertEqual(asdouble(-1.0), -1.0) - self.assertEqual(asdouble(42), 42.0) - self.assertEqual(asdouble(-1), -1.0) - self.assertEqual(asdouble(2**1000), float(2**1000)) - - self.assertEqual(asdouble(FloatSubclass(4.25)), 4.25) - self.assertEqual(asdouble(FloatSubclass2(4.25)), 4.25) - self.assertEqual(asdouble(Index()), 99.) - - self.assertRaises(TypeError, asdouble, BadIndex()) - self.assertRaises(TypeError, asdouble, BadFloat()) - self.assertRaises(RuntimeError, asdouble, BadFloat3()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(asdouble(BadIndex2()), 1.) - with self.assertWarns(DeprecationWarning): - self.assertEqual(asdouble(BadFloat2()), 4.25) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, asdouble, BadFloat2()) - self.assertRaises(TypeError, asdouble, object()) - self.assertRaises(TypeError, asdouble, NULL) - - def test_getinfo(self): - # Test PyFloat_GetInfo() - getinfo = _testlimitedcapi.float_getinfo - - self.assertEqual(getinfo(), sys.float_info) - - def test_getmax(self): - # Test PyFloat_GetMax() - getmax = _testlimitedcapi.float_getmax - - self.assertEqual(getmax(), sys.float_info.max) - - def test_getmin(self): - # Test PyFloat_GetMax() - getmin = _testlimitedcapi.float_getmin - - self.assertEqual(getmin(), sys.float_info.min) - - def test_pack(self): - # Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8() - pack = _testcapi.float_pack - - self.assertEqual(pack(2, 1.5, BIG_ENDIAN), b'>\x00') - self.assertEqual(pack(4, 1.5, BIG_ENDIAN), b'?\xc0\x00\x00') - self.assertEqual(pack(8, 1.5, BIG_ENDIAN), - b'?\xf8\x00\x00\x00\x00\x00\x00') - self.assertEqual(pack(2, 1.5, LITTLE_ENDIAN), b'\x00>') - self.assertEqual(pack(4, 1.5, LITTLE_ENDIAN), b'\x00\x00\xc0?') - self.assertEqual(pack(8, 1.5, LITTLE_ENDIAN), - b'\x00\x00\x00\x00\x00\x00\xf8?') - - def test_unpack(self): - # Test PyFloat_Unpack2(), PyFloat_Unpack4() and PyFloat_Unpack8() - unpack = _testcapi.float_unpack - - self.assertEqual(unpack(b'>\x00', BIG_ENDIAN), 1.5) - self.assertEqual(unpack(b'?\xc0\x00\x00', BIG_ENDIAN), 1.5) - self.assertEqual(unpack(b'?\xf8\x00\x00\x00\x00\x00\x00', BIG_ENDIAN), - 1.5) - self.assertEqual(unpack(b'\x00>', LITTLE_ENDIAN), 1.5) - self.assertEqual(unpack(b'\x00\x00\xc0?', LITTLE_ENDIAN), 1.5) - self.assertEqual(unpack(b'\x00\x00\x00\x00\x00\x00\xf8?', LITTLE_ENDIAN), - 1.5) - - def test_pack_unpack_roundtrip(self): - pack = _testcapi.float_pack - unpack = _testcapi.float_unpack - - large = 2.0 ** 100 - values = [1.0, 1.5, large, 1.0/7, math.pi] - if HAVE_IEEE_754: - values.extend((INF, NAN)) - for value in values: - for size in (2, 4, 8,): - if size == 2 and value == large: - # too large for 16-bit float - continue - rel_tol = EPSILON[size] - for endian in (BIG_ENDIAN, LITTLE_ENDIAN): - with self.subTest(value=value, size=size, endian=endian): - data = pack(size, value, endian) - value2 = unpack(data, endian) - if math.isnan(value): - self.assertTrue(math.isnan(value2), (value, value2)) - elif size < 8: - self.assertTrue(math.isclose(value2, value, rel_tol=rel_tol), - (value, value2)) - else: - self.assertEqual(value2, value) - - @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") - def test_pack_unpack_roundtrip_for_nans(self): - pack = _testcapi.float_pack - unpack = _testcapi.float_unpack - - for _ in range(10): - for size in (2, 4, 8): - sign = random.randint(0, 1) - if sys.maxsize != 2147483647: # not it 32-bit mode - signaling = random.randint(0, 1) - else: - # Skip sNaN's on x86 (32-bit). The problem is that sNaN - # doubles become qNaN doubles just by the C calling - # convention, there is no way to preserve sNaN doubles - # between C function calls with the current - # PyFloat_Pack/Unpack*() API. See also gh-130317 and - # e.g. https://developercommunity.visualstudio.com/t/155064 - signaling = 0 - if _testcapi.nan_msb_is_signaling: - # HP PA RISC and some MIPS CPUs use 0 for quiet, see: - # https://en.wikipedia.org/wiki/NaN#Encoding - signaling = 1 - i = make_nan(size, sign, not signaling) - data = bytes.fromhex(f'{i:x}') - for endian in (BIG_ENDIAN, LITTLE_ENDIAN): - with self.subTest(data=data, size=size, endian=endian): - data1 = data if endian == BIG_ENDIAN else data[::-1] - value = unpack(data1, endian) - data2 = pack(size, value, endian) - self.assertTrue(math.isnan(value)) - self.assertEqual(data1, data2) - - @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") - @unittest.skipUnless(sys.maxsize != 2147483647, "requires 64-bit mode") - def test_pack_unpack_nans_for_different_formats(self): - pack = _testcapi.float_pack - unpack = _testcapi.float_unpack - - for endian in (BIG_ENDIAN, LITTLE_ENDIAN): - with self.subTest(endian=endian): - byteorder = "big" if endian == BIG_ENDIAN else "little" - - # Convert sNaN to qNaN, if payload got truncated - data = make_nan(8, 0, False, 0x80001).to_bytes(8, byteorder) - snan_low = unpack(data, endian) - qnan4 = make_nan(4, 0, True, 0).to_bytes(4, byteorder) - qnan2 = make_nan(2, 0, True, 0).to_bytes(2, byteorder) - self.assertEqual(pack(4, snan_low, endian), qnan4) - self.assertEqual(pack(2, snan_low, endian), qnan2) - - # Preserve NaN type, if payload not truncated - data = make_nan(8, 0, False, 0x80000000001).to_bytes(8, byteorder) - snan_high = unpack(data, endian) - snan4 = make_nan(4, 0, False, 16384).to_bytes(4, byteorder) - snan2 = make_nan(2, 0, False, 2).to_bytes(2, byteorder) - self.assertEqual(pack(4, snan_high, endian), snan4) - self.assertEqual(pack(2, snan_high, endian), snan2) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_frame.py b/Lib/test/test_capi/test_frame.py deleted file mode 100644 index 23cb8e3dada..00000000000 --- a/Lib/test/test_capi/test_frame.py +++ /dev/null @@ -1,56 +0,0 @@ -import sys -import unittest -from test.support import import_helper - - -_testcapi = import_helper.import_module('_testcapi') - - -class FrameTest(unittest.TestCase): - def getframe(self): - return sys._getframe() - - def test_frame_getters(self): - frame = self.getframe() - self.assertEqual(frame.f_locals, _testcapi.frame_getlocals(frame)) - self.assertIs(frame.f_globals, _testcapi.frame_getglobals(frame)) - self.assertIs(frame.f_builtins, _testcapi.frame_getbuiltins(frame)) - self.assertEqual(frame.f_lasti, _testcapi.frame_getlasti(frame)) - - def test_getvar(self): - current_frame = sys._getframe() - x = 1 - self.assertEqual(_testcapi.frame_getvar(current_frame, "x"), 1) - self.assertEqual(_testcapi.frame_getvarstring(current_frame, b"x"), 1) - with self.assertRaises(NameError): - _testcapi.frame_getvar(current_frame, "y") - with self.assertRaises(NameError): - _testcapi.frame_getvarstring(current_frame, b"y") - - # wrong name type - with self.assertRaises(TypeError): - _testcapi.frame_getvar(current_frame, b'x') - with self.assertRaises(TypeError): - _testcapi.frame_getvar(current_frame, 123) - - def getgenframe(self): - yield sys._getframe() - - def test_frame_get_generator(self): - gen = self.getgenframe() - frame = next(gen) - self.assertIs(gen, _testcapi.frame_getgenerator(frame)) - - def test_frame_fback_api(self): - """Test that accessing `f_back` does not cause a segmentation fault on - a frame created with `PyFrame_New` (GH-99110).""" - def dummy(): - pass - - frame = _testcapi.frame_new(dummy.__code__, globals(), locals()) - # The following line should not cause a segmentation fault. - self.assertIsNone(frame.f_back) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_function.py b/Lib/test/test_capi/test_function.py deleted file mode 100644 index c1a278e5d4d..00000000000 --- a/Lib/test/test_capi/test_function.py +++ /dev/null @@ -1,340 +0,0 @@ -import unittest -from test.support import import_helper - - -_testcapi = import_helper.import_module('_testcapi') - - -class FunctionTest(unittest.TestCase): - def test_function_get_code(self): - # Test PyFunction_GetCode() - import types - - def some(): - pass - - code = _testcapi.function_get_code(some) - self.assertIsInstance(code, types.CodeType) - self.assertEqual(code, some.__code__) - - with self.assertRaises(SystemError): - _testcapi.function_get_code(None) # not a function - - def test_function_get_globals(self): - # Test PyFunction_GetGlobals() - def some(): - pass - - globals_ = _testcapi.function_get_globals(some) - self.assertIsInstance(globals_, dict) - self.assertEqual(globals_, some.__globals__) - - with self.assertRaises(SystemError): - _testcapi.function_get_globals(None) # not a function - - def test_function_get_module(self): - # Test PyFunction_GetModule() - def some(): - pass - - module = _testcapi.function_get_module(some) - self.assertIsInstance(module, str) - self.assertEqual(module, some.__module__) - - with self.assertRaises(SystemError): - _testcapi.function_get_module(None) # not a function - - def test_function_get_defaults(self): - # Test PyFunction_GetDefaults() - def some( - pos_only1, pos_only2='p', - /, - zero=0, optional=None, - *, - kw1, - kw2=True, - ): - pass - - defaults = _testcapi.function_get_defaults(some) - self.assertEqual(defaults, ('p', 0, None)) - self.assertEqual(defaults, some.__defaults__) - - with self.assertRaises(SystemError): - _testcapi.function_get_defaults(None) # not a function - - def test_function_set_defaults(self): - # Test PyFunction_SetDefaults() - def some( - pos_only1, pos_only2='p', - /, - zero=0, optional=None, - *, - kw1, - kw2=True, - ): - pass - - old_defaults = ('p', 0, None) - self.assertEqual(_testcapi.function_get_defaults(some), old_defaults) - self.assertEqual(some.__defaults__, old_defaults) - - with self.assertRaises(SystemError): - _testcapi.function_set_defaults(some, 1) # not tuple or None - self.assertEqual(_testcapi.function_get_defaults(some), old_defaults) - self.assertEqual(some.__defaults__, old_defaults) - - with self.assertRaises(SystemError): - _testcapi.function_set_defaults(1, ()) # not a function - self.assertEqual(_testcapi.function_get_defaults(some), old_defaults) - self.assertEqual(some.__defaults__, old_defaults) - - new_defaults = ('q', 1, None) - _testcapi.function_set_defaults(some, new_defaults) - self.assertEqual(_testcapi.function_get_defaults(some), new_defaults) - self.assertEqual(some.__defaults__, new_defaults) - - # Empty tuple is fine: - new_defaults = () - _testcapi.function_set_defaults(some, new_defaults) - self.assertEqual(_testcapi.function_get_defaults(some), new_defaults) - self.assertEqual(some.__defaults__, new_defaults) - - class tuplesub(tuple): ... # tuple subclasses must work - - new_defaults = tuplesub(((1, 2), ['a', 'b'], None)) - _testcapi.function_set_defaults(some, new_defaults) - self.assertEqual(_testcapi.function_get_defaults(some), new_defaults) - self.assertEqual(some.__defaults__, new_defaults) - - # `None` is special, it sets `defaults` to `NULL`, - # it needs special handling in `_testcapi`: - _testcapi.function_set_defaults(some, None) - self.assertEqual(_testcapi.function_get_defaults(some), None) - self.assertEqual(some.__defaults__, None) - - def test_function_get_kw_defaults(self): - # Test PyFunction_GetKwDefaults() - def some( - pos_only1, pos_only2='p', - /, - zero=0, optional=None, - *, - kw1, - kw2=True, - ): - pass - - defaults = _testcapi.function_get_kw_defaults(some) - self.assertEqual(defaults, {'kw2': True}) - self.assertEqual(defaults, some.__kwdefaults__) - - with self.assertRaises(SystemError): - _testcapi.function_get_kw_defaults(None) # not a function - - def test_function_set_kw_defaults(self): - # Test PyFunction_SetKwDefaults() - def some( - pos_only1, pos_only2='p', - /, - zero=0, optional=None, - *, - kw1, - kw2=True, - ): - pass - - old_defaults = {'kw2': True} - self.assertEqual(_testcapi.function_get_kw_defaults(some), old_defaults) - self.assertEqual(some.__kwdefaults__, old_defaults) - - with self.assertRaises(SystemError): - _testcapi.function_set_kw_defaults(some, 1) # not dict or None - self.assertEqual(_testcapi.function_get_kw_defaults(some), old_defaults) - self.assertEqual(some.__kwdefaults__, old_defaults) - - with self.assertRaises(SystemError): - _testcapi.function_set_kw_defaults(1, {}) # not a function - self.assertEqual(_testcapi.function_get_kw_defaults(some), old_defaults) - self.assertEqual(some.__kwdefaults__, old_defaults) - - new_defaults = {'kw2': (1, 2, 3)} - _testcapi.function_set_kw_defaults(some, new_defaults) - self.assertEqual(_testcapi.function_get_kw_defaults(some), new_defaults) - self.assertEqual(some.__kwdefaults__, new_defaults) - - # Empty dict is fine: - new_defaults = {} - _testcapi.function_set_kw_defaults(some, new_defaults) - self.assertEqual(_testcapi.function_get_kw_defaults(some), new_defaults) - self.assertEqual(some.__kwdefaults__, new_defaults) - - class dictsub(dict): ... # dict subclasses must work - - new_defaults = dictsub({'kw2': None}) - _testcapi.function_set_kw_defaults(some, new_defaults) - self.assertEqual(_testcapi.function_get_kw_defaults(some), new_defaults) - self.assertEqual(some.__kwdefaults__, new_defaults) - - # `None` is special, it sets `kwdefaults` to `NULL`, - # it needs special handling in `_testcapi`: - _testcapi.function_set_kw_defaults(some, None) - self.assertEqual(_testcapi.function_get_kw_defaults(some), None) - self.assertEqual(some.__kwdefaults__, None) - - def test_function_get_closure(self): - # Test PyFunction_GetClosure() - from types import CellType - - def regular_function(): ... - def unused_one_level(arg1): - def inner(arg2, arg3): ... - return inner - def unused_two_levels(arg1, arg2): - def decorator(arg3, arg4): - def inner(arg5, arg6): ... - return inner - return decorator - def with_one_level(arg1): - def inner(arg2, arg3): - return arg1 + arg2 + arg3 - return inner - def with_two_levels(arg1, arg2): - def decorator(arg3, arg4): - def inner(arg5, arg6): - return arg1 + arg2 + arg3 + arg4 + arg5 + arg6 - return inner - return decorator - - # Functions without closures: - self.assertIsNone(_testcapi.function_get_closure(regular_function)) - self.assertIsNone(regular_function.__closure__) - - func = unused_one_level(1) - closure = _testcapi.function_get_closure(func) - self.assertIsNone(closure) - self.assertIsNone(func.__closure__) - - func = unused_two_levels(1, 2)(3, 4) - closure = _testcapi.function_get_closure(func) - self.assertIsNone(closure) - self.assertIsNone(func.__closure__) - - # Functions with closures: - func = with_one_level(5) - closure = _testcapi.function_get_closure(func) - self.assertEqual(closure, func.__closure__) - self.assertIsInstance(closure, tuple) - self.assertEqual(len(closure), 1) - self.assertEqual(len(closure), len(func.__code__.co_freevars)) - for cell in closure: - self.assertIsInstance(cell, CellType) - self.assertTrue(closure[0].cell_contents, 5) - - func = with_two_levels(1, 2)(3, 4) - closure = _testcapi.function_get_closure(func) - self.assertEqual(closure, func.__closure__) - self.assertIsInstance(closure, tuple) - self.assertEqual(len(closure), 4) - self.assertEqual(len(closure), len(func.__code__.co_freevars)) - for cell in closure: - self.assertIsInstance(cell, CellType) - self.assertEqual([cell.cell_contents for cell in closure], - [1, 2, 3, 4]) - - def test_function_get_closure_error(self): - # Test PyFunction_GetClosure() - with self.assertRaises(SystemError): - _testcapi.function_get_closure(1) - with self.assertRaises(SystemError): - _testcapi.function_get_closure(None) - - def test_function_set_closure(self): - # Test PyFunction_SetClosure() - from types import CellType - - def function_without_closure(): ... - def function_with_closure(arg): - def inner(): - return arg - return inner - - func = function_without_closure - _testcapi.function_set_closure(func, (CellType(1), CellType(1))) - closure = _testcapi.function_get_closure(func) - self.assertEqual([c.cell_contents for c in closure], [1, 1]) - self.assertEqual([c.cell_contents for c in func.__closure__], [1, 1]) - - func = function_with_closure(1) - _testcapi.function_set_closure(func, - (CellType(1), CellType(2), CellType(3))) - closure = _testcapi.function_get_closure(func) - self.assertEqual([c.cell_contents for c in closure], [1, 2, 3]) - self.assertEqual([c.cell_contents for c in func.__closure__], [1, 2, 3]) - - def test_function_set_closure_none(self): - # Test PyFunction_SetClosure() - def function_without_closure(): ... - def function_with_closure(arg): - def inner(): - return arg - return inner - - _testcapi.function_set_closure(function_without_closure, None) - self.assertIsNone( - _testcapi.function_get_closure(function_without_closure)) - self.assertIsNone(function_without_closure.__closure__) - - _testcapi.function_set_closure(function_with_closure, None) - self.assertIsNone( - _testcapi.function_get_closure(function_with_closure)) - self.assertIsNone(function_with_closure.__closure__) - - def test_function_set_closure_errors(self): - # Test PyFunction_SetClosure() - def function_without_closure(): ... - - with self.assertRaises(SystemError): - _testcapi.function_set_closure(None, ()) # not a function - - with self.assertRaises(SystemError): - _testcapi.function_set_closure(function_without_closure, 1) - self.assertIsNone(function_without_closure.__closure__) # no change - - # NOTE: this works, but goes against the docs: - _testcapi.function_set_closure(function_without_closure, (1, 2)) - self.assertEqual( - _testcapi.function_get_closure(function_without_closure), (1, 2)) - self.assertEqual(function_without_closure.__closure__, (1, 2)) - - def test_function_get_annotations(self): - # Test PyFunction_GetAnnotations() - def normal(): - pass - - def annofn(arg: int) -> str: - return f'arg = {arg}' - - annotations = _testcapi.function_get_annotations(normal) - self.assertIsNone(annotations) - - annotations = _testcapi.function_get_annotations(annofn) - self.assertIsInstance(annotations, dict) - self.assertEqual(annotations, annofn.__annotations__) - - with self.assertRaises(SystemError): - _testcapi.function_get_annotations(None) - - # TODO: test PyFunction_New() - # TODO: test PyFunction_NewWithQualName() - # TODO: test PyFunction_SetVectorcall() - # TODO: test PyFunction_SetAnnotations() - # TODO: test PyClassMethod_New() - # TODO: test PyStaticMethod_New() - # - # PyFunction_AddWatcher() and PyFunction_ClearWatcher() are tested by - # test_capi.test_watchers. - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py deleted file mode 100644 index 2f047200daf..00000000000 --- a/Lib/test/test_capi/test_getargs.py +++ /dev/null @@ -1,1421 +0,0 @@ -import string -import sys -import unittest -from test import support -from test.support import import_helper -from test.support import script_helper -from test.support import warnings_helper -from test.support.testcase import FloatsAreIdenticalMixin -# Skip this test if the _testcapi module isn't available. -_testcapi = import_helper.import_module('_testcapi') -from _testcapi import getargs_keywords, getargs_keyword_only - -try: - import _testinternalcapi -except ImportError: - _testinternalcapi = None - -# > How about the following counterproposal. This also changes some of -# > the other format codes to be a little more regular. -# > -# > Code C type Range check -# > -# > b unsigned char 0..UCHAR_MAX -# > h signed short SHRT_MIN..SHRT_MAX -# > B unsigned char none ** -# > H unsigned short none ** -# > k * unsigned long none -# > I * unsigned int 0..UINT_MAX -# -# -# > i int INT_MIN..INT_MAX -# > l long LONG_MIN..LONG_MAX -# -# > K * unsigned long long none -# > L long long LLONG_MIN..LLONG_MAX -# -# > Notes: -# > -# > * New format codes. -# > -# > ** Changed from previous "range-and-a-half" to "none"; the -# > range-and-a-half checking wasn't particularly useful. -# -# Plus a C API or two, e.g. PyLong_AsUnsignedLongMask() -> -# unsigned long and PyLong_AsUnsignedLongLongMask() -> unsigned -# long long (if that exists). - -LARGE = 0x7FFFFFFF -VERY_LARGE = 0xFF0000121212121212121242 - -from _testcapi import UCHAR_MAX, USHRT_MAX, UINT_MAX, ULONG_MAX, INT_MAX, \ - INT_MIN, LONG_MIN, LONG_MAX, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, \ - SHRT_MIN, SHRT_MAX, FLT_MIN, FLT_MAX, DBL_MIN, DBL_MAX - -DBL_MAX_EXP = sys.float_info.max_exp -INF = float('inf') -NAN = float('nan') - -# fake, they are not defined in Python's header files -LLONG_MAX = 2**63-1 -LLONG_MIN = -2**63 -ULLONG_MAX = 2**64-1 - -NULL = None - -class CustomError(Exception): - pass - -class Index: - def __index__(self): - return 99 - -class IndexIntSubclass(int): - def __index__(self): - return 99 - -class BadIndex: - def __index__(self): - return 1.0 - -class BadIndex2: - def __index__(self): - return True - -class BadIndex3(int): - def __index__(self): - return True - - -class Int: - def __int__(self): - return 99 - -class IntSubclass(int): - def __int__(self): - return 99 - -class BadInt: - def __int__(self): - return 1.0 - -class BadInt2: - def __int__(self): - return True - -class BadInt3(int): - def __int__(self): - return True - - -class Float: - def __float__(self): - return 4.25 - -class FloatSubclass(float): - pass - -class FloatSubclass2(float): - def __float__(self): - return 4.25 - -class BadFloat: - def __float__(self): - return 687 - -class BadFloat2: - def __float__(self): - return FloatSubclass(4.25) - -class BadFloat3(float): - def __float__(self): - return FloatSubclass(4.25) - - -class Complex: - def __complex__(self): - return 4.25+0.5j - -class ComplexSubclass(complex): - pass - -class ComplexSubclass2(complex): - def __complex__(self): - return 4.25+0.5j - -class BadComplex: - def __complex__(self): - return 1.25 - -class BadComplex2: - def __complex__(self): - return ComplexSubclass(4.25+0.5j) - -class BadComplex3(complex): - def __complex__(self): - return ComplexSubclass(4.25+0.5j) - - -class TupleSubclass(tuple): - pass - -class DictSubclass(dict): - pass - -NONCONTIG_WRITABLE = memoryview(bytearray(b'noncontig'))[::-2] -NONCONTIG_READONLY = memoryview(b'noncontig')[::-2] - -class Unsigned_TestCase(unittest.TestCase): - def test_b(self): - from _testcapi import getargs_b - # b returns 'unsigned char', and does range checking (0 ... UCHAR_MAX) - self.assertRaises(TypeError, getargs_b, 3.14) - self.assertEqual(99, getargs_b(Index())) - self.assertEqual(0, getargs_b(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_b, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_b(BadIndex2())) - self.assertEqual(0, getargs_b(BadIndex3())) - self.assertRaises(TypeError, getargs_b, Int()) - self.assertEqual(0, getargs_b(IntSubclass())) - self.assertRaises(TypeError, getargs_b, BadInt()) - self.assertRaises(TypeError, getargs_b, BadInt2()) - self.assertEqual(0, getargs_b(BadInt3())) - - self.assertRaises(OverflowError, getargs_b, -1) - self.assertEqual(0, getargs_b(0)) - self.assertEqual(UCHAR_MAX, getargs_b(UCHAR_MAX)) - self.assertRaises(OverflowError, getargs_b, UCHAR_MAX + 1) - - self.assertEqual(42, getargs_b(42)) - self.assertRaises(OverflowError, getargs_b, VERY_LARGE) - - def test_B(self): - from _testcapi import getargs_B - # B returns 'unsigned char', no range checking - self.assertRaises(TypeError, getargs_B, 3.14) - self.assertEqual(99, getargs_B(Index())) - self.assertEqual(0, getargs_B(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_B, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_B(BadIndex2())) - self.assertEqual(0, getargs_B(BadIndex3())) - self.assertRaises(TypeError, getargs_B, Int()) - self.assertEqual(0, getargs_B(IntSubclass())) - self.assertRaises(TypeError, getargs_B, BadInt()) - self.assertRaises(TypeError, getargs_B, BadInt2()) - self.assertEqual(0, getargs_B(BadInt3())) - - self.assertEqual(UCHAR_MAX, getargs_B(-1)) - self.assertEqual(0, getargs_B(0)) - self.assertEqual(UCHAR_MAX, getargs_B(UCHAR_MAX)) - self.assertEqual(0, getargs_B(UCHAR_MAX+1)) - - self.assertEqual(42, getargs_B(42)) - self.assertEqual(UCHAR_MAX & VERY_LARGE, getargs_B(VERY_LARGE)) - - def test_H(self): - from _testcapi import getargs_H - # H returns 'unsigned short', no range checking - self.assertRaises(TypeError, getargs_H, 3.14) - self.assertEqual(99, getargs_H(Index())) - self.assertEqual(0, getargs_H(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_H, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_H(BadIndex2())) - self.assertEqual(0, getargs_H(BadIndex3())) - self.assertRaises(TypeError, getargs_H, Int()) - self.assertEqual(0, getargs_H(IntSubclass())) - self.assertRaises(TypeError, getargs_H, BadInt()) - self.assertRaises(TypeError, getargs_H, BadInt2()) - self.assertEqual(0, getargs_H(BadInt3())) - - self.assertEqual(USHRT_MAX, getargs_H(-1)) - self.assertEqual(0, getargs_H(0)) - self.assertEqual(USHRT_MAX, getargs_H(USHRT_MAX)) - self.assertEqual(0, getargs_H(USHRT_MAX+1)) - - self.assertEqual(42, getargs_H(42)) - - self.assertEqual(VERY_LARGE & USHRT_MAX, getargs_H(VERY_LARGE)) - - def test_I(self): - from _testcapi import getargs_I - # I returns 'unsigned int', no range checking - self.assertRaises(TypeError, getargs_I, 3.14) - self.assertEqual(99, getargs_I(Index())) - self.assertEqual(0, getargs_I(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_I, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_I(BadIndex2())) - self.assertEqual(0, getargs_I(BadIndex3())) - self.assertRaises(TypeError, getargs_I, Int()) - self.assertEqual(0, getargs_I(IntSubclass())) - self.assertRaises(TypeError, getargs_I, BadInt()) - self.assertRaises(TypeError, getargs_I, BadInt2()) - self.assertEqual(0, getargs_I(BadInt3())) - - self.assertEqual(UINT_MAX, getargs_I(-1)) - self.assertEqual(0, getargs_I(0)) - self.assertEqual(UINT_MAX, getargs_I(UINT_MAX)) - self.assertEqual(0, getargs_I(UINT_MAX+1)) - - self.assertEqual(42, getargs_I(42)) - - self.assertEqual(VERY_LARGE & UINT_MAX, getargs_I(VERY_LARGE)) - - def test_k(self): - from _testcapi import getargs_k - # k returns 'unsigned long', no range checking - self.assertRaises(TypeError, getargs_k, 3.14) - self.assertEqual(99, getargs_k(Index())) - self.assertEqual(0, getargs_k(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_k, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_k(BadIndex2())) - self.assertEqual(0, getargs_k(BadIndex3())) - self.assertRaises(TypeError, getargs_k, Int()) - self.assertEqual(0, getargs_k(IntSubclass())) - self.assertRaises(TypeError, getargs_k, BadInt()) - self.assertRaises(TypeError, getargs_k, BadInt2()) - self.assertEqual(0, getargs_k(BadInt3())) - - self.assertEqual(ULONG_MAX, getargs_k(-1)) - self.assertEqual(0, getargs_k(0)) - self.assertEqual(ULONG_MAX, getargs_k(ULONG_MAX)) - self.assertEqual(0, getargs_k(ULONG_MAX+1)) - - self.assertEqual(42, getargs_k(42)) - - self.assertEqual(VERY_LARGE & ULONG_MAX, getargs_k(VERY_LARGE)) - -class Signed_TestCase(unittest.TestCase): - def test_h(self): - from _testcapi import getargs_h - # h returns 'short', and does range checking (SHRT_MIN ... SHRT_MAX) - self.assertRaises(TypeError, getargs_h, 3.14) - self.assertEqual(99, getargs_h(Index())) - self.assertEqual(0, getargs_h(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_h, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_h(BadIndex2())) - self.assertEqual(0, getargs_h(BadIndex3())) - self.assertRaises(TypeError, getargs_h, Int()) - self.assertEqual(0, getargs_h(IntSubclass())) - self.assertRaises(TypeError, getargs_h, BadInt()) - self.assertRaises(TypeError, getargs_h, BadInt2()) - self.assertEqual(0, getargs_h(BadInt3())) - - self.assertRaises(OverflowError, getargs_h, SHRT_MIN-1) - self.assertEqual(SHRT_MIN, getargs_h(SHRT_MIN)) - self.assertEqual(SHRT_MAX, getargs_h(SHRT_MAX)) - self.assertRaises(OverflowError, getargs_h, SHRT_MAX+1) - - self.assertEqual(42, getargs_h(42)) - self.assertRaises(OverflowError, getargs_h, VERY_LARGE) - - def test_i(self): - from _testcapi import getargs_i - # i returns 'int', and does range checking (INT_MIN ... INT_MAX) - self.assertRaises(TypeError, getargs_i, 3.14) - self.assertEqual(99, getargs_i(Index())) - self.assertEqual(0, getargs_i(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_i, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_i(BadIndex2())) - self.assertEqual(0, getargs_i(BadIndex3())) - self.assertRaises(TypeError, getargs_i, Int()) - self.assertEqual(0, getargs_i(IntSubclass())) - self.assertRaises(TypeError, getargs_i, BadInt()) - self.assertRaises(TypeError, getargs_i, BadInt2()) - self.assertEqual(0, getargs_i(BadInt3())) - - self.assertRaises(OverflowError, getargs_i, INT_MIN-1) - self.assertEqual(INT_MIN, getargs_i(INT_MIN)) - self.assertEqual(INT_MAX, getargs_i(INT_MAX)) - self.assertRaises(OverflowError, getargs_i, INT_MAX+1) - - self.assertEqual(42, getargs_i(42)) - self.assertRaises(OverflowError, getargs_i, VERY_LARGE) - - def test_l(self): - from _testcapi import getargs_l - # l returns 'long', and does range checking (LONG_MIN ... LONG_MAX) - self.assertRaises(TypeError, getargs_l, 3.14) - self.assertEqual(99, getargs_l(Index())) - self.assertEqual(0, getargs_l(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_l, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_l(BadIndex2())) - self.assertEqual(0, getargs_l(BadIndex3())) - self.assertRaises(TypeError, getargs_l, Int()) - self.assertEqual(0, getargs_l(IntSubclass())) - self.assertRaises(TypeError, getargs_l, BadInt()) - self.assertRaises(TypeError, getargs_l, BadInt2()) - self.assertEqual(0, getargs_l(BadInt3())) - - self.assertRaises(OverflowError, getargs_l, LONG_MIN-1) - self.assertEqual(LONG_MIN, getargs_l(LONG_MIN)) - self.assertEqual(LONG_MAX, getargs_l(LONG_MAX)) - self.assertRaises(OverflowError, getargs_l, LONG_MAX+1) - - self.assertEqual(42, getargs_l(42)) - self.assertRaises(OverflowError, getargs_l, VERY_LARGE) - - def test_n(self): - from _testcapi import getargs_n - # n returns 'Py_ssize_t', and does range checking - # (PY_SSIZE_T_MIN ... PY_SSIZE_T_MAX) - self.assertRaises(TypeError, getargs_n, 3.14) - self.assertEqual(99, getargs_n(Index())) - self.assertEqual(0, getargs_n(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_n, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_n(BadIndex2())) - self.assertEqual(0, getargs_n(BadIndex3())) - self.assertRaises(TypeError, getargs_n, Int()) - self.assertEqual(0, getargs_n(IntSubclass())) - self.assertRaises(TypeError, getargs_n, BadInt()) - self.assertRaises(TypeError, getargs_n, BadInt2()) - self.assertEqual(0, getargs_n(BadInt3())) - - self.assertRaises(OverflowError, getargs_n, PY_SSIZE_T_MIN-1) - self.assertEqual(PY_SSIZE_T_MIN, getargs_n(PY_SSIZE_T_MIN)) - self.assertEqual(PY_SSIZE_T_MAX, getargs_n(PY_SSIZE_T_MAX)) - self.assertRaises(OverflowError, getargs_n, PY_SSIZE_T_MAX+1) - - self.assertEqual(42, getargs_n(42)) - self.assertRaises(OverflowError, getargs_n, VERY_LARGE) - - -class LongLong_TestCase(unittest.TestCase): - def test_L(self): - from _testcapi import getargs_L - # L returns 'long long', and does range checking (LLONG_MIN - # ... LLONG_MAX) - self.assertRaises(TypeError, getargs_L, 3.14) - self.assertRaises(TypeError, getargs_L, "Hello") - self.assertEqual(99, getargs_L(Index())) - self.assertEqual(0, getargs_L(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_L, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_L(BadIndex2())) - self.assertEqual(0, getargs_L(BadIndex3())) - self.assertRaises(TypeError, getargs_L, Int()) - self.assertEqual(0, getargs_L(IntSubclass())) - self.assertRaises(TypeError, getargs_L, BadInt()) - self.assertRaises(TypeError, getargs_L, BadInt2()) - self.assertEqual(0, getargs_L(BadInt3())) - - self.assertRaises(OverflowError, getargs_L, LLONG_MIN-1) - self.assertEqual(LLONG_MIN, getargs_L(LLONG_MIN)) - self.assertEqual(LLONG_MAX, getargs_L(LLONG_MAX)) - self.assertRaises(OverflowError, getargs_L, LLONG_MAX+1) - - self.assertEqual(42, getargs_L(42)) - self.assertRaises(OverflowError, getargs_L, VERY_LARGE) - - def test_K(self): - from _testcapi import getargs_K - # K return 'unsigned long long', no range checking - self.assertRaises(TypeError, getargs_K, 3.14) - self.assertEqual(99, getargs_K(Index())) - self.assertEqual(0, getargs_K(IndexIntSubclass())) - self.assertRaises(TypeError, getargs_K, BadIndex()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(1, getargs_K(BadIndex2())) - self.assertEqual(0, getargs_K(BadIndex3())) - self.assertRaises(TypeError, getargs_K, Int()) - self.assertEqual(0, getargs_K(IntSubclass())) - self.assertRaises(TypeError, getargs_K, BadInt()) - self.assertRaises(TypeError, getargs_K, BadInt2()) - self.assertEqual(0, getargs_K(BadInt3())) - - self.assertEqual(ULLONG_MAX, getargs_K(ULLONG_MAX)) - self.assertEqual(0, getargs_K(0)) - self.assertEqual(ULLONG_MAX, getargs_K(ULLONG_MAX)) - self.assertEqual(0, getargs_K(ULLONG_MAX+1)) - - self.assertEqual(42, getargs_K(42)) - - self.assertEqual(VERY_LARGE & ULLONG_MAX, getargs_K(VERY_LARGE)) - - -class Float_TestCase(unittest.TestCase, FloatsAreIdenticalMixin): - def test_f(self): - from _testcapi import getargs_f - self.assertEqual(getargs_f(4.25), 4.25) - self.assertEqual(getargs_f(4), 4.0) - self.assertRaises(TypeError, getargs_f, 4.25+0j) - self.assertEqual(getargs_f(Float()), 4.25) - self.assertEqual(getargs_f(FloatSubclass(7.5)), 7.5) - self.assertEqual(getargs_f(FloatSubclass2(7.5)), 7.5) - self.assertRaises(TypeError, getargs_f, BadFloat()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_f(BadFloat2()), 4.25) - self.assertEqual(getargs_f(BadFloat3(7.5)), 7.5) - self.assertEqual(getargs_f(Index()), 99.0) - self.assertRaises(TypeError, getargs_f, Int()) - - for x in (FLT_MIN, -FLT_MIN, FLT_MAX, -FLT_MAX, INF, -INF): - self.assertEqual(getargs_f(x), x) - if FLT_MAX < DBL_MAX: - self.assertEqual(getargs_f(DBL_MAX), INF) - self.assertEqual(getargs_f(-DBL_MAX), -INF) - if FLT_MIN > DBL_MIN: - self.assertFloatsAreIdentical(getargs_f(DBL_MIN), 0.0) - self.assertFloatsAreIdentical(getargs_f(-DBL_MIN), -0.0) - self.assertFloatsAreIdentical(getargs_f(0.0), 0.0) - self.assertFloatsAreIdentical(getargs_f(-0.0), -0.0) - r = getargs_f(NAN) - self.assertNotEqual(r, r) - - @support.requires_IEEE_754 - def test_f_rounding(self): - from _testcapi import getargs_f - self.assertEqual(getargs_f(3.40282356e38), FLT_MAX) - self.assertEqual(getargs_f(-3.40282356e38), -FLT_MAX) - - def test_d(self): - from _testcapi import getargs_d - self.assertEqual(getargs_d(4.25), 4.25) - self.assertEqual(getargs_d(4), 4.0) - self.assertRaises(TypeError, getargs_d, 4.25+0j) - self.assertEqual(getargs_d(Float()), 4.25) - self.assertEqual(getargs_d(FloatSubclass(7.5)), 7.5) - self.assertEqual(getargs_d(FloatSubclass2(7.5)), 7.5) - self.assertRaises(TypeError, getargs_d, BadFloat()) - with self.assertWarns(DeprecationWarning): - self.assertEqual(getargs_d(BadFloat2()), 4.25) - self.assertEqual(getargs_d(BadFloat3(7.5)), 7.5) - self.assertEqual(getargs_d(Index()), 99.0) - self.assertRaises(TypeError, getargs_d, Int()) - - for x in (DBL_MIN, -DBL_MIN, DBL_MAX, -DBL_MAX, INF, -INF): - self.assertEqual(getargs_d(x), x) - self.assertRaises(OverflowError, getargs_d, 1< 1 - self.assertEqual(getargs_c(b'a'), 97) - self.assertEqual(getargs_c(bytearray(b'a')), 97) - self.assertRaises(TypeError, getargs_c, memoryview(b'a')) - self.assertRaises(TypeError, getargs_c, 's') - self.assertRaises(TypeError, getargs_c, 97) - self.assertRaises(TypeError, getargs_c, None) - - def test_y(self): - from _testcapi import getargs_y - self.assertRaises(TypeError, getargs_y, 'abc\xe9') - self.assertEqual(getargs_y(b'bytes'), b'bytes') - self.assertRaises(ValueError, getargs_y, b'nul:\0') - self.assertRaises(TypeError, getargs_y, bytearray(b'bytearray')) - self.assertRaises(TypeError, getargs_y, memoryview(b'memoryview')) - self.assertRaises(TypeError, getargs_y, None) - - def test_y_star(self): - from _testcapi import getargs_y_star - self.assertRaises(TypeError, getargs_y_star, 'abc\xe9') - self.assertEqual(getargs_y_star(b'bytes'), b'bytes') - self.assertEqual(getargs_y_star(b'nul:\0'), b'nul:\0') - self.assertEqual(getargs_y_star(bytearray(b'bytearray')), b'bytearray') - self.assertEqual(getargs_y_star(memoryview(b'memoryview')), b'memoryview') - self.assertRaises(TypeError, getargs_y_star, None) - self.assertRaises(BufferError, getargs_y_star, NONCONTIG_WRITABLE) - self.assertRaises(BufferError, getargs_y_star, NONCONTIG_READONLY) - - def test_y_hash(self): - from _testcapi import getargs_y_hash - self.assertRaises(TypeError, getargs_y_hash, 'abc\xe9') - self.assertEqual(getargs_y_hash(b'bytes'), b'bytes') - self.assertEqual(getargs_y_hash(b'nul:\0'), b'nul:\0') - self.assertRaises(TypeError, getargs_y_hash, bytearray(b'bytearray')) - self.assertRaises(TypeError, getargs_y_hash, memoryview(b'memoryview')) - self.assertRaises(TypeError, getargs_y_hash, None) - # TypeError: must be read-only bytes-like object, not memoryview - self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_WRITABLE) - self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_READONLY) - - def test_w_star(self): - # getargs_w_star() modifies first and last byte - # getargs_w_star_opt() takes additional optional args: with one - # argument it should behave the same as getargs_w_star - from _testcapi import getargs_w_star, getargs_w_star_opt - for func in (getargs_w_star, getargs_w_star_opt): - with self.subTest(func=func): - self.assertRaises(TypeError, func, 'abc\xe9') - self.assertRaises(TypeError, func, b'bytes') - self.assertRaises(TypeError, func, b'nul:\0') - self.assertRaises(TypeError, func, memoryview(b'bytes')) - buf = bytearray(b'bytearray') - self.assertEqual(func(buf), b'[ytearra]') - self.assertEqual(buf, bytearray(b'[ytearra]')) - buf = bytearray(b'memoryview') - self.assertEqual(func(memoryview(buf)), b'[emoryvie]') - self.assertEqual(buf, bytearray(b'[emoryvie]')) - self.assertRaises(TypeError, func, None) - self.assertRaises(TypeError, func, NONCONTIG_WRITABLE) - self.assertRaises(TypeError, func, NONCONTIG_READONLY) - - def test_getargs_empty(self): - from _testcapi import getargs_empty - self.assertTrue(getargs_empty()) - self.assertRaises(TypeError, getargs_empty, 1) - self.assertRaises(TypeError, getargs_empty, 1, 2, 3) - self.assertRaises(TypeError, getargs_empty, a=1) - self.assertRaises(TypeError, getargs_empty, a=1, b=2) - self.assertRaises(TypeError, getargs_empty, 'x', 'y', 'z', a=1, b=2) - - -class String_TestCase(unittest.TestCase): - def test_C(self): - from _testcapi import getargs_C - self.assertRaises(TypeError, getargs_C, 'abc') # len > 1 - self.assertEqual(getargs_C('a'), 97) - self.assertEqual(getargs_C('\u20ac'), 0x20ac) - self.assertEqual(getargs_C('\U0001f40d'), 0x1f40d) - self.assertRaises(TypeError, getargs_C, b'a') - self.assertRaises(TypeError, getargs_C, bytearray(b'a')) - self.assertRaises(TypeError, getargs_C, memoryview(b'a')) - self.assertRaises(TypeError, getargs_C, 97) - self.assertRaises(TypeError, getargs_C, None) - - def test_s(self): - from _testcapi import getargs_s - self.assertEqual(getargs_s('abc\xe9'), b'abc\xc3\xa9') - self.assertRaises(ValueError, getargs_s, 'nul:\0') - self.assertRaises(TypeError, getargs_s, b'bytes') - self.assertRaises(TypeError, getargs_s, bytearray(b'bytearray')) - self.assertRaises(TypeError, getargs_s, memoryview(b'memoryview')) - self.assertRaises(TypeError, getargs_s, None) - - def test_s_star(self): - from _testcapi import getargs_s_star - self.assertEqual(getargs_s_star('abc\xe9'), b'abc\xc3\xa9') - self.assertEqual(getargs_s_star('nul:\0'), b'nul:\0') - self.assertEqual(getargs_s_star(b'bytes'), b'bytes') - self.assertEqual(getargs_s_star(bytearray(b'bytearray')), b'bytearray') - self.assertEqual(getargs_s_star(memoryview(b'memoryview')), b'memoryview') - self.assertRaises(TypeError, getargs_s_star, None) - self.assertRaises(BufferError, getargs_s_star, NONCONTIG_WRITABLE) - self.assertRaises(BufferError, getargs_s_star, NONCONTIG_READONLY) - - def test_s_hash(self): - from _testcapi import getargs_s_hash - self.assertEqual(getargs_s_hash('abc\xe9'), b'abc\xc3\xa9') - self.assertEqual(getargs_s_hash('nul:\0'), b'nul:\0') - self.assertEqual(getargs_s_hash(b'bytes'), b'bytes') - self.assertRaises(TypeError, getargs_s_hash, bytearray(b'bytearray')) - self.assertRaises(TypeError, getargs_s_hash, memoryview(b'memoryview')) - self.assertRaises(TypeError, getargs_s_hash, None) - # TypeError: must be read-only bytes-like object, not memoryview - self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_WRITABLE) - self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_READONLY) - - def test_z(self): - from _testcapi import getargs_z - self.assertEqual(getargs_z('abc\xe9'), b'abc\xc3\xa9') - self.assertRaises(ValueError, getargs_z, 'nul:\0') - self.assertRaises(TypeError, getargs_z, b'bytes') - self.assertRaises(TypeError, getargs_z, bytearray(b'bytearray')) - self.assertRaises(TypeError, getargs_z, memoryview(b'memoryview')) - self.assertIsNone(getargs_z(None)) - - def test_z_star(self): - from _testcapi import getargs_z_star - self.assertEqual(getargs_z_star('abc\xe9'), b'abc\xc3\xa9') - self.assertEqual(getargs_z_star('nul:\0'), b'nul:\0') - self.assertEqual(getargs_z_star(b'bytes'), b'bytes') - self.assertEqual(getargs_z_star(bytearray(b'bytearray')), b'bytearray') - self.assertEqual(getargs_z_star(memoryview(b'memoryview')), b'memoryview') - self.assertIsNone(getargs_z_star(None)) - self.assertRaises(BufferError, getargs_z_star, NONCONTIG_WRITABLE) - self.assertRaises(BufferError, getargs_z_star, NONCONTIG_READONLY) - - def test_z_hash(self): - from _testcapi import getargs_z_hash - self.assertEqual(getargs_z_hash('abc\xe9'), b'abc\xc3\xa9') - self.assertEqual(getargs_z_hash('nul:\0'), b'nul:\0') - self.assertEqual(getargs_z_hash(b'bytes'), b'bytes') - self.assertRaises(TypeError, getargs_z_hash, bytearray(b'bytearray')) - self.assertRaises(TypeError, getargs_z_hash, memoryview(b'memoryview')) - self.assertIsNone(getargs_z_hash(None)) - # TypeError: must be read-only bytes-like object, not memoryview - self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_WRITABLE) - self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_READONLY) - - def test_es(self): - from _testcapi import getargs_es - self.assertEqual(getargs_es('abc\xe9'), b'abc\xc3\xa9') - self.assertEqual(getargs_es('abc\xe9', 'latin1'), b'abc\xe9') - self.assertRaises(UnicodeEncodeError, getargs_es, 'abc\xe9', 'ascii') - self.assertRaises(LookupError, getargs_es, 'abc\xe9', 'spam') - self.assertRaises(TypeError, getargs_es, b'bytes', 'latin1') - self.assertRaises(TypeError, getargs_es, bytearray(b'bytearray'), 'latin1') - self.assertRaises(TypeError, getargs_es, memoryview(b'memoryview'), 'latin1') - self.assertRaises(TypeError, getargs_es, None, 'latin1') - self.assertRaises(TypeError, getargs_es, 'nul:\0', 'latin1') - - def test_et(self): - from _testcapi import getargs_et - self.assertEqual(getargs_et('abc\xe9'), b'abc\xc3\xa9') - self.assertEqual(getargs_et('abc\xe9', 'latin1'), b'abc\xe9') - self.assertRaises(UnicodeEncodeError, getargs_et, 'abc\xe9', 'ascii') - self.assertRaises(LookupError, getargs_et, 'abc\xe9', 'spam') - self.assertEqual(getargs_et(b'bytes', 'latin1'), b'bytes') - self.assertEqual(getargs_et(bytearray(b'bytearray'), 'latin1'), b'bytearray') - self.assertRaises(TypeError, getargs_et, memoryview(b'memoryview'), 'latin1') - self.assertRaises(TypeError, getargs_et, None, 'latin1') - self.assertRaises(TypeError, getargs_et, 'nul:\0', 'latin1') - self.assertRaises(TypeError, getargs_et, b'nul:\0', 'latin1') - self.assertRaises(TypeError, getargs_et, bytearray(b'nul:\0'), 'latin1') - - def test_es_hash(self): - from _testcapi import getargs_es_hash - self.assertEqual(getargs_es_hash('abc\xe9'), b'abc\xc3\xa9') - self.assertEqual(getargs_es_hash('abc\xe9', 'latin1'), b'abc\xe9') - self.assertRaises(UnicodeEncodeError, getargs_es_hash, 'abc\xe9', 'ascii') - self.assertRaises(LookupError, getargs_es_hash, 'abc\xe9', 'spam') - self.assertRaises(TypeError, getargs_es_hash, b'bytes', 'latin1') - self.assertRaises(TypeError, getargs_es_hash, bytearray(b'bytearray'), 'latin1') - self.assertRaises(TypeError, getargs_es_hash, memoryview(b'memoryview'), 'latin1') - self.assertRaises(TypeError, getargs_es_hash, None, 'latin1') - self.assertEqual(getargs_es_hash('nul:\0', 'latin1'), b'nul:\0') - - buf = bytearray(b'x'*8) - self.assertEqual(getargs_es_hash('abc\xe9', 'latin1', buf), b'abc\xe9') - self.assertEqual(buf, bytearray(b'abc\xe9\x00xxx')) - buf = bytearray(b'x'*5) - self.assertEqual(getargs_es_hash('abc\xe9', 'latin1', buf), b'abc\xe9') - self.assertEqual(buf, bytearray(b'abc\xe9\x00')) - buf = bytearray(b'x'*4) - self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) - self.assertEqual(buf, bytearray(b'x'*4)) - buf = bytearray() - self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) - - def test_et_hash(self): - from _testcapi import getargs_et_hash - self.assertEqual(getargs_et_hash('abc\xe9'), b'abc\xc3\xa9') - self.assertEqual(getargs_et_hash('abc\xe9', 'latin1'), b'abc\xe9') - self.assertRaises(UnicodeEncodeError, getargs_et_hash, 'abc\xe9', 'ascii') - self.assertRaises(LookupError, getargs_et_hash, 'abc\xe9', 'spam') - self.assertEqual(getargs_et_hash(b'bytes', 'latin1'), b'bytes') - self.assertEqual(getargs_et_hash(bytearray(b'bytearray'), 'latin1'), b'bytearray') - self.assertRaises(TypeError, getargs_et_hash, memoryview(b'memoryview'), 'latin1') - self.assertRaises(TypeError, getargs_et_hash, None, 'latin1') - self.assertEqual(getargs_et_hash('nul:\0', 'latin1'), b'nul:\0') - self.assertEqual(getargs_et_hash(b'nul:\0', 'latin1'), b'nul:\0') - self.assertEqual(getargs_et_hash(bytearray(b'nul:\0'), 'latin1'), b'nul:\0') - - buf = bytearray(b'x'*8) - self.assertEqual(getargs_et_hash('abc\xe9', 'latin1', buf), b'abc\xe9') - self.assertEqual(buf, bytearray(b'abc\xe9\x00xxx')) - buf = bytearray(b'x'*5) - self.assertEqual(getargs_et_hash('abc\xe9', 'latin1', buf), b'abc\xe9') - self.assertEqual(buf, bytearray(b'abc\xe9\x00')) - buf = bytearray(b'x'*4) - self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) - self.assertEqual(buf, bytearray(b'x'*4)) - buf = bytearray() - self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) - - def test_gh_99240_clear_args(self): - from _testcapi import gh_99240_clear_args - self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') - - -class Object_TestCase(unittest.TestCase): - def test_S(self): - from _testcapi import getargs_S - obj = b'bytes' - self.assertIs(getargs_S(obj), obj) - self.assertRaises(TypeError, getargs_S, bytearray(b'bytearray')) - self.assertRaises(TypeError, getargs_S, 'str') - self.assertRaises(TypeError, getargs_S, None) - self.assertRaises(TypeError, getargs_S, memoryview(obj)) - - def test_Y(self): - from _testcapi import getargs_Y - obj = bytearray(b'bytearray') - self.assertIs(getargs_Y(obj), obj) - self.assertRaises(TypeError, getargs_Y, b'bytes') - self.assertRaises(TypeError, getargs_Y, 'str') - self.assertRaises(TypeError, getargs_Y, None) - self.assertRaises(TypeError, getargs_Y, memoryview(obj)) - - def test_U(self): - from _testcapi import getargs_U - obj = 'str' - self.assertIs(getargs_U(obj), obj) - self.assertRaises(TypeError, getargs_U, b'bytes') - self.assertRaises(TypeError, getargs_U, bytearray(b'bytearray')) - self.assertRaises(TypeError, getargs_U, None) - - -# Bug #6012 -class Test6012(unittest.TestCase): - def test(self): - self.assertEqual(_testcapi.argparsing("Hello", "World"), 1) - - -class SkipitemTest(unittest.TestCase): - - # u, and Z raises DeprecationWarning - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_skipitem(self): - """ - If this test failed, you probably added a new "format unit" - in Python/getargs.c, but neglected to update our poor friend - skipitem() in the same file. (If so, shame on you!) - - With a few exceptions**, this function brute-force tests all - printable ASCII*** characters (32 to 126 inclusive) as format units, - checking to see that PyArg_ParseTupleAndKeywords() return consistent - errors both when the unit is attempted to be used and when it is - skipped. If the format unit doesn't exist, we'll get one of two - specific error messages (one for used, one for skipped); if it does - exist we *won't* get that error--we'll get either no error or some - other error. If we get the specific "does not exist" error for one - test and not for the other, there's a mismatch, and the test fails. - - ** Some format units have special funny semantics and it would - be difficult to accommodate them here. Since these are all - well-established and properly skipped in skipitem() we can - get away with not testing them--this test is really intended - to catch *new* format units. - - *** Python C source files must be ASCII. Therefore it's impossible - to have non-ASCII format units. - - """ - empty_tuple = () - tuple_1 = (0,) - dict_b = {'b':1} - keywords = ["a", "b"] - - for i in range(32, 127): - c = chr(i) - - # skip parentheses, the error reporting is inconsistent about them - # skip 'e' and 'w', they're always two-character codes - # skip '|' and '$', they don't represent arguments anyway - if c in '()ew|$': - continue - - # test the format unit when not skipped - format = c + "i" - try: - _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, - format, keywords) - when_not_skipped = False - except SystemError as e: - s = "argument 1 (impossible)" - when_not_skipped = (str(e) == s) - except TypeError: - when_not_skipped = False - - # test the format unit when skipped - optional_format = "|" + format - try: - _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, - optional_format, keywords) - when_skipped = False - except SystemError as e: - s = "impossible: '{}'".format(format) - when_skipped = (str(e) == s) - - message = ("test_skipitem_parity: " - "detected mismatch between convertsimple and skipitem " - "for format unit '{}' ({}), not skipped {}, skipped {}".format( - c, i, when_skipped, when_not_skipped)) - self.assertIs(when_skipped, when_not_skipped, message) - - def test_skipitem_with_suffix(self): - parse = _testcapi.parse_tuple_and_keywords - empty_tuple = () - tuple_1 = (0,) - dict_b = {'b':1} - keywords = ["a", "b"] - - supported = ('s#', 's*', 'z#', 'z*', 'y#', 'y*', 'w*') - for c in string.ascii_letters: - for c2 in '#*': - f = c + c2 - with self.subTest(format=f): - optional_format = "|" + f + "i" - if f in supported: - parse(empty_tuple, dict_b, optional_format, keywords) - else: - with self.assertRaisesRegex(SystemError, - 'impossible'): - parse(empty_tuple, dict_b, optional_format, keywords) - - for c in map(chr, range(32, 128)): - f = 'e' + c - optional_format = "|" + f + "i" - with self.subTest(format=f): - if c in 'st': - parse(empty_tuple, dict_b, optional_format, keywords) - else: - with self.assertRaisesRegex(SystemError, - 'impossible'): - parse(empty_tuple, dict_b, optional_format, keywords) - - -class ParseTupleAndKeywords_Test(unittest.TestCase): - - def test_parse_tuple_and_keywords(self): - # Test handling errors in the parse_tuple_and_keywords helper itself - self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords, - (), {}, 42, []) - self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, '', 42) - self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, '', [''] * 42) - self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, '', [42]) - - def test_basic(self): - parse = _testcapi.parse_tuple_and_keywords - - self.assertEqual(parse((), {'a': 1}, 'O', ['a']), (1,)) - self.assertEqual(parse((), {}, '|O', ['a']), (NULL,)) - self.assertEqual(parse((1, 2), {}, 'OO', ['a', 'b']), (1, 2)) - self.assertEqual(parse((1,), {'b': 2}, 'OO', ['a', 'b']), (1, 2)) - self.assertEqual(parse((), {'a': 1, 'b': 2}, 'OO', ['a', 'b']), (1, 2)) - self.assertEqual(parse((), {'b': 2}, '|OO', ['a', 'b']), (NULL, 2)) - - with self.assertRaisesRegex(TypeError, - "function missing required argument 'a'"): - parse((), {}, 'O', ['a']) - with self.assertRaisesRegex(TypeError, - "this function got an unexpected keyword argument 'b'"): - parse((), {'b': 1}, '|O', ['a']) - with self.assertRaisesRegex(TypeError, - fr"argument for function given by name \('a'\) " - fr"and position \(1\)"): - parse((1,), {'a': 2}, 'O|O', ['a', 'b']) - - def test_bad_use(self): - # Test handling invalid format and keywords in - # PyArg_ParseTupleAndKeywords() - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (1,), {}, '||O', ['a']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (1, 2), {}, '|O|O', ['a', 'b']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (), {'a': 1}, '$$O', ['a']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (), {'a': 1, 'b': 2}, '$O$O', ['a', 'b']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (), {'a': 1}, '$|O', ['a']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (), {'a': 1, 'b': 2}, '$O|O', ['a', 'b']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (1,), {}, '|O', ['a', 'b']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (1,), {}, '|OO', ['a']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (), {}, '|$O', ['']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (), {}, '|OO', ['a', '']) - - def test_positional_only(self): - parse = _testcapi.parse_tuple_and_keywords - - self.assertEqual(parse((1, 2, 3), {}, 'OOO', ['', '', 'a']), (1, 2, 3)) - self.assertEqual(parse((1, 2), {'a': 3}, 'OOO', ['', '', 'a']), (1, 2, 3)) - with self.assertRaisesRegex(TypeError, - r'function takes at least 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, 'OOO', ['', '', 'a']) - self.assertEqual(parse((1,), {}, 'O|OO', ['', '', 'a']), - (1, NULL, NULL)) - with self.assertRaisesRegex(TypeError, - r'function takes at least 1 positional argument \(0 given\)'): - parse((), {}, 'O|OO', ['', '', 'a']) - self.assertEqual(parse((1, 2), {'a': 3}, 'OO$O', ['', '', 'a']), - (1, 2, 3)) - with self.assertRaisesRegex(TypeError, - r'function takes exactly 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, 'OO$O', ['', '', 'a']) - self.assertEqual(parse((1,), {}, 'O|O$O', ['', '', 'a']), - (1, NULL, NULL)) - with self.assertRaisesRegex(TypeError, - r'function takes at least 1 positional argument \(0 given\)'): - parse((), {}, 'O|O$O', ['', '', 'a']) - with self.assertRaisesRegex(SystemError, r'Empty parameter name after \$'): - parse((1,), {}, 'O|$OO', ['', '', 'a']) - with self.assertRaisesRegex(SystemError, 'Empty keyword'): - parse((1,), {}, 'O|OO', ['', 'a', '']) - - def test_nonascii_keywords(self): - parse = _testcapi.parse_tuple_and_keywords - - for name in ('a', 'ä', 'ŷ', '㷷', '𐀀'): - with self.subTest(name=name): - self.assertEqual(parse((), {name: 1}, 'O', [name]), (1,)) - self.assertEqual(parse((), {}, '|O', [name]), (NULL,)) - with self.assertRaisesRegex(TypeError, - f"function missing required argument '{name}'"): - parse((), {}, 'O', [name]) - with self.assertRaisesRegex(TypeError, - fr"argument for function given by name \('{name}'\) " - fr"and position \(1\)"): - parse((1,), {name: 2}, 'O|O', [name, 'b']) - with self.assertRaisesRegex(TypeError, - f"this function got an unexpected keyword argument '{name}'"): - parse((), {name: 1}, '|O', ['b']) - with self.assertRaisesRegex(TypeError, - "this function got an unexpected keyword argument 'b'"): - parse((), {'b': 1}, '|O', [name]) - - invalid = name.encode() + (name.encode()[:-1] or b'\x80') - self.assertEqual(parse((), {}, '|O', [invalid]), (NULL,)) - self.assertEqual(parse((1,), {'b': 2}, 'O|O', [invalid, 'b']), - (1, 2)) - with self.assertRaisesRegex(TypeError, - f"function missing required argument '{name}\ufffd'"): - parse((), {}, 'O', [invalid]) - with self.assertRaisesRegex(UnicodeDecodeError, - f"'utf-8' codec can't decode bytes? "): - parse((), {'b': 1}, '|OO', [invalid, 'b']) - with self.assertRaisesRegex(UnicodeDecodeError, - f"'utf-8' codec can't decode bytes? "): - parse((), {'b': 1}, '|O', [invalid]) - - for name2 in ('b', 'ë', 'ĉ', 'Ɐ', '𐀁'): - with self.subTest(name2=name2): - with self.assertRaisesRegex(TypeError, - f"this function got an unexpected keyword argument '{name2}'"): - parse((), {name2: 1}, '|O', [name]) - - name2 = name.encode().decode('latin1') - if name2 != name: - with self.assertRaisesRegex(TypeError, - f"this function got an unexpected keyword argument '{name2}'"): - parse((), {name2: 1}, '|O', [name]) - name3 = name + '3' - with self.assertRaisesRegex(TypeError, - f"this function got an unexpected keyword argument '{name2}'"): - parse((), {name2: 1, name3: 2}, '|OO', [name, name3]) - - def test_nested_sequence(self): - parse = _testcapi.parse_tuple_and_keywords - - self.assertEqual(parse(((1, 2, 3),), {}, '(OOO)', ['a']), (1, 2, 3)) - self.assertEqual(parse((1, (2, 3), 4), {}, 'O(OO)O', ['a', 'b', 'c']), - (1, 2, 3, 4)) - parse(((1, 2, 3),), {}, '(iii)', ['a']) - parse(([1, 2, 3],), {}, '(iii)', ['a']) - - with self.assertRaisesRegex(TypeError, - "argument 1 must be tuple of length 2, not 3"): - parse(((1, 2, 3),), {}, '(ii)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be tuple of length 2, not 1"): - parse(((1,),), {}, '(ii)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be sequence of length 2, not 3"): - parse(([1, 2, 3],), {}, '(ii)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be sequence of length 2, not 1"): - parse(([1,],), {}, '(ii)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be 2-item tuple, not int"): - parse((1,), {}, '(ii)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be 2-item tuple, not None$"): - parse((None,), {}, '(ii)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be 2-item tuple, not str"): - parse(('ab',), {}, '(CC)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be 2-item tuple, not bytes"): - parse((b'ab',), {}, '(ii)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be 2-item tuple, not bytearray"): - parse((bytearray(b'ab'),), {}, '(ii)', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be 2-item tuple, not dict"): - parse(({},), {}, '(ii)', ['a']) - - with self.assertWarnsRegex(DeprecationWarning, - "argument must be 3-item tuple, not list"): - self.assertEqual(parse(([1, 2, 3],), {}, '(OOO)', ['a']), (1, 2, 3)) - with self.assertWarnsRegex(DeprecationWarning, - "argument must be 2-item tuple, not list"): - with self.assertRaisesRegex(TypeError, - "argument 1 must be tuple of length 2, not 3"): - parse(([1, 2, 3],), {}, '(OO)', ['a']) - with self.assertWarnsRegex(DeprecationWarning, - "argument must be 2-item tuple, not list"): - with self.assertRaisesRegex(TypeError, - "argument 1 must be tuple of length 2, not 1"): - parse(([1,],), {}, '(OO)', ['a']) - - for f in 'es', 'et', 'es#', 'et#': - with self.assertRaises(LookupError): # empty encoding "" - parse((('a',),), {}, '(' + f + ')', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be tuple of length 1, not 0"): - parse(((),), {}, '(' + f + ')', ['a']) - with self.assertRaisesRegex(TypeError, - "argument 1 must be sequence of length 1, not 0"): - parse(([],), {}, '(' + f + ')', ['a']) - - @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') - def test_gh_119213(self): - rc, out, err = script_helper.assert_python_ok("-c", """if True: - from test import support - script = '''if True: - import _testinternalcapi - _testinternalcapi.gh_119213_getargs(spam='eggs') - ''' - config = dict( - allow_fork=False, - allow_exec=False, - allow_threads=True, - allow_daemon_threads=False, - use_main_obmalloc=False, - gil=2, - check_multi_interp_extensions=True, - ) - rc = support.run_in_subinterp_with_config(script, **config) - assert rc == 0 - - # The crash is different if the interpreter was not destroyed first. - #interpid = _testinternalcapi.create_interpreter() - #rc = _testinternalcapi.exec_interpreter(interpid, script) - #assert rc == 0 - """) - self.assertEqual(rc, 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_hash.py b/Lib/test/test_capi/test_hash.py deleted file mode 100644 index f553ffb0d90..00000000000 --- a/Lib/test/test_capi/test_hash.py +++ /dev/null @@ -1,93 +0,0 @@ -import sys -import unittest -from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') - - -SIZEOF_VOID_P = _testcapi.SIZEOF_VOID_P -SIZEOF_PY_HASH_T = SIZEOF_VOID_P - - -class CAPITest(unittest.TestCase): - def test_hash_getfuncdef(self): - # Test PyHash_GetFuncDef() - hash_getfuncdef = _testcapi.hash_getfuncdef - func_def = hash_getfuncdef() - - match func_def.name: - case "fnv": - self.assertEqual(func_def.hash_bits, 8 * SIZEOF_PY_HASH_T) - self.assertEqual(func_def.seed_bits, 16 * SIZEOF_PY_HASH_T) - case "siphash13": - self.assertEqual(func_def.hash_bits, 64) - self.assertEqual(func_def.seed_bits, 128) - case "siphash24": - self.assertEqual(func_def.hash_bits, 64) - self.assertEqual(func_def.seed_bits, 128) - case _: - self.fail(f"unknown function name: {func_def.name!r}") - - # compare with sys.hash_info - hash_info = sys.hash_info - self.assertEqual(func_def.name, hash_info.algorithm) - self.assertEqual(func_def.hash_bits, hash_info.hash_bits) - self.assertEqual(func_def.seed_bits, hash_info.seed_bits) - - def test_hash_pointer(self): - # Test Py_HashPointer() - hash_pointer = _testcapi.hash_pointer - - UHASH_T_MASK = ((2 ** (8 * SIZEOF_PY_HASH_T)) - 1) - HASH_T_MAX = (2 ** (8 * SIZEOF_PY_HASH_T - 1) - 1) - - def python_hash_pointer(x): - # Py_HashPointer() rotates the pointer bits by 4 bits to the right - x = (x >> 4) | ((x & 15) << (8 * SIZEOF_VOID_P - 4)) - - # Convert unsigned uintptr_t (Py_uhash_t) to signed Py_hash_t - if HASH_T_MAX < x: - x = (~x) + 1 - x &= UHASH_T_MASK - x = (~x) + 1 - return x - - if SIZEOF_VOID_P == 8: - values = ( - 0xABCDEF1234567890, - 0x1234567890ABCDEF, - 0xFEE4ABEDD1CECA5E, - ) - else: - values = ( - 0x12345678, - 0x1234ABCD, - 0xDEADCAFE, - ) - - for value in values: - expected = python_hash_pointer(value) - with self.subTest(value=value): - self.assertEqual(hash_pointer(value), expected, - f"hash_pointer({value:x}) = " - f"{hash_pointer(value):x} != {expected:x}") - - # Py_HashPointer(NULL) returns 0 - self.assertEqual(hash_pointer(0), 0) - - # Py_HashPointer((void*)(uintptr_t)-1) doesn't return -1 but -2 - VOID_P_MAX = -1 & (2 ** (8 * SIZEOF_VOID_P) - 1) - self.assertEqual(hash_pointer(VOID_P_MAX), -2) - - def test_hash_buffer(self): - hash_buffer = _testcapi.hash_buffer - - def check(data): - self.assertEqual(hash_buffer(data), hash(data)) - - check(b'') - check(b'abc') - check(b'x' * 1024) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py deleted file mode 100644 index 660e8a0e789..00000000000 --- a/Lib/test/test_capi/test_immortal.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest -from test.support import import_helper - -_testcapi = import_helper.import_module('_testcapi') -_testinternalcapi = import_helper.import_module('_testinternalcapi') - - -class TestUnstableCAPI(unittest.TestCase): - def test_immortal(self): - # Not extensive - known_immortals = (True, False, None, 0, ()) - for immortal in known_immortals: - with self.subTest(immortal=immortal): - self.assertTrue(_testcapi.is_immortal(immortal)) - - # Some arbitrary mutable objects - non_immortals = (object(), self, [object()]) - for non_immortal in non_immortals: - with self.subTest(non_immortal=non_immortal): - self.assertFalse(_testcapi.is_immortal(non_immortal)) - - # CRASHES _testcapi.is_immortal(NULL) - - -class TestInternalCAPI(unittest.TestCase): - - def test_immortal_builtins(self): - for obj in range(-5, 256): - self.assertTrue(_testinternalcapi.is_static_immortal(obj)) - self.assertTrue(_testinternalcapi.is_static_immortal(None)) - self.assertTrue(_testinternalcapi.is_static_immortal(False)) - self.assertTrue(_testinternalcapi.is_static_immortal(True)) - self.assertTrue(_testinternalcapi.is_static_immortal(...)) - self.assertTrue(_testinternalcapi.is_static_immortal(())) - for obj in range(300, 400): - self.assertFalse(_testinternalcapi.is_static_immortal(obj)) - for obj in ([], {}, set()): - self.assertFalse(_testinternalcapi.is_static_immortal(obj)) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_import.py b/Lib/test/test_capi/test_import.py deleted file mode 100644 index 25136624ca4..00000000000 --- a/Lib/test/test_capi/test_import.py +++ /dev/null @@ -1,381 +0,0 @@ -import importlib.util -import os.path -import sys -import types -import unittest -from test.support import os_helper -from test.support import import_helper -from test.support.warnings_helper import check_warnings - -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') -NULL = None - - -class ImportTests(unittest.TestCase): - def test_getmagicnumber(self): - # Test PyImport_GetMagicNumber() - magic = _testlimitedcapi.PyImport_GetMagicNumber() - self.assertEqual(magic, - int.from_bytes(importlib.util.MAGIC_NUMBER, 'little')) - - def test_getmagictag(self): - # Test PyImport_GetMagicTag() - tag = _testlimitedcapi.PyImport_GetMagicTag() - self.assertEqual(tag, sys.implementation.cache_tag) - - def test_getmoduledict(self): - # Test PyImport_GetModuleDict() - modules = _testlimitedcapi.PyImport_GetModuleDict() - self.assertIs(modules, sys.modules) - - def check_import_loaded_module(self, import_module): - for name in ('os', 'sys', 'test', 'unittest'): - with self.subTest(name=name): - self.assertIn(name, sys.modules) - old_module = sys.modules[name] - module = import_module(name) - self.assertIsInstance(module, types.ModuleType) - self.assertIs(module, old_module) - - def check_import_fresh_module(self, import_module): - old_modules = dict(sys.modules) - try: - for name in ('colorsys', 'math'): - with self.subTest(name=name): - sys.modules.pop(name, None) - module = import_module(name) - self.assertIsInstance(module, types.ModuleType) - self.assertIs(module, sys.modules[name]) - self.assertEqual(module.__name__, name) - finally: - sys.modules.clear() - sys.modules.update(old_modules) - - def test_getmodule(self): - # Test PyImport_GetModule() - getmodule = _testlimitedcapi.PyImport_GetModule - self.check_import_loaded_module(getmodule) - - nonexistent = 'nonexistent' - self.assertNotIn(nonexistent, sys.modules) - self.assertIs(getmodule(nonexistent), KeyError) - self.assertIs(getmodule(''), KeyError) - self.assertIs(getmodule(object()), KeyError) - - self.assertRaises(TypeError, getmodule, []) # unhashable - # CRASHES getmodule(NULL) - - def check_addmodule(self, add_module, accept_nonstr=False): - # create a new module - names = ['nonexistent'] - if accept_nonstr: - names.append(b'\xff') # non-UTF-8 - # PyImport_AddModuleObject() accepts non-string names - names.append(tuple(['hashable non-string'])) - for name in names: - with self.subTest(name=name): - self.assertNotIn(name, sys.modules) - try: - module = add_module(name) - self.assertIsInstance(module, types.ModuleType) - self.assertEqual(module.__name__, name) - self.assertIs(module, sys.modules[name]) - finally: - sys.modules.pop(name, None) - - # get an existing module - self.check_import_loaded_module(add_module) - - def test_addmoduleobject(self): - # Test PyImport_AddModuleObject() - addmoduleobject = _testlimitedcapi.PyImport_AddModuleObject - self.check_addmodule(addmoduleobject, accept_nonstr=True) - - self.assertRaises(TypeError, addmoduleobject, []) # unhashable - # CRASHES addmoduleobject(NULL) - - def test_addmodule(self): - # Test PyImport_AddModule() - addmodule = _testlimitedcapi.PyImport_AddModule - self.check_addmodule(addmodule) - - self.assertRaises(UnicodeDecodeError, addmodule, b'\xff') - # CRASHES addmodule(NULL) - - def test_addmoduleref(self): - # Test PyImport_AddModuleRef() - addmoduleref = _testlimitedcapi.PyImport_AddModuleRef - self.check_addmodule(addmoduleref) - - self.assertRaises(UnicodeDecodeError, addmoduleref, b'\xff') - # CRASHES addmoduleref(NULL) - - def check_import_func(self, import_module): - self.check_import_loaded_module(import_module) - self.check_import_fresh_module(import_module) - self.assertRaises(ModuleNotFoundError, import_module, 'nonexistent') - self.assertRaises(ValueError, import_module, '') - - def test_import(self): - # Test PyImport_Import() - import_ = _testlimitedcapi.PyImport_Import - self.check_import_func(import_) - - self.assertRaises(TypeError, import_, b'os') - self.assertRaises(SystemError, import_, NULL) - - def test_importmodule(self): - # Test PyImport_ImportModule() - importmodule = _testlimitedcapi.PyImport_ImportModule - self.check_import_func(importmodule) - - self.assertRaises(UnicodeDecodeError, importmodule, b'\xff') - # CRASHES importmodule(NULL) - - def test_importmodulenoblock(self): - # Test deprecated PyImport_ImportModuleNoBlock() - importmodulenoblock = _testlimitedcapi.PyImport_ImportModuleNoBlock - with check_warnings(('', DeprecationWarning)): - self.check_import_func(importmodulenoblock) - self.assertRaises(UnicodeDecodeError, importmodulenoblock, b'\xff') - - # CRASHES importmodulenoblock(NULL) - - def check_frozen_import(self, import_frozen_module): - # Importing a frozen module executes its code, so start by unloading - # the module to execute the code in a new (temporary) module. - old_zipimport = sys.modules.pop('zipimport') - try: - self.assertEqual(import_frozen_module('zipimport'), 1) - - # import zipimport again - self.assertEqual(import_frozen_module('zipimport'), 1) - finally: - sys.modules['zipimport'] = old_zipimport - - # not a frozen module - self.assertEqual(import_frozen_module('sys'), 0) - self.assertEqual(import_frozen_module('nonexistent'), 0) - self.assertEqual(import_frozen_module(''), 0) - - def test_importfrozenmodule(self): - # Test PyImport_ImportFrozenModule() - importfrozenmodule = _testlimitedcapi.PyImport_ImportFrozenModule - self.check_frozen_import(importfrozenmodule) - - self.assertRaises(UnicodeDecodeError, importfrozenmodule, b'\xff') - # CRASHES importfrozenmodule(NULL) - - def test_importfrozenmoduleobject(self): - # Test PyImport_ImportFrozenModuleObject() - importfrozenmoduleobject = _testlimitedcapi.PyImport_ImportFrozenModuleObject - self.check_frozen_import(importfrozenmoduleobject) - self.assertEqual(importfrozenmoduleobject(b'zipimport'), 0) - self.assertEqual(importfrozenmoduleobject(NULL), 0) - - def test_importmoduleex(self): - # Test PyImport_ImportModuleEx() - importmoduleex = _testlimitedcapi.PyImport_ImportModuleEx - self.check_import_func(lambda name: importmoduleex(name, NULL, NULL, NULL)) - - self.assertRaises(ModuleNotFoundError, importmoduleex, 'nonexistent', NULL, NULL, NULL) - self.assertRaises(ValueError, importmoduleex, '', NULL, NULL, NULL) - self.assertRaises(UnicodeDecodeError, importmoduleex, b'\xff', NULL, NULL, NULL) - # CRASHES importmoduleex(NULL, NULL, NULL, NULL) - - def check_importmodulelevel(self, importmodulelevel): - self.check_import_func(lambda name: importmodulelevel(name, NULL, NULL, NULL, 0)) - - self.assertRaises(ModuleNotFoundError, importmodulelevel, 'nonexistent', NULL, NULL, NULL, 0) - self.assertRaises(ValueError, importmodulelevel, '', NULL, NULL, NULL, 0) - - if __package__: - self.assertIs(importmodulelevel('test_import', globals(), NULL, NULL, 1), - sys.modules['test.test_capi.test_import']) - self.assertIs(importmodulelevel('test_capi', globals(), NULL, NULL, 2), - sys.modules['test.test_capi']) - self.assertRaises(ValueError, importmodulelevel, 'os', NULL, NULL, NULL, -1) - with self.assertWarns(ImportWarning): - self.assertRaises(KeyError, importmodulelevel, 'test_import', {}, NULL, NULL, 1) - self.assertRaises(TypeError, importmodulelevel, 'test_import', [], NULL, NULL, 1) - - def test_importmodulelevel(self): - # Test PyImport_ImportModuleLevel() - importmodulelevel = _testlimitedcapi.PyImport_ImportModuleLevel - self.check_importmodulelevel(importmodulelevel) - - self.assertRaises(UnicodeDecodeError, importmodulelevel, b'\xff', NULL, NULL, NULL, 0) - # CRASHES importmodulelevel(NULL, NULL, NULL, NULL, 0) - - def test_importmodulelevelobject(self): - # Test PyImport_ImportModuleLevelObject() - importmodulelevel = _testlimitedcapi.PyImport_ImportModuleLevelObject - self.check_importmodulelevel(importmodulelevel) - - self.assertRaises(TypeError, importmodulelevel, b'os', NULL, NULL, NULL, 0) - self.assertRaises(ValueError, importmodulelevel, NULL, NULL, NULL, NULL, 0) - - def check_executecodemodule(self, execute_code, *args): - name = 'test_import_executecode' - try: - # Create a temporary module where the code will be executed - self.assertNotIn(name, sys.modules) - module = _testlimitedcapi.PyImport_AddModuleRef(name) - self.assertNotHasAttr(module, 'attr') - - # Execute the code - code = compile('attr = 1', '', 'exec') - module2 = execute_code(name, code, *args) - self.assertIs(module2, module) - - # Check the function side effects - self.assertEqual(module.attr, 1) - finally: - sys.modules.pop(name, None) - return module.__spec__.origin - - def test_executecodemodule(self): - # Test PyImport_ExecCodeModule() - execcodemodule = _testlimitedcapi.PyImport_ExecCodeModule - self.check_executecodemodule(execcodemodule) - - code = compile('attr = 1', '', 'exec') - self.assertRaises(UnicodeDecodeError, execcodemodule, b'\xff', code) - # CRASHES execcodemodule(NULL, code) - # CRASHES execcodemodule(name, NULL) - - def test_executecodemoduleex(self): - # Test PyImport_ExecCodeModuleEx() - execcodemoduleex = _testlimitedcapi.PyImport_ExecCodeModuleEx - - # Test NULL path (it should not crash) - self.check_executecodemodule(execcodemoduleex, NULL) - - # Test non-NULL path - pathname = b'pathname' - origin = self.check_executecodemodule(execcodemoduleex, pathname) - self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) - - pathname = os_helper.TESTFN_UNDECODABLE - if pathname: - origin = self.check_executecodemodule(execcodemoduleex, pathname) - self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) - - code = compile('attr = 1', '', 'exec') - self.assertRaises(UnicodeDecodeError, execcodemoduleex, b'\xff', code, NULL) - # CRASHES execcodemoduleex(NULL, code, NULL) - # CRASHES execcodemoduleex(name, NULL, NULL) - - def check_executecode_pathnames(self, execute_code_func, object=False): - # Test non-NULL pathname and NULL cpathname - - # Test NULL paths (it should not crash) - self.check_executecodemodule(execute_code_func, NULL, NULL) - - pathname = 'pathname' - origin = self.check_executecodemodule(execute_code_func, pathname, NULL) - self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) - origin = self.check_executecodemodule(execute_code_func, NULL, pathname) - if not object: - self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) - - pathname = os_helper.TESTFN_UNDECODABLE - if pathname: - if object: - pathname = os.fsdecode(pathname) - origin = self.check_executecodemodule(execute_code_func, pathname, NULL) - self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) - self.check_executecodemodule(execute_code_func, NULL, pathname) - - # Test NULL pathname and non-NULL cpathname - pyc_filename = importlib.util.cache_from_source(__file__) - py_filename = importlib.util.source_from_cache(pyc_filename) - origin = self.check_executecodemodule(execute_code_func, NULL, pyc_filename) - if not object: - self.assertEqual(origin, py_filename) - - def test_executecodemodulewithpathnames(self): - # Test PyImport_ExecCodeModuleWithPathnames() - execute_code_func = _testlimitedcapi.PyImport_ExecCodeModuleWithPathnames - self.check_executecode_pathnames(execute_code_func) - - code = compile('attr = 1', '', 'exec') - self.assertRaises(UnicodeDecodeError, execute_code_func, b'\xff', code, NULL, NULL) - # CRASHES execute_code_func(NULL, code, NULL, NULL) - # CRASHES execute_code_func(name, NULL, NULL, NULL) - - def test_executecodemoduleobject(self): - # Test PyImport_ExecCodeModuleObject() - execute_code_func = _testlimitedcapi.PyImport_ExecCodeModuleObject - self.check_executecode_pathnames(execute_code_func, object=True) - - code = compile('attr = 1', '', 'exec') - self.assertRaises(TypeError, execute_code_func, [], code, NULL, NULL) - nonstring = tuple(['hashable non-string']) - self.assertRaises(AttributeError, execute_code_func, nonstring, code, NULL, NULL) - sys.modules.pop(nonstring, None) - # CRASHES execute_code_func(NULL, code, NULL, NULL) - # CRASHES execute_code_func(name, NULL, NULL, NULL) - - def check_importmoduleattr(self, importmoduleattr): - self.assertIs(importmoduleattr('sys', 'argv'), sys.argv) - self.assertIs(importmoduleattr('types', 'ModuleType'), types.ModuleType) - - # module name containing a dot - attr = importmoduleattr('email.message', 'Message') - from email.message import Message - self.assertIs(attr, Message) - - with self.assertRaises(ImportError): - # nonexistent module - importmoduleattr('nonexistentmodule', 'attr') - with self.assertRaises(AttributeError): - # nonexistent attribute - importmoduleattr('sys', 'nonexistentattr') - with self.assertRaises(AttributeError): - # attribute name containing a dot - importmoduleattr('sys', 'implementation.name') - - def test_importmoduleattr(self): - # Test PyImport_ImportModuleAttr() - importmoduleattr = _testcapi.PyImport_ImportModuleAttr - self.check_importmoduleattr(importmoduleattr) - - # Invalid module name type - for mod_name in (object(), 123, b'bytes'): - with self.subTest(mod_name=mod_name): - with self.assertRaises(TypeError): - importmoduleattr(mod_name, "attr") - - # Invalid attribute name type - for attr_name in (object(), 123, b'bytes'): - with self.subTest(attr_name=attr_name): - with self.assertRaises(TypeError): - importmoduleattr("sys", attr_name) - - with self.assertRaises(SystemError): - importmoduleattr(NULL, "argv") - # CRASHES importmoduleattr("sys", NULL) - - def test_importmoduleattrstring(self): - # Test PyImport_ImportModuleAttrString() - importmoduleattr = _testcapi.PyImport_ImportModuleAttrString - self.check_importmoduleattr(importmoduleattr) - - with self.assertRaises(UnicodeDecodeError): - importmoduleattr(b"sys\xff", "argv") - with self.assertRaises(UnicodeDecodeError): - importmoduleattr("sys", b"argv\xff") - - # CRASHES importmoduleattr(NULL, "argv") - # CRASHES importmoduleattr("sys", NULL) - - # TODO: test PyImport_GetImporter() - # TODO: test PyImport_ReloadModule() - # TODO: test PyImport_ExtendInittab() - # PyImport_AppendInittab() is tested by test_embed - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py deleted file mode 100644 index b95b8ba960b..00000000000 --- a/Lib/test/test_capi/test_list.py +++ /dev/null @@ -1,359 +0,0 @@ -import unittest -from test.support import import_helper -from collections import UserList -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') - -NULL = None -PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN -PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX - -class ListSubclass(list): - pass - - -class DelAppend: - def __init__(self, lst, item): - self.lst = lst - self.item = item - - def __del__(self): - self.lst.append(self.item) - - -class CAPITest(unittest.TestCase): - def test_check(self): - # Test PyList_Check() - check = _testlimitedcapi.list_check - self.assertTrue(check([1, 2])) - self.assertTrue(check([])) - self.assertTrue(check(ListSubclass([1, 2]))) - self.assertFalse(check({1: 2})) - self.assertFalse(check((1, 2))) - self.assertFalse(check(42)) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - - def test_list_check_exact(self): - # Test PyList_CheckExact() - check = _testlimitedcapi.list_check_exact - self.assertTrue(check([1])) - self.assertTrue(check([])) - self.assertFalse(check(ListSubclass([1]))) - self.assertFalse(check(UserList([1, 2]))) - self.assertFalse(check({1: 2})) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_list_new(self): - # Test PyList_New() - list_new = _testlimitedcapi.list_new - lst = list_new(0) - self.assertEqual(lst, []) - self.assertIs(type(lst), list) - lst2 = list_new(0) - self.assertIsNot(lst2, lst) - self.assertRaises(SystemError, list_new, NULL) - self.assertRaises(SystemError, list_new, -1) - - def test_list_size(self): - # Test PyList_Size() - size = _testlimitedcapi.list_size - self.assertEqual(size([]), 0) - self.assertEqual(size([1, 2]), 2) - self.assertEqual(size(ListSubclass([1, 2])), 2) - self.assertRaises(SystemError, size, UserList()) - self.assertRaises(SystemError, size, {}) - self.assertRaises(SystemError, size, 23) - self.assertRaises(SystemError, size, object()) - # CRASHES size(NULL) - - def test_list_get_size(self): - # Test PyList_GET_SIZE() - size = _testcapi.list_get_size - self.assertEqual(size([]), 0) - self.assertEqual(size([1, 2]), 2) - self.assertEqual(size(ListSubclass([1, 2])), 2) - # CRASHES size(object()) - # CRASHES size(23) - # CRASHES size({}) - # CRASHES size(UserList()) - # CRASHES size(NULL) - - def check_list_get_item(self, getitem, exctype): - # Common test cases for PyList_GetItem() and PyList_GetItemRef() - lst = [1, 2, 3] - self.assertEqual(getitem(lst, 0), 1) - self.assertEqual(getitem(lst, 2), 3) - self.assertRaises(IndexError, getitem, lst, 3) - self.assertRaises(IndexError, getitem, lst, -1) - self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MIN) - self.assertRaises(IndexError, getitem, lst, PY_SSIZE_T_MAX) - self.assertRaises(exctype, getitem, 42, 1) - self.assertRaises(exctype, getitem, (1, 2, 3), 1) - self.assertRaises(exctype, getitem, {1: 2}, 1) - # CRASHES getitem(NULL, 1) - - def test_list_getitem(self): - # Test PyList_GetItem() - self.check_list_get_item(_testlimitedcapi.list_getitem, SystemError) - - def test_list_get_item_ref(self): - # Test PyList_GetItemRef() - self.check_list_get_item(_testlimitedcapi.list_get_item_ref, TypeError) - - def test_list_get_item(self): - # Test PyList_GET_ITEM() - get_item = _testcapi.list_get_item - lst = [1, 2, [1, 2, 3]] - self.assertEqual(get_item(lst, 0), 1) - self.assertEqual(get_item(lst, 2), [1, 2, 3]) - - # CRASHES for out of index: get_item(lst, 3) - # CRASHES for get_item(lst, PY_SSIZE_T_MIN) - # CRASHES for get_item(lst, PY_SSIZE_T_MAX) - # CRASHES get_item(21, 2) - # CRASHES get_item(NULL, 1) - - def test_list_setitem(self): - # Test PyList_SetItem() - setitem = _testlimitedcapi.list_setitem - lst = [1, 2, 3] - setitem(lst, 0, 10) - self.assertEqual(lst, [10, 2, 3]) - setitem(lst, 2, 12) - self.assertEqual(lst, [10, 2, 12]) - self.assertRaises(IndexError, setitem, lst, 3 , 5) - self.assertRaises(IndexError, setitem, lst, -1, 5) - self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MIN, 5) - self.assertRaises(IndexError, setitem, lst, PY_SSIZE_T_MAX, 5) - self.assertRaises(SystemError, setitem, (1, 2, 3), 1, 5) - self.assertRaises(SystemError, setitem, {1: 2}, 1, 5) - - # CRASHES setitem(NULL, 'a', 5) - - def test_list_set_item(self): - # Test PyList_SET_ITEM() - set_item = _testcapi.list_set_item - lst = [1, 2, 3] - set_item(lst, 1, 10) - set_item(lst, 2, [1, 2, 3]) - self.assertEqual(lst, [1, 10, [1, 2, 3]]) - - # CRASHES for set_item([1], -1, 5) - # CRASHES for set_item([1], PY_SSIZE_T_MIN, 5) - # CRASHES for set_item([1], PY_SSIZE_T_MAX, 5) - # CRASHES for set_item([], 0, 1) - # CRASHES for set_item(NULL, 0, 1) - - - def test_list_insert(self): - # Test PyList_Insert() - insert = _testlimitedcapi.list_insert - lst = [1, 2, 3] - insert(lst, 0, 23) - self.assertEqual(lst, [23, 1, 2, 3]) - insert(lst, -1, 22) - self.assertEqual(lst, [23, 1, 2, 22, 3]) - insert(lst, PY_SSIZE_T_MIN, 1) - self.assertEqual(lst[0], 1) - insert(lst, len(lst), 123) - self.assertEqual(lst[-1], 123) - insert(lst, len(lst)-1, 124) - self.assertEqual(lst[-2], 124) - insert(lst, PY_SSIZE_T_MAX, 223) - self.assertEqual(lst[-1], 223) - - self.assertRaises(SystemError, insert, (1, 2, 3), 1, 5) - self.assertRaises(SystemError, insert, {1: 2}, 1, 5) - - # CRASHES insert(NULL, 1, 5) - - def test_list_append(self): - # Test PyList_Append() - append = _testlimitedcapi.list_append - lst = [1, 2, 3] - append(lst, 10) - self.assertEqual(lst, [1, 2, 3, 10]) - append(lst, [4, 5]) - self.assertEqual(lst, [1, 2, 3, 10, [4, 5]]) - self.assertRaises(SystemError, append, lst, NULL) - self.assertRaises(SystemError, append, (), 0) - self.assertRaises(SystemError, append, 42, 0) - # CRASHES append(NULL, 0) - - def test_list_getslice(self): - # Test PyList_GetSlice() - getslice = _testlimitedcapi.list_getslice - lst = [1, 2, 3] - - # empty - self.assertEqual(getslice(lst, PY_SSIZE_T_MIN, 0), []) - self.assertEqual(getslice(lst, -1, 0), []) - self.assertEqual(getslice(lst, 3, PY_SSIZE_T_MAX), []) - - # slice - self.assertEqual(getslice(lst, 1, 3), [2, 3]) - - # whole - self.assertEqual(getslice(lst, 0, len(lst)), lst) - self.assertEqual(getslice(lst, 0, 100), lst) - self.assertEqual(getslice(lst, -100, 100), lst) - - self.assertRaises(SystemError, getslice, (1, 2, 3), 0, 0) - self.assertRaises(SystemError, getslice, 'abc', 0, 0) - self.assertRaises(SystemError, getslice, 42, 0, 0) - - # CRASHES getslice(NULL, 0, 0) - - def test_list_setslice(self): - # Test PyList_SetSlice() - list_setslice = _testlimitedcapi.list_setslice - def set_slice(lst, low, high, value): - lst = lst.copy() - self.assertEqual(list_setslice(lst, low, high, value), 0) - return lst - - # insert items - self.assertEqual(set_slice([], 0, 0, list("abc")), list("abc")) - self.assertEqual(set_slice([], PY_SSIZE_T_MIN, PY_SSIZE_T_MIN, list("abc")), list("abc")) - self.assertEqual(set_slice([], PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, list("abc")), list("abc")) - lst = list("abc") - self.assertEqual(set_slice(lst, 0, 0, ["X"]), list("Xabc")) - self.assertEqual(set_slice(lst, 1, 1, list("XY")), list("aXYbc")) - self.assertEqual(set_slice(lst, len(lst), len(lst), ["X"]), list("abcX")) - # self.assertEqual(set_slice(lst, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, ["X"]), list("abcX")) - - # replace items - lst = list("abc") - self.assertEqual(set_slice(lst, -100, 1, list("X")), list("Xbc")) - self.assertEqual(set_slice(lst, 1, 2, list("X")), list("aXc")) - self.assertEqual(set_slice(lst, 1, 3, list("XY")), list("aXY")) - self.assertEqual(set_slice(lst, 0, 3, list("XYZ")), list("XYZ")) - - # delete items - lst = list("abcdef") - self.assertEqual(set_slice(lst, 0, len(lst), []), []) - self.assertEqual(set_slice(lst, -100, 100, []), []) - self.assertEqual(set_slice(lst, 1, 5, []), list("af")) - self.assertEqual(set_slice(lst, 3, len(lst), []), list("abc")) - - # delete items with NULL - lst = list("abcdef") - self.assertEqual(set_slice(lst, 0, len(lst), NULL), []) - self.assertEqual(set_slice(lst, 3, len(lst), NULL), list("abc")) - - self.assertRaises(SystemError, list_setslice, (), 0, 0, []) - self.assertRaises(SystemError, list_setslice, 42, 0, 0, []) - - # Item finalizer modify the list (clear the list) - lst = [] - lst.append(DelAppend(lst, 'zombie')) - self.assertEqual(list_setslice(lst, 0, len(lst), NULL), 0) - self.assertEqual(lst, ['zombie']) - - # Item finalizer modify the list (remove an list item) - lst = [] - lst.append(DelAppend(lst, 'zombie')) - lst.extend("abc") - self.assertEqual(list_setslice(lst, 0, 1, NULL), 0) - self.assertEqual(lst, ['a', 'b', 'c', 'zombie']) - - # CRASHES setslice(NULL, 0, 0, []) - - def test_list_sort(self): - # Test PyList_Sort() - sort = _testlimitedcapi.list_sort - lst = [4, 6, 7, 3, 1, 5, 9, 2, 0, 8] - sort(lst) - self.assertEqual(lst, list(range(10))) - - lst2 = ListSubclass([4, 6, 7, 3, 1, 5, 9, 2, 0, 8]) - sort(lst2) - self.assertEqual(lst2, list(range(10))) - - self.assertRaises(SystemError, sort, ()) - self.assertRaises(SystemError, sort, object()) - self.assertRaises(SystemError, sort, NULL) - - - def test_list_reverse(self): - # Test PyList_Reverse() - reverse = _testlimitedcapi.list_reverse - def list_reverse(lst): - self.assertEqual(reverse(lst), 0) - return lst - - self.assertEqual(list_reverse([]), []) - self.assertEqual(list_reverse([2, 5, 10]), [10, 5, 2]) - self.assertEqual(list_reverse(list_reverse([2, 5, 10])), [2, 5, 10]) - - self.assertRaises(SystemError, reverse, ()) - self.assertRaises(SystemError, reverse, object()) - self.assertRaises(SystemError, reverse, NULL) - - def test_list_astuple(self): - # Test PyList_AsTuple() - astuple = _testlimitedcapi.list_astuple - self.assertEqual(astuple([]), ()) - self.assertEqual(astuple([[]]), ([],)) - self.assertEqual(astuple([2, 5, 10]), (2, 5, 10)) - - self.assertRaises(SystemError, astuple, ()) - self.assertRaises(SystemError, astuple, object()) - self.assertRaises(SystemError, astuple, NULL) - - def test_list_clear(self): - # Test PyList_Clear() - list_clear = _testcapi.list_clear - - lst = [1, 2, 3] - self.assertEqual(list_clear(lst), 0) - self.assertEqual(lst, []) - - lst = [] - self.assertEqual(list_clear(lst), 0) - self.assertEqual(lst, []) - - self.assertRaises(SystemError, list_clear, ()) - self.assertRaises(SystemError, list_clear, object()) - - # Item finalizer modify the list - lst = [] - lst.append(DelAppend(lst, 'zombie')) - list_clear(lst) - self.assertEqual(lst, ['zombie']) - - # CRASHES list_clear(NULL) - - def test_list_extend(self): - # Test PyList_Extend() - list_extend = _testcapi.list_extend - - for other_type in (list, tuple, str, iter): - lst = list("ab") - arg = other_type("def") - self.assertEqual(list_extend(lst, arg), 0) - self.assertEqual(lst, list("abdef")) - - # PyList_Extend(lst, lst) - lst = list("abc") - self.assertEqual(list_extend(lst, lst), 0) - self.assertEqual(lst, list("abcabc")) - - self.assertRaises(TypeError, list_extend, [], object()) - self.assertRaises(SystemError, list_extend, (), list("abc")) - - # CRASHES list_extend(NULL, []) - # CRASHES list_extend([], NULL) - - def test_uninitialized_list_repr(self): - lst = _testlimitedcapi.list_new(3) - self.assertEqual(repr(lst), '[, , ]') - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py deleted file mode 100644 index fc0454b71cb..00000000000 --- a/Lib/test/test_capi/test_long.py +++ /dev/null @@ -1,818 +0,0 @@ -import unittest -import sys -import test.support as support - -from test.support import import_helper - -# Skip this test if the _testcapi and _testlimitedcapi modules isn't available. -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') - -NULL = None - - -class IntSubclass(int): - pass - -class Index: - def __init__(self, value): - self.value = value - - def __index__(self): - return self.value - -# use __index__(), not __int__() -class MyIndexAndInt: - def __index__(self): - return 10 - def __int__(self): - return 22 - - -class LongTests(unittest.TestCase): - - def test_compact(self): - for n in { - # Edge cases - *(2**n for n in range(66)), - *(-2**n for n in range(66)), - *(2**n - 1 for n in range(66)), - *(-2**n + 1 for n in range(66)), - # Essentially random - *(37**n for n in range(14)), - *(-37**n for n in range(14)), - }: - with self.subTest(n=n): - is_compact, value = _testcapi.call_long_compact_api(n) - if is_compact: - self.assertEqual(n, value) - - def test_compact_known(self): - # Sanity-check some implementation details (we don't guarantee - # that these are/aren't compact) - self.assertEqual(_testcapi.call_long_compact_api(-1), (True, -1)) - self.assertEqual(_testcapi.call_long_compact_api(0), (True, 0)) - self.assertEqual(_testcapi.call_long_compact_api(256), (True, 256)) - self.assertEqual(_testcapi.call_long_compact_api(sys.maxsize), - (False, -1)) - - def test_long_check(self): - # Test PyLong_Check() - check = _testlimitedcapi.pylong_check - self.assertTrue(check(1)) - self.assertTrue(check(123456789012345678901234567890)) - self.assertTrue(check(-1)) - self.assertTrue(check(True)) - self.assertTrue(check(IntSubclass(1))) - self.assertFalse(check(1.0)) - self.assertFalse(check(object())) - # CRASHES check(NULL) - - def test_long_checkexact(self): - # Test PyLong_CheckExact() - check = _testlimitedcapi.pylong_checkexact - self.assertTrue(check(1)) - self.assertTrue(check(123456789012345678901234567890)) - self.assertTrue(check(-1)) - self.assertFalse(check(True)) - self.assertFalse(check(IntSubclass(1))) - self.assertFalse(check(1.0)) - self.assertFalse(check(object())) - # CRASHES check(NULL) - - def test_long_fromdouble(self): - # Test PyLong_FromDouble() - fromdouble = _testlimitedcapi.pylong_fromdouble - float_max = sys.float_info.max - for value in (5.0, 5.1, 5.9, -5.1, -5.9, 0.0, -0.0, float_max, -float_max): - with self.subTest(value=value): - self.assertEqual(fromdouble(value), int(value)) - self.assertRaises(OverflowError, fromdouble, float('inf')) - self.assertRaises(OverflowError, fromdouble, float('-inf')) - self.assertRaises(ValueError, fromdouble, float('nan')) - - def test_long_fromvoidptr(self): - # Test PyLong_FromVoidPtr() - fromvoidptr = _testlimitedcapi.pylong_fromvoidptr - obj = object() - x = fromvoidptr(obj) - y = fromvoidptr(NULL) - self.assertIsInstance(x, int) - self.assertGreaterEqual(x, 0) - self.assertIsInstance(y, int) - self.assertEqual(y, 0) - self.assertNotEqual(x, y) - - def test_long_fromstring(self): - # Test PyLong_FromString() - fromstring = _testlimitedcapi.pylong_fromstring - self.assertEqual(fromstring(b'123', 10), (123, 3)) - self.assertEqual(fromstring(b'cafe', 16), (0xcafe, 4)) - self.assertEqual(fromstring(b'xyz', 36), (44027, 3)) - self.assertEqual(fromstring(b'123', 0), (123, 3)) - self.assertEqual(fromstring(b'0xcafe', 0), (0xcafe, 6)) - self.assertRaises(ValueError, fromstring, b'cafe', 0) - self.assertEqual(fromstring(b'-123', 10), (-123, 4)) - self.assertEqual(fromstring(b' -123 ', 10), (-123, 6)) - self.assertEqual(fromstring(b'1_23', 10), (123, 4)) - self.assertRaises(ValueError, fromstring, b'- 123', 10) - self.assertRaises(ValueError, fromstring, b'', 10) - - self.assertRaises(ValueError, fromstring, b'123', 1) - self.assertRaises(ValueError, fromstring, b'123', -1) - self.assertRaises(ValueError, fromstring, b'123', 37) - - self.assertRaises(ValueError, fromstring, '١٢٣٤٥٦٧٨٩٠'.encode(), 0) - self.assertRaises(ValueError, fromstring, '١٢٣٤٥٦٧٨٩٠'.encode(), 16) - - self.assertEqual(fromstring(b'123\x00', 0), (123, 3)) - self.assertEqual(fromstring(b'123\x00456', 0), (123, 3)) - self.assertEqual(fromstring(b'123\x00', 16), (0x123, 3)) - self.assertEqual(fromstring(b'123\x00456', 16), (0x123, 3)) - - # CRASHES fromstring(NULL, 0) - # CRASHES fromstring(NULL, 16) - - def test_long_fromunicodeobject(self): - # Test PyLong_FromUnicodeObject() - fromunicodeobject = _testcapi.pylong_fromunicodeobject - self.assertEqual(fromunicodeobject('123', 10), 123) - self.assertEqual(fromunicodeobject('cafe', 16), 0xcafe) - self.assertEqual(fromunicodeobject('xyz', 36), 44027) - self.assertEqual(fromunicodeobject('123', 0), 123) - self.assertEqual(fromunicodeobject('0xcafe', 0), 0xcafe) - self.assertRaises(ValueError, fromunicodeobject, 'cafe', 0) - self.assertEqual(fromunicodeobject('-123', 10), -123) - self.assertEqual(fromunicodeobject(' -123 ', 10), -123) - self.assertEqual(fromunicodeobject('1_23', 10), 123) - self.assertRaises(ValueError, fromunicodeobject, '- 123', 10) - self.assertRaises(ValueError, fromunicodeobject, '', 10) - - self.assertRaises(ValueError, fromunicodeobject, '123', 1) - self.assertRaises(ValueError, fromunicodeobject, '123', -1) - self.assertRaises(ValueError, fromunicodeobject, '123', 37) - - self.assertEqual(fromunicodeobject('١٢٣٤٥٦٧٨٩٠', 0), 1234567890) - self.assertEqual(fromunicodeobject('١٢٣٤٥٦٧٨٩٠', 16), 0x1234567890) - - self.assertRaises(ValueError, fromunicodeobject, '123\x00', 0) - self.assertRaises(ValueError, fromunicodeobject, '123\x00456', 0) - self.assertRaises(ValueError, fromunicodeobject, '123\x00', 16) - self.assertRaises(ValueError, fromunicodeobject, '123\x00456', 16) - - # CRASHES fromunicodeobject(NULL, 0) - # CRASHES fromunicodeobject(NULL, 16) - - def check_long_asint(self, func, min_val, max_val, *, - use_index=True, - mask=False, - negative_value_error=OverflowError): - # round trip (object -> C integer -> object) - values = (0, 1, 512, 1234, max_val) - if min_val < 0: - values += (-1, -512, -1234, min_val) - for value in values: - with self.subTest(value=value): - self.assertEqual(func(value), value) - self.assertEqual(func(IntSubclass(value)), value) - if use_index: - self.assertEqual(func(Index(value)), value) - - if use_index: - self.assertEqual(func(MyIndexAndInt()), 10) - else: - self.assertRaises(TypeError, func, Index(42)) - self.assertRaises(TypeError, func, MyIndexAndInt()) - - if mask: - self.assertEqual(func(min_val - 1), max_val) - self.assertEqual(func(max_val + 1), min_val) - self.assertEqual(func(-1 << 1000), 0) - self.assertEqual(func(1 << 1000), 0) - else: - self.assertRaises(negative_value_error, func, min_val - 1) - self.assertRaises(negative_value_error, func, -1 << 1000) - self.assertRaises(OverflowError, func, max_val + 1) - self.assertRaises(OverflowError, func, 1 << 1000) - self.assertRaises(TypeError, func, 1.0) - self.assertRaises(TypeError, func, b'2') - self.assertRaises(TypeError, func, '3') - self.assertRaises(SystemError, func, NULL) - - def check_long_asintandoverflow(self, func, min_val, max_val): - # round trip (object -> C integer -> object) - for value in (min_val, max_val, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(func(value), (value, 0)) - self.assertEqual(func(IntSubclass(value)), (value, 0)) - self.assertEqual(func(Index(value)), (value, 0)) - - self.assertEqual(func(MyIndexAndInt()), (10, 0)) - - self.assertEqual(func(min_val - 1), (-1, -1)) - self.assertEqual(func(max_val + 1), (-1, +1)) - self.assertRaises(SystemError, func, None) - self.assertRaises(TypeError, func, 1.0) - - def test_long_asint(self): - # Test PyLong_AsInt() - PyLong_AsInt = _testlimitedcapi.PyLong_AsInt - from _testcapi import INT_MIN, INT_MAX - self.check_long_asint(PyLong_AsInt, INT_MIN, INT_MAX) - - def test_long_aslong(self): - # Test PyLong_AsLong() and PyLong_FromLong() - aslong = _testlimitedcapi.pylong_aslong - from _testcapi import LONG_MIN, LONG_MAX - self.check_long_asint(aslong, LONG_MIN, LONG_MAX) - - def test_long_aslongandoverflow(self): - # Test PyLong_AsLongAndOverflow() - aslongandoverflow = _testlimitedcapi.pylong_aslongandoverflow - from _testcapi import LONG_MIN, LONG_MAX - self.check_long_asintandoverflow(aslongandoverflow, LONG_MIN, LONG_MAX) - - def test_long_asunsignedlong(self): - # Test PyLong_AsUnsignedLong() and PyLong_FromUnsignedLong() - asunsignedlong = _testlimitedcapi.pylong_asunsignedlong - from _testcapi import ULONG_MAX - self.check_long_asint(asunsignedlong, 0, ULONG_MAX, - use_index=False) - - def test_long_asunsignedlongmask(self): - # Test PyLong_AsUnsignedLongMask() - asunsignedlongmask = _testlimitedcapi.pylong_asunsignedlongmask - from _testcapi import ULONG_MAX - self.check_long_asint(asunsignedlongmask, 0, ULONG_MAX, mask=True) - - def test_long_aslonglong(self): - # Test PyLong_AsLongLong() and PyLong_FromLongLong() - aslonglong = _testlimitedcapi.pylong_aslonglong - from _testcapi import LLONG_MIN, LLONG_MAX - self.check_long_asint(aslonglong, LLONG_MIN, LLONG_MAX) - - def test_long_aslonglongandoverflow(self): - # Test PyLong_AsLongLongAndOverflow() - aslonglongandoverflow = _testlimitedcapi.pylong_aslonglongandoverflow - from _testcapi import LLONG_MIN, LLONG_MAX - self.check_long_asintandoverflow(aslonglongandoverflow, LLONG_MIN, LLONG_MAX) - - def test_long_asunsignedlonglong(self): - # Test PyLong_AsUnsignedLongLong() and PyLong_FromUnsignedLongLong() - asunsignedlonglong = _testlimitedcapi.pylong_asunsignedlonglong - from _testcapi import ULLONG_MAX - self.check_long_asint(asunsignedlonglong, 0, ULLONG_MAX, use_index=False) - - def test_long_asunsignedlonglongmask(self): - # Test PyLong_AsUnsignedLongLongMask() - asunsignedlonglongmask = _testlimitedcapi.pylong_asunsignedlonglongmask - from _testcapi import ULLONG_MAX - self.check_long_asint(asunsignedlonglongmask, 0, ULLONG_MAX, mask=True) - - def test_long_as_ssize_t(self): - # Test PyLong_AsSsize_t() and PyLong_FromSsize_t() - as_ssize_t = _testlimitedcapi.pylong_as_ssize_t - from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX - self.check_long_asint(as_ssize_t, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, - use_index=False) - - def test_long_as_size_t(self): - # Test PyLong_AsSize_t() and PyLong_FromSize_t() - as_size_t = _testlimitedcapi.pylong_as_size_t - from _testcapi import SIZE_MAX - self.check_long_asint(as_size_t, 0, SIZE_MAX, use_index=False) - - def test_long_asdouble(self): - # Test PyLong_AsDouble() - asdouble = _testlimitedcapi.pylong_asdouble - MAX = int(sys.float_info.max) - for value in (-MAX, MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asdouble(value), float(value)) - self.assertIsInstance(asdouble(value), float) - - self.assertEqual(asdouble(IntSubclass(42)), 42.0) - self.assertRaises(TypeError, asdouble, Index(42)) - self.assertRaises(TypeError, asdouble, MyIndexAndInt()) - - self.assertRaises(OverflowError, asdouble, 2 * MAX) - self.assertRaises(OverflowError, asdouble, -2 * MAX) - self.assertRaises(TypeError, asdouble, 1.0) - self.assertRaises(TypeError, asdouble, b'2') - self.assertRaises(TypeError, asdouble, '3') - self.assertRaises(SystemError, asdouble, NULL) - - def test_long_asvoidptr(self): - # Test PyLong_AsVoidPtr() - fromvoidptr = _testlimitedcapi.pylong_fromvoidptr - asvoidptr = _testlimitedcapi.pylong_asvoidptr - obj = object() - x = fromvoidptr(obj) - y = fromvoidptr(NULL) - self.assertIs(asvoidptr(x), obj) - self.assertIs(asvoidptr(y), NULL) - self.assertIs(asvoidptr(IntSubclass(x)), obj) - - # negative values - M = (1 << _testcapi.SIZEOF_VOID_P * 8) - if x >= M//2: - self.assertIs(asvoidptr(x - M), obj) - if y >= M//2: - self.assertIs(asvoidptr(y - M), NULL) - - self.assertRaises(TypeError, asvoidptr, Index(x)) - self.assertRaises(TypeError, asvoidptr, object()) - self.assertRaises(OverflowError, asvoidptr, 2**1000) - self.assertRaises(OverflowError, asvoidptr, -2**1000) - # CRASHES asvoidptr(NULL) - - def _test_long_aspid(self, aspid): - # Test PyLong_AsPid() - from _testcapi import SIZEOF_PID_T - bits = 8 * SIZEOF_PID_T - PID_T_MIN = -2**(bits-1) - PID_T_MAX = 2**(bits-1) - 1 - self.check_long_asint(aspid, PID_T_MIN, PID_T_MAX) - - def test_long_aspid(self): - self._test_long_aspid(_testcapi.pylong_aspid) - - def test_long_aspid_limited(self): - self._test_long_aspid(_testlimitedcapi.pylong_aspid) - - @support.bigmemtest(2**32, memuse=0.35) - def test_long_asnativebytes_huge(self, size): - asnativebytes = _testcapi.pylong_asnativebytes - v = 1 << size - buffer = bytearray(size * 2 // 15 + 10) - r = asnativebytes(v, buffer, 0, -1) - self.assertEqual(r, size // 8 + 1) - self.assertEqual(buffer.count(0), len(buffer)) - r = asnativebytes(v, buffer, len(buffer), -1) - self.assertEqual(r, size // 8 + 1) - self.assertEqual(buffer.count(0), len(buffer) - 1) - - def test_long_asnativebytes(self): - import math - from _testcapi import ( - pylong_asnativebytes as asnativebytes, - SIZE_MAX, - ) - - # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot - SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) - MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 - MAX_USIZE = 2 ** (SZ * 8) - 1 - if support.verbose: - print(f"SIZEOF_SIZE={SZ}\n{MAX_SSIZE=:016X}\n{MAX_USIZE=:016X}") - - # These tests check that the requested buffer size is correct. - # This matches our current implementation: We only specify that the - # return value is a size *sufficient* to hold the result when queried - # using n_bytes=0. If our implementation changes, feel free to update - # the expectations here -- or loosen them to be range checks. - # (i.e. 0 *could* be stored in 1 byte and 512 in 2) - for v, expect in [ - (0, SZ), - (512, SZ), - (-512, SZ), - (MAX_SSIZE, SZ), - (MAX_USIZE, SZ + 1), - (-MAX_SSIZE, SZ), - (-MAX_USIZE, SZ + 1), - (2**255-1, 32), - (-(2**255-1), 32), - (2**255, 33), - (-(2**255), 33), # if you ask, we'll say 33, but 32 would do - (2**256-1, 33), - (-(2**256-1), 33), - (2**256, 33), - (-(2**256), 33), - ]: - with self.subTest(f"sizeof-{v:X}"): - buffer = bytearray(b"\x5a") - self.assertEqual(expect, asnativebytes(v, buffer, 0, -1), - "PyLong_AsNativeBytes(v, , 0, -1)") - self.assertEqual(buffer, b"\x5a", - "buffer overwritten when it should not have been") - # Also check via the __index__ path. - # We pass Py_ASNATIVEBYTES_NATIVE_ENDIAN | ALLOW_INDEX - self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, 3 | 16), - "PyLong_AsNativeBytes(Index(v), , 0, -1)") - self.assertEqual(buffer, b"\x5a", - "buffer overwritten when it should not have been") - - # Test that we populate n=2 bytes but do not overwrite more. - buffer = bytearray(b"\x99"*3) - self.assertEqual(2, asnativebytes(4, buffer, 2, 0), # BE - "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 0) // BE") - self.assertEqual(buffer, b"\x00\x04\x99") - self.assertEqual(2, asnativebytes(4, buffer, 2, 1), # LE - "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 1) // LE") - self.assertEqual(buffer, b"\x04\x00\x99") - - # We request as many bytes as `expect_be` contains, and always check - # the result (both big and little endian). We check the return value - # independently, since the buffer should always be filled correctly even - # if we need more bytes - for v, expect_be, expect_n in [ - (0, b'\x00', 1), - (0, b'\x00' * 2, 2), - (0, b'\x00' * 8, min(8, SZ)), - (1, b'\x01', 1), - (1, b'\x00' * 10 + b'\x01', min(11, SZ)), - (42, b'\x2a', 1), - (42, b'\x00' * 10 + b'\x2a', min(11, SZ)), - (-1, b'\xff', 1), - (-1, b'\xff' * 10, min(11, SZ)), - (-42, b'\xd6', 1), - (-42, b'\xff' * 10 + b'\xd6', min(11, SZ)), - # Extracts 255 into a single byte, but requests 2 - # (this is currently a special case, and "should" request SZ) - (255, b'\xff', 2), - (255, b'\x00\xff', 2), - (256, b'\x01\x00', 2), - (0x80, b'\x00' * 7 + b'\x80', min(8, SZ)), - # Extracts successfully (unsigned), but requests 9 bytes - (2**63, b'\x80' + b'\x00' * 7, 9), - (2**63, b'\x00\x80' + b'\x00' * 7, 9), - # Extracts into 8 bytes, but if you provide 9 we'll say 9 - (-2**63, b'\x80' + b'\x00' * 7, 8), - (-2**63, b'\xff\x80' + b'\x00' * 7, 9), - - (2**255-1, b'\x7f' + b'\xff' * 31, 32), - (-(2**255-1), b'\x80' + b'\x00' * 30 + b'\x01', 32), - # Request extra bytes, but result says we only needed 32 - (-(2**255-1), b'\xff\x80' + b'\x00' * 30 + b'\x01', 32), - (-(2**255-1), b'\xff\xff\x80' + b'\x00' * 30 + b'\x01', 32), - - # Extracting 256 bits of integer will request 33 bytes, but still - # copy as many bits as possible into the buffer. So we *can* copy - # into a 32-byte buffer, though negative number may be unrecoverable - (2**256-1, b'\xff' * 32, 33), - (2**256-1, b'\x00' + b'\xff' * 32, 33), - (-(2**256-1), b'\x00' * 31 + b'\x01', 33), - (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 33), - (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 33), - # However, -2**255 precisely will extract into 32 bytes and return - # success. For bigger buffers, it will still succeed, but will - # return 33 - (-(2**255), b'\x80' + b'\x00' * 31, 32), - (-(2**255), b'\xff\x80' + b'\x00' * 31, 33), - - # The classic "Windows HRESULT as negative number" case - # HRESULT hr; - # PyLong_AsNativeBytes(<-2147467259>, &hr, sizeof(HRESULT), -1) - # assert(hr == E_FAIL) - (-2147467259, b'\x80\x00\x40\x05', 4), - ]: - with self.subTest(f"{v:X}-{len(expect_be)}bytes"): - n = len(expect_be) - # Fill the buffer with dummy data to ensure all bytes - # are overwritten. - buffer = bytearray(b"\xa5"*n) - expect_le = expect_be[::-1] - - self.assertEqual(expect_n, asnativebytes(v, buffer, n, 0), - f"PyLong_AsNativeBytes(v, buffer, {n}, )") - self.assertEqual(expect_be, buffer[:n], "") - self.assertEqual(expect_n, asnativebytes(v, buffer, n, 1), - f"PyLong_AsNativeBytes(v, buffer, {n}, )") - self.assertEqual(expect_le, buffer[:n], "") - - # Test cases that do not request size for a sign bit when we pass the - # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag - for v, expect_be, expect_n in [ - (255, b'\xff', 1), - # We pass a 2 byte buffer so it just uses the whole thing - (255, b'\x00\xff', 2), - - (2**63, b'\x80' + b'\x00' * 7, 8), - # We pass a 9 byte buffer so it uses the whole thing - (2**63, b'\x00\x80' + b'\x00' * 7, 9), - - (2**256-1, b'\xff' * 32, 32), - # We pass a 33 byte buffer so it uses the whole thing - (2**256-1, b'\x00' + b'\xff' * 32, 33), - ]: - with self.subTest(f"{v:X}-{len(expect_be)}bytes-unsigned"): - n = len(expect_be) - buffer = bytearray(b"\xa5"*n) - self.assertEqual(expect_n, asnativebytes(v, buffer, n, 4), - f"PyLong_AsNativeBytes(v, buffer, {n}, )") - self.assertEqual(expect_n, asnativebytes(v, buffer, n, 5), - f"PyLong_AsNativeBytes(v, buffer, {n}, )") - - # Ensure Py_ASNATIVEBYTES_REJECT_NEGATIVE raises on negative value - with self.assertRaises(ValueError): - asnativebytes(-1, buffer, 0, 8) - - # Ensure omitting Py_ASNATIVEBYTES_ALLOW_INDEX raises on __index__ value - with self.assertRaises(TypeError): - asnativebytes(Index(1), buffer, 0, -1) - with self.assertRaises(TypeError): - asnativebytes(Index(1), buffer, 0, 3) - - # Check a few error conditions. These are validated in code, but are - # unspecified in docs, so if we make changes to the implementation, it's - # fine to just update these tests rather than preserve the behaviour. - with self.assertRaises(TypeError): - asnativebytes('not a number', buffer, 0, -1) - - def test_long_asnativebytes_fuzz(self): - import math - from random import Random - from _testcapi import ( - pylong_asnativebytes as asnativebytes, - SIZE_MAX, - ) - - # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot - SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) - - rng = Random() - # Allocate bigger buffer than actual values are going to be - buffer = bytearray(260) - - for _ in range(1000): - n = rng.randrange(1, 256) - bytes_be = bytes([ - # Ensure the most significant byte is nonzero - rng.randrange(1, 256), - *[rng.randrange(256) for _ in range(n - 1)] - ]) - bytes_le = bytes_be[::-1] - v = int.from_bytes(bytes_le, 'little') - - expect_1 = expect_2 = (SZ, n) - if bytes_be[0] & 0x80: - # All values are positive, so if MSB is set, expect extra bit - # when we request the size or have a large enough buffer - expect_1 = (SZ, n + 1) - # When passing Py_ASNATIVEBYTES_UNSIGNED_BUFFER, we expect the - # return to be exactly the right size. - expect_2 = (n,) - - try: - actual = asnativebytes(v, buffer, 0, -1) - self.assertIn(actual, expect_1) - - actual = asnativebytes(v, buffer, len(buffer), 0) - self.assertIn(actual, expect_1) - self.assertEqual(bytes_be, buffer[-n:]) - - actual = asnativebytes(v, buffer, len(buffer), 1) - self.assertIn(actual, expect_1) - self.assertEqual(bytes_le, buffer[:n]) - - actual = asnativebytes(v, buffer, n, 4) - self.assertIn(actual, expect_2, bytes_be.hex()) - actual = asnativebytes(v, buffer, n, 5) - self.assertIn(actual, expect_2, bytes_be.hex()) - except AssertionError as ex: - value_hex = ''.join(reversed([ - f'{b:02X}{"" if i % 8 else "_"}' - for i, b in enumerate(bytes_le, start=1) - ])).strip('_') - if support.verbose: - print() - print(n, 'bytes') - print('hex =', value_hex) - print('int =', v) - raise - raise AssertionError(f"Value: 0x{value_hex}") from ex - - def test_long_fromnativebytes(self): - import math - from _testcapi import ( - pylong_fromnativebytes as fromnativebytes, - SIZE_MAX, - ) - - # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot - SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) - MAX_SSIZE = 2 ** (SZ * 8 - 1) - 1 - MAX_USIZE = 2 ** (SZ * 8) - 1 - - for v_be, expect_s, expect_u in [ - (b'\x00', 0, 0), - (b'\x01', 1, 1), - (b'\xff', -1, 255), - (b'\x00\xff', 255, 255), - (b'\xff\xff', -1, 65535), - ]: - with self.subTest(f"{expect_s}-{expect_u:X}-{len(v_be)}bytes"): - n = len(v_be) - v_le = v_be[::-1] - - self.assertEqual(expect_s, fromnativebytes(v_be, n, 0, 1), - f"PyLong_FromNativeBytes(buffer, {n}, )") - self.assertEqual(expect_s, fromnativebytes(v_le, n, 1, 1), - f"PyLong_FromNativeBytes(buffer, {n}, )") - self.assertEqual(expect_u, fromnativebytes(v_be, n, 0, 0), - f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") - self.assertEqual(expect_u, fromnativebytes(v_le, n, 1, 0), - f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") - - # Check native endian when the result would be the same either - # way and we can test it. - if v_be == v_le: - self.assertEqual(expect_s, fromnativebytes(v_be, n, -1, 1), - f"PyLong_FromNativeBytes(buffer, {n}, )") - self.assertEqual(expect_u, fromnativebytes(v_be, n, -1, 0), - f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") - - # Swap the unsigned request for tests and use the - # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag instead - self.assertEqual(expect_u, fromnativebytes(v_be, n, 4, 1), - f"PyLong_FromNativeBytes(buffer, {n}, )") - - def test_long_getsign(self): - # Test PyLong_GetSign() - getsign = _testcapi.pylong_getsign - self.assertEqual(getsign(1), 1) - self.assertEqual(getsign(123456), 1) - self.assertEqual(getsign(-2), -1) - self.assertEqual(getsign(0), 0) - self.assertEqual(getsign(True), 1) - self.assertEqual(getsign(IntSubclass(-11)), -1) - self.assertEqual(getsign(False), 0) - - self.assertRaises(TypeError, getsign, 1.0) - self.assertRaises(TypeError, getsign, Index(123)) - - # CRASHES getsign(NULL) - - def test_long_ispositive(self): - # Test PyLong_IsPositive() - ispositive = _testcapi.pylong_ispositive - self.assertEqual(ispositive(1), 1) - self.assertEqual(ispositive(123), 1) - self.assertEqual(ispositive(-1), 0) - self.assertEqual(ispositive(0), 0) - self.assertEqual(ispositive(True), 1) - self.assertEqual(ispositive(False), 0) - self.assertEqual(ispositive(IntSubclass(-1)), 0) - self.assertRaises(TypeError, ispositive, 1.0) - self.assertRaises(TypeError, ispositive, Index(123)) - - # CRASHES ispositive(NULL) - - def test_long_isnegative(self): - # Test PyLong_IsNegative() - isnegative = _testcapi.pylong_isnegative - self.assertEqual(isnegative(1), 0) - self.assertEqual(isnegative(123), 0) - self.assertEqual(isnegative(-1), 1) - self.assertEqual(isnegative(0), 0) - self.assertEqual(isnegative(True), 0) - self.assertEqual(isnegative(False), 0) - self.assertEqual(isnegative(IntSubclass(-1)), 1) - self.assertRaises(TypeError, isnegative, 1.0) - self.assertRaises(TypeError, isnegative, Index(123)) - - # CRASHES isnegative(NULL) - - def test_long_iszero(self): - # Test PyLong_IsZero() - iszero = _testcapi.pylong_iszero - self.assertEqual(iszero(1), 0) - self.assertEqual(iszero(-1), 0) - self.assertEqual(iszero(0), 1) - self.assertEqual(iszero(True), 0) - self.assertEqual(iszero(False), 1) - self.assertEqual(iszero(IntSubclass(-1)), 0) - self.assertEqual(iszero(IntSubclass(0)), 1) - self.assertRaises(TypeError, iszero, 1.0) - self.assertRaises(TypeError, iszero, Index(123)) - - # CRASHES iszero(NULL) - - def test_long_asint32(self): - # Test PyLong_AsInt32() and PyLong_FromInt32() - to_int32 = _testlimitedcapi.pylong_asint32 - from _testcapi import INT32_MIN, INT32_MAX - self.check_long_asint(to_int32, INT32_MIN, INT32_MAX) - - def test_long_asint64(self): - # Test PyLong_AsInt64() and PyLong_FromInt64() - as_int64 = _testlimitedcapi.pylong_asint64 - from _testcapi import INT64_MIN, INT64_MAX - self.check_long_asint(as_int64, INT64_MIN, INT64_MAX) - - def test_long_asuint32(self): - # Test PyLong_AsUInt32() and PyLong_FromUInt32() - as_uint32 = _testlimitedcapi.pylong_asuint32 - from _testcapi import UINT32_MAX - self.check_long_asint(as_uint32, 0, UINT32_MAX, - negative_value_error=ValueError) - - def test_long_asuint64(self): - # Test PyLong_AsUInt64() and PyLong_FromUInt64() - as_uint64 = _testlimitedcapi.pylong_asuint64 - from _testcapi import UINT64_MAX - self.check_long_asint(as_uint64, 0, UINT64_MAX, - negative_value_error=ValueError) - - def test_long_layout(self): - # Test PyLong_GetNativeLayout() - int_info = sys.int_info - layout = _testcapi.get_pylong_layout() - expected = { - 'bits_per_digit': int_info.bits_per_digit, - 'digit_size': int_info.sizeof_digit, - 'digits_order': -1, - 'digit_endianness': -1 if sys.byteorder == 'little' else 1, - } - self.assertEqual(layout, expected) - - def test_long_export(self): - # Test PyLong_Export() - layout = _testcapi.get_pylong_layout() - base = 2 ** layout['bits_per_digit'] - - pylong_export = _testcapi.pylong_export - - # value fits into int64_t - self.assertEqual(pylong_export(0), 0) - self.assertEqual(pylong_export(123), 123) - self.assertEqual(pylong_export(-123), -123) - self.assertEqual(pylong_export(IntSubclass(123)), 123) - - # use an array, doesn't fit into int64_t - self.assertEqual(pylong_export(base**10 * 2 + 1), - (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) - self.assertEqual(pylong_export(-(base**10 * 2 + 1)), - (1, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) - self.assertEqual(pylong_export(IntSubclass(base**10 * 2 + 1)), - (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) - - self.assertRaises(TypeError, pylong_export, 1.0) - self.assertRaises(TypeError, pylong_export, 0+1j) - self.assertRaises(TypeError, pylong_export, "abc") - - def test_longwriter_create(self): - # Test PyLongWriter_Create() - layout = _testcapi.get_pylong_layout() - base = 2 ** layout['bits_per_digit'] - - pylongwriter_create = _testcapi.pylongwriter_create - self.assertRaises(ValueError, pylongwriter_create, 0, []) - self.assertRaises(ValueError, pylongwriter_create, -123, []) - self.assertEqual(pylongwriter_create(0, [0]), 0) - self.assertEqual(pylongwriter_create(0, [123]), 123) - self.assertEqual(pylongwriter_create(1, [123]), -123) - self.assertEqual(pylongwriter_create(1, [1, 2]), - -(base * 2 + 1)) - self.assertEqual(pylongwriter_create(0, [1, 2, 3]), - base**2 * 3 + base * 2 + 1) - max_digit = base - 1 - self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit]), - base**2 * max_digit + base * max_digit + max_digit) - - # normalize - self.assertEqual(pylongwriter_create(0, [123, 0, 0]), 123) - - # test singletons + normalize - for num in (-2, 0, 1, 5, 42, 100): - self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0]), - num) - - def to_digits(num): - digits = [] - while True: - num, digit = divmod(num, base) - digits.append(digit) - if not num: - break - return digits - - # round trip: Python int -> export -> Python int - pylong_export = _testcapi.pylong_export - numbers = [*range(0, 10), 12345, 0xdeadbeef, 2**100, 2**100-1] - numbers.extend(-num for num in list(numbers)) - for num in numbers: - with self.subTest(num=num): - data = pylong_export(num) - if isinstance(data, tuple): - negative, digits = data - else: - value = data - negative = int(value < 0) - digits = to_digits(abs(value)) - self.assertEqual(pylongwriter_create(negative, digits), num, - (negative, digits)) - - def test_bug_143050(self): - with support.adjust_int_max_str_digits(0): - # Bug coming from using _pylong.int_from_string(), that - # currently requires > 6000 decimal digits. - int('-' + '0' * 7000, 10) - _testcapi.test_immortal_small_ints() - # Test also nonzero small int - int('-' + '0' * 7000 + '123', 10) - _testcapi.test_immortal_small_ints() - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py deleted file mode 100644 index 5035b2b4829..00000000000 --- a/Lib/test/test_capi/test_mem.py +++ /dev/null @@ -1,183 +0,0 @@ -import re -import textwrap -import unittest - - -from test import support -from test.support import import_helper, requires_subprocess -from test.support.script_helper import assert_python_failure, assert_python_ok - - -# Skip this test if the _testcapi and _testinternalcapi extensions are not -# available. -_testcapi = import_helper.import_module('_testcapi') -_testinternalcapi = import_helper.import_module('_testinternalcapi') - -@requires_subprocess() -class PyMemDebugTests(unittest.TestCase): - PYTHONMALLOC = 'debug' - # '0x04c06e0' or '04C06E0' - PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' - - def check(self, code): - with support.SuppressCrashReport(): - out = assert_python_failure( - '-c', code, - PYTHONMALLOC=self.PYTHONMALLOC, - # FreeBSD: instruct jemalloc to not fill freed() memory - # with junk byte 0x5a, see JEMALLOC(3) - MALLOC_CONF="junk:false", - ) - stderr = out.err - return stderr.decode('ascii', 'replace') - - def test_buffer_overflow(self): - out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') - regex = (r"Debug memory block at address p={ptr}: API 'm'\n" - r" 16 bytes originally requested\n" - r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" - r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" - r" at tail\+0: 0x78 \*\*\* OUCH\n" - r" at tail\+1: 0xfd\n" - r" at tail\+2: 0xfd\n" - r" .*\n" - r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" - r" Data at p: cd cd cd .*\n" - r"\n" - r"Enable tracemalloc to get the memory block allocation traceback\n" - r"\n" - r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") - regex = regex.format(ptr=self.PTR_REGEX) - regex = re.compile(regex, flags=re.DOTALL) - self.assertRegex(out, regex) - - def test_api_misuse(self): - out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') - regex = (r"Debug memory block at address p={ptr}: API 'm'\n" - r" 16 bytes originally requested\n" - r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" - r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" - r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" - r" Data at p: cd cd cd .*\n" - r"\n" - r"Enable tracemalloc to get the memory block allocation traceback\n" - r"\n" - r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") - regex = regex.format(ptr=self.PTR_REGEX) - self.assertRegex(out, regex) - - def check_malloc_without_gil(self, code): - out = self.check(code) - if not support.Py_GIL_DISABLED: - expected = ('Fatal Python error: _PyMem_DebugMalloc: ' - 'Python memory allocator called without holding the GIL') - else: - expected = ('Fatal Python error: _PyMem_DebugMalloc: ' - 'Python memory allocator called without an active thread state. ' - 'Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?') - self.assertIn(expected, out) - - def test_pymem_malloc_without_gil(self): - # Debug hooks must raise an error if PyMem_Malloc() is called - # without holding the GIL - code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' - self.check_malloc_without_gil(code) - - def test_pyobject_malloc_without_gil(self): - # Debug hooks must raise an error if PyObject_Malloc() is called - # without holding the GIL - code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' - self.check_malloc_without_gil(code) - - def check_pyobject_is_freed(self, func_name): - code = textwrap.dedent(f''' - import gc, os, sys, _testinternalcapi - # Disable the GC to avoid crash on GC collection - gc.disable() - _testinternalcapi.{func_name}() - # Exit immediately to avoid a crash while deallocating - # the invalid object - os._exit(0) - ''') - assert_python_ok( - '-c', code, - PYTHONMALLOC=self.PYTHONMALLOC, - MALLOC_CONF="junk:false", - ) - - def test_pyobject_null_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_null_is_freed') - - def test_pyobject_uninitialized_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') - - def test_pyobject_forbidden_bytes_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') - - def test_pyobject_freed_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_freed_is_freed') - - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') - def test_set_nomemory(self): - code = """if 1: - import _testcapi - - class C(): pass - - # The first loop tests both functions and that remove_mem_hooks() - # can be called twice in a row. The second loop checks a call to - # set_nomemory() after a call to remove_mem_hooks(). The third - # loop checks the start and stop arguments of set_nomemory(). - for outer_cnt in range(1, 4): - start = 10 * outer_cnt - for j in range(100): - if j == 0: - if outer_cnt != 3: - _testcapi.set_nomemory(start) - else: - _testcapi.set_nomemory(start, start + 1) - try: - C() - except MemoryError as e: - if outer_cnt != 3: - _testcapi.remove_mem_hooks() - print('MemoryError', outer_cnt, j) - _testcapi.remove_mem_hooks() - break - """ - rc, out, err = assert_python_ok('-c', code) - lines = out.splitlines() - for i, line in enumerate(lines, 1): - self.assertIn(b'MemoryError', out) - *_, count = line.split(b' ') - count = int(count) - self.assertLessEqual(count, i*10) - self.assertGreaterEqual(count, i*10-4) - - -# free-threading requires mimalloc (not malloc) -@support.requires_gil_enabled() -class PyMemMallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'malloc_debug' - - -@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') -class PyMemPymallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'pymalloc_debug' - - -@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc') -class PyMemMimallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'mimalloc_debug' - - -@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') -class PyMemDefaultTests(PyMemDebugTests): - # test default allocator of Python compiled in debug mode - PYTHONMALLOC = '' - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py deleted file mode 100644 index e71c1f6f554..00000000000 --- a/Lib/test/test_capi/test_misc.py +++ /dev/null @@ -1,2922 +0,0 @@ -# Run the _testcapi module tests (tests for the Python/C API): by defn, -# these are all functions _testcapi exports whose name begins with 'test_'. - -import _thread -from collections import deque -import contextlib -import importlib.machinery -import importlib.util -import json -import os -import pickle -import queue -import random -import sys -import textwrap -import threading -import time -import types -import unittest -import weakref -import operator -from test import support -from test.support import MISSING_C_DOCSTRINGS -from test.support import import_helper -from test.support import threading_helper -from test.support import warnings_helper -from test.support import requires_limited_api -from test.support import expected_failure_if_gil_disabled -from test.support import Py_GIL_DISABLED -from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end -try: - import _posixsubprocess -except ImportError: - _posixsubprocess = None -try: - import _testmultiphase -except ImportError: - _testmultiphase = None -try: - import _testsinglephase -except ImportError: - _testsinglephase = None -try: - import _interpreters -except ModuleNotFoundError: - _interpreters = None - -# Skip this test if the _testcapi module isn't available. -_testcapi = import_helper.import_module('_testcapi') - -from _testcapi import HeapCTypeSubclass, HeapCTypeSubclassWithFinalizer - -import _testlimitedcapi -import _testinternalcapi - - -NULL = None - -def decode_stderr(err): - return err.decode('utf-8', 'replace').replace('\r', '') - - -def requires_subinterpreters(meth): - """Decorator to skip a test if subinterpreters are not supported.""" - return unittest.skipIf(_interpreters is None, - 'subinterpreters required')(meth) - - -def testfunction(self): - """some doc""" - return self - - -class InstanceMethod: - id = _testcapi.instancemethod(id) - testfunction = _testcapi.instancemethod(testfunction) - - -CURRENT_THREAD_REGEX = r'Current thread.*:\n' if not support.Py_GIL_DISABLED else r'Stack .*:\n' - - -@support.force_not_colorized_test_class -class CAPITest(unittest.TestCase): - - def test_instancemethod(self): - inst = InstanceMethod() - self.assertEqual(id(inst), inst.id()) - self.assertTrue(inst.testfunction() is inst) - self.assertEqual(inst.testfunction.__doc__, testfunction.__doc__) - self.assertEqual(InstanceMethod.testfunction.__doc__, testfunction.__doc__) - - InstanceMethod.testfunction.attribute = "test" - self.assertEqual(testfunction.attribute, "test") - self.assertRaises(AttributeError, setattr, inst.testfunction, "attribute", "test") - - @support.requires_subprocess() - def test_no_FatalError_infinite_loop(self): - code = textwrap.dedent(""" - import _testcapi - from test import support - - with support.SuppressCrashReport(): - _testcapi.crash_no_current_thread() - """) - - run_result, _cmd_line = run_python_until_end('-c', code) - _rc, out, err = run_result - self.assertEqual(out, b'') - # This used to cause an infinite loop. - if not support.Py_GIL_DISABLED: - msg = ("Fatal Python error: PyThreadState_Get: " - "the function must be called with the GIL held, " - "after Python initialization and before Python finalization, " - "but the GIL is released " - "(the current Python thread state is NULL)").encode() - else: - msg = ("Fatal Python error: PyThreadState_Get: " - "the function must be called with an active thread state, " - "after Python initialization and before Python finalization, " - "but it was called without an active thread state. " - "Are you trying to call the C API inside of a Py_BEGIN_ALLOW_THREADS block?").encode() - self.assertStartsWith(err.rstrip(), msg) - - def test_memoryview_from_NULL_pointer(self): - self.assertRaises(ValueError, _testcapi.make_memoryview_from_NULL_pointer) - - @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') - def test_seq_bytes_to_charp_array(self): - # Issue #15732: crash in _PySequence_BytesToCharpArray() - class Z(object): - def __len__(self): - return 1 - with self.assertRaisesRegex(TypeError, 'indexing'): - _posixsubprocess.fork_exec( - 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) - # Issue #15736: overflow in _PySequence_BytesToCharpArray() - class Z(object): - def __len__(self): - return sys.maxsize - def __getitem__(self, i): - return b'x' - self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) - - @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') - def test_subprocess_fork_exec(self): - class Z(object): - def __len__(self): - return 1 - - # Issue #15738: crash in subprocess_fork_exec() - self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) - - @unittest.skipIf(MISSING_C_DOCSTRINGS, - "Signature information for builtins requires docstrings") - def test_docstring_signature_parsing(self): - - self.assertEqual(_testcapi.no_docstring.__doc__, None) - self.assertEqual(_testcapi.no_docstring.__text_signature__, None) - - self.assertEqual(_testcapi.docstring_empty.__doc__, None) - self.assertEqual(_testcapi.docstring_empty.__text_signature__, None) - - self.assertEqual(_testcapi.docstring_no_signature.__doc__, - "This docstring has no signature.") - self.assertEqual(_testcapi.docstring_no_signature.__text_signature__, None) - - self.assertEqual(_testcapi.docstring_with_invalid_signature.__doc__, - "docstring_with_invalid_signature($module, /, boo)\n" - "\n" - "This docstring has an invalid signature." - ) - self.assertEqual(_testcapi.docstring_with_invalid_signature.__text_signature__, None) - - self.assertEqual(_testcapi.docstring_with_invalid_signature2.__doc__, - "docstring_with_invalid_signature2($module, /, boo)\n" - "\n" - "--\n" - "\n" - "This docstring also has an invalid signature." - ) - self.assertEqual(_testcapi.docstring_with_invalid_signature2.__text_signature__, None) - - self.assertEqual(_testcapi.docstring_with_signature.__doc__, - "This docstring has a valid signature.") - self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "($module, /, sig)") - - self.assertEqual(_testcapi.docstring_with_signature_but_no_doc.__doc__, None) - self.assertEqual(_testcapi.docstring_with_signature_but_no_doc.__text_signature__, - "($module, /, sig)") - - self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__doc__, - "\nThis docstring has a valid signature and some extra newlines.") - self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__text_signature__, - "($module, /, parameter)") - - def test_c_type_with_matrix_multiplication(self): - M = _testcapi.matmulType - m1 = M() - m2 = M() - self.assertEqual(m1 @ m2, ("matmul", m1, m2)) - self.assertEqual(m1 @ 42, ("matmul", m1, 42)) - self.assertEqual(42 @ m1, ("matmul", 42, m1)) - o = m1 - o @= m2 - self.assertEqual(o, ("imatmul", m1, m2)) - o = m1 - o @= 42 - self.assertEqual(o, ("imatmul", m1, 42)) - o = 42 - o @= m1 - self.assertEqual(o, ("matmul", 42, m1)) - - def test_c_type_with_ipow(self): - # When the __ipow__ method of a type was implemented in C, using the - # modulo param would cause segfaults. - o = _testcapi.ipowType() - self.assertEqual(o.__ipow__(1), (1, None)) - self.assertEqual(o.__ipow__(2, 2), (2, 2)) - - def test_return_null_without_error(self): - # Issue #23571: A function must not return NULL without setting an - # error - if support.Py_DEBUG: - code = textwrap.dedent(""" - import _testcapi - from test import support - - with support.SuppressCrashReport(): - _testcapi.return_null_without_error() - """) - rc, out, err = assert_python_failure('-c', code) - err = decode_stderr(err) - self.assertRegex(err, - r'Fatal Python error: _Py_CheckFunctionResult: ' - r'a function returned NULL without setting an exception\n' - r'Python runtime state: initialized\n' - r'SystemError: ' - r'returned NULL without setting an exception\n' - r'\n' + - CURRENT_THREAD_REGEX + - r' File .*", line 6 in \n') - else: - with self.assertRaises(SystemError) as cm: - _testcapi.return_null_without_error() - self.assertRegex(str(cm.exception), - 'return_null_without_error.* ' - 'returned NULL without setting an exception') - - def test_return_result_with_error(self): - # Issue #23571: A function must not return a result with an error set - if support.Py_DEBUG: - code = textwrap.dedent(""" - import _testcapi - from test import support - - with support.SuppressCrashReport(): - _testcapi.return_result_with_error() - """) - rc, out, err = assert_python_failure('-c', code) - err = decode_stderr(err) - self.assertRegex(err, - r'Fatal Python error: _Py_CheckFunctionResult: ' - r'a function returned a result with an exception set\n' - r'Python runtime state: initialized\n' - r'ValueError\n' - r'\n' - r'The above exception was the direct cause ' - r'of the following exception:\n' - r'\n' - r'SystemError: ' - r'returned a result with an exception set\n' - r'\n' + - CURRENT_THREAD_REGEX + - r' File .*, line 6 in \n') - else: - with self.assertRaises(SystemError) as cm: - _testcapi.return_result_with_error() - self.assertRegex(str(cm.exception), - 'return_result_with_error.* ' - 'returned a result with an exception set') - - def test_getitem_with_error(self): - # Test _Py_CheckSlotResult(). Raise an exception and then calls - # PyObject_GetItem(): check that the assertion catches the bug. - # PyObject_GetItem() must not be called with an exception set. - code = textwrap.dedent(""" - import _testcapi - from test import support - - with support.SuppressCrashReport(): - _testcapi.getitem_with_error({1: 2}, 1) - """) - rc, out, err = assert_python_failure('-c', code) - err = decode_stderr(err) - if 'SystemError: ' not in err: - self.assertRegex(err, - r'Fatal Python error: _Py_CheckSlotResult: ' - r'Slot __getitem__ of type dict succeeded ' - r'with an exception set\n' - r'Python runtime state: initialized\n' - r'ValueError: bug\n' - r'\n' + - CURRENT_THREAD_REGEX + - r' File .*, line 6 in \n' - r'\n' - r'Extension modules: _testcapi \(total: 1\)\n') - else: - # Python built with NDEBUG macro defined: - # test _Py_CheckFunctionResult() instead. - self.assertIn('returned a result with an exception set', err) - - def test_buildvalue(self): - # Test Py_BuildValue() with object arguments - buildvalue = _testcapi.py_buildvalue - self.assertEqual(buildvalue(''), None) - self.assertEqual(buildvalue('()'), ()) - self.assertEqual(buildvalue('[]'), []) - self.assertEqual(buildvalue('{}'), {}) - self.assertEqual(buildvalue('()[]{}'), ((), [], {})) - self.assertEqual(buildvalue('O', 1), 1) - self.assertEqual(buildvalue('(O)', 1), (1,)) - self.assertEqual(buildvalue('[O]', 1), [1]) - self.assertRaises(SystemError, buildvalue, '{O}', 1) - self.assertEqual(buildvalue('OO', 1, 2), (1, 2)) - self.assertEqual(buildvalue('(OO)', 1, 2), (1, 2)) - self.assertEqual(buildvalue('[OO]', 1, 2), [1, 2]) - self.assertEqual(buildvalue('{OO}', 1, 2), {1: 2}) - self.assertEqual(buildvalue('{OOOO}', 1, 2, 3, 4), {1: 2, 3: 4}) - self.assertEqual(buildvalue('((O))', 1), ((1,),)) - self.assertEqual(buildvalue('((OO))', 1, 2), ((1, 2),)) - - self.assertEqual(buildvalue(' \t,:'), None) - self.assertEqual(buildvalue('O,', 1), 1) - self.assertEqual(buildvalue(' O ', 1), 1) - self.assertEqual(buildvalue('\tO\t', 1), 1) - self.assertEqual(buildvalue('O,O', 1, 2), (1, 2)) - self.assertEqual(buildvalue('O, O', 1, 2), (1, 2)) - self.assertEqual(buildvalue('O,\tO', 1, 2), (1, 2)) - self.assertEqual(buildvalue('O O', 1, 2), (1, 2)) - self.assertEqual(buildvalue('O\tO', 1, 2), (1, 2)) - self.assertEqual(buildvalue('(O,O)', 1, 2), (1, 2)) - self.assertEqual(buildvalue('(O, O,)', 1, 2), (1, 2)) - self.assertEqual(buildvalue(' ( O O ) ', 1, 2), (1, 2)) - self.assertEqual(buildvalue('\t(\tO\tO\t)\t', 1, 2), (1, 2)) - self.assertEqual(buildvalue('[O,O]', 1, 2), [1, 2]) - self.assertEqual(buildvalue('[O, O,]', 1, 2), [1, 2]) - self.assertEqual(buildvalue(' [ O O ] ', 1, 2), [1, 2]) - self.assertEqual(buildvalue(' [\tO\tO\t] ', 1, 2), [1, 2]) - self.assertEqual(buildvalue('{O:O}', 1, 2), {1: 2}) - self.assertEqual(buildvalue('{O:O,O:O}', 1, 2, 3, 4), {1: 2, 3: 4}) - self.assertEqual(buildvalue('{O: O, O: O,}', 1, 2, 3, 4), {1: 2, 3: 4}) - self.assertEqual(buildvalue(' { O O O O } ', 1, 2, 3, 4), {1: 2, 3: 4}) - self.assertEqual(buildvalue('\t{\tO\tO\tO\tO\t}\t', 1, 2, 3, 4), {1: 2, 3: 4}) - - self.assertRaises(SystemError, buildvalue, 'O', NULL) - self.assertRaises(SystemError, buildvalue, '(O)', NULL) - self.assertRaises(SystemError, buildvalue, '[O]', NULL) - self.assertRaises(SystemError, buildvalue, '{O}', NULL) - self.assertRaises(SystemError, buildvalue, 'OO', 1, NULL) - self.assertRaises(SystemError, buildvalue, 'OO', NULL, 2) - self.assertRaises(SystemError, buildvalue, '(OO)', 1, NULL) - self.assertRaises(SystemError, buildvalue, '(OO)', NULL, 2) - self.assertRaises(SystemError, buildvalue, '[OO]', 1, NULL) - self.assertRaises(SystemError, buildvalue, '[OO]', NULL, 2) - self.assertRaises(SystemError, buildvalue, '{OO}', 1, NULL) - self.assertRaises(SystemError, buildvalue, '{OO}', NULL, 2) - - def test_buildvalue_ints(self): - # Test Py_BuildValue() with integer arguments - buildvalue = _testcapi.py_buildvalue_ints - from _testcapi import SHRT_MIN, SHRT_MAX, USHRT_MAX, INT_MIN, INT_MAX, UINT_MAX - self.assertEqual(buildvalue('i', INT_MAX), INT_MAX) - self.assertEqual(buildvalue('i', INT_MIN), INT_MIN) - self.assertEqual(buildvalue('I', UINT_MAX), UINT_MAX) - - self.assertEqual(buildvalue('h', SHRT_MAX), SHRT_MAX) - self.assertEqual(buildvalue('h', SHRT_MIN), SHRT_MIN) - self.assertEqual(buildvalue('H', USHRT_MAX), USHRT_MAX) - - self.assertEqual(buildvalue('b', 127), 127) - self.assertEqual(buildvalue('b', -128), -128) - self.assertEqual(buildvalue('B', 255), 255) - - self.assertEqual(buildvalue('c', ord('A')), b'A') - self.assertEqual(buildvalue('c', 255), b'\xff') - self.assertEqual(buildvalue('c', 256), b'\x00') - self.assertEqual(buildvalue('c', -1), b'\xff') - - self.assertEqual(buildvalue('C', 255), chr(255)) - self.assertEqual(buildvalue('C', 256), chr(256)) - self.assertEqual(buildvalue('C', sys.maxunicode), chr(sys.maxunicode)) - self.assertRaises(ValueError, buildvalue, 'C', -1) - self.assertRaises(ValueError, buildvalue, 'C', sys.maxunicode+1) - - # gh-84489 - self.assertRaises(ValueError, buildvalue, '(C )i', -1, 2) - self.assertRaises(ValueError, buildvalue, '[C ]i', -1, 2) - self.assertRaises(ValueError, buildvalue, '{Ci }i', -1, 2, 3) - - def test_buildvalue_N(self): - _testcapi.test_buildvalue_N() - - def test_trashcan_subclass(self): - # bpo-35983: Check that the trashcan mechanism for "list" is NOT - # activated when its tp_dealloc is being called by a subclass - from _testcapi import MyList - L = None - for i in range(100): - L = MyList((L,)) - - @support.requires_resource('cpu') - @support.skip_emscripten_stack_overflow() - @support.skip_wasi_stack_overflow() - def test_trashcan_python_class1(self): - self.do_test_trashcan_python_class(list) - - @support.requires_resource('cpu') - @support.skip_emscripten_stack_overflow() - @support.skip_wasi_stack_overflow() - def test_trashcan_python_class2(self): - from _testcapi import MyList - self.do_test_trashcan_python_class(MyList) - - def do_test_trashcan_python_class(self, base): - # Check that the trashcan mechanism works properly for a Python - # subclass of a class using the trashcan (this specific test assumes - # that the base class "base" behaves like list) - class PyList(base): - # Count the number of PyList instances to verify that there is - # no memory leak - num = 0 - def __init__(self, *args): - __class__.num += 1 - super().__init__(*args) - def __del__(self): - __class__.num -= 1 - - for parity in (0, 1): - L = None - # We need in the order of 2**20 iterations here such that a - # typical 8MB stack would overflow without the trashcan. - for i in range(2**20): - L = PyList((L,)) - L.attr = i - if parity: - # Add one additional nesting layer - L = (L,) - self.assertGreater(PyList.num, 0) - del L - self.assertEqual(PyList.num, 0) - - @unittest.skipIf(MISSING_C_DOCSTRINGS, - "Signature information for builtins requires docstrings") - def test_heap_ctype_doc_and_text_signature(self): - self.assertEqual(_testcapi.HeapDocCType.__doc__, "somedoc") - self.assertEqual(_testcapi.HeapDocCType.__text_signature__, "(arg1, arg2)") - - def test_null_type_doc(self): - self.assertEqual(_testcapi.NullTpDocType.__doc__, None) - - def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self): - class HeapGcCTypeSubclass(_testcapi.HeapGcCType): - def __init__(self): - self.value2 = 20 - super().__init__() - - subclass_instance = HeapGcCTypeSubclass() - type_refcnt = sys.getrefcount(HeapGcCTypeSubclass) - - # Test that subclass instance was fully created - self.assertEqual(subclass_instance.value, 10) - self.assertEqual(subclass_instance.value2, 20) - - # Test that the type reference count is only decremented once - del subclass_instance - self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapGcCTypeSubclass)) - - def test_subclass_of_heap_gc_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): - class A(_testcapi.HeapGcCType): - def __init__(self): - self.value2 = 20 - super().__init__() - - class B(A): - def __init__(self): - super().__init__() - - def __del__(self): - self.__class__ = A - A.refcnt_in_del = sys.getrefcount(A) - B.refcnt_in_del = sys.getrefcount(B) - - subclass_instance = B() - type_refcnt = sys.getrefcount(B) - new_type_refcnt = sys.getrefcount(A) - - # Test that subclass instance was fully created - self.assertEqual(subclass_instance.value, 10) - self.assertEqual(subclass_instance.value2, 20) - - del subclass_instance - - # Test that setting __class__ modified the reference counts of the types - if support.Py_DEBUG: - # gh-89373: In debug mode, _Py_Dealloc() keeps a strong reference - # to the type while calling tp_dealloc() - self.assertEqual(type_refcnt, B.refcnt_in_del) - else: - self.assertEqual(type_refcnt - 1, B.refcnt_in_del) - self.assertEqual(new_type_refcnt + 1, A.refcnt_in_del) - - # Test that the original type already has decreased its refcnt - self.assertEqual(type_refcnt - 1, sys.getrefcount(B)) - - # Test that subtype_dealloc decref the newly assigned __class__ only once - self.assertEqual(new_type_refcnt, sys.getrefcount(A)) - - def test_heaptype_with_dict(self): - for cls in ( - _testcapi.HeapCTypeWithDict, - _testlimitedcapi.HeapCTypeWithRelativeDict, - ): - with self.subTest(cls=cls): - inst = cls() - inst.foo = 42 - self.assertEqual(inst.foo, 42) - self.assertEqual(inst.dictobj, inst.__dict__) - self.assertEqual(inst.dictobj, {"foo": 42}) - - inst = cls() - self.assertEqual({}, inst.__dict__) - - def test_heaptype_with_managed_dict(self): - inst = _testcapi.HeapCTypeWithManagedDict() - inst.foo = 42 - self.assertEqual(inst.foo, 42) - self.assertEqual(inst.__dict__, {"foo": 42}) - - inst = _testcapi.HeapCTypeWithManagedDict() - self.assertEqual({}, inst.__dict__) - - a = _testcapi.HeapCTypeWithManagedDict() - b = _testcapi.HeapCTypeWithManagedDict() - a.b = b - b.a = a - del a, b - - def test_sublclassing_managed_dict(self): - - class C(_testcapi.HeapCTypeWithManagedDict): - pass - - i = C() - i.spam = i - del i - - def test_heaptype_with_negative_dict(self): - inst = _testcapi.HeapCTypeWithNegativeDict() - inst.foo = 42 - self.assertEqual(inst.foo, 42) - self.assertEqual(inst.dictobj, inst.__dict__) - self.assertEqual(inst.dictobj, {"foo": 42}) - - inst = _testcapi.HeapCTypeWithNegativeDict() - self.assertEqual({}, inst.__dict__) - - def test_heaptype_with_weakref(self): - for cls in ( - _testcapi.HeapCTypeWithWeakref, - _testlimitedcapi.HeapCTypeWithRelativeWeakref, - ): - with self.subTest(cls=cls): - inst = cls() - ref = weakref.ref(inst) - self.assertEqual(ref(), inst) - self.assertEqual(inst.weakreflist, ref) - - def test_heaptype_with_managed_weakref(self): - inst = _testcapi.HeapCTypeWithManagedWeakref() - ref = weakref.ref(inst) - self.assertEqual(ref(), inst) - - def test_sublclassing_managed_weakref(self): - - class C(_testcapi.HeapCTypeWithManagedWeakref): - pass - - inst = C() - ref = weakref.ref(inst) - self.assertEqual(ref(), inst) - - def test_sublclassing_managed_both(self): - - class C1(_testcapi.HeapCTypeWithManagedWeakref, _testcapi.HeapCTypeWithManagedDict): - pass - - class C2(_testcapi.HeapCTypeWithManagedDict, _testcapi.HeapCTypeWithManagedWeakref): - pass - - for cls in (C1, C2): - inst = cls() - ref = weakref.ref(inst) - self.assertEqual(ref(), inst) - inst.spam = inst - del inst - ref = weakref.ref(cls()) - self.assertIs(ref(), None) - - def test_heaptype_with_buffer(self): - inst = _testcapi.HeapCTypeWithBuffer() - b = bytes(inst) - self.assertEqual(b, b"1234") - - def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self): - subclass_instance = _testcapi.HeapCTypeSubclass() - type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass) - - # Test that subclass instance was fully created - self.assertEqual(subclass_instance.value, 10) - self.assertEqual(subclass_instance.value2, 20) - - # Test that the type reference count is only decremented once - del subclass_instance - self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclass)) - - def test_c_subclass_of_heap_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): - subclass_instance = HeapCTypeSubclassWithFinalizer() - type_refcnt = sys.getrefcount(HeapCTypeSubclassWithFinalizer) - new_type_refcnt = sys.getrefcount(HeapCTypeSubclass) - - # Test that subclass instance was fully created - self.assertEqual(subclass_instance.value, 10) - self.assertEqual(subclass_instance.value2, 20) - - # The tp_finalize slot will set __class__ to HeapCTypeSubclass - del subclass_instance - - # Test that setting __class__ modified the reference counts of the types - # - # This is highly sensitive to implementation details and may break in the future. - # - # We expect the refcount on the old type, HeapCTypeSubclassWithFinalizer, to - # remain the same: the finalizer gets a strong reference (+1) when it gets the - # type from the module and setting __class__ decrements the refcount (-1). - # - # We expect the refcount on the new type, HeapCTypeSubclass, to increase by 2: - # the finalizer get a strong reference (+1) when it gets the type from the - # module and setting __class__ increments the refcount (+1). - expected_type_refcnt = type_refcnt - expected_new_type_refcnt = new_type_refcnt + 2 - - if not Py_GIL_DISABLED: - # In default builds the result returned from sys.getrefcount - # includes a temporary reference that is created by the interpreter - # when it pushes its argument on the operand stack. This temporary - # reference is not included in the result returned by Py_REFCNT, which - # is used in the finalizer. - # - # In free-threaded builds the result returned from sys.getrefcount - # does not include the temporary reference. Types use deferred - # refcounting and the interpreter will not create a new reference - # for deferred values on the operand stack. - expected_type_refcnt -= 1 - expected_new_type_refcnt -= 1 - - if support.Py_DEBUG: - # gh-89373: In debug mode, _Py_Dealloc() keeps a strong reference - # to the type while calling tp_dealloc() - expected_type_refcnt += 1 - - self.assertEqual(expected_type_refcnt, HeapCTypeSubclassWithFinalizer.refcnt_in_del) - self.assertEqual(expected_new_type_refcnt, HeapCTypeSubclass.refcnt_in_del) - - # Test that the original type already has decreased its refcnt - self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapCTypeSubclassWithFinalizer)) - - # Test that subtype_dealloc decref the newly assigned __class__ only once - self.assertEqual(new_type_refcnt, sys.getrefcount(HeapCTypeSubclass)) - - def test_heaptype_with_setattro(self): - obj = _testcapi.HeapCTypeSetattr() - self.assertEqual(obj.pvalue, 10) - obj.value = 12 - self.assertEqual(obj.pvalue, 12) - del obj.value - self.assertEqual(obj.pvalue, 0) - - def test_heaptype_with_custom_metaclass(self): - metaclass = _testcapi.HeapCTypeMetaclass - self.assertIsSubclass(metaclass, type) - - # Class creation from C - t = _testcapi.pytype_fromspec_meta(metaclass) - self.assertIsInstance(t, type) - self.assertEqual(t.__name__, "HeapCTypeViaMetaclass") - self.assertIs(type(t), metaclass) - - # Class creation from Python - t = metaclass("PyClassViaMetaclass", (), {}) - self.assertIsInstance(t, type) - self.assertEqual(t.__name__, "PyClassViaMetaclass") - - def test_heaptype_with_custom_metaclass_null_new(self): - metaclass = _testcapi.HeapCTypeMetaclassNullNew - - self.assertIsSubclass(metaclass, type) - - # Class creation from C - t = _testcapi.pytype_fromspec_meta(metaclass) - self.assertIsInstance(t, type) - self.assertEqual(t.__name__, "HeapCTypeViaMetaclass") - self.assertIs(type(t), metaclass) - - # Class creation from Python - with self.assertRaisesRegex(TypeError, "cannot create .* instances"): - metaclass("PyClassViaMetaclass", (), {}) - - def test_heaptype_with_custom_metaclass_custom_new(self): - metaclass = _testcapi.HeapCTypeMetaclassCustomNew - - self.assertIsSubclass(_testcapi.HeapCTypeMetaclassCustomNew, type) - - msg = "Metaclasses with custom tp_new are not supported." - with self.assertRaisesRegex(TypeError, msg): - t = _testcapi.pytype_fromspec_meta(metaclass) - - def test_heaptype_base_with_custom_metaclass(self): - metaclass = _testcapi.HeapCTypeMetaclassCustomNew - - class Base(metaclass=metaclass): - pass - - # Class creation from C - msg = "Metaclasses with custom tp_new are not supported." - with self.assertRaisesRegex(TypeError, msg): - sub = _testcapi.make_type_with_base(Base) - - def test_heaptype_with_tp_vectorcall(self): - tp = _testcapi.HeapCTypeVectorcall - v0 = tp.__new__(tp) - v0.__init__() - v1 = tp() - self.assertEqual(v0.value, 2) - self.assertEqual(v1.value, 1) - - def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): - for weakref_cls in (_testcapi.HeapCTypeWithWeakref, - _testlimitedcapi.HeapCTypeWithRelativeWeakref): - for dict_cls in (_testcapi.HeapCTypeWithDict, - _testlimitedcapi.HeapCTypeWithRelativeDict): - with self.subTest(weakref_cls=weakref_cls, dict_cls=dict_cls): - - with self.assertRaises(TypeError): - class Both1(weakref_cls, dict_cls): - pass - with self.assertRaises(TypeError): - class Both2(dict_cls, weakref_cls): - pass - - def test_multiple_inheritance_ctypes_with_weakref_or_dict_and_other_builtin(self): - for dict_cls in (_testcapi.HeapCTypeWithDict, - _testlimitedcapi.HeapCTypeWithRelativeDict): - for weakref_cls in (_testcapi.HeapCTypeWithWeakref, - _testlimitedcapi.HeapCTypeWithRelativeWeakref): - with self.subTest(dict_cls=dict_cls, weakref_cls=weakref_cls): - - with self.assertRaises(TypeError): - class C1(dict_cls, list): - pass - - with self.assertRaises(TypeError): - class C2(weakref_cls, list): - pass - - class C3(_testcapi.HeapCTypeWithManagedDict, list): - pass - class C4(_testcapi.HeapCTypeWithManagedWeakref, list): - pass - - inst = C3() - inst.append(0) - str(inst.__dict__) - - inst = C4() - inst.append(0) - str(inst.__weakref__) - - for cls in (_testcapi.HeapCTypeWithManagedDict, - _testcapi.HeapCTypeWithManagedWeakref): - for cls2 in (dict_cls, weakref_cls): - class S(cls, cls2): - pass - class B1(C3, cls): - pass - class B2(C4, cls): - pass - - def test_pytype_fromspec_with_repeated_slots(self): - for variant in range(2): - with self.subTest(variant=variant): - with self.assertRaises(SystemError): - _testcapi.create_type_from_repeated_slots(variant) - - def test_immutable_type_with_mutable_base(self): - class MutableBase: ... - - with self.assertRaisesRegex(TypeError, 'Creating immutable type'): - _testcapi.make_immutable_type_with_base(MutableBase) - - def test_pynumber_tobase(self): - from _testcapi import pynumber_tobase - small_number = 123 - large_number = 2**64 - class IDX: - def __init__(self, val): - self.val = val - def __index__(self): - return self.val - - test_cases = ((2, '0b1111011', '0b10000000000000000000000000000000000000000000000000000000000000000'), - (8, '0o173', '0o2000000000000000000000'), - (10, '123', '18446744073709551616'), - (16, '0x7b', '0x10000000000000000')) - for base, small_target, large_target in test_cases: - with self.subTest(base=base, st=small_target, lt=large_target): - # Test for small number - self.assertEqual(pynumber_tobase(small_number, base), small_target) - self.assertEqual(pynumber_tobase(-small_number, base), '-' + small_target) - self.assertEqual(pynumber_tobase(IDX(small_number), base), small_target) - # Test for large number(out of range of a longlong,i.e.[-2**63, 2**63-1]) - self.assertEqual(pynumber_tobase(large_number, base), large_target) - self.assertEqual(pynumber_tobase(-large_number, base), '-' + large_target) - self.assertEqual(pynumber_tobase(IDX(large_number), base), large_target) - self.assertRaises(TypeError, pynumber_tobase, IDX(123.0), 10) - self.assertRaises(TypeError, pynumber_tobase, IDX('123'), 10) - self.assertRaises(TypeError, pynumber_tobase, 123.0, 10) - self.assertRaises(TypeError, pynumber_tobase, '123', 10) - self.assertRaises(SystemError, pynumber_tobase, 123, 0) - - def test_pyobject_repr_from_null(self): - s = _testcapi.pyobject_repr_from_null() - self.assertEqual(s, '') - - def test_pyobject_str_from_null(self): - s = _testcapi.pyobject_str_from_null() - self.assertEqual(s, '') - - def test_pyobject_bytes_from_null(self): - s = _testcapi.pyobject_bytes_from_null() - self.assertEqual(s, b'') - - def test_Py_CompileString(self): - # Check that Py_CompileString respects the coding cookie - _compile = _testcapi.Py_CompileString - code = b"# -*- coding: latin1 -*-\nprint('\xc2\xa4')\n" - result = _compile(code) - expected = compile(code, "", "exec") - self.assertEqual(result.co_consts, expected.co_consts) - - def test_export_symbols(self): - # bpo-44133: Ensure that the "Py_FrozenMain" and - # "PyThread_get_thread_native_id" symbols are exported by the Python - # (directly by the binary, or via by the Python dynamic library). - ctypes = import_helper.import_module('ctypes') - names = [] - - # Test if the PY_HAVE_THREAD_NATIVE_ID macro is defined - if hasattr(_thread, 'get_native_id'): - names.append('PyThread_get_thread_native_id') - - # Python/frozenmain.c fails to build on Windows when the symbols are - # missing: - # - PyWinFreeze_ExeInit - # - PyWinFreeze_ExeTerm - # - PyInitFrozenExtensions - if os.name != 'nt': - names.append('Py_FrozenMain') - - for name in names: - self.assertHasAttr(ctypes.pythonapi, name) - - def test_clear_managed_dict(self): - - class C: - def __init__(self): - self.a = 1 - - c = C() - _testcapi.clear_managed_dict(c) - self.assertEqual(c.__dict__, {}) - c = C() - self.assertEqual(c.__dict__, {'a':1}) - _testcapi.clear_managed_dict(c) - self.assertEqual(c.__dict__, {}) - - def test_unstable_gc_new_with_extra_data(self): - class Data(_testcapi.ObjExtraData): - __slots__ = ('x', 'y') - - d = Data() - d.x = 10 - d.y = 20 - d.extra = 30 - self.assertEqual(d.x, 10) - self.assertEqual(d.y, 20) - self.assertEqual(d.extra, 30) - del d.extra - self.assertIsNone(d.extra) - - def test_gen_get_code(self): - def genf(): yield - gen = genf() - self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) - - def test_tp_bases_slot(self): - cls = _testcapi.HeapCTypeWithBasesSlot - self.assertEqual(cls.__bases__, (int,)) - self.assertEqual(cls.__base__, int) - - def test_tp_bases_slot_none(self): - self.assertRaisesRegex( - SystemError, - "Py_tp_bases is not a tuple", - _testcapi.create_heapctype_with_none_bases_slot - ) - - -@requires_limited_api -class TestHeapTypeRelative(unittest.TestCase): - """Test API for extending opaque types (PEP 697)""" - - @requires_limited_api - def test_heaptype_relative_sizes(self): - # Test subclassing using "relative" basicsize, see PEP 697 - def check(extra_base_size, extra_size): - Base, Sub, instance, data_ptr, data_offset, data_size = ( - _testlimitedcapi.make_sized_heaptypes( - extra_base_size, -extra_size)) - - # no alignment shenanigans when inheriting directly - if extra_size == 0: - self.assertEqual(Base.__basicsize__, Sub.__basicsize__) - self.assertEqual(data_size, 0) - - else: - # The following offsets should be in increasing order: - offsets = [ - (0, 'start of object'), - (Base.__basicsize__, 'end of base data'), - (data_offset, 'subclass data'), - (data_offset + extra_size, 'end of requested subcls data'), - (data_offset + data_size, 'end of reserved subcls data'), - (Sub.__basicsize__, 'end of object'), - ] - ordered_offsets = sorted(offsets, key=operator.itemgetter(0)) - self.assertEqual( - offsets, ordered_offsets, - msg=f'Offsets not in expected order, got: {ordered_offsets}') - - # end of reserved subcls data == end of object - self.assertEqual(Sub.__basicsize__, data_offset + data_size) - - # we don't reserve (requested + alignment) or more data - self.assertLess(data_size - extra_size, - _testlimitedcapi.ALIGNOF_MAX_ALIGN_T) - - # The offsets/sizes we calculated should be aligned. - self.assertEqual(data_offset % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) - self.assertEqual(data_size % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) - - sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, - object.__basicsize__, - object.__basicsize__-1, - object.__basicsize__+1}) - for extra_base_size in sizes: - for extra_size in sizes: - args = dict(extra_base_size=extra_base_size, - extra_size=extra_size) - with self.subTest(**args): - check(**args) - - def test_HeapCCollection(self): - """Make sure HeapCCollection works properly by itself""" - collection = _testcapi.HeapCCollection(1, 2, 3) - self.assertEqual(list(collection), [1, 2, 3]) - - def test_heaptype_inherit_itemsize(self): - """Test HeapCCollection subclasses work properly""" - sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, - object.__basicsize__, - object.__basicsize__-1, - object.__basicsize__+1}) - for extra_size in sizes: - with self.subTest(extra_size=extra_size): - Sub = _testlimitedcapi.subclass_var_heaptype( - _testcapi.HeapCCollection, -extra_size, 0, 0) - collection = Sub(1, 2, 3) - collection.set_data_to_3s() - - self.assertEqual(list(collection), [1, 2, 3]) - mem = collection.get_data() - self.assertGreaterEqual(len(mem), extra_size) - self.assertTrue(set(mem) <= {3}, f'got {mem!r}') - - def test_heaptype_invalid_inheritance(self): - with self.assertRaises(SystemError, - msg="Cannot extend variable-size class without " - + "Py_TPFLAGS_ITEMS_AT_END"): - _testlimitedcapi.subclass_heaptype(int, -8, 0) - - def test_heaptype_relative_members(self): - """Test HeapCCollection subclasses work properly""" - sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, - object.__basicsize__, - object.__basicsize__-1, - object.__basicsize__+1}) - for extra_base_size in sizes: - for extra_size in sizes: - for offset in sizes: - with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size, offset=offset): - if offset < extra_size: - Sub = _testlimitedcapi.make_heaptype_with_member( - extra_base_size, -extra_size, offset, True) - Base = Sub.mro()[1] - instance = Sub() - self.assertEqual(instance.memb, instance.get_memb()) - instance.set_memb(13) - self.assertEqual(instance.memb, instance.get_memb()) - self.assertEqual(instance.get_memb(), 13) - instance.memb = 14 - self.assertEqual(instance.memb, instance.get_memb()) - self.assertEqual(instance.get_memb(), 14) - self.assertGreaterEqual(instance.get_memb_offset(), Base.__basicsize__) - self.assertLess(instance.get_memb_offset(), Sub.__basicsize__) - with self.assertRaises(SystemError): - instance.get_memb_relative() - with self.assertRaises(SystemError): - instance.set_memb_relative(0) - else: - with self.assertRaises(SystemError): - Sub = _testlimitedcapi.make_heaptype_with_member( - extra_base_size, -extra_size, offset, True) - with self.assertRaises(SystemError): - Sub = _testlimitedcapi.make_heaptype_with_member( - extra_base_size, extra_size, offset, True) - with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size): - with self.assertRaises(SystemError): - Sub = _testlimitedcapi.make_heaptype_with_member( - extra_base_size, -extra_size, -1, True) - - def test_heaptype_relative_members_errors(self): - with self.assertRaisesRegex( - SystemError, - r"With Py_RELATIVE_OFFSET, basicsize must be negative"): - _testlimitedcapi.make_heaptype_with_member(0, 1234, 0, True) - with self.assertRaisesRegex( - SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): - _testlimitedcapi.make_heaptype_with_member(0, -8, 1234, True) - with self.assertRaisesRegex( - SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): - _testlimitedcapi.make_heaptype_with_member(0, -8, -1, True) - - Sub = _testlimitedcapi.make_heaptype_with_member(0, -8, 0, True) - instance = Sub() - with self.assertRaisesRegex( - SystemError, r"PyMember_GetOne used with Py_RELATIVE_OFFSET"): - instance.get_memb_relative() - with self.assertRaisesRegex( - SystemError, r"PyMember_SetOne used with Py_RELATIVE_OFFSET"): - instance.set_memb_relative(0) - - def test_heaptype_relative_special_members_errors(self): - for member_name in "__vectorcalloffset__", "__dictoffset__", "__weaklistoffset__": - with self.subTest(member_name=member_name): - with self.assertRaisesRegex( - SystemError, - r"With Py_RELATIVE_OFFSET, basicsize must be negative."): - _testlimitedcapi.make_heaptype_with_member( - basicsize=sys.getsizeof(object()) + 100, - add_relative_flag=True, - member_name=member_name, - member_offset=0, - member_type=_testlimitedcapi.Py_T_PYSSIZET, - member_flags=_testlimitedcapi.Py_READONLY, - ) - with self.assertRaisesRegex( - SystemError, - r"Member offset out of range \(0\.\.-basicsize\)"): - _testlimitedcapi.make_heaptype_with_member( - basicsize=-8, - add_relative_flag=True, - member_name=member_name, - member_offset=-1, - member_type=_testlimitedcapi.Py_T_PYSSIZET, - member_flags=_testlimitedcapi.Py_READONLY, - ) - with self.assertRaisesRegex( - SystemError, - r"type of %s must be Py_T_PYSSIZET" % member_name): - _testlimitedcapi.make_heaptype_with_member( - basicsize=-100, - add_relative_flag=True, - member_name=member_name, - member_offset=0, - member_flags=_testlimitedcapi.Py_READONLY, - ) - with self.assertRaisesRegex( - SystemError, - r"flags for %s must be " % member_name): - _testlimitedcapi.make_heaptype_with_member( - basicsize=-100, - add_relative_flag=True, - member_name=member_name, - member_offset=0, - member_type=_testlimitedcapi.Py_T_PYSSIZET, - member_flags=0, - ) - - def test_pyobject_getitemdata_error(self): - """Test PyObject_GetItemData fails on unsupported types""" - with self.assertRaises(TypeError): - # None is not variable-length - _testcapi.pyobject_getitemdata(None) - with self.assertRaises(TypeError): - # int is variable-length, but doesn't have the - # Py_TPFLAGS_ITEMS_AT_END layout (and flag) - _testcapi.pyobject_getitemdata(0) - - -class TestPendingCalls(unittest.TestCase): - - # See the comment in ceval.c (at the "handle_eval_breaker" label) - # about when pending calls get run. This is especially relevant - # here for creating deterministic tests. - - def main_pendingcalls_submit(self, l, n): - def callback(): - #this function can be interrupted by thread switching so let's - #use an atomic operation - l.append(None) - - for i in range(n): - time.sleep(random.random()*0.02) #0.01 secs on average - #try submitting callback until successful. - #rely on regular interrupt to flush queue if we are - #unsuccessful. - while True: - if _testcapi._pending_threadfunc(callback): - break - - def pendingcalls_submit(self, l, n, *, main=True, ensure=False): - def callback(): - #this function can be interrupted by thread switching so let's - #use an atomic operation - l.append(None) - - if main: - return _testcapi._pending_threadfunc(callback, n, - blocking=False, - ensure_added=ensure) - else: - return _testinternalcapi.pending_threadfunc(callback, n, - blocking=False, - ensure_added=ensure) - - def pendingcalls_wait(self, l, numadded, context = None): - #now, stick around until l[0] has grown to 10 - count = 0 - while len(l) != numadded: - #this busy loop is where we expect to be interrupted to - #run our callbacks. Note that some callbacks are only run on the - #main thread - if False and support.verbose: - print("(%i)"%(len(l),),) - for i in range(1000): - a = i*i - if context and not context.event.is_set(): - continue - count += 1 - self.assertTrue(count < 10000, - "timeout waiting for %i callbacks, got %i"%(numadded, len(l))) - if False and support.verbose: - print("(%i)"%(len(l),)) - - @threading_helper.requires_working_threading() - def test_main_pendingcalls_threaded(self): - - #do every callback on a separate thread - n = 32 #total callbacks - threads = [] - class foo(object):pass - context = foo() - context.l = [] - context.n = 2 #submits per thread - context.nThreads = n // context.n - context.nFinished = 0 - context.lock = threading.Lock() - context.event = threading.Event() - - threads = [threading.Thread(target=self.main_pendingcalls_thread, - args=(context,)) - for i in range(context.nThreads)] - with threading_helper.start_threads(threads): - self.pendingcalls_wait(context.l, n, context) - - def main_pendingcalls_thread(self, context): - try: - self.main_pendingcalls_submit(context.l, context.n) - finally: - with context.lock: - context.nFinished += 1 - nFinished = context.nFinished - if False and support.verbose: - print("finished threads: ", nFinished) - if nFinished == context.nThreads: - context.event.set() - - def test_main_pendingcalls_non_threaded(self): - #again, just using the main thread, likely they will all be dispatched at - #once. It is ok to ask for too many, because we loop until we find a slot. - #the loop can be interrupted to dispatch. - #there are only 32 dispatch slots, so we go for twice that! - l = [] - n = 64 - self.main_pendingcalls_submit(l, n) - self.pendingcalls_wait(l, n) - - def test_max_pending(self): - with self.subTest('main-only'): - maxpending = 32 - - l = [] - added = self.pendingcalls_submit(l, 1, main=True) - self.pendingcalls_wait(l, added) - self.assertEqual(added, 1) - - l = [] - added = self.pendingcalls_submit(l, maxpending, main=True) - self.pendingcalls_wait(l, added) - self.assertEqual(added, maxpending) - - l = [] - added = self.pendingcalls_submit(l, maxpending+1, main=True) - self.pendingcalls_wait(l, added) - self.assertEqual(added, maxpending) - - with self.subTest('not main-only'): - # Per-interpreter pending calls has a much higher limit - # on how many may be pending at a time. - maxpending = 300 - - l = [] - added = self.pendingcalls_submit(l, 1, main=False) - self.pendingcalls_wait(l, added) - self.assertEqual(added, 1) - - l = [] - added = self.pendingcalls_submit(l, maxpending, main=False) - self.pendingcalls_wait(l, added) - self.assertEqual(added, maxpending) - - l = [] - added = self.pendingcalls_submit(l, maxpending+1, main=False) - self.pendingcalls_wait(l, added) - self.assertEqual(added, maxpending) - - class PendingTask(types.SimpleNamespace): - - _add_pending = _testinternalcapi.pending_threadfunc - - def __init__(self, req, taskid=None, notify_done=None): - self.id = taskid - self.req = req - self.notify_done = notify_done - - self.creator_tid = threading.get_ident() - self.requester_tid = None - self.runner_tid = None - self.result = None - - def run(self): - assert self.result is None - self.runner_tid = threading.get_ident() - self._run() - if self.notify_done is not None: - self.notify_done() - - def _run(self): - self.result = self.req - - def run_in_pending_call(self, worker_tids): - assert self._add_pending is _testinternalcapi.pending_threadfunc - self.requester_tid = threading.get_ident() - def callback(): - assert self.result is None - # It can be tricky to control which thread handles - # the eval breaker, so we take a naive approach to - # make sure. - if threading.get_ident() not in worker_tids: - self._add_pending(callback, ensure_added=True) - return - self.run() - self._add_pending(callback, ensure_added=True) - - def create_thread(self, worker_tids): - return threading.Thread( - target=self.run_in_pending_call, - args=(worker_tids,), - ) - - def wait_for_result(self): - while self.result is None: - time.sleep(0.01) - - @threading_helper.requires_working_threading() - def test_subthreads_can_handle_pending_calls(self): - payload = 'Spam spam spam spam. Lovely spam! Wonderful spam!' - - task = self.PendingTask(payload) - def do_the_work(): - tid = threading.get_ident() - t = task.create_thread({tid}) - with threading_helper.start_threads([t]): - task.wait_for_result() - t = threading.Thread(target=do_the_work) - with threading_helper.start_threads([t]): - pass - - self.assertEqual(task.result, payload) - - @threading_helper.requires_working_threading() - def test_many_subthreads_can_handle_pending_calls(self): - main_tid = threading.get_ident() - self.assertEqual(threading.main_thread().ident, main_tid) - - # We can't use queue.Queue since it isn't reentrant relative - # to pending calls. - _queue = deque() - _active = deque() - _done_lock = threading.Lock() - def queue_put(task): - _queue.append(task) - _active.append(True) - def queue_get(): - try: - task = _queue.popleft() - except IndexError: - raise queue.Empty - return task - def queue_task_done(): - _active.pop() - if not _active: - try: - _done_lock.release() - except RuntimeError: - assert not _done_lock.locked() - def queue_empty(): - return not _queue - def queue_join(): - _done_lock.acquire() - _done_lock.release() - - tasks = [] - for i in range(20): - task = self.PendingTask( - req=f'request {i}', - taskid=i, - notify_done=queue_task_done, - ) - tasks.append(task) - queue_put(task) - # This will be released once all the tasks have finished. - _done_lock.acquire() - - def add_tasks(worker_tids): - while True: - if done: - return - try: - task = queue_get() - except queue.Empty: - break - task.run_in_pending_call(worker_tids) - - done = False - def run_tasks(): - while not queue_empty(): - if done: - return - time.sleep(0.01) - # Give the worker a chance to handle any remaining pending calls. - while not done: - time.sleep(0.01) - - # Start the workers and wait for them to finish. - worker_threads = [threading.Thread(target=run_tasks) - for _ in range(3)] - with threading_helper.start_threads(worker_threads): - try: - # Add a pending call for each task. - worker_tids = [t.ident for t in worker_threads] - threads = [threading.Thread(target=add_tasks, args=(worker_tids,)) - for _ in range(3)] - with threading_helper.start_threads(threads): - try: - pass - except BaseException: - done = True - raise # re-raise - # Wait for the pending calls to finish. - queue_join() - # Notify the workers that they can stop. - done = True - except BaseException: - done = True - raise # re-raise - runner_tids = [t.runner_tid for t in tasks] - - self.assertNotIn(main_tid, runner_tids) - for task in tasks: - with self.subTest(f'task {task.id}'): - self.assertNotEqual(task.requester_tid, main_tid) - self.assertNotEqual(task.requester_tid, task.runner_tid) - self.assertNotIn(task.requester_tid, runner_tids) - - @requires_subinterpreters - @support.skip_if_sanitizer("gh-129824: race on assign_version_tag", thread=True) - def test_isolated_subinterpreter(self): - # We exercise the most important permutations. - - # This test relies on pending calls getting called - # (eval breaker tripped) at each loop iteration - # and at each call. - - maxtext = 250 - main_interpid = 0 - interpid = _interpreters.create() - self.addCleanup(lambda: _interpreters.destroy(interpid)) - _interpreters.run_string(interpid, f"""if True: - import json - import os - import threading - import time - import _testinternalcapi - from test.support import threading_helper - """) - - def create_pipe(): - r, w = os.pipe() - self.addCleanup(lambda: os.close(r)) - self.addCleanup(lambda: os.close(w)) - return r, w - - with self.subTest('add in main, run in subinterpreter'): - r_ready, w_ready = create_pipe() - r_done, w_done= create_pipe() - timeout = time.time() + 30 # seconds - - def do_work(): - _interpreters.run_string(interpid, f"""if True: - # Wait until this interp has handled the pending call. - waiting = False - done = False - def wait(os_read=os.read): - global done, waiting - waiting = True - os_read({r_done}, 1) - done = True - t = threading.Thread(target=wait) - with threading_helper.start_threads([t]): - while not waiting: - pass - os.write({w_ready}, b'\\0') - # Loop to trigger the eval breaker. - while not done: - time.sleep(0.01) - if time.time() > {timeout}: - raise Exception('timed out!') - """) - t = threading.Thread(target=do_work) - with threading_helper.start_threads([t]): - os.read(r_ready, 1) - # Add the pending call and wait for it to finish. - actual = _testinternalcapi.pending_identify(interpid) - # Signal the subinterpreter to stop. - os.write(w_done, b'\0') - - self.assertEqual(actual, int(interpid)) - - with self.subTest('add in main, run in subinterpreter sub-thread'): - r_ready, w_ready = create_pipe() - r_done, w_done= create_pipe() - timeout = time.time() + 30 # seconds - - def do_work(): - _interpreters.run_string(interpid, f"""if True: - waiting = False - done = False - def subthread(): - while not waiting: - pass - os.write({w_ready}, b'\\0') - # Loop to trigger the eval breaker. - while not done: - time.sleep(0.01) - if time.time() > {timeout}: - raise Exception('timed out!') - t = threading.Thread(target=subthread) - with threading_helper.start_threads([t]): - # Wait until this interp has handled the pending call. - waiting = True - os.read({r_done}, 1) - done = True - """) - t = threading.Thread(target=do_work) - with threading_helper.start_threads([t]): - os.read(r_ready, 1) - # Add the pending call and wait for it to finish. - actual = _testinternalcapi.pending_identify(interpid) - # Signal the subinterpreter to stop. - os.write(w_done, b'\0') - - self.assertEqual(actual, int(interpid)) - - with self.subTest('add in subinterpreter, run in main'): - r_ready, w_ready = create_pipe() - r_done, w_done= create_pipe() - r_data, w_data= create_pipe() - timeout = time.time() + 30 # seconds - - def add_job(): - os.read(r_ready, 1) - _interpreters.run_string(interpid, f"""if True: - # Add the pending call and wait for it to finish. - actual = _testinternalcapi.pending_identify({main_interpid}) - # Signal the subinterpreter to stop. - os.write({w_done}, b'\\0') - os.write({w_data}, actual.to_bytes(1, 'little')) - """) - # Wait until this interp has handled the pending call. - waiting = False - done = False - def wait(os_read=os.read): - nonlocal done, waiting - waiting = True - os_read(r_done, 1) - done = True - t1 = threading.Thread(target=add_job) - t2 = threading.Thread(target=wait) - with threading_helper.start_threads([t1, t2]): - while not waiting: - pass - os.write(w_ready, b'\0') - # Loop to trigger the eval breaker. - while not done: - time.sleep(0.01) - if time.time() > timeout: - raise Exception('timed out!') - text = os.read(r_data, 1) - actual = int.from_bytes(text, 'little') - - self.assertEqual(actual, int(main_interpid)) - - with self.subTest('add in subinterpreter, run in sub-thread'): - r_ready, w_ready = create_pipe() - r_done, w_done= create_pipe() - r_data, w_data= create_pipe() - timeout = time.time() + 30 # seconds - - def add_job(): - os.read(r_ready, 1) - _interpreters.run_string(interpid, f"""if True: - # Add the pending call and wait for it to finish. - actual = _testinternalcapi.pending_identify({main_interpid}) - # Signal the subinterpreter to stop. - os.write({w_done}, b'\\0') - os.write({w_data}, actual.to_bytes(1, 'little')) - """) - # Wait until this interp has handled the pending call. - waiting = False - done = False - def wait(os_read=os.read): - nonlocal done, waiting - waiting = True - os_read(r_done, 1) - done = True - def subthread(): - while not waiting: - pass - os.write(w_ready, b'\0') - # Loop to trigger the eval breaker. - while not done: - time.sleep(0.01) - if time.time() > timeout: - raise Exception('timed out!') - t1 = threading.Thread(target=add_job) - t2 = threading.Thread(target=wait) - t3 = threading.Thread(target=subthread) - with threading_helper.start_threads([t1, t2, t3]): - pass - text = os.read(r_data, 1) - actual = int.from_bytes(text, 'little') - - self.assertEqual(actual, int(main_interpid)) - - # XXX We can't use the rest until gh-105716 is fixed. - return - - with self.subTest('add in subinterpreter, run in subinterpreter sub-thread'): - r_ready, w_ready = create_pipe() - r_done, w_done= create_pipe() - r_data, w_data= create_pipe() - timeout = time.time() + 30 # seconds - - def do_work(): - _interpreters.run_string(interpid, f"""if True: - waiting = False - done = False - def subthread(): - while not waiting: - pass - os.write({w_ready}, b'\\0') - # Loop to trigger the eval breaker. - while not done: - time.sleep(0.01) - if time.time() > {timeout}: - raise Exception('timed out!') - t = threading.Thread(target=subthread) - with threading_helper.start_threads([t]): - # Wait until this interp has handled the pending call. - waiting = True - os.read({r_done}, 1) - done = True - """) - t = threading.Thread(target=do_work) - #with threading_helper.start_threads([t]): - t.start() - if True: - os.read(r_ready, 1) - _interpreters.run_string(interpid, f"""if True: - # Add the pending call and wait for it to finish. - actual = _testinternalcapi.pending_identify({interpid}) - # Signal the subinterpreter to stop. - os.write({w_done}, b'\\0') - os.write({w_data}, actual.to_bytes(1, 'little')) - """) - t.join() - text = os.read(r_data, 1) - actual = int.from_bytes(text, 'little') - - self.assertEqual(actual, int(interpid)) - - -class SubinterpreterTest(unittest.TestCase): - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_subinterps(self): - import builtins - r, w = os.pipe() - code = """if 1: - import sys, builtins, pickle - with open({:d}, "wb") as f: - pickle.dump(id(sys.modules), f) - pickle.dump(id(builtins), f) - """.format(w) - with open(r, "rb") as f: - ret = support.run_in_subinterp(code) - self.assertEqual(ret, 0) - self.assertNotEqual(pickle.load(f), id(sys.modules)) - self.assertNotEqual(pickle.load(f), id(builtins)) - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_subinterps_recent_language_features(self): - r, w = os.pipe() - code = """if 1: - import pickle - with open({:d}, "wb") as f: - - @(lambda x:x) # Py 3.9 - def noop(x): return x - - a = (b := f'1{{2}}3') + noop('x') # Py 3.8 (:=) / 3.6 (f'') - - async def foo(arg): return await arg # Py 3.5 - - pickle.dump(dict(a=a, b=b), f) - """.format(w) - - with open(r, "rb") as f: - ret = support.run_in_subinterp(code) - self.assertEqual(ret, 0) - self.assertEqual(pickle.load(f), {'a': '123x', 'b': '123'}) - - # _testcapi cannot be imported in a subinterpreter on a Free Threaded build - @support.requires_gil_enabled() - def test_py_config_isoloated_per_interpreter(self): - # A config change in one interpreter must not leak to out to others. - # - # This test could verify ANY config value, it just happens to have been - # written around the time of int_max_str_digits. Refactoring is okay. - code = """if 1: - import sys, _testcapi - - # Any config value would do, this happens to be the one being - # double checked at the time this test was written. - _testcapi.config_set('int_max_str_digits', 55555) - sub_value = _testcapi.config_get('int_max_str_digits') - assert sub_value == 55555, sub_value - """ - before_config = _testcapi.config_get('int_max_str_digits') - assert before_config != 55555 - self.assertEqual(support.run_in_subinterp(code), 0, - 'subinterp code failure, check stderr.') - after_config = _testcapi.config_get('int_max_str_digits') - self.assertIsNot( - before_config, after_config, - "Expected get_config() to return a new dict on each call") - self.assertEqual(before_config, after_config, - "CAUTION: Tests executed after this may be " - "running under an altered config.") - # try:...finally: calling set_config(before_config) not done - # as that results in sys.argv, sys.path, and sys.warnoptions - # "being modified by test_capi" per test.regrtest. So if this - # test fails, assume that the environment in this process may - # be altered and suspect. - - @requires_subinterpreters - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_configured_settings(self): - """ - The config with which an interpreter is created corresponds - 1-to-1 with the new interpreter's settings. This test verifies - that they match. - """ - - OBMALLOC = 1<<5 - EXTENSIONS = 1<<8 - THREADS = 1<<10 - DAEMON_THREADS = 1<<11 - FORK = 1<<15 - EXEC = 1<<16 - ALL_FLAGS = (OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS - | EXTENSIONS); - - features = [ - 'obmalloc', - 'fork', - 'exec', - 'threads', - 'daemon_threads', - 'extensions', - 'own_gil', - ] - kwlist = [f'allow_{n}' for n in features] - kwlist[0] = 'use_main_obmalloc' - kwlist[-2] = 'check_multi_interp_extensions' - kwlist[-1] = 'own_gil' - - expected_to_work = { - (True, True, True, True, True, True, True): - (ALL_FLAGS, True), - (True, False, False, False, False, False, False): - (OBMALLOC, False), - (False, False, False, True, False, True, False): - (THREADS | EXTENSIONS, False), - } - - expected_to_fail = { - (False, False, False, False, False, False, False), - } - - # gh-117649: The free-threaded build does not currently allow - # setting check_multi_interp_extensions to False. - if Py_GIL_DISABLED: - for config in list(expected_to_work.keys()): - kwargs = dict(zip(kwlist, config)) - if not kwargs['check_multi_interp_extensions']: - del expected_to_work[config] - expected_to_fail.add(config) - - # expected to work - for config, expected in expected_to_work.items(): - kwargs = dict(zip(kwlist, config)) - exp_flags, exp_gil = expected - expected = { - 'feature_flags': exp_flags, - 'own_gil': exp_gil, - } - with self.subTest(config): - r, w = os.pipe() - script = textwrap.dedent(f''' - import _testinternalcapi, json, os - settings = _testinternalcapi.get_interp_settings() - with os.fdopen({w}, "w") as stdin: - json.dump(settings, stdin) - ''') - with os.fdopen(r) as stdout: - ret = support.run_in_subinterp_with_config(script, **kwargs) - self.assertEqual(ret, 0) - out = stdout.read() - settings = json.loads(out) - - self.assertEqual(settings, expected) - - # expected to fail - for config in expected_to_fail: - kwargs = dict(zip(kwlist, config)) - with self.subTest(config): - script = textwrap.dedent(f''' - import _testinternalcapi - _testinternalcapi.get_interp_settings() - raise NotImplementedError('unreachable') - ''') - with self.assertRaises(_interpreters.InterpreterError): - support.run_in_subinterp_with_config(script, **kwargs) - - @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - # gh-117649: The free-threaded build does not currently allow overriding - # the check_multi_interp_extensions setting. - @expected_failure_if_gil_disabled() - def test_overridden_setting_extensions_subinterp_check(self): - """ - PyInterpreterConfig.check_multi_interp_extensions can be overridden - with PyInterpreterState.override_multi_interp_extensions_check. - This verifies that the override works but does not modify - the underlying setting. - """ - - OBMALLOC = 1<<5 - EXTENSIONS = 1<<8 - THREADS = 1<<10 - DAEMON_THREADS = 1<<11 - FORK = 1<<15 - EXEC = 1<<16 - BASE_FLAGS = OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS - base_kwargs = { - 'use_main_obmalloc': True, - 'allow_fork': True, - 'allow_exec': True, - 'allow_threads': True, - 'allow_daemon_threads': True, - 'own_gil': False, - } - - def check(enabled, override): - kwargs = dict( - base_kwargs, - check_multi_interp_extensions=enabled, - ) - flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS - settings = { - 'feature_flags': flags, - 'own_gil': False, - } - - expected = { - 'requested': override, - 'override__initial': 0, - 'override_after': override, - 'override_restored': 0, - # The override should not affect the config or settings. - 'settings__initial': settings, - 'settings_after': settings, - 'settings_restored': settings, - # These are the most likely values to be wrong. - 'allowed__initial': not enabled, - 'allowed_after': not ((override > 0) if override else enabled), - 'allowed_restored': not enabled, - } - - r, w = os.pipe() - if Py_GIL_DISABLED: - # gh-117649: The test fails before `w` is closed - self.addCleanup(os.close, w) - script = textwrap.dedent(f''' - from test.test_capi.check_config import run_singlephase_check - run_singlephase_check({override}, {w}) - ''') - with os.fdopen(r) as stdout: - ret = support.run_in_subinterp_with_config(script, **kwargs) - self.assertEqual(ret, 0) - out = stdout.read() - results = json.loads(out) - - self.assertEqual(results, expected) - - self.maxDiff = None - - # setting: check disabled - with self.subTest('config: check disabled; override: disabled'): - check(False, -1) - with self.subTest('config: check disabled; override: use config'): - check(False, 0) - with self.subTest('config: check disabled; override: enabled'): - check(False, 1) - - # setting: check enabled - with self.subTest('config: check enabled; override: disabled'): - check(True, -1) - with self.subTest('config: check enabled; override: use config'): - check(True, 0) - with self.subTest('config: check enabled; override: enabled'): - check(True, 1) - - def test_mutate_exception(self): - """ - Exceptions saved in global module state get shared between - individual module instances. This test checks whether or not - a change in one interpreter's module gets reflected into the - other ones. - """ - import binascii - - support.run_in_subinterp("import binascii; binascii.Error.foobar = 'foobar'") - - self.assertNotHasAttr(binascii.Error, "foobar") - - @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") - # gh-117649: The free-threaded build does not currently support sharing - # extension module state between interpreters. - @expected_failure_if_gil_disabled() - def test_module_state_shared_in_global(self): - """ - bpo-44050: Extension module state should be shared between interpreters - when it doesn't support sub-interpreters. - """ - r, w = os.pipe() - self.addCleanup(os.close, r) - self.addCleanup(os.close, w) - - # Apple extensions must be distributed as frameworks. This requires - # a specialist loader. - if support.is_apple_mobile: - loader = "AppleFrameworkLoader" - else: - loader = "ExtensionFileLoader" - - script = textwrap.dedent(f""" - import importlib.machinery - import importlib.util - import os - - fullname = '_test_module_state_shared' - origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.{loader}(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - module = importlib.util.module_from_spec(spec) - attr_id = str(id(module.Error)).encode() - - os.write({w}, attr_id) - """) - exec(script) - main_attr_id = os.read(r, 100) - - ret = support.run_in_subinterp(script) - self.assertEqual(ret, 0) - subinterp_attr_id = os.read(r, 100) - self.assertEqual(main_attr_id, subinterp_attr_id) - - -@requires_subinterpreters -class InterpreterConfigTests(unittest.TestCase): - - supported = { - 'isolated': types.SimpleNamespace( - use_main_obmalloc=False, - allow_fork=False, - allow_exec=False, - allow_threads=True, - allow_daemon_threads=False, - check_multi_interp_extensions=True, - gil='own', - ), - 'legacy': types.SimpleNamespace( - use_main_obmalloc=True, - allow_fork=True, - allow_exec=True, - allow_threads=True, - allow_daemon_threads=True, - check_multi_interp_extensions=bool(Py_GIL_DISABLED), - gil='shared', - ), - 'empty': types.SimpleNamespace( - use_main_obmalloc=False, - allow_fork=False, - allow_exec=False, - allow_threads=False, - allow_daemon_threads=False, - check_multi_interp_extensions=False, - gil='default', - ), - } - gil_supported = ['default', 'shared', 'own'] - - def iter_all_configs(self): - for use_main_obmalloc in (True, False): - for allow_fork in (True, False): - for allow_exec in (True, False): - for allow_threads in (True, False): - for allow_daemon in (True, False): - for checkext in (True, False): - for gil in ('shared', 'own', 'default'): - yield types.SimpleNamespace( - use_main_obmalloc=use_main_obmalloc, - allow_fork=allow_fork, - allow_exec=allow_exec, - allow_threads=allow_threads, - allow_daemon_threads=allow_daemon, - check_multi_interp_extensions=checkext, - gil=gil, - ) - - def assert_ns_equal(self, ns1, ns2, msg=None): - # This is mostly copied from TestCase.assertDictEqual. - self.assertEqual(type(ns1), type(ns2)) - if ns1 == ns2: - return - - import difflib - import pprint - from unittest.util import _common_shorten_repr - standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) - diff = ('\n' + '\n'.join(difflib.ndiff( - pprint.pformat(vars(ns1)).splitlines(), - pprint.pformat(vars(ns2)).splitlines()))) - diff = f'namespace({diff})' - standardMsg = self._truncateMessage(standardMsg, diff) - self.fail(self._formatMessage(msg, standardMsg)) - - def test_predefined_config(self): - def check(name, expected): - expected = self.supported[expected] - args = (name,) if name else () - - config1 = _interpreters.new_config(*args) - self.assert_ns_equal(config1, expected) - self.assertIsNot(config1, expected) - - config2 = _interpreters.new_config(*args) - self.assert_ns_equal(config2, expected) - self.assertIsNot(config2, expected) - self.assertIsNot(config2, config1) - - with self.subTest('default'): - check(None, 'isolated') - - for name in self.supported: - with self.subTest(name): - check(name, name) - - def test_update_from_dict(self): - for name, vanilla in self.supported.items(): - with self.subTest(f'noop ({name})'): - expected = vanilla - overrides = vars(vanilla) - config = _interpreters.new_config(name, **overrides) - self.assert_ns_equal(config, expected) - - with self.subTest(f'change all ({name})'): - overrides = {k: not v for k, v in vars(vanilla).items()} - for gil in self.gil_supported: - if vanilla.gil == gil: - continue - overrides['gil'] = gil - expected = types.SimpleNamespace(**overrides) - config = _interpreters.new_config( - name, **overrides) - self.assert_ns_equal(config, expected) - - # Override individual fields. - for field, old in vars(vanilla).items(): - if field == 'gil': - values = [v for v in self.gil_supported if v != old] - else: - values = [not old] - for val in values: - with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): - overrides = {field: val} - expected = types.SimpleNamespace( - **dict(vars(vanilla), **overrides), - ) - config = _interpreters.new_config( - name, **overrides) - self.assert_ns_equal(config, expected) - - with self.subTest('unsupported field'): - for name in self.supported: - with self.assertRaises(ValueError): - _interpreters.new_config(name, spam=True) - - # Bad values for bool fields. - for field, value in vars(self.supported['empty']).items(): - if field == 'gil': - continue - assert isinstance(value, bool) - for value in [1, '', 'spam', 1.0, None, object()]: - with self.subTest(f'unsupported value ({field}={value!r})'): - with self.assertRaises(TypeError): - _interpreters.new_config(**{field: value}) - - # Bad values for .gil. - for value in [True, 1, 1.0, None, object()]: - with self.subTest(f'unsupported value(gil={value!r})'): - with self.assertRaises(TypeError): - _interpreters.new_config(gil=value) - for value in ['', 'spam']: - with self.subTest(f'unsupported value (gil={value!r})'): - with self.assertRaises(ValueError): - _interpreters.new_config(gil=value) - - def test_interp_init(self): - questionable = [ - # strange - dict( - allow_fork=True, - allow_exec=False, - ), - dict( - gil='shared', - use_main_obmalloc=False, - ), - # risky - dict( - allow_fork=True, - allow_threads=True, - ), - # ought to be invalid? - dict( - allow_threads=False, - allow_daemon_threads=True, - ), - dict( - gil='own', - use_main_obmalloc=True, - ), - ] - invalid = [ - dict( - use_main_obmalloc=False, - check_multi_interp_extensions=False - ), - ] - if Py_GIL_DISABLED: - invalid.append(dict(check_multi_interp_extensions=False)) - def match(config, override_cases): - ns = vars(config) - for overrides in override_cases: - if dict(ns, **overrides) == ns: - return True - return False - - def check(config): - script = 'pass' - rc = _testinternalcapi.run_in_subinterp_with_config(script, config) - self.assertEqual(rc, 0) - - for config in self.iter_all_configs(): - if config.gil == 'default': - continue - if match(config, invalid): - with self.subTest(f'invalid: {config}'): - with self.assertRaises(_interpreters.InterpreterError): - check(config) - elif match(config, questionable): - with self.subTest(f'questionable: {config}'): - check(config) - else: - with self.subTest(f'valid: {config}'): - check(config) - - def test_get_config(self): - @contextlib.contextmanager - def new_interp(config): - interpid = _interpreters.create(config, reqrefs=False) - try: - yield interpid - finally: - try: - _interpreters.destroy(interpid) - except _interpreters.InterpreterNotFoundError: - pass - - with self.subTest('main'): - expected = _interpreters.new_config('legacy') - expected.gil = 'own' - if Py_GIL_DISABLED: - expected.check_multi_interp_extensions = False - interpid, *_ = _interpreters.get_main() - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('isolated'): - expected = _interpreters.new_config('isolated') - with new_interp('isolated') as interpid: - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('legacy'): - expected = _interpreters.new_config('legacy') - with new_interp('legacy') as interpid: - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('custom'): - orig = _interpreters.new_config( - 'empty', - use_main_obmalloc=True, - gil='shared', - check_multi_interp_extensions=bool(Py_GIL_DISABLED), - ) - with new_interp(orig) as interpid: - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, orig) - - -@requires_subinterpreters -class InterpreterIDTests(unittest.TestCase): - - def add_interp_cleanup(self, interpid): - def ensure_destroyed(): - try: - _interpreters.destroy(interpid) - except _interpreters.InterpreterNotFoundError: - pass - self.addCleanup(ensure_destroyed) - - def new_interpreter(self): - id = _interpreters.create() - self.add_interp_cleanup(id) - return id - - def test_conversion_int(self): - convert = _testinternalcapi.normalize_interp_id - interpid = convert(10) - self.assertEqual(interpid, 10) - - def test_conversion_coerced(self): - convert = _testinternalcapi.normalize_interp_id - class MyInt(str): - def __index__(self): - return 10 - interpid = convert(MyInt()) - self.assertEqual(interpid, 10) - - def test_conversion_from_interpreter(self): - convert = _testinternalcapi.normalize_interp_id - interpid = self.new_interpreter() - converted = convert(interpid) - self.assertEqual(converted, interpid) - - def test_conversion_bad(self): - convert = _testinternalcapi.normalize_interp_id - - for badid in [ - object(), - 10.0, - '10', - b'10', - ]: - with self.subTest(f'bad: {badid!r}'): - with self.assertRaises(TypeError): - convert(badid) - - badid = -1 - with self.subTest(f'bad: {badid!r}'): - with self.assertRaises(ValueError): - convert(badid) - - badid = 2**64 - with self.subTest(f'bad: {badid!r}'): - with self.assertRaises(OverflowError): - convert(badid) - - def test_lookup_exists(self): - interpid = self.new_interpreter() - self.assertTrue( - _testinternalcapi.interpreter_exists(interpid)) - - def test_lookup_does_not_exist(self): - interpid = _testinternalcapi.unused_interpreter_id() - self.assertFalse( - _testinternalcapi.interpreter_exists(interpid)) - - def test_lookup_destroyed(self): - interpid = _interpreters.create() - _interpreters.destroy(interpid) - self.assertFalse( - _testinternalcapi.interpreter_exists(interpid)) - - def get_refcount_helpers(self): - return ( - _testinternalcapi.get_interpreter_refcount, - (lambda id: _interpreters.incref(id, implieslink=False)), - _interpreters.decref, - ) - - def test_linked_lifecycle_does_not_exist(self): - exists = _testinternalcapi.interpreter_exists - is_linked = _testinternalcapi.interpreter_refcount_linked - link = _testinternalcapi.link_interpreter_refcount - unlink = _testinternalcapi.unlink_interpreter_refcount - get_refcount, incref, decref = self.get_refcount_helpers() - - with self.subTest('never existed'): - interpid = _testinternalcapi.unused_interpreter_id() - self.assertFalse( - exists(interpid)) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - is_linked(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - link(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - unlink(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - get_refcount(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - incref(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - decref(interpid) - - with self.subTest('destroyed'): - interpid = _interpreters.create() - _interpreters.destroy(interpid) - self.assertFalse( - exists(interpid)) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - is_linked(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - link(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - unlink(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - get_refcount(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - incref(interpid) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - decref(interpid) - - def test_linked_lifecycle_initial(self): - is_linked = _testinternalcapi.interpreter_refcount_linked - get_refcount, _, _ = self.get_refcount_helpers() - - # A new interpreter will start out not linked, with a refcount of 0. - interpid = self.new_interpreter() - linked = is_linked(interpid) - refcount = get_refcount(interpid) - - self.assertFalse(linked) - self.assertEqual(refcount, 0) - - def test_linked_lifecycle_never_linked(self): - exists = _testinternalcapi.interpreter_exists - is_linked = _testinternalcapi.interpreter_refcount_linked - get_refcount, incref, decref = self.get_refcount_helpers() - - interpid = self.new_interpreter() - - # Incref will not automatically link it. - incref(interpid) - self.assertFalse( - is_linked(interpid)) - self.assertEqual( - 1, get_refcount(interpid)) - - # It isn't linked so it isn't destroyed. - decref(interpid) - self.assertTrue( - exists(interpid)) - self.assertFalse( - is_linked(interpid)) - self.assertEqual( - 0, get_refcount(interpid)) - - def test_linked_lifecycle_link_unlink(self): - exists = _testinternalcapi.interpreter_exists - is_linked = _testinternalcapi.interpreter_refcount_linked - link = _testinternalcapi.link_interpreter_refcount - unlink = _testinternalcapi.unlink_interpreter_refcount - - interpid = self.new_interpreter() - - # Linking at refcount 0 does not destroy the interpreter. - link(interpid) - self.assertTrue( - exists(interpid)) - self.assertTrue( - is_linked(interpid)) - - # Unlinking at refcount 0 does not destroy the interpreter. - unlink(interpid) - self.assertTrue( - exists(interpid)) - self.assertFalse( - is_linked(interpid)) - - def test_linked_lifecycle_link_incref_decref(self): - exists = _testinternalcapi.interpreter_exists - is_linked = _testinternalcapi.interpreter_refcount_linked - link = _testinternalcapi.link_interpreter_refcount - get_refcount, incref, decref = self.get_refcount_helpers() - - interpid = self.new_interpreter() - - # Linking it will not change the refcount. - link(interpid) - self.assertTrue( - is_linked(interpid)) - self.assertEqual( - 0, get_refcount(interpid)) - - # Decref with a refcount of 0 is not allowed. - incref(interpid) - self.assertEqual( - 1, get_refcount(interpid)) - - # When linked, decref back to 0 destroys the interpreter. - decref(interpid) - self.assertFalse( - exists(interpid)) - - def test_linked_lifecycle_incref_link(self): - is_linked = _testinternalcapi.interpreter_refcount_linked - link = _testinternalcapi.link_interpreter_refcount - get_refcount, incref, _ = self.get_refcount_helpers() - - interpid = self.new_interpreter() - - incref(interpid) - self.assertEqual( - 1, get_refcount(interpid)) - - # Linking it will not reset the refcount. - link(interpid) - self.assertTrue( - is_linked(interpid)) - self.assertEqual( - 1, get_refcount(interpid)) - - def test_linked_lifecycle_link_incref_unlink_decref(self): - exists = _testinternalcapi.interpreter_exists - is_linked = _testinternalcapi.interpreter_refcount_linked - link = _testinternalcapi.link_interpreter_refcount - unlink = _testinternalcapi.unlink_interpreter_refcount - get_refcount, incref, decref = self.get_refcount_helpers() - - interpid = self.new_interpreter() - - link(interpid) - self.assertTrue( - is_linked(interpid)) - - incref(interpid) - self.assertEqual( - 1, get_refcount(interpid)) - - # Unlinking it will not change the refcount. - unlink(interpid) - self.assertFalse( - is_linked(interpid)) - self.assertEqual( - 1, get_refcount(interpid)) - - # Unlinked: decref back to 0 does not destroys the interpreter. - decref(interpid) - self.assertTrue( - exists(interpid)) - self.assertEqual( - 0, get_refcount(interpid)) - - -class TestStaticTypes(unittest.TestCase): - - _has_run = False - - @classmethod - def setUpClass(cls): - # The tests here don't play nice with our approach to refleak - # detection, so we bail out in that case. - if cls._has_run: - raise unittest.SkipTest('these tests do not support re-running') - cls._has_run = True - - @contextlib.contextmanager - def basic_static_type(self, *args): - cls = _testcapi.get_basic_static_type(*args) - yield cls - - def test_pytype_ready_always_sets_tp_type(self): - # The point of this test is to prevent something like - # https://github.com/python/cpython/issues/104614 - # from happening again. - - # First check when tp_base/tp_bases is *not* set before PyType_Ready(). - with self.basic_static_type() as cls: - self.assertIs(cls.__base__, object); - self.assertEqual(cls.__bases__, (object,)); - self.assertIs(type(cls), type(object)); - - # Then check when we *do* set tp_base/tp_bases first. - with self.basic_static_type(object) as cls: - self.assertIs(cls.__base__, object); - self.assertEqual(cls.__bases__, (object,)); - self.assertIs(type(cls), type(object)); - - -class TestThreadState(unittest.TestCase): - - @threading_helper.reap_threads - @threading_helper.requires_working_threading() - def test_thread_state(self): - # some extra thread-state tests driven via _testcapi - def target(): - idents = [] - - def callback(): - idents.append(threading.get_ident()) - - _testcapi._test_thread_state(callback) - a = b = callback - time.sleep(1) - # Check our main thread is in the list exactly 3 times. - self.assertEqual(idents.count(threading.get_ident()), 3, - "Couldn't find main thread correctly in the list") - - target() - t = threading.Thread(target=target) - t.start() - t.join() - - @threading_helper.reap_threads - @threading_helper.requires_working_threading() - def test_thread_gilstate_in_clear(self): - # See https://github.com/python/cpython/issues/119585 - class C: - def __del__(self): - _testcapi.gilstate_ensure_release() - - # Thread-local variables are destroyed in `PyThreadState_Clear()`. - local_var = threading.local() - - def callback(): - local_var.x = C() - - _testcapi._test_thread_state(callback) - - @threading_helper.reap_threads - @threading_helper.requires_working_threading() - def test_gilstate_ensure_no_deadlock(self): - # See https://github.com/python/cpython/issues/96071 - code = textwrap.dedent(""" - import _testcapi - - def callback(): - print('callback called') - - _testcapi._test_thread_state(callback) - """) - ret = assert_python_ok('-X', 'tracemalloc', '-c', code) - self.assertIn(b'callback called', ret.out) - - def test_gilstate_matches_current(self): - _testcapi.test_current_tstate_matches() - - -def get_test_funcs(mod, exclude_prefix=None): - funcs = {} - for name in dir(mod): - if not name.startswith('test_'): - continue - if exclude_prefix is not None and name.startswith(exclude_prefix): - continue - funcs[name] = getattr(mod, name) - return funcs - - -class Test_testcapi(unittest.TestCase): - locals().update(get_test_funcs(_testcapi)) - - # Suppress warning from PyUnicode_FromUnicode(). - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_widechar(self): - _testlimitedcapi.test_widechar() - - def test_version_api_data(self): - self.assertEqual(_testcapi.Py_Version, sys.hexversion) - - -class Test_testlimitedcapi(unittest.TestCase): - locals().update(get_test_funcs(_testlimitedcapi)) - - -class Test_testinternalcapi(unittest.TestCase): - locals().update(get_test_funcs(_testinternalcapi, - exclude_prefix='test_lock_')) - - -@threading_helper.requires_working_threading() -class Test_PyLock(unittest.TestCase): - locals().update((name, getattr(_testinternalcapi, name)) - for name in dir(_testinternalcapi) - if name.startswith('test_lock_')) - - -@unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") -class Test_ModuleStateAccess(unittest.TestCase): - """Test access to module start (PEP 573)""" - - # The C part of the tests lives in _testmultiphase, in a module called - # _testmultiphase_meth_state_access. - # This module has multi-phase initialization, unlike _testcapi. - - def setUp(self): - fullname = '_testmultiphase_meth_state_access' # XXX - origin = importlib.util.find_spec('_testmultiphase').origin - # Apple extensions must be distributed as frameworks. This requires - # a specialist loader. - if support.is_apple_mobile: - loader = importlib.machinery.AppleFrameworkLoader(fullname, origin) - else: - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - module = importlib.util.module_from_spec(spec) - loader.exec_module(module) - self.module = module - - def test_subclass_get_module(self): - """PyType_GetModule for defining_class""" - class StateAccessType_Subclass(self.module.StateAccessType): - pass - - instance = StateAccessType_Subclass() - self.assertIs(instance.get_defining_module(), self.module) - - def test_subclass_get_module_with_super(self): - class StateAccessType_Subclass(self.module.StateAccessType): - def get_defining_module(self): - return super().get_defining_module() - - instance = StateAccessType_Subclass() - self.assertIs(instance.get_defining_module(), self.module) - - def test_state_access(self): - """Checks methods defined with and without argument clinic - - This tests a no-arg method (get_count) and a method with - both a positional and keyword argument. - """ - - a = self.module.StateAccessType() - b = self.module.StateAccessType() - - methods = { - 'clinic': a.increment_count_clinic, - 'noclinic': a.increment_count_noclinic, - } - - for name, increment_count in methods.items(): - with self.subTest(name): - self.assertEqual(a.get_count(), b.get_count()) - self.assertEqual(a.get_count(), 0) - - increment_count() - self.assertEqual(a.get_count(), b.get_count()) - self.assertEqual(a.get_count(), 1) - - increment_count(3) - self.assertEqual(a.get_count(), b.get_count()) - self.assertEqual(a.get_count(), 4) - - increment_count(-2, twice=True) - self.assertEqual(a.get_count(), b.get_count()) - self.assertEqual(a.get_count(), 0) - - with self.assertRaises(TypeError): - increment_count(thrice=3) - - with self.assertRaises(TypeError): - increment_count(1, 2, 3) - - def test_get_module_bad_def(self): - # PyType_GetModuleByDef fails gracefully if it doesn't - # find what it's looking for. - # see bpo-46433 - instance = self.module.StateAccessType() - with self.assertRaises(TypeError): - instance.getmodulebydef_bad_def() - - def test_get_module_static_in_mro(self): - # Here, the class PyType_GetModuleByDef is looking for - # appears in the MRO after a static type (Exception). - # see bpo-46433 - class Subclass(BaseException, self.module.StateAccessType): - pass - self.assertIs(Subclass().get_defining_module(), self.module) - - -class TestInternalFrameApi(unittest.TestCase): - - @staticmethod - def func(): - return sys._getframe() - - def test_code(self): - frame = self.func() - code = _testinternalcapi.iframe_getcode(frame) - self.assertIs(code, self.func.__code__) - - def test_lasti(self): - frame = self.func() - lasti = _testinternalcapi.iframe_getlasti(frame) - self.assertGreater(lasti, 0) - self.assertLess(lasti, len(self.func.__code__.co_code)) - - def test_line(self): - frame = self.func() - line = _testinternalcapi.iframe_getline(frame) - firstline = self.func.__code__.co_firstlineno - self.assertEqual(line, firstline + 2) - - -SUFFICIENT_TO_DEOPT_AND_SPECIALIZE = 100 - -class Test_Pep523API(unittest.TestCase): - - def do_test(self, func, names): - actual_calls = [] - start = SUFFICIENT_TO_DEOPT_AND_SPECIALIZE - count = start + SUFFICIENT_TO_DEOPT_AND_SPECIALIZE - try: - for i in range(count): - if i == start: - _testinternalcapi.set_eval_frame_record(actual_calls) - func() - finally: - _testinternalcapi.set_eval_frame_default() - expected_calls = names * SUFFICIENT_TO_DEOPT_AND_SPECIALIZE - self.assertEqual(len(expected_calls), len(actual_calls)) - for expected, actual in zip(expected_calls, actual_calls, strict=True): - self.assertEqual(expected, actual) - - def test_inlined_binary_subscr(self): - class C: - def __getitem__(self, other): - return None - def func(): - C()[42] - names = ["func", "__getitem__"] - self.do_test(func, names) - - def test_inlined_call(self): - def inner(x=42): - pass - def func(): - inner() - inner(42) - names = ["func", "inner", "inner"] - self.do_test(func, names) - - def test_inlined_call_function_ex(self): - def inner(x): - pass - def func(): - inner(*[42]) - names = ["func", "inner"] - self.do_test(func, names) - - def test_inlined_for_iter(self): - def gen(): - yield 42 - def func(): - for _ in gen(): - pass - names = ["func", "gen", "gen", "gen"] - self.do_test(func, names) - - def test_inlined_load_attr(self): - class C: - @property - def a(self): - return 42 - class D: - def __getattribute__(self, name): - return 42 - def func(): - C().a - D().a - names = ["func", "a", "__getattribute__"] - self.do_test(func, names) - - def test_inlined_send(self): - def inner(): - yield 42 - def outer(): - yield from inner() - def func(): - list(outer()) - names = ["func", "outer", "outer", "inner", "inner", "outer", "inner"] - self.do_test(func, names) - - -@unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') -class TestPyThreadId(unittest.TestCase): - def test_py_thread_id(self): - # gh-112535: Test _Py_ThreadId(): make sure that thread identifiers - # in a few threads are unique - py_thread_id = _testinternalcapi.py_thread_id - short_sleep = 0.010 - - class GetThreadId(threading.Thread): - def __init__(self): - super().__init__() - self.get_lock = threading.Lock() - self.get_lock.acquire() - self.started_lock = threading.Event() - self.py_tid = None - - def run(self): - self.started_lock.set() - self.get_lock.acquire() - self.py_tid = py_thread_id() - time.sleep(short_sleep) - self.py_tid2 = py_thread_id() - - nthread = 5 - threads = [GetThreadId() for _ in range(nthread)] - - # first make run sure that all threads are running - for thread in threads: - thread.start() - for thread in threads: - thread.started_lock.wait() - - # call _Py_ThreadId() in the main thread - py_thread_ids = [py_thread_id()] - - # now call _Py_ThreadId() in each thread - for thread in threads: - thread.get_lock.release() - - # call _Py_ThreadId() in each thread and wait until threads complete - for thread in threads: - thread.join() - py_thread_ids.append(thread.py_tid) - # _PyThread_Id() should not change for a given thread. - # For example, it should remain the same after a short sleep. - self.assertEqual(thread.py_tid2, thread.py_tid) - - # make sure that all _Py_ThreadId() are unique - for tid in py_thread_ids: - self.assertIsInstance(tid, int) - self.assertGreater(tid, 0) - self.assertEqual(len(set(py_thread_ids)), len(py_thread_ids), - py_thread_ids) - -class TestVersions(unittest.TestCase): - full_cases = ( - (3, 4, 1, 0xA, 2, 0x030401a2), - (3, 10, 0, 0xF, 0, 0x030a00f0), - (0x103, 0x10B, 0xFF00, -1, 0xF0, 0x030b00f0), # test masking - ) - xy_cases = ( - (3, 4, 0x03040000), - (3, 10, 0x030a0000), - (0x103, 0x10B, 0x030b0000), # test masking - ) - - def test_pack_full_version(self): - for *args, expected in self.full_cases: - with self.subTest(hexversion=hex(expected)): - result = _testlimitedcapi.pack_full_version(*args) - self.assertEqual(result, expected) - - def test_pack_version(self): - for *args, expected in self.xy_cases: - with self.subTest(hexversion=hex(expected)): - result = _testlimitedcapi.pack_version(*args) - self.assertEqual(result, expected) - - def test_pack_full_version_ctypes(self): - ctypes = import_helper.import_module('ctypes') - ctypes_func = ctypes.pythonapi.Py_PACK_FULL_VERSION - ctypes_func.restype = ctypes.c_uint32 - ctypes_func.argtypes = [ctypes.c_int] * 5 - for *args, expected in self.full_cases: - with self.subTest(hexversion=hex(expected)): - result = ctypes_func(*args) - self.assertEqual(result, expected) - - def test_pack_version_ctypes(self): - ctypes = import_helper.import_module('ctypes') - ctypes_func = ctypes.pythonapi.Py_PACK_VERSION - ctypes_func.restype = ctypes.c_uint32 - ctypes_func.argtypes = [ctypes.c_int] * 2 - for *args, expected in self.xy_cases: - with self.subTest(hexversion=hex(expected)): - result = ctypes_func(*args) - self.assertEqual(result, expected) - - -class TestCEval(unittest.TestCase): - def test_ceval_decref(self): - code = textwrap.dedent(""" - import _testcapi - _testcapi.toggle_reftrace_printer(True) - l1 = [] - l2 = [] - del l1 - del l2 - _testcapi.toggle_reftrace_printer(False) - """) - _, out, _ = assert_python_ok("-c", code) - lines = out.decode("utf-8").splitlines() - self.assertEqual(lines.count("CREATE list"), 2) - self.assertEqual(lines.count("DESTROY list"), 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_number.py b/Lib/test/test_capi/test_number.py deleted file mode 100644 index bdd8868529f..00000000000 --- a/Lib/test/test_capi/test_number.py +++ /dev/null @@ -1,352 +0,0 @@ -import itertools -import operator -import unittest -import warnings - -from test.support import import_helper - -_testcapi = import_helper.import_module('_testcapi') -from _testcapi import PY_SSIZE_T_MAX, PY_SSIZE_T_MIN - -try: - from _testbuffer import ndarray -except ImportError: - ndarray = None - -NULL = None - -class BadDescr: - def __get__(self, obj, objtype=None): - raise RuntimeError - -class WithDunder: - def _meth(self, *args): - if self.val: - return self.val - if self.exc: - raise self.exc - @classmethod - def with_val(cls, val): - obj = super().__new__(cls) - obj.val = val - obj.exc = None - setattr(cls, cls.methname, cls._meth) - return obj - - @classmethod - def with_exc(cls, exc): - obj = super().__new__(cls) - obj.val = None - obj.exc = exc - setattr(cls, cls.methname, cls._meth) - return obj - -class HasBadAttr: - def __new__(cls): - obj = super().__new__(cls) - setattr(cls, cls.methname, BadDescr()) - return obj - - -class IndexLike(WithDunder): - methname = '__index__' - -class IntLike(WithDunder): - methname = '__int__' - -class FloatLike(WithDunder): - methname = '__float__' - - -def subclassof(base): - return type(base.__name__ + 'Subclass', (base,), {}) - - -class SomeError(Exception): - pass - -class OtherError(Exception): - pass - - -class CAPITest(unittest.TestCase): - def test_check(self): - # Test PyNumber_Check() - check = _testcapi.number_check - - self.assertTrue(check(1)) - self.assertTrue(check(IndexLike.with_val(1))) - self.assertTrue(check(IntLike.with_val(99))) - self.assertTrue(check(0.5)) - self.assertTrue(check(FloatLike.with_val(4.25))) - self.assertTrue(check(1+2j)) - - self.assertFalse(check([])) - self.assertFalse(check("abc")) - self.assertFalse(check(object())) - self.assertFalse(check(NULL)) - - def test_unary_ops(self): - methmap = {'__neg__': _testcapi.number_negative, # PyNumber_Negative() - '__pos__': _testcapi.number_positive, # PyNumber_Positive() - '__abs__': _testcapi.number_absolute, # PyNumber_Absolute() - '__invert__': _testcapi.number_invert} # PyNumber_Invert() - - for name, func in methmap.items(): - # Generic object, has no tp_as_number structure - self.assertRaises(TypeError, func, object()) - - # C-API function accepts NULL - self.assertRaises(SystemError, func, NULL) - - # Behave as corresponding unary operation - op = getattr(operator, name) - for x in [0, 42, -1, 3.14, 1+2j]: - try: - op(x) - except TypeError: - self.assertRaises(TypeError, func, x) - else: - self.assertEqual(func(x), op(x)) - - def test_binary_ops(self): - methmap = {'__add__': _testcapi.number_add, # PyNumber_Add() - '__sub__': _testcapi.number_subtract, # PyNumber_Subtract() - '__mul__': _testcapi.number_multiply, # PyNumber_Multiply() - '__matmul__': _testcapi.number_matrixmultiply, # PyNumber_MatrixMultiply() - '__floordiv__': _testcapi.number_floordivide, # PyNumber_FloorDivide() - '__truediv__': _testcapi.number_truedivide, # PyNumber_TrueDivide() - '__mod__': _testcapi.number_remainder, # PyNumber_Remainder() - '__divmod__': _testcapi.number_divmod, # PyNumber_Divmod() - '__lshift__': _testcapi.number_lshift, # PyNumber_Lshift() - '__rshift__': _testcapi.number_rshift, # PyNumber_Rshift() - '__and__': _testcapi.number_and, # PyNumber_And() - '__xor__': _testcapi.number_xor, # PyNumber_Xor() - '__or__': _testcapi.number_or, # PyNumber_Or() - '__pow__': _testcapi.number_power, # PyNumber_Power() - '__iadd__': _testcapi.number_inplaceadd, # PyNumber_InPlaceAdd() - '__isub__': _testcapi.number_inplacesubtract, # PyNumber_InPlaceSubtract() - '__imul__': _testcapi.number_inplacemultiply, # PyNumber_InPlaceMultiply() - '__imatmul__': _testcapi.number_inplacematrixmultiply, # PyNumber_InPlaceMatrixMultiply() - '__ifloordiv__': _testcapi.number_inplacefloordivide, # PyNumber_InPlaceFloorDivide() - '__itruediv__': _testcapi.number_inplacetruedivide, # PyNumber_InPlaceTrueDivide() - '__imod__': _testcapi.number_inplaceremainder, # PyNumber_InPlaceRemainder() - '__ilshift__': _testcapi.number_inplacelshift, # PyNumber_InPlaceLshift() - '__irshift__': _testcapi.number_inplacershift, # PyNumber_InPlaceRshift() - '__iand__': _testcapi.number_inplaceand, # PyNumber_InPlaceAnd() - '__ixor__': _testcapi.number_inplacexor, # PyNumber_InPlaceXor() - '__ior__': _testcapi.number_inplaceor, # PyNumber_InPlaceOr() - '__ipow__': _testcapi.number_inplacepower, # PyNumber_InPlacePower() - } - - for name, func in methmap.items(): - cases = [0, 42, 3.14, -1, 123, 1+2j] - - # Generic object, has no tp_as_number structure - for x in cases: - self.assertRaises(TypeError, func, object(), x) - self.assertRaises(TypeError, func, x, object()) - - # Behave as corresponding binary operation - op = getattr(operator, name, divmod) - for x, y in itertools.combinations(cases, 2): - try: - op(x, y) - except (TypeError, ValueError, ZeroDivisionError) as exc: - self.assertRaises(exc.__class__, func, x, y) - else: - self.assertEqual(func(x, y), op(x, y)) - - # CRASHES func(NULL, object()) - # CRASHES func(object(), NULL) - - @unittest.skipIf(ndarray is None, "needs _testbuffer") - def test_misc_add(self): - # PyNumber_Add(), PyNumber_InPlaceAdd() - add = _testcapi.number_add - inplaceadd = _testcapi.number_inplaceadd - - # test sq_concat/sq_inplace_concat slots - a, b, r = [1, 2], [3, 4], [1, 2, 3, 4] - self.assertEqual(add(a, b), r) - self.assertEqual(a, [1, 2]) - self.assertRaises(TypeError, add, ndarray([1], (1,)), 2) - a, b, r = [1, 2], [3, 4], [1, 2, 3, 4] - self.assertEqual(inplaceadd(a, b), r) - self.assertEqual(a, r) - self.assertRaises(TypeError, inplaceadd, ndarray([1], (1,)), 2) - - @unittest.skipIf(ndarray is None, "needs _testbuffer") - def test_misc_multiply(self): - # PyNumber_Multiply(), PyNumber_InPlaceMultiply() - multiply = _testcapi.number_multiply - inplacemultiply = _testcapi.number_inplacemultiply - - # test sq_repeat/sq_inplace_repeat slots - a, b, r = [1], 2, [1, 1] - self.assertEqual(multiply(a, b), r) - self.assertEqual((a, b), ([1], 2)) - self.assertEqual(multiply(b, a), r) - self.assertEqual((a, b), ([1], 2)) - self.assertEqual(multiply([1], -1), []) - self.assertRaises(TypeError, multiply, ndarray([1], (1,)), 2) - self.assertRaises(TypeError, multiply, [1], 0.5) - self.assertRaises(OverflowError, multiply, [1], PY_SSIZE_T_MAX + 1) - self.assertRaises(MemoryError, multiply, [1, 2], PY_SSIZE_T_MAX//2 + 1) - a, b, r = [1], 2, [1, 1] - self.assertEqual(inplacemultiply(a, b), r) - self.assertEqual((a, b), (r, 2)) - a = [1] - self.assertEqual(inplacemultiply(b, a), r) - self.assertEqual((a, b), ([1], 2)) - self.assertRaises(TypeError, inplacemultiply, ndarray([1], (1,)), 2) - self.assertRaises(OverflowError, inplacemultiply, [1], PY_SSIZE_T_MAX + 1) - self.assertRaises(MemoryError, inplacemultiply, [1, 2], PY_SSIZE_T_MAX//2 + 1) - - def test_misc_power(self): - # PyNumber_Power(), PyNumber_InPlacePower() - power = _testcapi.number_power - inplacepower = _testcapi.number_inplacepower - - class HasPow(WithDunder): - methname = '__pow__' - - # ternary op - self.assertEqual(power(4, 11, 5), pow(4, 11, 5)) - self.assertRaises(TypeError, power, 4, 11, 1.25) - self.assertRaises(TypeError, power, 4, 11, HasPow.with_val(NotImplemented)) - self.assertRaises(TypeError, power, 4, 11, object()) - self.assertEqual(inplacepower(4, 11, 5), pow(4, 11, 5)) - self.assertRaises(TypeError, inplacepower, 4, 11, 1.25) - self.assertRaises(TypeError, inplacepower, 4, 11, object()) - - class X: - def __pow__(*args): - return args - - x = X() - self.assertEqual(power(x, 11), (x, 11)) - self.assertEqual(inplacepower(x, 11), (x, 11)) - self.assertEqual(power(x, 11, 5), (x, 11, 5)) - self.assertEqual(inplacepower(x, 11, 5), (x, 11, 5)) - - class X: - def __rpow__(*args): - return args - - x = X() - self.assertEqual(power(4, x), (x, 4)) - self.assertEqual(inplacepower(4, x), (x, 4)) - self.assertEqual(power(4, x, 5), (x, 4, 5)) - self.assertEqual(inplacepower(4, x, 5), (x, 4, 5)) - - class X: - def __ipow__(*args): - return args - - x = X() - self.assertEqual(inplacepower(x, 11), (x, 11)) - # XXX: In-place power doesn't pass the third arg to __ipow__. - self.assertEqual(inplacepower(x, 11, 5), (x, 11)) - - def test_long(self): - # Test PyNumber_Long() - long = _testcapi.number_long - - self.assertEqual(long(42), 42) - self.assertEqual(long(1.25), 1) - self.assertEqual(long("42"), 42) - self.assertEqual(long(b"42"), 42) - self.assertEqual(long(bytearray(b"42")), 42) - self.assertEqual(long(memoryview(b"42")), 42) - self.assertEqual(long(IndexLike.with_val(99)), 99) - self.assertEqual(long(IntLike.with_val(99)), 99) - - self.assertRaises(TypeError, long, IntLike.with_val(1.0)) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, long, IntLike.with_val(True)) - with self.assertWarns(DeprecationWarning): - self.assertEqual(long(IntLike.with_val(True)), 1) - self.assertRaises(RuntimeError, long, IntLike.with_exc(RuntimeError)) - - self.assertRaises(TypeError, long, 1j) - self.assertRaises(TypeError, long, object()) - self.assertRaises(SystemError, long, NULL) - - def test_float(self): - # Test PyNumber_Float() - float_ = _testcapi.number_float - - self.assertEqual(float_(1.25), 1.25) - self.assertEqual(float_(123), 123.) - self.assertEqual(float_("1.25"), 1.25) - - self.assertEqual(float_(FloatLike.with_val(4.25)), 4.25) - self.assertEqual(float_(IndexLike.with_val(99)), 99.0) - self.assertEqual(float_(IndexLike.with_val(-1)), -1.0) - - self.assertRaises(TypeError, float_, FloatLike.with_val(687)) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, float_, FloatLike.with_val(subclassof(float)(4.25))) - with self.assertWarns(DeprecationWarning): - self.assertEqual(float_(FloatLike.with_val(subclassof(float)(4.25))), 4.25) - self.assertRaises(RuntimeError, float_, FloatLike.with_exc(RuntimeError)) - - self.assertRaises(TypeError, float_, IndexLike.with_val(1.25)) - self.assertRaises(OverflowError, float_, IndexLike.with_val(2**2000)) - - self.assertRaises(TypeError, float_, 1j) - self.assertRaises(TypeError, float_, object()) - self.assertRaises(SystemError, float_, NULL) - - def test_index(self): - # Test PyNumber_Index() - index = _testcapi.number_index - - self.assertEqual(index(11), 11) - - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, index, IndexLike.with_val(True)) - with self.assertWarns(DeprecationWarning): - self.assertEqual(index(IndexLike.with_val(True)), 1) - self.assertRaises(TypeError, index, IndexLike.with_val(1.0)) - self.assertRaises(RuntimeError, index, IndexLike.with_exc(RuntimeError)) - - self.assertRaises(TypeError, index, 1.25) - self.assertRaises(TypeError, index, "42") - self.assertRaises(TypeError, index, object()) - self.assertRaises(SystemError, index, NULL) - - def test_tobase(self): - # Test PyNumber_ToBase() - tobase = _testcapi.number_tobase - - self.assertEqual(tobase(10, 2), bin(10)) - self.assertEqual(tobase(11, 8), oct(11)) - self.assertEqual(tobase(16, 10), str(16)) - self.assertEqual(tobase(13, 16), hex(13)) - - self.assertRaises(SystemError, tobase, NULL, 2) - self.assertRaises(SystemError, tobase, 2, 3) - self.assertRaises(TypeError, tobase, 1.25, 2) - self.assertRaises(TypeError, tobase, "42", 2) - - def test_asssizet(self): - # Test PyNumber_AsSsize_t() - asssizet = _testcapi.number_asssizet - - for n in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: - self.assertEqual(asssizet(n, OverflowError), n) - self.assertEqual(asssizet(PY_SSIZE_T_MAX+10, NULL), PY_SSIZE_T_MAX) - self.assertEqual(asssizet(PY_SSIZE_T_MIN-10, NULL), PY_SSIZE_T_MIN) - - self.assertRaises(OverflowError, asssizet, PY_SSIZE_T_MAX + 10, OverflowError) - self.assertRaises(RuntimeError, asssizet, PY_SSIZE_T_MAX + 10, RuntimeError) - self.assertRaises(SystemError, asssizet, NULL, TypeError) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py deleted file mode 100644 index 807b62c1986..00000000000 --- a/Lib/test/test_capi/test_object.py +++ /dev/null @@ -1,258 +0,0 @@ -import enum -import sys -import textwrap -import unittest -from test import support -from test.support import import_helper -from test.support import os_helper -from test.support import threading_helper -from test.support.script_helper import assert_python_failure - - -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') -_testcapi = import_helper.import_module('_testcapi') -_testinternalcapi = import_helper.import_module('_testinternalcapi') - - -class Constant(enum.IntEnum): - Py_CONSTANT_NONE = 0 - Py_CONSTANT_FALSE = 1 - Py_CONSTANT_TRUE = 2 - Py_CONSTANT_ELLIPSIS = 3 - Py_CONSTANT_NOT_IMPLEMENTED = 4 - Py_CONSTANT_ZERO = 5 - Py_CONSTANT_ONE = 6 - Py_CONSTANT_EMPTY_STR = 7 - Py_CONSTANT_EMPTY_BYTES = 8 - Py_CONSTANT_EMPTY_TUPLE = 9 - - INVALID_CONSTANT = Py_CONSTANT_EMPTY_TUPLE + 1 - - -class GetConstantTest(unittest.TestCase): - def check_get_constant(self, get_constant): - self.assertIs(get_constant(Constant.Py_CONSTANT_NONE), None) - self.assertIs(get_constant(Constant.Py_CONSTANT_FALSE), False) - self.assertIs(get_constant(Constant.Py_CONSTANT_TRUE), True) - self.assertIs(get_constant(Constant.Py_CONSTANT_ELLIPSIS), Ellipsis) - self.assertIs(get_constant(Constant.Py_CONSTANT_NOT_IMPLEMENTED), NotImplemented) - - for constant_id, constant_type, value in ( - (Constant.Py_CONSTANT_ZERO, int, 0), - (Constant.Py_CONSTANT_ONE, int, 1), - (Constant.Py_CONSTANT_EMPTY_STR, str, ""), - (Constant.Py_CONSTANT_EMPTY_BYTES, bytes, b""), - (Constant.Py_CONSTANT_EMPTY_TUPLE, tuple, ()), - ): - with self.subTest(constant_id=constant_id): - obj = get_constant(constant_id) - self.assertEqual(type(obj), constant_type, obj) - self.assertEqual(obj, value) - - with self.assertRaises(SystemError): - get_constant(Constant.INVALID_CONSTANT) - - def test_get_constant(self): - self.check_get_constant(_testlimitedcapi.get_constant) - - def test_get_constant_borrowed(self): - self.check_get_constant(_testlimitedcapi.get_constant_borrowed) - - -class PrintTest(unittest.TestCase): - def testPyObjectPrintObject(self): - - class PrintableObject: - - def __repr__(self): - return "spam spam spam" - - def __str__(self): - return "egg egg egg" - - obj = PrintableObject() - output_filename = os_helper.TESTFN - self.addCleanup(os_helper.unlink, output_filename) - - # Test repr printing - _testcapi.call_pyobject_print(obj, output_filename, False) - with open(output_filename, 'r') as output_file: - self.assertEqual(output_file.read(), repr(obj)) - - # Test str printing - _testcapi.call_pyobject_print(obj, output_filename, True) - with open(output_filename, 'r') as output_file: - self.assertEqual(output_file.read(), str(obj)) - - def testPyObjectPrintNULL(self): - output_filename = os_helper.TESTFN - self.addCleanup(os_helper.unlink, output_filename) - - # Test repr printing - _testcapi.pyobject_print_null(output_filename) - with open(output_filename, 'r') as output_file: - self.assertEqual(output_file.read(), '') - - def testPyObjectPrintNoRefObject(self): - output_filename = os_helper.TESTFN - self.addCleanup(os_helper.unlink, output_filename) - - # Test repr printing - correct_output = _testcapi.pyobject_print_noref_object(output_filename) - with open(output_filename, 'r') as output_file: - self.assertEqual(output_file.read(), correct_output) - - def testPyObjectPrintOSError(self): - output_filename = os_helper.TESTFN - self.addCleanup(os_helper.unlink, output_filename) - - open(output_filename, "w+").close() - with self.assertRaises(OSError): - _testcapi.pyobject_print_os_error(output_filename) - - -class ClearWeakRefsNoCallbacksTest(unittest.TestCase): - """Test PyUnstable_Object_ClearWeakRefsNoCallbacks""" - def test_ClearWeakRefsNoCallbacks(self): - """Ensure PyUnstable_Object_ClearWeakRefsNoCallbacks works""" - import weakref - import gc - class C: - pass - obj = C() - messages = [] - ref = weakref.ref(obj, lambda: messages.append("don't add this")) - self.assertIs(ref(), obj) - self.assertFalse(messages) - _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) - self.assertIsNone(ref()) - gc.collect() - self.assertFalse(messages) - - def test_ClearWeakRefsNoCallbacks_no_weakref_support(self): - """Don't fail on objects that don't support weakrefs""" - import weakref - obj = object() - with self.assertRaises(TypeError): - ref = weakref.ref(obj) - _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) - - -@threading_helper.requires_working_threading() -class EnableDeferredRefcountingTest(unittest.TestCase): - """Test PyUnstable_Object_EnableDeferredRefcount""" - @support.requires_resource("cpu") - def test_enable_deferred_refcount(self): - from threading import Thread - - self.assertEqual(_testcapi.pyobject_enable_deferred_refcount("not tracked"), 0) - foo = [] - self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(foo), int(support.Py_GIL_DISABLED)) - - # Make sure reference counting works on foo now - self.assertEqual(foo, []) - if support.Py_GIL_DISABLED: - self.assertTrue(_testinternalcapi.has_deferred_refcount(foo)) - - # Make sure that PyUnstable_Object_EnableDeferredRefcount is thread safe - def silly_func(obj): - self.assertIn( - _testcapi.pyobject_enable_deferred_refcount(obj), - (0, 1) - ) - - silly_list = [1, 2, 3] - threads = [ - Thread(target=silly_func, args=(silly_list,)) for _ in range(4) - ] - - with threading_helper.start_threads(threads): - for i in range(10): - silly_list.append(i) - - if support.Py_GIL_DISABLED: - self.assertTrue(_testinternalcapi.has_deferred_refcount(silly_list)) - - -class IsUniquelyReferencedTest(unittest.TestCase): - """Test PyUnstable_Object_IsUniquelyReferenced""" - def test_is_uniquely_referenced(self): - self.assertTrue(_testcapi.is_uniquely_referenced(object())) - self.assertTrue(_testcapi.is_uniquely_referenced([])) - # Immortals - self.assertFalse(_testcapi.is_uniquely_referenced("spanish inquisition")) - self.assertFalse(_testcapi.is_uniquely_referenced(42)) - # CRASHES is_uniquely_referenced(NULL) - -class CAPITest(unittest.TestCase): - def check_negative_refcount(self, code): - # bpo-35059: Check that Py_DECREF() reports the correct filename - # when calling _Py_NegativeRefcount() to abort Python. - code = textwrap.dedent(code) - rc, out, err = assert_python_failure('-c', code) - self.assertRegex(err, - br'object\.c:[0-9]+: ' - br'_Py_NegativeRefcount: Assertion failed: ' - br'object has negative ref count') - - @unittest.skipUnless(hasattr(_testcapi, 'negative_refcount'), - 'need _testcapi.negative_refcount()') - def test_negative_refcount(self): - code = """ - import _testcapi - from test import support - - with support.SuppressCrashReport(): - _testcapi.negative_refcount() - """ - self.check_negative_refcount(code) - - @unittest.skipUnless(hasattr(_testcapi, 'decref_freed_object'), - 'need _testcapi.decref_freed_object()') - @support.skip_if_sanitizer("use after free on purpose", - address=True, memory=True, ub=True) - def test_decref_freed_object(self): - code = """ - import _testcapi - from test import support - - with support.SuppressCrashReport(): - _testcapi.decref_freed_object() - """ - self.check_negative_refcount(code) - - @support.requires_resource('cpu') - def test_decref_delayed(self): - # gh-130519: Test that _PyObject_XDecRefDelayed() and QSBR code path - # handles destructors that are possibly re-entrant or trigger a GC. - import gc - - class MyObj: - def __del__(self): - gc.collect() - - for _ in range(1000): - obj = MyObj() - _testinternalcapi.incref_decref_delayed(obj) - - def test_is_unique_temporary(self): - self.assertTrue(_testcapi.pyobject_is_unique_temporary(object())) - obj = object() - self.assertFalse(_testcapi.pyobject_is_unique_temporary(obj)) - - def func(x): - # This relies on the LOAD_FAST_BORROW optimization (gh-130704) - self.assertEqual(sys.getrefcount(x), 1) - self.assertFalse(_testcapi.pyobject_is_unique_temporary(x)) - - func(object()) - - # Test that a newly created object in C is not considered - # a uniquely referenced temporary, because it's not on the stack. - # gh-142586: do the test in a loop over a list to test for handling - # tagged ints on the stack. - for i in [0, 1, 2]: - self.assertFalse(_testcapi.pyobject_is_unique_temporary_new_object()) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py deleted file mode 100644 index be3f8895db5..00000000000 --- a/Lib/test/test_capi/test_opt.py +++ /dev/null @@ -1,2021 +0,0 @@ -import contextlib -import itertools -import sys -import textwrap -import unittest -import gc -import os - -import _opcode - -from test.support import (script_helper, requires_specialization, - import_helper, Py_GIL_DISABLED, requires_jit_enabled, - reset_code) - -_testinternalcapi = import_helper.import_module("_testinternalcapi") - -from _testinternalcapi import TIER2_THRESHOLD - - -@contextlib.contextmanager -def clear_executors(func): - # Clear executors in func before and after running a block - reset_code(func) - try: - yield - finally: - reset_code(func) - - -def get_first_executor(func): - code = func.__code__ - co_code = code.co_code - for i in range(0, len(co_code), 2): - try: - return _opcode.get_executor(code, i) - except ValueError: - pass - return None - - -def iter_opnames(ex): - for item in ex: - yield item[0] - - -def get_opnames(ex): - return list(iter_opnames(ex)) - - -@requires_specialization -@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@requires_jit_enabled -class TestExecutorInvalidation(unittest.TestCase): - - def test_invalidate_object(self): - # Generate a new set of functions at each call - ns = {} - func_src = "\n".join( - f""" - def f{n}(): - for _ in range({TIER2_THRESHOLD}): - pass - """ for n in range(5) - ) - exec(textwrap.dedent(func_src), ns, ns) - funcs = [ ns[f'f{n}'] for n in range(5)] - objects = [object() for _ in range(5)] - - for f in funcs: - f() - executors = [get_first_executor(f) for f in funcs] - # Set things up so each executor depends on the objects - # with an equal or lower index. - for i, exe in enumerate(executors): - self.assertTrue(exe.is_valid()) - for obj in objects[:i+1]: - _testinternalcapi.add_executor_dependency(exe, obj) - self.assertTrue(exe.is_valid()) - # Assert that the correct executors are invalidated - # and check that nothing crashes when we invalidate - # an executor multiple times. - for i in (4,3,2,1,0): - _testinternalcapi.invalidate_executors(objects[i]) - for exe in executors[i:]: - self.assertFalse(exe.is_valid()) - for exe in executors[:i]: - self.assertTrue(exe.is_valid()) - - def test_uop_optimizer_invalidation(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(f""" - def f(): - for i in range({TIER2_THRESHOLD}): - pass - """), ns, ns) - f = ns['f'] - f() - exe = get_first_executor(f) - self.assertIsNotNone(exe) - self.assertTrue(exe.is_valid()) - _testinternalcapi.invalidate_executors(f.__code__) - self.assertFalse(exe.is_valid()) - - def test_sys__clear_internal_caches(self): - def f(): - for _ in range(TIER2_THRESHOLD): - pass - f() - exe = get_first_executor(f) - self.assertIsNotNone(exe) - self.assertTrue(exe.is_valid()) - sys._clear_internal_caches() - self.assertFalse(exe.is_valid()) - exe = get_first_executor(f) - self.assertIsNone(exe) - - -@requires_specialization -@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@requires_jit_enabled -@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") -class TestUops(unittest.TestCase): - - def test_basic_loop(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_JUMP_TO_TOP", uops) - self.assertIn("_LOAD_FAST_BORROW_0", uops) - - def test_extended_arg(self): - "Check EXTENDED_ARG handling in superblock creation" - ns = {} - exec(textwrap.dedent(f""" - def many_vars(): - # 260 vars, so z9 should have index 259 - a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 - b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 - c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 - d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 - e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 - f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 - g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 - h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 - i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 - j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 - k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 - l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 - m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 - n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 - o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 - p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 - q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 - r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 - s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 - t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 - u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 - v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 - w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 - x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 - y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = {TIER2_THRESHOLD} - while z9 > 0: - z9 = z9 - 1 - +z9 - """), ns, ns) - many_vars = ns["many_vars"] - - ex = get_first_executor(many_vars) - self.assertIsNone(ex) - many_vars() - - ex = get_first_executor(many_vars) - self.assertIsNotNone(ex) - self.assertTrue(any((opcode, oparg, operand) == ("_LOAD_FAST_BORROW", 259, 0) - for opcode, oparg, _, operand in list(ex))) - - def test_unspecialized_unpack(self): - # An example of an unspecialized opcode - def testfunc(x): - i = 0 - while i < x: - i += 1 - a, b = {1: 2, 3: 3} - assert a == 1 and b == 3 - i = 0 - while i < x: - i += 1 - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_UNPACK_SEQUENCE", uops) - - def test_pop_jump_if_false(self): - def testfunc(n): - i = 0 - while i < n: - i += 1 - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_GUARD_IS_TRUE_POP", uops) - - def test_pop_jump_if_none(self): - def testfunc(a): - for x in a: - if x is None: - x = 0 - - testfunc(range(TIER2_THRESHOLD)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_IS_NONE_POP", uops) - self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) - - def test_pop_jump_if_not_none(self): - def testfunc(a): - for x in a: - x = None - if x is not None: - x = 0 - - testfunc(range(TIER2_THRESHOLD)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_IS_NONE_POP", uops) - self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) - - def test_pop_jump_if_true(self): - def testfunc(n): - i = 0 - while not i >= n: - i += 1 - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_GUARD_IS_FALSE_POP", uops) - - def test_jump_backward(self): - def testfunc(n): - i = 0 - while i < n: - i += 1 - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_JUMP_TO_TOP", uops) - - def test_jump_forward(self): - def testfunc(n): - a = 0 - while a < n: - if a < 0: - a = -a - else: - a = +a - a += 1 - return a - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - # Since there is no JUMP_FORWARD instruction, - # look for indirect evidence: the += operator - self.assertIn("_BINARY_OP_ADD_INT", uops) - - def test_for_iter_range(self): - def testfunc(n): - total = 0 - for i in range(n): - total += i - return total - - total = testfunc(TIER2_THRESHOLD) - self.assertEqual(total, sum(range(TIER2_THRESHOLD))) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = get_opnames(ex) - self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_for_iter_list(self): - def testfunc(a): - total = 0 - for i in a: - total += i - return total - - a = list(range(TIER2_THRESHOLD)) - total = testfunc(a) - self.assertEqual(total, sum(a)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = get_opnames(ex) - self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_for_iter_tuple(self): - def testfunc(a): - total = 0 - for i in a: - total += i - return total - - a = tuple(range(TIER2_THRESHOLD)) - total = testfunc(a) - self.assertEqual(total, sum(a)) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - # for i, (opname, oparg) in enumerate(ex): - # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = get_opnames(ex) - self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) - # Verification that the jump goes past END_FOR - # is done by manual inspection of the output - - def test_list_edge_case(self): - def testfunc(it): - for x in it: - pass - - a = [1, 2, 3] - it = iter(a) - testfunc(it) - a.append(4) - with self.assertRaises(StopIteration): - next(it) - - def test_call_py_exact_args(self): - def testfunc(n): - def dummy(x): - return x+1 - for i in range(n): - dummy(i) - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_PUSH_FRAME", uops) - self.assertIn("_BINARY_OP_ADD_INT", uops) - - def test_branch_taken(self): - def testfunc(n): - for i in range(n): - if i < 0: - i = 0 - else: - i = 1 - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_GUARD_IS_FALSE_POP", uops) - - def test_for_iter_tier_two(self): - class MyIter: - def __init__(self, n): - self.n = n - def __iter__(self): - return self - def __next__(self): - self.n -= 1 - if self.n < 0: - raise StopIteration - return self.n - - def testfunc(n, m): - x = 0 - for i in range(m): - for j in MyIter(n): - x += 1000*i + j - return x - - x = testfunc(TIER2_THRESHOLD, TIER2_THRESHOLD) - - self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * TIER2_THRESHOLD * 1001) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_FOR_ITER_TIER_TWO", uops) - - def test_confidence_score(self): - def testfunc(n): - bits = 0 - for i in range(n): - if i & 0x01: - bits += 1 - if i & 0x02: - bits += 1 - if i&0x04: - bits += 1 - if i&0x08: - bits += 1 - if i&0x10: - bits += 1 - return bits - - x = testfunc(TIER2_THRESHOLD * 2) - - self.assertEqual(x, TIER2_THRESHOLD * 5) - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - ops = list(iter_opnames(ex)) - #Since branch is 50/50 the trace could go either way. - count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") - self.assertLessEqual(count, 2) - - -@requires_specialization -@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@requires_jit_enabled -@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") -class TestUopsOptimization(unittest.TestCase): - - def _run_with_optimizer(self, testfunc, arg): - res = testfunc(arg) - - ex = get_first_executor(testfunc) - return res, ex - - - def test_int_type_propagation(self): - def testfunc(loops): - num = 0 - for i in range(loops): - x = num + num - a = x + 1 - num += 1 - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - self.assertEqual(res, (TIER2_THRESHOLD - 1) * 2 + 1) - binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] - guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] - guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] - self.assertGreaterEqual(len(binop_count), 3) - self.assertLessEqual(len(guard_tos_int_count), 1) - self.assertLessEqual(len(guard_nos_int_count), 1) - - def test_int_type_propagation_through_frame(self): - def double(x): - return x + x - def testfunc(loops): - num = 0 - for i in range(loops): - x = num + num - a = double(x) - num += 1 - return a - - res = testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - self.assertEqual(res, (TIER2_THRESHOLD - 1) * 4) - binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] - guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] - guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] - self.assertGreaterEqual(len(binop_count), 3) - self.assertLessEqual(len(guard_tos_int_count), 1) - self.assertLessEqual(len(guard_nos_int_count), 1) - - def test_int_type_propagation_from_frame(self): - def double(x): - return x + x - def testfunc(loops): - num = 0 - for i in range(loops): - a = double(num) - x = a + a - num += 1 - return x - - res = testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - self.assertEqual(res, (TIER2_THRESHOLD - 1) * 4) - binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] - guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] - guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] - self.assertGreaterEqual(len(binop_count), 3) - self.assertLessEqual(len(guard_tos_int_count), 1) - self.assertLessEqual(len(guard_nos_int_count), 1) - - def test_int_impure_region(self): - def testfunc(loops): - num = 0 - while num < loops: - x = num + num - y = 1 - x // 2 - a = x + y - num += 1 - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] - self.assertGreaterEqual(len(binop_count), 3) - - def test_call_py_exact_args(self): - def testfunc(n): - def dummy(x): - return x+1 - for i in range(n): - dummy(i) - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_PUSH_FRAME", uops) - self.assertIn("_BINARY_OP_ADD_INT", uops) - self.assertNotIn("_CHECK_PEP_523", uops) - - def test_int_type_propagate_through_range(self): - def testfunc(n): - - for i in range(n): - x = i + i - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, (TIER2_THRESHOLD - 1) * 2) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - - def test_int_value_numbering(self): - def testfunc(n): - - y = 1 - for i in range(n): - x = y - z = x - a = z - b = a - res = x + z + a + b - return res - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, 4) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - guard_tos_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] - self.assertEqual(len(guard_tos_count), 1) - - def test_comprehension(self): - def testfunc(n): - for _ in range(n): - return [i for i in range(n)] - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, list(range(TIER2_THRESHOLD))) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_BINARY_OP_ADD_INT", uops) - - def test_call_py_exact_args_disappearing(self): - def dummy(x): - return x+1 - - def testfunc(n): - for i in range(n): - dummy(i) - - # Trigger specialization - testfunc(8) - del dummy - gc.collect() - - def dummy(x): - return x + 2 - testfunc(32) - - ex = get_first_executor(testfunc) - # Honestly as long as it doesn't crash it's fine. - # Whether we get an executor or not is non-deterministic, - # because it's decided by when the function is freed. - # This test is a little implementation specific. - - def test_promote_globals_to_constants(self): - - result = script_helper.run_python_until_end('-c', textwrap.dedent(""" - import _testinternalcapi - import opcode - import _opcode - - def get_first_executor(func): - code = func.__code__ - co_code = code.co_code - for i in range(0, len(co_code), 2): - try: - return _opcode.get_executor(code, i) - except ValueError: - pass - return None - - def get_opnames(ex): - return {item[0] for item in ex} - - def testfunc(n): - for i in range(n): - x = range(i) - return x - - testfunc(_testinternalcapi.TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - assert ex is not None - uops = get_opnames(ex) - assert "_LOAD_GLOBAL_BUILTINS" not in uops - assert "_LOAD_CONST_INLINE_BORROW" in uops - """), PYTHON_JIT="1") - self.assertEqual(result[0].rc, 0, result) - - def test_float_add_constant_propagation(self): - def testfunc(n): - a = 1.0 - for _ in range(n): - a = a + 0.25 - a = a + 0.25 - a = a + 0.25 - a = a + 0.25 - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertAlmostEqual(res, TIER2_THRESHOLD + 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] - guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] - self.assertLessEqual(len(guard_tos_float_count), 1) - self.assertLessEqual(len(guard_nos_float_count), 1) - # TODO gh-115506: this assertion may change after propagating constants. - # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_ADD_FLOAT", uops) - - def test_float_subtract_constant_propagation(self): - def testfunc(n): - a = 1.0 - for _ in range(n): - a = a - 0.25 - a = a - 0.25 - a = a - 0.25 - a = a - 0.25 - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertAlmostEqual(res, -TIER2_THRESHOLD + 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] - guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] - self.assertLessEqual(len(guard_tos_float_count), 1) - self.assertLessEqual(len(guard_nos_float_count), 1) - # TODO gh-115506: this assertion may change after propagating constants. - # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) - - def test_float_multiply_constant_propagation(self): - def testfunc(n): - a = 1.0 - for _ in range(n): - a = a * 1.0 - a = a * 1.0 - a = a * 1.0 - a = a * 1.0 - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertAlmostEqual(res, 1.0) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] - guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] - self.assertLessEqual(len(guard_tos_float_count), 1) - self.assertLessEqual(len(guard_nos_float_count), 1) - # TODO gh-115506: this assertion may change after propagating constants. - # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) - - def test_add_unicode_propagation(self): - def testfunc(n): - a = "" - for _ in range(n): - a + a - a + a - a + a - a + a - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, "") - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_tos_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_UNICODE"] - guard_nos_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_UNICODE"] - self.assertLessEqual(len(guard_tos_unicode_count), 1) - self.assertLessEqual(len(guard_nos_unicode_count), 1) - self.assertIn("_BINARY_OP_ADD_UNICODE", uops) - - def test_compare_op_type_propagation_float(self): - def testfunc(n): - a = 1.0 - for _ in range(n): - x = a == a - x = a == a - x = a == a - x = a == a - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertTrue(res) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] - guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] - self.assertLessEqual(len(guard_tos_float_count), 1) - self.assertLessEqual(len(guard_nos_float_count), 1) - self.assertIn("_COMPARE_OP_FLOAT", uops) - - def test_compare_op_type_propagation_int(self): - def testfunc(n): - a = 1 - for _ in range(n): - x = a == a - x = a == a - x = a == a - x = a == a - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertTrue(res) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] - guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] - self.assertLessEqual(len(guard_tos_int_count), 1) - self.assertLessEqual(len(guard_nos_int_count), 1) - self.assertIn("_COMPARE_OP_INT", uops) - - def test_compare_op_type_propagation_int_partial(self): - def testfunc(n): - a = 1 - for _ in range(n): - if a > 2: - x = 0 - if a < 2: - x = 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] - guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] - self.assertLessEqual(len(guard_nos_int_count), 1) - self.assertEqual(len(guard_tos_int_count), 0) - self.assertIn("_COMPARE_OP_INT", uops) - - def test_compare_op_type_propagation_float_partial(self): - def testfunc(n): - a = 1.0 - for _ in range(n): - if a > 2.0: - x = 0 - if a < 2.0: - x = 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_nos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] - guard_tos_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_FLOAT"] - self.assertLessEqual(len(guard_nos_float_count), 1) - self.assertEqual(len(guard_tos_float_count), 0) - self.assertIn("_COMPARE_OP_FLOAT", uops) - - def test_compare_op_type_propagation_unicode(self): - def testfunc(n): - a = "" - for _ in range(n): - x = a == a - x = a == a - x = a == a - x = a == a - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertTrue(res) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - guard_tos_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_UNICODE"] - guard_nos_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_UNICODE"] - self.assertLessEqual(len(guard_tos_unicode_count), 1) - self.assertLessEqual(len(guard_nos_unicode_count), 1) - self.assertIn("_COMPARE_OP_STR", uops) - - def test_type_inconsistency(self): - ns = {} - src = textwrap.dedent(""" - def testfunc(n): - for i in range(n): - x = _test_global + _test_global - """) - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertIn("_BINARY_OP_ADD_INT", uops) - # Try again, but between the runs, set the global to a float. - # This should result in no executor the second time. - ns = {} - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - - def test_combine_stack_space_checks_sequential(self): - def dummy12(x): - return x - 1 - def dummy13(y): - z = y + 2 - return y, z - def testfunc(n): - a = 0 - for _ in range(n): - b = dummy12(7) - c, d = dummy13(9) - a += b + c + d - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD * 26) - self.assertIsNotNone(ex) - - uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] - uop_names = [uop[0] for uop in uops_and_operands] - self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) - self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) - # sequential calls: max(12, 13) == 13 - largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) - self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) - - def test_combine_stack_space_checks_nested(self): - def dummy12(x): - return x + 3 - def dummy15(y): - z = dummy12(y) - return y, z - def testfunc(n): - a = 0 - for _ in range(n): - b, c = dummy15(2) - a += b + c - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD * 7) - self.assertIsNotNone(ex) - - uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] - uop_names = [uop[0] for uop in uops_and_operands] - self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) - self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) - # nested calls: 15 + 12 == 27 - largest_stack = ( - _testinternalcapi.get_co_framesize(dummy15.__code__) + - _testinternalcapi.get_co_framesize(dummy12.__code__) - ) - self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) - - def test_combine_stack_space_checks_several_calls(self): - def dummy12(x): - return x + 3 - def dummy13(y): - z = y + 2 - return y, z - def dummy18(y): - z = dummy12(y) - x, w = dummy13(z) - return z, x, w - def testfunc(n): - a = 0 - for _ in range(n): - b = dummy12(5) - c, d, e = dummy18(2) - a += b + c + d + e - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD * 25) - self.assertIsNotNone(ex) - - uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] - uop_names = [uop[0] for uop in uops_and_operands] - self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) - self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) - # max(12, 18 + max(12, 13)) == 31 - largest_stack = ( - _testinternalcapi.get_co_framesize(dummy18.__code__) + - _testinternalcapi.get_co_framesize(dummy13.__code__) - ) - self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) - - def test_combine_stack_space_checks_several_calls_different_order(self): - # same as `several_calls` but with top-level calls reversed - def dummy12(x): - return x + 3 - def dummy13(y): - z = y + 2 - return y, z - def dummy18(y): - z = dummy12(y) - x, w = dummy13(z) - return z, x, w - def testfunc(n): - a = 0 - for _ in range(n): - c, d, e = dummy18(2) - b = dummy12(5) - a += b + c + d + e - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD * 25) - self.assertIsNotNone(ex) - - uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] - uop_names = [uop[0] for uop in uops_and_operands] - self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) - self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) - # max(18 + max(12, 13), 12) == 31 - largest_stack = ( - _testinternalcapi.get_co_framesize(dummy18.__code__) + - _testinternalcapi.get_co_framesize(dummy13.__code__) - ) - self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) - - def test_combine_stack_space_complex(self): - def dummy0(x): - return x - def dummy1(x): - return dummy0(x) - def dummy2(x): - return dummy1(x) - def dummy3(x): - return dummy0(x) - def dummy4(x): - y = dummy0(x) - return dummy3(y) - def dummy5(x): - return dummy2(x) - def dummy6(x): - y = dummy5(x) - z = dummy0(y) - return dummy4(z) - def testfunc(n): - a = 0 - for _ in range(n): - b = dummy5(1) - c = dummy0(1) - d = dummy6(1) - a += b + c + d - return a - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD * 3) - self.assertIsNotNone(ex) - - uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] - uop_names = [uop[0] for uop in uops_and_operands] - self.assertEqual(uop_names.count("_PUSH_FRAME"), 15) - self.assertEqual(uop_names.count("_RETURN_VALUE"), 15) - - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) - largest_stack = ( - _testinternalcapi.get_co_framesize(dummy6.__code__) + - _testinternalcapi.get_co_framesize(dummy5.__code__) + - _testinternalcapi.get_co_framesize(dummy2.__code__) + - _testinternalcapi.get_co_framesize(dummy1.__code__) + - _testinternalcapi.get_co_framesize(dummy0.__code__) - ) - self.assertIn( - ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands - ) - - def test_combine_stack_space_checks_large_framesize(self): - # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is - # actually doing its job. Note that the resulting trace hits - # UOP_MAX_TRACE_LENGTH, but since all _CHECK_STACK_SPACEs happen early, this - # test is still meaningful. - repetitions = 10000 - ns = {} - header = """ - def dummy_large(a0): - """ - body = "".join([f""" - a{n+1} = a{n} + 1 - """ for n in range(repetitions)]) - return_ = f""" - return a{repetitions-1} - """ - exec(textwrap.dedent(header + body + return_), ns, ns) - dummy_large = ns['dummy_large'] - - # this is something like: - # - # def dummy_large(a0): - # a1 = a0 + 1 - # a2 = a1 + 1 - # .... - # a9999 = a9998 + 1 - # return a9999 - - def dummy15(z): - y = dummy_large(z) - return y + 3 - - def testfunc(n): - b = 0 - for _ in range(n): - b += dummy15(7) - return b - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD * (repetitions + 9)) - self.assertIsNotNone(ex) - - uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] - uop_names = [uop[0] for uop in uops_and_operands] - self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) - - # this hits a different case during trace projection in refcount test runs only, - # so we need to account for both possibilities - self.assertIn(uop_names.count("_CHECK_STACK_SPACE"), [0, 1]) - if uop_names.count("_CHECK_STACK_SPACE") == 0: - largest_stack = ( - _testinternalcapi.get_co_framesize(dummy15.__code__) + - _testinternalcapi.get_co_framesize(dummy_large.__code__) - ) - else: - largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) - self.assertIn( - ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands - ) - - def test_combine_stack_space_checks_recursion(self): - def dummy15(x): - while x > 0: - return dummy15(x - 1) - return 42 - def testfunc(n): - a = 0 - for _ in range(n): - a += dummy15(n) - return a - - recursion_limit = sys.getrecursionlimit() - try: - sys.setrecursionlimit(TIER2_THRESHOLD + recursion_limit) - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - finally: - sys.setrecursionlimit(recursion_limit) - self.assertEqual(res, TIER2_THRESHOLD * 42) - self.assertIsNotNone(ex) - - uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] - uop_names = [uop[0] for uop in uops_and_operands] - self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) - self.assertEqual(uop_names.count("_RETURN_VALUE"), 0) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 1) - self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) - largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) - self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) - - def test_many_nested(self): - # overflow the trace_stack - def dummy_a(x): - return x - def dummy_b(x): - return dummy_a(x) - def dummy_c(x): - return dummy_b(x) - def dummy_d(x): - return dummy_c(x) - def dummy_e(x): - return dummy_d(x) - def dummy_f(x): - return dummy_e(x) - def dummy_g(x): - return dummy_f(x) - def dummy_h(x): - return dummy_g(x) - def testfunc(n): - a = 0 - for _ in range(n): - a += dummy_h(n) - return a - - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 32 * 32) - self.assertIsNone(ex) - - def test_return_generator(self): - def gen(): - yield None - def testfunc(n): - for i in range(n): - gen() - return i - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD - 1) - self.assertIsNotNone(ex) - self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) - - @unittest.skip("Tracing into generators currently isn't supported.") - def test_for_iter_gen(self): - def gen(n): - for i in range(n): - yield i - def testfunc(n): - g = gen(n) - s = 0 - for i in g: - s += i - return s - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, sum(range(TIER2_THRESHOLD))) - self.assertIsNotNone(ex) - self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex)) - - def test_modified_local_is_seen_by_optimized_code(self): - l = sys._getframe().f_locals - a = 1 - s = 0 - for j in range(1 << 10): - a + a - l["xa"[j >> 9]] = 1.0 - s += a - self.assertIs(type(a), float) - self.assertIs(type(s), float) - self.assertEqual(s, 1024.0) - - def test_guard_type_version_removed(self): - def thing(a): - x = 0 - for _ in range(TIER2_THRESHOLD): - x += a.attr - x += a.attr - return x - - class Foo: - attr = 1 - - res, ex = self._run_with_optimizer(thing, Foo()) - opnames = list(iter_opnames(ex)) - self.assertIsNotNone(ex) - self.assertEqual(res, TIER2_THRESHOLD * 2) - guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") - self.assertEqual(guard_type_version_count, 1) - - def test_guard_type_version_removed_inlined(self): - """ - Verify that the guard type version if we have an inlined function - """ - - def fn(): - pass - - def thing(a): - x = 0 - for _ in range(TIER2_THRESHOLD): - x += a.attr - fn() - x += a.attr - return x - - class Foo: - attr = 1 - - res, ex = self._run_with_optimizer(thing, Foo()) - opnames = list(iter_opnames(ex)) - self.assertIsNotNone(ex) - self.assertEqual(res, TIER2_THRESHOLD * 2) - guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") - self.assertEqual(guard_type_version_count, 1) - - def test_guard_type_version_removed_invalidation(self): - - def thing(a): - x = 0 - for i in range(TIER2_THRESHOLD * 2 + 1): - x += a.attr - # The first TIER2_THRESHOLD iterations we set the attribute on - # this dummy class, which shouldn't trigger the type watcher. - # Note that the code needs to be in this weird form so it's - # optimized inline without any control flow: - setattr((Bar, Foo)[i == TIER2_THRESHOLD + 1], "attr", 2) - x += a.attr - return x - - class Foo: - attr = 1 - - class Bar: - pass - - res, ex = self._run_with_optimizer(thing, Foo()) - opnames = list(iter_opnames(ex)) - self.assertIsNotNone(ex) - self.assertEqual(res, TIER2_THRESHOLD * 6 + 1) - call = opnames.index("_CALL_BUILTIN_FAST") - load_attr_top = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", 0, call) - load_attr_bottom = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", call) - self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) - self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2) - - def test_guard_type_version_removed_escaping(self): - - def thing(a): - x = 0 - for i in range(TIER2_THRESHOLD): - x += a.attr - # eval should be escaping - eval("None") - x += a.attr - return x - - class Foo: - attr = 1 - res, ex = self._run_with_optimizer(thing, Foo()) - opnames = list(iter_opnames(ex)) - self.assertIsNotNone(ex) - self.assertEqual(res, TIER2_THRESHOLD * 2) - call = opnames.index("_CALL_BUILTIN_FAST_WITH_KEYWORDS") - load_attr_top = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", 0, call) - load_attr_bottom = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", call) - self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) - self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2) - - def test_guard_type_version_executor_invalidated(self): - """ - Verify that the executor is invalided on a type change. - """ - - def thing(a): - x = 0 - for i in range(TIER2_THRESHOLD): - x += a.attr - x += a.attr - return x - - class Foo: - attr = 1 - - res, ex = self._run_with_optimizer(thing, Foo()) - self.assertEqual(res, TIER2_THRESHOLD * 2) - self.assertIsNotNone(ex) - self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1) - self.assertTrue(ex.is_valid()) - Foo.attr = 0 - self.assertFalse(ex.is_valid()) - - def test_type_version_doesnt_segfault(self): - """ - Tests that setting a type version doesn't cause a segfault when later looking at the stack. - """ - - # Minimized from mdp.py benchmark - - class A: - def __init__(self): - self.attr = {} - - def method(self, arg): - self.attr[arg] = None - - def fn(a): - for _ in range(100): - (_ for _ in []) - (_ for _ in [a.method(None)]) - - fn(A()) - - def test_func_guards_removed_or_reduced(self): - def testfunc(n): - for i in range(n): - # Only works on functions promoted to constants - global_identity(i) - - testfunc(TIER2_THRESHOLD) - - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_PUSH_FRAME", uops) - # Strength reduced version - self.assertIn("_CHECK_FUNCTION_VERSION_INLINE", uops) - self.assertNotIn("_CHECK_FUNCTION_VERSION", uops) - # Removed guard - self.assertNotIn("_CHECK_FUNCTION_EXACT_ARGS", uops) - - def test_jit_error_pops(self): - """ - Tests that the correct number of pops are inserted into the - exit stub - """ - items = 17 * [None] + [[]] - with self.assertRaises(TypeError): - {item for item in items} - - def test_power_type_depends_on_input_values(self): - template = textwrap.dedent(""" - import _testinternalcapi - - L, R, X, Y = {l}, {r}, {x}, {y} - - def check(actual: complex, expected: complex) -> None: - assert actual == expected, (actual, expected) - assert type(actual) is type(expected), (actual, expected) - - def f(l: complex, r: complex) -> None: - expected_local_local = pow(l, r) + pow(l, r) - expected_const_local = pow(L, r) + pow(L, r) - expected_local_const = pow(l, R) + pow(l, R) - expected_const_const = pow(L, R) + pow(L, R) - for _ in range(_testinternalcapi.TIER2_THRESHOLD): - # Narrow types: - l + l, r + r - # The powers produce results, and the addition is unguarded: - check(l ** r + l ** r, expected_local_local) - check(L ** r + L ** r, expected_const_local) - check(l ** R + l ** R, expected_local_const) - check(L ** R + L ** R, expected_const_const) - - # JIT for one pair of values... - f(L, R) - # ...then run with another: - f(X, Y) - """) - interesting = [ - (1, 1), # int ** int -> int - (1, -1), # int ** int -> float - (1.0, 1), # float ** int -> float - (1, 1.0), # int ** float -> float - (-1, 0.5), # int ** float -> complex - (1.0, 1.0), # float ** float -> float - (-1.0, 0.5), # float ** float -> complex - ] - for (l, r), (x, y) in itertools.product(interesting, repeat=2): - s = template.format(l=l, r=r, x=x, y=y) - with self.subTest(l=l, r=r, x=x, y=y): - script_helper.assert_python_ok("-c", s) - - def test_symbols_flow_through_tuples(self): - def testfunc(n): - for _ in range(n): - a = 1 - b = 2 - t = a, b - x, y = t - r = x + y - return r - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, 3) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_BINARY_OP_ADD_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertNotIn("_GUARD_TOS_INT", uops) - - def test_decref_escapes(self): - class Convert9999ToNone: - def __del__(self): - ns = sys._getframe(1).f_locals - if ns["i"] == _testinternalcapi.TIER2_THRESHOLD: - ns["i"] = None - - def crash_addition(): - try: - for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): - n = Convert9999ToNone() - i + i # Remove guards for i. - n = None # Change i. - i + i # This crashed when we didn't treat DECREF as escaping (gh-124483) - except TypeError: - pass - - crash_addition() - - def test_narrow_type_to_constant_bool_false(self): - def f(n): - trace = [] - for i in range(n): - # false is always False, but we can only prove that it's a bool: - false = i == TIER2_THRESHOLD - trace.append("A") - if not false: # Kept. - trace.append("B") - if not false: # Removed! - trace.append("C") - trace.append("D") - if false: # Removed! - trace.append("X") - trace.append("E") - trace.append("F") - if false: # Removed! - trace.append("X") - trace.append("G") - return trace - - trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - # Only one guard remains: - self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 1) - self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 0) - # But all of the appends we care about are still there: - self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) - - def test_narrow_type_to_constant_bool_true(self): - def f(n): - trace = [] - for i in range(n): - # true always True, but we can only prove that it's a bool: - true = i != TIER2_THRESHOLD - trace.append("A") - if true: # Kept. - trace.append("B") - if not true: # Removed! - trace.append("X") - trace.append("C") - if true: # Removed! - trace.append("D") - trace.append("E") - trace.append("F") - if not true: # Removed! - trace.append("X") - trace.append("G") - return trace - - trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - # Only one guard remains: - self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 0) - self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 1) - # But all of the appends we care about are still there: - self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) - - def test_narrow_type_to_constant_int_zero(self): - def f(n): - trace = [] - for i in range(n): - # zero is always (int) 0, but we can only prove that it's a integer: - false = i == TIER2_THRESHOLD # this will always be false, while hopefully still fooling optimizer improvements - zero = false + 0 # this should always set the variable zero equal to 0 - trace.append("A") - if not zero: # Kept. - trace.append("B") - if not zero: # Removed! - trace.append("C") - trace.append("D") - if zero: # Removed! - trace.append("X") - trace.append("E") - trace.append("F") - if zero: # Removed! - trace.append("X") - trace.append("G") - return trace - - trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - # Only one guard remains: - self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 1) - self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 0) - # But all of the appends we care about are still there: - self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) - - def test_narrow_type_to_constant_str_empty(self): - def f(n): - trace = [] - for i in range(n): - # Hopefully the optimizer can't guess what the value is. - # empty is always "", but we can only prove that it's a string: - false = i == TIER2_THRESHOLD - empty = "X"[:false] - trace.append("A") - if not empty: # Kept. - trace.append("B") - if not empty: # Removed! - trace.append("C") - trace.append("D") - if empty: # Removed! - trace.append("X") - trace.append("E") - trace.append("F") - if empty: # Removed! - trace.append("X") - trace.append("G") - return trace - - trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - # Only one guard remains: - self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 1) - self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 0) - # But all of the appends we care about are still there: - self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) - - def test_compare_pop_two_load_const_inline_borrow(self): - def testfunc(n): - x = 0 - for _ in range(n): - a = 10 - b = 10 - if a == b: - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_COMPARE_OP_INT", uops) - self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) - - def test_to_bool_bool_contains_op_set(self): - """ - Test that _TO_BOOL_BOOL is removed from code like: - - res = foo in some_set - if res: - .... - - """ - def testfunc(n): - x = 0 - s = {1, 2, 3} - for _ in range(n): - a = 2 - in_set = a in s - if in_set: - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CONTAINS_OP_SET", uops) - self.assertNotIn("_TO_BOOL_BOOL", uops) - - def test_to_bool_bool_contains_op_dict(self): - """ - Test that _TO_BOOL_BOOL is removed from code like: - - res = foo in some_dict - if res: - .... - - """ - def testfunc(n): - x = 0 - s = {1: 1, 2: 2, 3: 3} - for _ in range(n): - a = 2 - in_dict = a in s - if in_dict: - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CONTAINS_OP_DICT", uops) - self.assertNotIn("_TO_BOOL_BOOL", uops) - - - def test_remove_guard_for_known_type_str(self): - def f(n): - for i in range(n): - false = i == TIER2_THRESHOLD - empty = "X"[:false] - empty += "" # Make JIT realize this is a string. - if empty: - return 1 - return 0 - - res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(res, 0) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_TO_BOOL_STR", uops) - self.assertNotIn("_GUARD_TOS_UNICODE", uops) - - def test_remove_guard_for_known_type_dict(self): - def f(n): - x = 0 - for _ in range(n): - d = {} - d["Spam"] = 1 # unguarded! - x += d["Spam"] # ...unguarded! - return x - - res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertEqual(uops.count("_GUARD_NOS_DICT"), 0) - self.assertEqual(uops.count("_STORE_SUBSCR_DICT"), 1) - self.assertEqual(uops.count("_BINARY_OP_SUBSCR_DICT"), 1) - - def test_remove_guard_for_known_type_list(self): - def f(n): - x = 0 - for _ in range(n): - l = [0] - l[0] = 1 # unguarded! - [a] = l # ...unguarded! - b = l[0] # ...unguarded! - if l: # ...unguarded! - x += a + b - return x - - res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(res, 2 * TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertEqual(uops.count("_GUARD_NOS_LIST"), 0) - self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT"), 1) - self.assertEqual(uops.count("_GUARD_TOS_LIST"), 0) - self.assertEqual(uops.count("_UNPACK_SEQUENCE_LIST"), 1) - self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT"), 1) - self.assertEqual(uops.count("_TO_BOOL_LIST"), 1) - - def test_remove_guard_for_known_type_set(self): - def f(n): - x = 0 - for _ in range(n): - x += "Spam" in {"Spam"} # Unguarded! - return x - - res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_ANY_SET", uops) - self.assertIn("_CONTAINS_OP_SET", uops) - - def test_remove_guard_for_known_type_tuple(self): - def f(n): - x = 0 - for _ in range(n): - t = (1, 2, (3, (4,))) - t_0, t_1, (t_2_0, t_2_1) = t # Unguarded! - t_2_1_0 = t_2_1[0] # Unguarded! - x += t_0 + t_1 + t_2_0 + t_2_1_0 - return x - - res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) - self.assertEqual(res, 10 * TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_TUPLE", uops) - self.assertIn("_UNPACK_SEQUENCE_TUPLE", uops) - self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) - self.assertNotIn("_GUARD_NOS_TUPLE", uops) - self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) - - def test_binary_subcsr_str_int_narrows_to_str(self): - def testfunc(n): - x = [] - s = "foo" - for _ in range(n): - y = s[0] # _BINARY_OP_SUBSCR_STR_INT - z = "bar" + y # (_GUARD_TOS_UNICODE) + _BINARY_OP_ADD_UNICODE - x.append(z) - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, ["barf"] * TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_BINARY_OP_SUBSCR_STR_INT", uops) - # _BINARY_OP_SUBSCR_STR_INT narrows the result to 'str' so - # the unicode guard before _BINARY_OP_ADD_UNICODE is removed. - self.assertNotIn("_GUARD_TOS_UNICODE", uops) - self.assertIn("_BINARY_OP_ADD_UNICODE", uops) - - def test_call_type_1(self): - def testfunc(n): - x = 0 - for _ in range(n): - x += type(42) is int - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CALL_TYPE_1", uops) - self.assertNotIn("_GUARD_NOS_NULL", uops) - self.assertNotIn("_GUARD_CALLABLE_TYPE_1", uops) - - def test_call_type_1_result_is_const(self): - def testfunc(n): - x = 0 - for _ in range(n): - t = type(42) - if t is not None: # guard is removed - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CALL_TYPE_1", uops) - self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) - - def test_call_str_1(self): - def testfunc(n): - x = 0 - for _ in range(n): - y = str(42) - if y == '42': - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CALL_STR_1", uops) - self.assertNotIn("_GUARD_NOS_NULL", uops) - self.assertNotIn("_GUARD_CALLABLE_STR_1", uops) - - def test_call_str_1_result_is_str(self): - def testfunc(n): - x = 0 - for _ in range(n): - y = str(42) + 'foo' - if y == '42foo': - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CALL_STR_1", uops) - self.assertIn("_BINARY_OP_ADD_UNICODE", uops) - self.assertNotIn("_GUARD_NOS_UNICODE", uops) - self.assertNotIn("_GUARD_TOS_UNICODE", uops) - - def test_call_str_1_result_is_const_for_str_input(self): - # Test a special case where the argument of str(arg) - # is known to be a string. The information about the - # argument being a string should be propagated to the - # result of str(arg). - def testfunc(n): - x = 0 - for _ in range(n): - y = str('foo') # string argument - if y: # _TO_BOOL_STR + _GUARD_IS_TRUE_POP are removed - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CALL_STR_1", uops) - self.assertNotIn("_TO_BOOL_STR", uops) - self.assertNotIn("_GUARD_IS_TRUE_POP", uops) - - def test_call_tuple_1(self): - def testfunc(n): - x = 0 - for _ in range(n): - y = tuple([1, 2]) # _CALL_TUPLE_1 - if y == (1, 2): - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CALL_TUPLE_1", uops) - self.assertNotIn("_GUARD_NOS_NULL", uops) - self.assertNotIn("_GUARD_CALLABLE_TUPLE_1", uops) - - def test_call_tuple_1_result_is_tuple(self): - def testfunc(n): - x = 0 - for _ in range(n): - y = tuple([1, 2]) # _CALL_TUPLE_1 - if y[0] == 1: # _BINARY_OP_SUBSCR_TUPLE_INT - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CALL_TUPLE_1", uops) - self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) - self.assertNotIn("_GUARD_NOS_TUPLE", uops) - - def test_call_tuple_1_result_propagates_for_tuple_input(self): - # Test a special case where the argument of tuple(arg) - # is known to be a tuple. The information about the - # argument being a tuple should be propagated to the - # result of tuple(arg). - def testfunc(n): - x = 0 - for _ in range(n): - y = tuple((1, 2)) # tuple argument - a, _ = y # _UNPACK_SEQUENCE_TWO_TUPLE - if a == 1: # _COMPARE_OP_INT + _GUARD_IS_TRUE_POP are removed - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_CALL_TUPLE_1", uops) - self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) - self.assertNotIn("_COMPARE_OP_INT", uops) - self.assertNotIn("_GUARD_IS_TRUE_POP", uops) - - def test_call_len(self): - def testfunc(n): - a = [1, 2, 3, 4] - for _ in range(n): - _ = len(a) - 1 - - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_NOS_NULL", uops) - self.assertNotIn("_GUARD_CALLABLE_LEN", uops) - self.assertIn("_CALL_LEN", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertNotIn("_GUARD_TOS_INT", uops) - - def test_binary_op_subscr_tuple_int(self): - def testfunc(n): - x = 0 - for _ in range(n): - y = (1, 2) - if y[0] == 1: # _COMPARE_OP_INT + _GUARD_IS_TRUE_POP are removed - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) - self.assertNotIn("_COMPARE_OP_INT", uops) - self.assertNotIn("_GUARD_IS_TRUE_POP", uops) - - def test_attr_promotion_failure(self): - # We're not testing for any specific uops here, just - # testing it doesn't crash. - script_helper.assert_python_ok('-c', textwrap.dedent(""" - import _testinternalcapi - import _opcode - import email - - def get_first_executor(func): - code = func.__code__ - co_code = code.co_code - for i in range(0, len(co_code), 2): - try: - return _opcode.get_executor(code, i) - except ValueError: - pass - return None - - def testfunc(n): - for _ in range(n): - email.jit_testing = None - prompt = email.jit_testing - del email.jit_testing - - - testfunc(_testinternalcapi.TIER2_THRESHOLD) - ex = get_first_executor(testfunc) - assert ex is not None - """)) - - def test_interpreter_finalization_with_generator_alive(self): - script_helper.assert_python_ok("-c", textwrap.dedent(""" - import sys - t = tuple(range(%d)) - def simple_for(): - for x in t: - x - - def gen(): - try: - yield - except: - simple_for() - - sys.settrace(lambda *args: None) - simple_for() - g = gen() - next(g) - """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) - - def test_next_instr_for_exception_handler_set(self): - # gh-140104: We just want the exception to be caught properly. - def f(): - for i in range(TIER2_THRESHOLD + 3): - try: - undefined_variable(i) - except Exception: - pass - - f() - - def test_next_instr_for_exception_handler_set_lasts_instr(self): - # gh-140104: We just want the exception to be caught properly. - def f(): - a_list = [] - for _ in range(TIER2_THRESHOLD + 3): - try: - a_list[""] = 0 - except Exception: - pass - - -def global_identity(x): - return x - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_pyatomic.py b/Lib/test/test_capi/test_pyatomic.py deleted file mode 100644 index 846d6d50c25..00000000000 --- a/Lib/test/test_capi/test_pyatomic.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest -from test.support import import_helper - -# Skip this test if the _testcapi module isn't available. -_testcapi = import_helper.import_module('_testcapi') - -class PyAtomicTests(unittest.TestCase): - pass - -for name in sorted(dir(_testcapi)): - if name.startswith('test_atomic'): - setattr(PyAtomicTests, name, getattr(_testcapi, name)) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py deleted file mode 100644 index 894f66b437a..00000000000 --- a/Lib/test/test_capi/test_run.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import unittest -from collections import UserDict -from test.support import import_helper -from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE - -NULL = None -_testcapi = import_helper.import_module('_testcapi') -Py_single_input = _testcapi.Py_single_input -Py_file_input = _testcapi.Py_file_input -Py_eval_input = _testcapi.Py_eval_input - - -class DictSubclass(dict): - pass - - -class CAPITest(unittest.TestCase): - # TODO: Test the following functions: - # - # PyRun_SimpleStringFlags - # PyRun_AnyFileExFlags - # PyRun_SimpleFileExFlags - # PyRun_InteractiveOneFlags - # PyRun_InteractiveOneObject - # PyRun_InteractiveLoopFlags - # PyRun_String (may be a macro) - # PyRun_AnyFile (may be a macro) - # PyRun_AnyFileEx (may be a macro) - # PyRun_AnyFileFlags (may be a macro) - # PyRun_SimpleString (may be a macro) - # PyRun_SimpleFile (may be a macro) - # PyRun_SimpleFileEx (may be a macro) - # PyRun_InteractiveOne (may be a macro) - # PyRun_InteractiveLoop (may be a macro) - # PyRun_File (may be a macro) - # PyRun_FileEx (may be a macro) - # PyRun_FileFlags (may be a macro) - - def test_run_stringflags(self): - # Test PyRun_StringFlags(). - # XXX: fopen() uses different path encoding than Python on Windows. - def run(s, *args): - return _testcapi.run_stringflags(s, Py_file_input, *args) - source = b'a\n' - - self.assertIsNone(run(b'a\n', dict(a=1))) - self.assertIsNone(run(b'a\n', dict(a=1), {})) - self.assertIsNone(run(b'a\n', {}, dict(a=1))) - self.assertIsNone(run(b'a\n', {}, UserDict(a=1))) - - self.assertRaises(NameError, run, b'a\n', {}) - self.assertRaises(NameError, run, b'a\n', {}, {}) - self.assertRaises(TypeError, run, b'a\n', dict(a=1), []) - self.assertRaises(TypeError, run, b'a\n', dict(a=1), 1) - - self.assertIsNone(run(b'a\n', DictSubclass(a=1))) - self.assertIsNone(run(b'a\n', DictSubclass(), dict(a=1))) - self.assertRaises(NameError, run, b'a\n', DictSubclass()) - - self.assertIsNone(run(b'\xc3\xa4\n', {'\xe4': 1})) - self.assertRaises(SyntaxError, run, b'\xe4\n', {}) - - self.assertRaises(SystemError, run, b'a\n', NULL) - self.assertRaises(SystemError, run, b'a\n', NULL, {}) - self.assertRaises(SystemError, run, b'a\n', NULL, dict(a=1)) - self.assertRaises(SystemError, run, b'a\n', UserDict()) - self.assertRaises(SystemError, run, b'a\n', UserDict(), {}) - self.assertRaises(SystemError, run, b'a\n', UserDict(), dict(a=1)) - - # CRASHES run(NULL, {}) - - def test_run_fileexflags(self): - # Test PyRun_FileExFlags(). - filename = os.fsencode(TESTFN if os.name != 'nt' else TESTFN_ASCII) - with open(filename, 'wb') as fp: - fp.write(b'a\n') - self.addCleanup(unlink, filename) - def run(*args): - return _testcapi.run_fileexflags(filename, Py_file_input, *args) - - self.assertIsNone(run(dict(a=1))) - self.assertIsNone(run(dict(a=1), {})) - self.assertIsNone(run({}, dict(a=1))) - self.assertIsNone(run({}, UserDict(a=1))) - self.assertIsNone(run(dict(a=1), {}, 1)) # closeit = True - - self.assertRaises(NameError, run, {}) - self.assertRaises(NameError, run, {}, {}) - self.assertRaises(TypeError, run, dict(a=1), []) - self.assertRaises(TypeError, run, dict(a=1), 1) - - self.assertIsNone(run(DictSubclass(a=1))) - self.assertIsNone(run(DictSubclass(), dict(a=1))) - self.assertRaises(NameError, run, DictSubclass()) - - self.assertRaises(SystemError, run, NULL) - self.assertRaises(SystemError, run, NULL, {}) - self.assertRaises(SystemError, run, NULL, dict(a=1)) - self.assertRaises(SystemError, run, UserDict()) - self.assertRaises(SystemError, run, UserDict(), {}) - self.assertRaises(SystemError, run, UserDict(), dict(a=1)) - - @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') - @unittest.skipIf(os.name == 'nt', 'does not work on Windows') - def test_run_fileexflags_with_undecodable_filename(self): - run = _testcapi.run_fileexflags - try: - with open(TESTFN_UNDECODABLE, 'wb') as fp: - fp.write(b'a\n') - self.addCleanup(unlink, TESTFN_UNDECODABLE) - except OSError: - self.skipTest('undecodable paths are not supported') - self.assertIsNone(run(TESTFN_UNDECODABLE, Py_file_input, dict(a=1))) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_capi/test_set.py b/Lib/test/test_capi/test_set.py deleted file mode 100644 index 62d90a3f943..00000000000 --- a/Lib/test/test_capi/test_set.py +++ /dev/null @@ -1,271 +0,0 @@ -import unittest - -from test.support import import_helper - -# Skip this test if the _testcapi, _testlimitedcapi or _testinternalcapi -# modules aren't available. -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') -_testinternalcapi = import_helper.import_module('_testinternalcapi') - -class set_subclass(set): - pass - -class frozenset_subclass(frozenset): - pass - - -class BaseSetTests: - def assertImmutable(self, action, *args): - self.assertRaises(SystemError, action, frozenset(), *args) - self.assertRaises(SystemError, action, frozenset({1}), *args) - self.assertRaises(SystemError, action, frozenset_subclass(), *args) - self.assertRaises(SystemError, action, frozenset_subclass({1}), *args) - - -class TestSetCAPI(BaseSetTests, unittest.TestCase): - def test_set_check(self): - check = _testlimitedcapi.set_check - self.assertTrue(check(set())) - self.assertTrue(check({1, 2})) - self.assertFalse(check(frozenset())) - self.assertTrue(check(set_subclass())) - self.assertFalse(check(frozenset_subclass())) - self.assertFalse(check(object())) - # CRASHES: check(NULL) - - def test_set_check_exact(self): - check = _testlimitedcapi.set_checkexact - self.assertTrue(check(set())) - self.assertTrue(check({1, 2})) - self.assertFalse(check(frozenset())) - self.assertFalse(check(set_subclass())) - self.assertFalse(check(frozenset_subclass())) - self.assertFalse(check(object())) - # CRASHES: check(NULL) - - def test_frozenset_check(self): - check = _testlimitedcapi.frozenset_check - self.assertFalse(check(set())) - self.assertTrue(check(frozenset())) - self.assertTrue(check(frozenset({1, 2}))) - self.assertFalse(check(set_subclass())) - self.assertTrue(check(frozenset_subclass())) - self.assertFalse(check(object())) - # CRASHES: check(NULL) - - def test_frozenset_check_exact(self): - check = _testlimitedcapi.frozenset_checkexact - self.assertFalse(check(set())) - self.assertTrue(check(frozenset())) - self.assertTrue(check(frozenset({1, 2}))) - self.assertFalse(check(set_subclass())) - self.assertFalse(check(frozenset_subclass())) - self.assertFalse(check(object())) - # CRASHES: check(NULL) - - def test_anyset_check(self): - check = _testlimitedcapi.anyset_check - self.assertTrue(check(set())) - self.assertTrue(check({1, 2})) - self.assertTrue(check(frozenset())) - self.assertTrue(check(frozenset({1, 2}))) - self.assertTrue(check(set_subclass())) - self.assertTrue(check(frozenset_subclass())) - self.assertFalse(check(object())) - # CRASHES: check(NULL) - - def test_anyset_check_exact(self): - check = _testlimitedcapi.anyset_checkexact - self.assertTrue(check(set())) - self.assertTrue(check({1, 2})) - self.assertTrue(check(frozenset())) - self.assertTrue(check(frozenset({1, 2}))) - self.assertFalse(check(set_subclass())) - self.assertFalse(check(frozenset_subclass())) - self.assertFalse(check(object())) - # CRASHES: check(NULL) - - def test_set_new(self): - set_new = _testlimitedcapi.set_new - self.assertEqual(set_new().__class__, set) - self.assertEqual(set_new(), set()) - self.assertEqual(set_new((1, 1, 2)), {1, 2}) - self.assertEqual(set_new([1, 1, 2]), {1, 2}) - with self.assertRaisesRegex(TypeError, 'object is not iterable'): - set_new(object()) - with self.assertRaisesRegex(TypeError, 'object is not iterable'): - set_new(1) - with self.assertRaisesRegex(TypeError, "unhashable type: 'dict'"): - set_new((1, {})) - - def test_frozenset_new(self): - frozenset_new = _testlimitedcapi.frozenset_new - self.assertEqual(frozenset_new().__class__, frozenset) - self.assertEqual(frozenset_new(), frozenset()) - self.assertEqual(frozenset_new((1, 1, 2)), frozenset({1, 2})) - self.assertEqual(frozenset_new([1, 1, 2]), frozenset({1, 2})) - with self.assertRaisesRegex(TypeError, 'object is not iterable'): - frozenset_new(object()) - with self.assertRaisesRegex(TypeError, 'object is not iterable'): - frozenset_new(1) - with self.assertRaisesRegex(TypeError, "unhashable type: 'dict'"): - frozenset_new((1, {})) - - def test_set_size(self): - get_size = _testlimitedcapi.set_size - self.assertEqual(get_size(set()), 0) - self.assertEqual(get_size(frozenset()), 0) - self.assertEqual(get_size({1, 1, 2}), 2) - self.assertEqual(get_size(frozenset({1, 1, 2})), 2) - self.assertEqual(get_size(set_subclass((1, 2, 3))), 3) - self.assertEqual(get_size(frozenset_subclass((1, 2, 3))), 3) - with self.assertRaises(SystemError): - get_size(object()) - # CRASHES: get_size(NULL) - - def test_set_get_size(self): - get_size = _testcapi.set_get_size - self.assertEqual(get_size(set()), 0) - self.assertEqual(get_size(frozenset()), 0) - self.assertEqual(get_size({1, 1, 2}), 2) - self.assertEqual(get_size(frozenset({1, 1, 2})), 2) - self.assertEqual(get_size(set_subclass((1, 2, 3))), 3) - self.assertEqual(get_size(frozenset_subclass((1, 2, 3))), 3) - # CRASHES: get_size(NULL) - # CRASHES: get_size(object()) - - def test_set_contains(self): - contains = _testlimitedcapi.set_contains - for cls in (set, frozenset, set_subclass, frozenset_subclass): - with self.subTest(cls=cls): - instance = cls((1, 2)) - self.assertTrue(contains(instance, 1)) - self.assertFalse(contains(instance, 'missing')) - with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): - contains(instance, []) - # CRASHES: contains(instance, NULL) - # CRASHES: contains(NULL, object()) - # CRASHES: contains(NULL, NULL) - - def test_add(self): - add = _testlimitedcapi.set_add - for cls in (set, set_subclass): - with self.subTest(cls=cls): - instance = cls((1, 2)) - self.assertEqual(add(instance, 1), 0) - self.assertEqual(instance, {1, 2}) - self.assertEqual(add(instance, 3), 0) - self.assertEqual(instance, {1, 2, 3}) - with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): - add(instance, []) - with self.assertRaises(SystemError): - add(object(), 1) - self.assertImmutable(add, 1) - # CRASHES: add(NULL, object()) - # CRASHES: add(instance, NULL) - # CRASHES: add(NULL, NULL) - - def test_discard(self): - discard = _testlimitedcapi.set_discard - for cls in (set, set_subclass): - with self.subTest(cls=cls): - instance = cls((1, 2)) - self.assertEqual(discard(instance, 3), 0) - self.assertEqual(instance, {1, 2}) - self.assertEqual(discard(instance, 1), 1) - self.assertEqual(instance, {2}) - self.assertEqual(discard(instance, 2), 1) - self.assertEqual(instance, set()) - self.assertEqual(discard(instance, 2), 0) - self.assertEqual(instance, set()) - with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): - discard(instance, []) - with self.assertRaises(SystemError): - discard(object(), 1) - self.assertImmutable(discard, 1) - # CRASHES: discard(NULL, object()) - # CRASHES: discard(instance, NULL) - # CRASHES: discard(NULL, NULL) - - def test_pop(self): - pop = _testlimitedcapi.set_pop - orig = (1, 2) - for cls in (set, set_subclass): - with self.subTest(cls=cls): - instance = cls(orig) - self.assertIn(pop(instance), orig) - self.assertEqual(len(instance), 1) - self.assertIn(pop(instance), orig) - self.assertEqual(len(instance), 0) - with self.assertRaises(KeyError): - pop(instance) - with self.assertRaises(SystemError): - pop(object()) - self.assertImmutable(pop) - # CRASHES: pop(NULL) - - def test_clear(self): - clear = _testlimitedcapi.set_clear - for cls in (set, set_subclass): - with self.subTest(cls=cls): - instance = cls((1, 2)) - self.assertEqual(clear(instance), 0) - self.assertEqual(instance, set()) - self.assertEqual(clear(instance), 0) - self.assertEqual(instance, set()) - with self.assertRaises(SystemError): - clear(object()) - self.assertImmutable(clear) - # CRASHES: clear(NULL) - - -class TestInternalCAPI(BaseSetTests, unittest.TestCase): - def test_set_update(self): - update = _testinternalcapi.set_update - for cls in (set, set_subclass): - for it in ('ab', ('a', 'b'), ['a', 'b'], - set('ab'), set_subclass('ab'), - frozenset('ab'), frozenset_subclass('ab')): - with self.subTest(cls=cls, it=it): - instance = cls() - self.assertEqual(update(instance, it), 0) - self.assertEqual(instance, {'a', 'b'}) - instance = cls(it) - self.assertEqual(update(instance, it), 0) - self.assertEqual(instance, {'a', 'b'}) - with self.assertRaisesRegex(TypeError, 'object is not iterable'): - update(cls(), 1) - with self.assertRaisesRegex(TypeError, "unhashable type: 'dict'"): - update(cls(), [{}]) - with self.assertRaises(SystemError): - update(object(), 'ab') - self.assertImmutable(update, 'ab') - # CRASHES: update(NULL, object()) - # CRASHES: update(instance, NULL) - # CRASHES: update(NULL, NULL) - - def test_set_next_entry(self): - set_next = _testinternalcapi.set_next_entry - for cls in (set, set_subclass, frozenset, frozenset_subclass): - with self.subTest(cls=cls): - instance = cls('abc') - pos = 0 - items = [] - while True: - res = set_next(instance, pos) - if res is None: - break - rc, pos, hash_, item = res - items.append(item) - self.assertEqual(rc, 1) - self.assertIn(item, instance) - self.assertEqual(hash(item), hash_) - self.assertEqual(items, list(instance)) - with self.assertRaises(SystemError): - set_next(object(), 0) - # CRASHES: set_next(NULL, 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py deleted file mode 100644 index f14ad9a9a5f..00000000000 --- a/Lib/test/test_capi/test_structmembers.py +++ /dev/null @@ -1,186 +0,0 @@ -import unittest -from test.support import import_helper - -# Skip this test if the _testcapi module isn't available. -import_helper.import_module('_testcapi') -from _testcapi import (_test_structmembersType_OldAPI, - _test_structmembersType_NewAPI, - CHAR_MAX, CHAR_MIN, UCHAR_MAX, - SHRT_MAX, SHRT_MIN, USHRT_MAX, - INT_MAX, INT_MIN, UINT_MAX, - LONG_MAX, LONG_MIN, ULONG_MAX, - LLONG_MAX, LLONG_MIN, ULLONG_MAX, - PY_SSIZE_T_MAX, PY_SSIZE_T_MIN, - ) - - -class Index: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value - -# There are two classes: one using and another using -# `Py_`-prefixed API. They should behave the same in Python - -def _make_test_object(cls): - return cls(False, # T_BOOL - 1, # T_BYTE - 2, # T_UBYTE - 3, # T_SHORT - 4, # T_USHORT - 5, # T_INT - 6, # T_UINT - 7, # T_LONG - 8, # T_ULONG - 23, # T_PYSSIZET - 9.99999,# T_FLOAT - 10.1010101010, # T_DOUBLE - "hi", # T_STRING_INPLACE - 12, # T_LONGLONG - 13, # T_ULONGLONG - b"c", # T_CHAR - ) - - -class ReadWriteTests: - def setUp(self): - self.ts = _make_test_object(self.cls) - - def _test_write(self, name, value, expected=None): - if expected is None: - expected = value - ts = self.ts - setattr(ts, name, value) - self.assertEqual(getattr(ts, name), expected) - - def _test_warn(self, name, value, expected=None): - ts = self.ts - self.assertWarns(RuntimeWarning, setattr, ts, name, value) - if expected is not None: - self.assertEqual(getattr(ts, name), expected) - - def _test_overflow(self, name, value): - ts = self.ts - self.assertRaises(OverflowError, setattr, ts, name, value) - - def _test_int_range(self, name, minval, maxval, *, hardlimit=None, - indexlimit=None): - if hardlimit is None: - hardlimit = (minval, maxval) - ts = self.ts - self._test_write(name, minval) - self._test_write(name, maxval) - hardminval, hardmaxval = hardlimit - self._test_overflow(name, hardminval-1) - self._test_overflow(name, hardmaxval+1) - self._test_overflow(name, 2**1000) - self._test_overflow(name, -2**1000) - if hardminval < minval: - self._test_warn(name, hardminval) - self._test_warn(name, minval-1, maxval) - if maxval < hardmaxval: - self._test_warn(name, maxval+1, minval) - self._test_warn(name, hardmaxval) - - if indexlimit is False: - self.assertRaises(TypeError, setattr, ts, name, Index(minval)) - self.assertRaises(TypeError, setattr, ts, name, Index(maxval)) - else: - self._test_write(name, Index(minval), minval) - self._test_write(name, Index(maxval), maxval) - self._test_overflow(name, Index(hardminval-1)) - self._test_overflow(name, Index(hardmaxval+1)) - self._test_overflow(name, Index(2**1000)) - self._test_overflow(name, Index(-2**1000)) - if hardminval < minval: - self._test_warn(name, Index(hardminval)) - self._test_warn(name, Index(minval-1), maxval) - if maxval < hardmaxval: - self._test_warn(name, Index(maxval+1), minval) - self._test_warn(name, Index(hardmaxval)) - - def test_bool(self): - ts = self.ts - ts.T_BOOL = True - self.assertIs(ts.T_BOOL, True) - ts.T_BOOL = False - self.assertIs(ts.T_BOOL, False) - self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 1) - self.assertRaises(TypeError, setattr, ts, 'T_BOOL', 0) - self.assertRaises(TypeError, setattr, ts, 'T_BOOL', None) - - def test_byte(self): - self._test_int_range('T_BYTE', CHAR_MIN, CHAR_MAX, - hardlimit=(LONG_MIN, LONG_MAX)) - self._test_int_range('T_UBYTE', 0, UCHAR_MAX, - hardlimit=(LONG_MIN, LONG_MAX)) - - def test_short(self): - self._test_int_range('T_SHORT', SHRT_MIN, SHRT_MAX, - hardlimit=(LONG_MIN, LONG_MAX)) - self._test_int_range('T_USHORT', 0, USHRT_MAX, - hardlimit=(LONG_MIN, LONG_MAX)) - - def test_int(self): - self._test_int_range('T_INT', INT_MIN, INT_MAX, - hardlimit=(LONG_MIN, LONG_MAX)) - self._test_int_range('T_UINT', 0, UINT_MAX, - hardlimit=(LONG_MIN, ULONG_MAX)) - - def test_long(self): - self._test_int_range('T_LONG', LONG_MIN, LONG_MAX) - self._test_int_range('T_ULONG', 0, ULONG_MAX, - hardlimit=(LONG_MIN, ULONG_MAX)) - - def test_py_ssize_t(self): - self._test_int_range('T_PYSSIZET', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, indexlimit=False) - - def test_longlong(self): - self._test_int_range('T_LONGLONG', LLONG_MIN, LLONG_MAX) - self._test_int_range('T_ULONGLONG', 0, ULLONG_MAX, - hardlimit=(LONG_MIN, ULLONG_MAX)) - - def test_bad_assignments(self): - ts = self.ts - integer_attributes = [ - 'T_BOOL', - 'T_BYTE', 'T_UBYTE', - 'T_SHORT', 'T_USHORT', - 'T_INT', 'T_UINT', - 'T_LONG', 'T_ULONG', - 'T_LONGLONG', 'T_ULONGLONG', - 'T_PYSSIZET' - ] - - # issue8014: this produced 'bad argument to internal function' - # internal error - for nonint in None, 3.2j, "full of eels", {}, []: - for attr in integer_attributes: - self.assertRaises(TypeError, setattr, ts, attr, nonint) - - def test_inplace_string(self): - ts = self.ts - self.assertEqual(ts.T_STRING_INPLACE, "hi") - self.assertRaises(TypeError, setattr, ts, "T_STRING_INPLACE", "s") - self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") - - def test_char(self): - ts = self.ts - self.assertEqual(ts.T_CHAR, "c") - ts.T_CHAR = "z" - self.assertEqual(ts.T_CHAR, "z") - self.assertRaises(TypeError, setattr, ts, "T_CHAR", "") - self.assertRaises(TypeError, setattr, ts, "T_CHAR", b"a") - self.assertRaises(TypeError, setattr, ts, "T_CHAR", bytearray(b"b")) - self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") - -class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): - cls = _test_structmembersType_OldAPI - -class ReadWriteTests_NewAPI(ReadWriteTests, unittest.TestCase): - cls = _test_structmembersType_NewAPI - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py deleted file mode 100644 index d3a9b378e77..00000000000 --- a/Lib/test/test_capi/test_sys.py +++ /dev/null @@ -1,154 +0,0 @@ -import unittest -import contextlib -import sys -from test import support -from test.support import import_helper - -try: - import _testlimitedcapi -except ImportError: - _testlimitedcapi = None - -NULL = None - -class CAPITest(unittest.TestCase): - # TODO: Test the following functions: - # - # PySys_Audit() - # PySys_AuditTuple() - - maxDiff = None - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_sys_getobject(self): - # Test PySys_GetObject() - getobject = _testlimitedcapi.sys_getobject - - self.assertIs(getobject(b'stdout'), sys.stdout) - with support.swap_attr(sys, '\U0001f40d', 42): - self.assertEqual(getobject('\U0001f40d'.encode()), 42) - - self.assertIs(getobject(b'nonexisting'), AttributeError) - with support.catch_unraisable_exception() as cm: - self.assertIs(getobject(b'\xff'), AttributeError) - self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) - self.assertRegex(str(cm.unraisable.exc_value), - "'utf-8' codec can't decode") - # CRASHES getobject(NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_sys_setobject(self): - # Test PySys_SetObject() - setobject = _testlimitedcapi.sys_setobject - - value = ['value'] - value2 = ['value2'] - try: - self.assertEqual(setobject(b'newattr', value), 0) - self.assertIs(sys.newattr, value) - self.assertEqual(setobject(b'newattr', value2), 0) - self.assertIs(sys.newattr, value2) - self.assertEqual(setobject(b'newattr', NULL), 0) - self.assertNotHasAttr(sys, 'newattr') - self.assertEqual(setobject(b'newattr', NULL), 0) - finally: - with contextlib.suppress(AttributeError): - del sys.newattr - try: - self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) - self.assertIs(getattr(sys, '\U0001f40d'), value) - self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) - self.assertNotHasAttr(sys, '\U0001f40d') - finally: - with contextlib.suppress(AttributeError): - delattr(sys, '\U0001f40d') - - with self.assertRaises(UnicodeDecodeError): - setobject(b'\xff', value) - # CRASHES setobject(NULL, value) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_sys_getxoptions(self): - # Test PySys_GetXOptions() - getxoptions = _testlimitedcapi.sys_getxoptions - - self.assertIs(getxoptions(), sys._xoptions) - - xoptions = sys._xoptions - try: - sys._xoptions = 'non-dict' - self.assertEqual(getxoptions(), {}) - self.assertIs(getxoptions(), sys._xoptions) - - del sys._xoptions - self.assertEqual(getxoptions(), {}) - self.assertIs(getxoptions(), sys._xoptions) - finally: - sys._xoptions = xoptions - self.assertIs(getxoptions(), sys._xoptions) - - def _test_sys_formatstream(self, funname, streamname): - import_helper.import_module('ctypes') - from ctypes import pythonapi, c_char_p, py_object - func = getattr(pythonapi, funname) - func.argtypes = (c_char_p,) - - # Supports plain C types. - with support.captured_output(streamname) as stream: - func(b'Hello, %s!', c_char_p(b'world')) - self.assertEqual(stream.getvalue(), 'Hello, world!') - - # Supports Python objects. - with support.captured_output(streamname) as stream: - func(b'Hello, %R!', py_object('world')) - self.assertEqual(stream.getvalue(), "Hello, 'world'!") - - # The total length is not limited. - with support.captured_output(streamname) as stream: - func(b'Hello, %s!', c_char_p(b'world'*200)) - self.assertEqual(stream.getvalue(), 'Hello, ' + 'world'*200 + '!') - - def test_sys_formatstdout(self): - # Test PySys_FormatStdout() - self._test_sys_formatstream('PySys_FormatStdout', 'stdout') - - def test_sys_formatstderr(self): - # Test PySys_FormatStderr() - self._test_sys_formatstream('PySys_FormatStderr', 'stderr') - - def _test_sys_writestream(self, funname, streamname): - import_helper.import_module('ctypes') - from ctypes import pythonapi, c_char_p - func = getattr(pythonapi, funname) - func.argtypes = (c_char_p,) - - # Supports plain C types. - with support.captured_output(streamname) as stream: - func(b'Hello, %s!', c_char_p(b'world')) - self.assertEqual(stream.getvalue(), 'Hello, world!') - - # There is a limit on the total length. - with support.captured_output(streamname) as stream: - func(b'Hello, %s!', c_char_p(b'world'*100)) - self.assertEqual(stream.getvalue(), 'Hello, ' + 'world'*100 + '!') - with support.captured_output(streamname) as stream: - func(b'Hello, %s!', c_char_p(b'world'*200)) - out = stream.getvalue() - self.assertEqual(out[:20], 'Hello, worldworldwor') - self.assertEqual(out[-13:], '... truncated') - self.assertGreater(len(out), 1000) - - def test_sys_writestdout(self): - # Test PySys_WriteStdout() - self._test_sys_writestream('PySys_WriteStdout', 'stdout') - - def test_sys_writestderr(self): - # Test PySys_WriteStderr() - self._test_sys_writestream('PySys_WriteStderr', 'stderr') - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_time.py b/Lib/test/test_capi/test_time.py deleted file mode 100644 index 989c158818c..00000000000 --- a/Lib/test/test_capi/test_time.py +++ /dev/null @@ -1,78 +0,0 @@ -import time -import unittest -from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') - - -PyTime_MIN = _testcapi.PyTime_MIN -PyTime_MAX = _testcapi.PyTime_MAX -SEC_TO_NS = 10 ** 9 -DAY_TO_SEC = (24 * 60 * 60) -# Worst clock resolution: maximum delta between two clock reads. -CLOCK_RES = 0.050 - - -class CAPITest(unittest.TestCase): - def test_min_max(self): - # PyTime_t is just int64_t - self.assertEqual(PyTime_MIN, -2**63) - self.assertEqual(PyTime_MAX, 2**63 - 1) - - def test_assecondsdouble(self): - # Test PyTime_AsSecondsDouble() - def ns_to_sec(ns): - if abs(ns) % SEC_TO_NS == 0: - return float(ns // SEC_TO_NS) - else: - return float(ns) / SEC_TO_NS - - seconds = ( - 0, - 1, - DAY_TO_SEC, - 365 * DAY_TO_SEC, - ) - values = { - PyTime_MIN, - PyTime_MIN + 1, - PyTime_MAX - 1, - PyTime_MAX, - } - for second in seconds: - ns = second * SEC_TO_NS - values.add(ns) - # test nanosecond before/after to test rounding - values.add(ns - 1) - values.add(ns + 1) - for ns in list(values): - if (-ns) > PyTime_MAX: - continue - values.add(-ns) - for ns in sorted(values): - with self.subTest(ns=ns): - self.assertEqual(_testcapi.PyTime_AsSecondsDouble(ns), - ns_to_sec(ns)) - - def check_clock(self, c_func, py_func): - t1 = c_func() - t2 = py_func() - self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) - - def test_monotonic(self): - # Test PyTime_Monotonic() and PyTime_MonotonicRaw() - self.check_clock(_testcapi.PyTime_Monotonic, time.monotonic) - self.check_clock(_testcapi.PyTime_MonotonicRaw, time.monotonic) - - def test_perf_counter(self): - # Test PyTime_PerfCounter() and PyTime_PerfCounterRaw() - self.check_clock(_testcapi.PyTime_PerfCounter, time.perf_counter) - self.check_clock(_testcapi.PyTime_PerfCounterRaw, time.perf_counter) - - def test_time(self): - # Test PyTime_Time() and PyTime_TimeRaw() - self.check_clock(_testcapi.PyTime_Time, time.time) - self.check_clock(_testcapi.PyTime_TimeRaw, time.time) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py deleted file mode 100644 index 3683c008dbc..00000000000 --- a/Lib/test/test_capi/test_tuple.py +++ /dev/null @@ -1,301 +0,0 @@ -import unittest -import gc -from test.support import import_helper - -_testcapi = import_helper.import_module('_testcapi') -_testlimitedcapi = import_helper.import_module('_testlimitedcapi') - -NULL = None -PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN -PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX - -class TupleSubclass(tuple): - pass - - -class CAPITest(unittest.TestCase): - def _not_tracked(self, t): - self.assertFalse(gc.is_tracked(t), t) - - def _tracked(self, t): - self.assertTrue(gc.is_tracked(t), t) - - def test_check(self): - # Test PyTuple_Check() - check = _testlimitedcapi.tuple_check - - self.assertTrue(check((1, 2))) - self.assertTrue(check(())) - self.assertTrue(check(TupleSubclass((1, 2)))) - self.assertFalse(check({1: 2})) - self.assertFalse(check([1, 2])) - self.assertFalse(check(42)) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_tuple_checkexact(self): - # Test PyTuple_CheckExact() - check = _testlimitedcapi.tuple_checkexact - - self.assertTrue(check((1, 2))) - self.assertTrue(check(())) - self.assertFalse(check(TupleSubclass((1, 2)))) - self.assertFalse(check({1: 2})) - self.assertFalse(check([1, 2])) - self.assertFalse(check(42)) - self.assertFalse(check(object())) - - # CRASHES check(NULL) - - def test_tuple_new(self): - # Test PyTuple_New() - tuple_new = _testlimitedcapi.tuple_new - size = _testlimitedcapi.tuple_size - checknull = _testcapi._check_tuple_item_is_NULL - - tup1 = tuple_new(0) - self.assertEqual(tup1, ()) - self.assertEqual(size(tup1), 0) - self.assertIs(type(tup1), tuple) - self._not_tracked(tup1) - - tup2 = tuple_new(1) - self.assertIs(type(tup2), tuple) - self.assertEqual(size(tup2), 1) - self.assertIsNot(tup2, tup1) - self.assertTrue(checknull(tup2, 0)) - self._tracked(tup2) - - self.assertRaises(SystemError, tuple_new, -1) - self.assertRaises(SystemError, tuple_new, PY_SSIZE_T_MIN) - self.assertRaises(MemoryError, tuple_new, PY_SSIZE_T_MAX) - - def test_tuple_pack(self): - # Test PyTuple_Pack() - pack = _testlimitedcapi.tuple_pack - - self.assertEqual(pack(0), ()) - self.assertEqual(pack(1, [1]), ([1],)) - self.assertEqual(pack(2, [1], [2]), ([1], [2])) - - self._tracked(pack(1, [1])) - self._tracked(pack(2, [1], b'abc')) - self._not_tracked(pack(2, 42, b'abc')) - - self.assertRaises(SystemError, pack, PY_SSIZE_T_MIN) - self.assertRaises(SystemError, pack, -1) - self.assertRaises(MemoryError, pack, PY_SSIZE_T_MAX) - - # CRASHES pack(1, NULL) - # CRASHES pack(2, [1]) - - def test_tuple_size(self): - # Test PyTuple_Size() - size = _testlimitedcapi.tuple_size - - self.assertEqual(size(()), 0) - self.assertEqual(size((1, 2)), 2) - self.assertEqual(size(TupleSubclass((1, 2))), 2) - - self.assertRaises(SystemError, size, []) - self.assertRaises(SystemError, size, 42) - self.assertRaises(SystemError, size, object()) - - # CRASHES size(NULL) - - def test_tuple_get_size(self): - # Test PyTuple_GET_SIZE() - size = _testcapi.tuple_get_size - - self.assertEqual(size(()), 0) - self.assertEqual(size((1, 2)), 2) - self.assertEqual(size(TupleSubclass((1, 2))), 2) - - def test_tuple_getitem(self): - # Test PyTuple_GetItem() - getitem = _testlimitedcapi.tuple_getitem - - tup = ([1], [2], [3]) - self.assertEqual(getitem(tup, 0), [1]) - self.assertEqual(getitem(tup, 2), [3]) - - tup2 = TupleSubclass(([1], [2], [3])) - self.assertEqual(getitem(tup2, 0), [1]) - self.assertEqual(getitem(tup2, 2), [3]) - - self.assertRaises(IndexError, getitem, tup, PY_SSIZE_T_MIN) - self.assertRaises(IndexError, getitem, tup, -1) - self.assertRaises(IndexError, getitem, tup, len(tup)) - self.assertRaises(IndexError, getitem, tup, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, getitem, [1, 2, 3], 1) - self.assertRaises(SystemError, getitem, 42, 1) - - # CRASHES getitem(NULL, 0) - - def test_tuple_get_item(self): - # Test PyTuple_GET_ITEM() - get_item = _testcapi.tuple_get_item - - tup = ([1], [2], [3]) - self.assertEqual(get_item(tup, 0), [1]) - self.assertEqual(get_item(tup, 2), [3]) - - tup2 = TupleSubclass(([1], [2], [3])) - self.assertEqual(get_item(tup2, 0), [1]) - self.assertEqual(get_item(tup2, 2), [3]) - - # CRASHES get_item(NULL, 0) - - def test_tuple_getslice(self): - # Test PyTuple_GetSlice() - getslice = _testlimitedcapi.tuple_getslice - - # empty - tup = ([1], [2], [3]) - self.assertEqual(getslice(tup, PY_SSIZE_T_MIN, 0), ()) - self.assertEqual(getslice(tup, -1, 0), ()) - self.assertEqual(getslice(tup, 3, PY_SSIZE_T_MAX), ()) - self.assertEqual(getslice(tup, 1, 1), ()) - self.assertEqual(getslice(tup, 2, 1), ()) - tup = TupleSubclass(([1], [2], [3])) - self.assertEqual(getslice(tup, PY_SSIZE_T_MIN, 0), ()) - self.assertEqual(getslice(tup, -1, 0), ()) - self.assertEqual(getslice(tup, 3, PY_SSIZE_T_MAX), ()) - self.assertEqual(getslice(tup, 1, 1), ()) - self.assertEqual(getslice(tup, 2, 1), ()) - - # slice - tup = ([1], [2], [3], [4]) - self.assertEqual(getslice(tup, 1, 3), ([2], [3])) - tup = TupleSubclass(([1], [2], [3], [4])) - self.assertEqual(getslice(tup, 1, 3), ([2], [3])) - - # whole - tup = ([1], [2], [3]) - self.assertEqual(getslice(tup, 0, 3), tup) - self.assertEqual(getslice(tup, 0, 100), tup) - self.assertEqual(getslice(tup, -100, 100), tup) - tup = TupleSubclass(([1], [2], [3])) - self.assertEqual(getslice(tup, 0, 3), tup) - self.assertEqual(getslice(tup, 0, 100), tup) - self.assertEqual(getslice(tup, -100, 100), tup) - - self.assertRaises(SystemError, getslice, [[1], [2], [3]], 0, 0) - self.assertRaises(SystemError, getslice, 42, 0, 0) - - # CRASHES getslice(NULL, 0, 0) - - def test_tuple_setitem(self): - # Test PyTuple_SetItem() - setitem = _testlimitedcapi.tuple_setitem - checknull = _testcapi._check_tuple_item_is_NULL - - tup = ([1], [2]) - self.assertEqual(setitem(tup, 0, []), ([], [2])) - self.assertEqual(setitem(tup, 1, []), ([1], [])) - - tup2 = setitem(tup, 1, NULL) - self.assertTrue(checknull(tup2, 1)) - - tup2 = TupleSubclass(([1], [2])) - self.assertRaises(SystemError, setitem, tup2, 0, []) - - self.assertRaises(IndexError, setitem, tup, PY_SSIZE_T_MIN, []) - self.assertRaises(IndexError, setitem, tup, -1, []) - self.assertRaises(IndexError, setitem, tup, len(tup), []) - self.assertRaises(IndexError, setitem, tup, PY_SSIZE_T_MAX, []) - self.assertRaises(SystemError, setitem, [1], 0, []) - self.assertRaises(SystemError, setitem, 42, 0, []) - - # CRASHES setitem(NULL, 0, []) - - def test_tuple_set_item(self): - # Test PyTuple_SET_ITEM() - set_item = _testcapi.tuple_set_item - checknull = _testcapi._check_tuple_item_is_NULL - - tup = ([1], [2]) - self.assertEqual(set_item(tup, 0, []), ([], [2])) - self.assertEqual(set_item(tup, 1, []), ([1], [])) - - tup2 = set_item(tup, 1, NULL) - self.assertTrue(checknull(tup2, 1)) - - tup2 = TupleSubclass(([1], [2])) - self.assertIs(set_item(tup2, 0, []), tup2) - self.assertEqual(tup2, ([], [2])) - - # CRASHES set_item(tup, -1, []) - # CRASHES set_item(tup, len(tup), []) - # CRASHES set_item([1], 0, []) - # CRASHES set_item(NULL, 0, []) - - def test__tuple_resize(self): - # Test _PyTuple_Resize() - resize = _testcapi._tuple_resize - checknull = _testcapi._check_tuple_item_is_NULL - - a = () - b = resize(a, 0, False) - self.assertEqual(len(a), 0) - self.assertEqual(len(b), 0) - b = resize(a, 2, False) - self.assertEqual(len(a), 0) - self.assertEqual(len(b), 2) - self.assertTrue(checknull(b, 0)) - self.assertTrue(checknull(b, 1)) - - a = ([1], [2], [3]) - b = resize(a, 3) - self.assertEqual(b, a) - b = resize(a, 2) - self.assertEqual(b, a[:2]) - b = resize(a, 5) - self.assertEqual(len(b), 5) - self.assertEqual(b[:3], a) - self.assertTrue(checknull(b, 3)) - self.assertTrue(checknull(b, 4)) - - a = () - self.assertRaises(MemoryError, resize, a, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, resize, a, -1) - self.assertRaises(SystemError, resize, a, PY_SSIZE_T_MIN) - # refcount > 1 - a = (1, 2, 3) - self.assertRaises(SystemError, resize, a, 3, False) - self.assertRaises(SystemError, resize, a, 0, False) - # non-tuple - self.assertRaises(SystemError, resize, [1, 2, 3], 0, False) - self.assertRaises(SystemError, resize, NULL, 0, False) - - def test_bug_59313(self): - # Before 3.14, the C-API function PySequence_Tuple - # would create incomplete tuples which were visible to - # the cycle GC, and this test would crash the interpeter. - TAG = object() - tuples = [] - - def referrer_tuples(): - return [x for x in gc.get_referrers(TAG) - if isinstance(x, tuple)] - - def my_iter(): - nonlocal tuples - yield TAG # 'tag' gets stored in the result tuple - tuples += referrer_tuples() - for x in range(10): - tuples += referrer_tuples() - # Prior to 3.13 would raise a SystemError when the tuple needs to be resized - yield x - - self.assertEqual(tuple(my_iter()), (TAG, *range(10))) - self.assertEqual(tuples, []) - - def test_uninitialized_tuple_repr(self): - tup = _testlimitedcapi.tuple_new(3) - self.assertEqual(repr(tup), '(, , )') - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py deleted file mode 100644 index 15fb4a93e2a..00000000000 --- a/Lib/test/test_capi/test_type.py +++ /dev/null @@ -1,276 +0,0 @@ -from test.support import import_helper, Py_GIL_DISABLED, refleak_helper -import unittest - -_testcapi = import_helper.import_module('_testcapi') - - -class BuiltinStaticTypesTests(unittest.TestCase): - - TYPES = [ - object, - type, - int, - str, - dict, - type(None), - bool, - BaseException, - Exception, - Warning, - DeprecationWarning, # Warning subclass - ] - - def test_tp_bases_is_set(self): - # PyTypeObject.tp_bases is documented as public API. - # See https://github.com/python/cpython/issues/105020. - for typeobj in self.TYPES: - with self.subTest(typeobj): - bases = _testcapi.type_get_tp_bases(typeobj) - self.assertIsNot(bases, None) - - def test_tp_mro_is_set(self): - # PyTypeObject.tp_bases is documented as public API. - # See https://github.com/python/cpython/issues/105020. - for typeobj in self.TYPES: - with self.subTest(typeobj): - mro = _testcapi.type_get_tp_mro(typeobj) - self.assertIsNot(mro, None) - - -class TypeTests(unittest.TestCase): - def test_get_type_name(self): - class MyType: - pass - - from _testcapi import ( - get_type_name, get_type_qualname, - get_type_fullyqualname, get_type_module_name) - - from collections import OrderedDict - ht = _testcapi.get_heaptype_for_name() - for cls, fullname, modname, qualname, name in ( - (int, - 'int', - 'builtins', - 'int', - 'int'), - (OrderedDict, - 'collections.OrderedDict', - 'collections', - 'OrderedDict', - 'OrderedDict'), - (ht, - '_testcapi.HeapTypeNameType', - '_testcapi', - 'HeapTypeNameType', - 'HeapTypeNameType'), - (MyType, - f'{__name__}.TypeTests.test_get_type_name..MyType', - __name__, - 'TypeTests.test_get_type_name..MyType', - 'MyType'), - ): - with self.subTest(cls=repr(cls)): - self.assertEqual(get_type_fullyqualname(cls), fullname) - self.assertEqual(get_type_module_name(cls), modname) - self.assertEqual(get_type_qualname(cls), qualname) - self.assertEqual(get_type_name(cls), name) - - # override __module__ - ht.__module__ = 'test_module' - self.assertEqual(get_type_fullyqualname(ht), 'test_module.HeapTypeNameType') - self.assertEqual(get_type_module_name(ht), 'test_module') - self.assertEqual(get_type_qualname(ht), 'HeapTypeNameType') - self.assertEqual(get_type_name(ht), 'HeapTypeNameType') - - # override __name__ and __qualname__ - MyType.__name__ = 'my_name' - MyType.__qualname__ = 'my_qualname' - self.assertEqual(get_type_fullyqualname(MyType), f'{__name__}.my_qualname') - self.assertEqual(get_type_module_name(MyType), __name__) - self.assertEqual(get_type_qualname(MyType), 'my_qualname') - self.assertEqual(get_type_name(MyType), 'my_name') - - # override also __module__ - MyType.__module__ = 'my_module' - self.assertEqual(get_type_fullyqualname(MyType), 'my_module.my_qualname') - self.assertEqual(get_type_module_name(MyType), 'my_module') - self.assertEqual(get_type_qualname(MyType), 'my_qualname') - self.assertEqual(get_type_name(MyType), 'my_name') - - # PyType_GetFullyQualifiedName() ignores the module if it's "builtins" - # or "__main__" of it is not a string - MyType.__module__ = 'builtins' - self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') - MyType.__module__ = '__main__' - self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') - MyType.__module__ = 123 - self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') - - def test_get_base_by_token(self): - def get_base_by_token(src, key, comparable=True): - def run(use_mro): - find_first = _testcapi.pytype_getbasebytoken - ret1, result = find_first(src, key, use_mro, True) - ret2, no_result = find_first(src, key, use_mro, False) - self.assertIn(ret1, (0, 1)) - self.assertEqual(ret1, result is not None) - self.assertEqual(ret1, ret2) - self.assertIsNone(no_result) - return result - - found_in_mro = run(True) - found_in_bases = run(False) - if comparable: - self.assertIs(found_in_mro, found_in_bases) - return found_in_mro - return found_in_mro, found_in_bases - - create_type = _testcapi.create_type_with_token - get_token = _testcapi.get_tp_token - - Py_TP_USE_SPEC = _testcapi.Py_TP_USE_SPEC - self.assertEqual(Py_TP_USE_SPEC, 0) - - A1 = create_type('_testcapi.A1', Py_TP_USE_SPEC) - self.assertTrue(get_token(A1) != Py_TP_USE_SPEC) - - B1 = create_type('_testcapi.B1', id(self)) - self.assertTrue(get_token(B1) == id(self)) - - tokenA1 = get_token(A1) - # find A1 from A1 - found = get_base_by_token(A1, tokenA1) - self.assertIs(found, A1) - - # no token in static types - STATIC = type(1) - self.assertEqual(get_token(STATIC), 0) - found = get_base_by_token(STATIC, tokenA1) - self.assertIs(found, None) - - # no token in pure subtypes - class A2(A1): pass - self.assertEqual(get_token(A2), 0) - # find A1 - class Z(STATIC, B1, A2): pass - found = get_base_by_token(Z, tokenA1) - self.assertIs(found, A1) - - # searching for NULL token is an error - with self.assertRaises(SystemError): - get_base_by_token(Z, 0) - with self.assertRaises(SystemError): - get_base_by_token(STATIC, 0) - - # share the token with A1 - C1 = create_type('_testcapi.C1', tokenA1) - self.assertTrue(get_token(C1) == tokenA1) - - # find C1 first by shared token - class Z(C1, A2): pass - found = get_base_by_token(Z, tokenA1) - self.assertIs(found, C1) - # B1 not found - found = get_base_by_token(Z, get_token(B1)) - self.assertIs(found, None) - - with self.assertRaises(TypeError): - _testcapi.pytype_getbasebytoken( - 'not a type', id(self), True, False) - - def test_get_module_by_def(self): - heaptype = _testcapi.create_type_with_token('_testcapi.H', 0) - mod = _testcapi.pytype_getmodulebydef(heaptype) - self.assertIs(mod, _testcapi) - - class H1(heaptype): pass - mod = _testcapi.pytype_getmodulebydef(H1) - self.assertIs(mod, _testcapi) - - with self.assertRaises(TypeError): - _testcapi.pytype_getmodulebydef(int) - - class H2(int): pass - with self.assertRaises(TypeError): - _testcapi.pytype_getmodulebydef(H2) - - def test_freeze(self): - # test PyType_Freeze() - type_freeze = _testcapi.type_freeze - - # simple case, no inherante - class MyType: - pass - MyType.attr = "mutable" - - type_freeze(MyType) - err_msg = "cannot set 'attr' attribute of immutable type 'MyType'" - with self.assertRaisesRegex(TypeError, err_msg): - # the class is now immutable - MyType.attr = "immutable" - - # test MRO: PyType_Freeze() requires base classes to be immutable - class A: pass - class B: pass - class C(B): pass - class D(A, C): pass - - self.assertEqual(D.mro(), [D, A, C, B, object]) - with self.assertRaises(TypeError): - type_freeze(D) - - type_freeze(A) - type_freeze(B) - type_freeze(C) - # all parent classes are now immutable, so D can be made immutable - # as well - type_freeze(D) - - @unittest.skipIf( - Py_GIL_DISABLED and refleak_helper.hunting_for_refleaks(), - "Specialization failure triggers gh-127773") - def test_freeze_meta(self): - """test PyType_Freeze() with overridden MRO""" - type_freeze = _testcapi.type_freeze - - class Base: - value = 1 - - class Meta(type): - def mro(cls): - return (cls, Base, object) - - class FreezeThis(metaclass=Meta): - """This has `Base` in the MRO, but not tp_bases""" - - self.assertEqual(FreezeThis.value, 1) - - with self.assertRaises(TypeError): - type_freeze(FreezeThis) - - Base.value = 2 - self.assertEqual(FreezeThis.value, 2) - - type_freeze(Base) - with self.assertRaises(TypeError): - Base.value = 3 - type_freeze(FreezeThis) - self.assertEqual(FreezeThis.value, 2) - - def test_manual_heap_type(self): - # gh-128923: test that a manually allocated and initailized heap type - # works correctly - ManualHeapType = _testcapi.ManualHeapType - for i in range(100): - self.assertIsInstance(ManualHeapType(), ManualHeapType) - - def test_extension_managed_dict_type(self): - ManagedDictType = _testcapi.ManagedDictType - obj = ManagedDictType() - obj.foo = 42 - self.assertEqual(obj.foo, 42) - self.assertEqual(obj.__dict__, {'foo': 42}) - obj.__dict__ = {'bar': 3} - self.assertEqual(obj.__dict__, {'bar': 3}) - self.assertEqual(obj.bar, 3) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py deleted file mode 100644 index 7e5f4c9dac9..00000000000 --- a/Lib/test/test_capi/test_unicode.py +++ /dev/null @@ -1,2039 +0,0 @@ -import unittest -import sys -from test import support -from test.support import threading_helper - -try: - import _testcapi - from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX -except ImportError: - _testcapi = None -try: - import _testlimitedcapi -except ImportError: - _testlimitedcapi = None -try: - import _testinternalcapi -except ImportError: - _testinternalcapi = None -try: - import ctypes -except ImportError: - ctypes = None - - -NULL = None - -class Str(str): - pass - - -class CAPITest(unittest.TestCase): - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_new(self): - """Test PyUnicode_New()""" - from _testcapi import unicode_new as new - - for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff: - self.assertEqual(new(0, maxchar), '') - self.assertEqual(new(5, maxchar), chr(maxchar)*5) - self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX, maxchar) - self.assertEqual(new(0, 0x110000), '') - self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x4f60) - self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x4f60) - self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x1f600) - self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x1f600) - self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4, 0x1f600) - self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4+1, 0x1f600) - self.assertRaises(SystemError, new, 5, 0x110000) - self.assertRaises(SystemError, new, -1, 0) - self.assertRaises(SystemError, new, PY_SSIZE_T_MIN, 0) - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_fill(self): - """Test PyUnicode_Fill()""" - from _testcapi import unicode_fill as fill - - strings = [ - # all strings have exactly 5 characters - 'abcde', '\xa1\xa2\xa3\xa4\xa5', - '\u4f60\u597d\u4e16\u754c\uff01', - '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' - ] - chars = [0x78, 0xa9, 0x20ac, 0x1f638] - - for idx, fill_char in enumerate(chars): - # wide -> narrow: exceed maxchar limitation - for to in strings[:idx]: - self.assertRaises(ValueError, fill, to, 0, 0, fill_char) - for to in strings[idx:]: - for start in [*range(7), PY_SSIZE_T_MAX]: - for length in [*range(-1, 7 - start), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: - filled = max(min(length, 5 - start), 0) - if filled == 5 and to != strings[idx]: - # narrow -> wide - # Tests omitted since this creates invalid strings. - continue - expected = to[:start] + chr(fill_char) * filled + to[start + filled:] - self.assertEqual(fill(to, start, length, fill_char), - (expected, filled)) - - s = strings[0] - self.assertRaises(IndexError, fill, s, -1, 0, 0x78) - self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78) - self.assertRaises(ValueError, fill, s, 0, 0, 0x110000) - self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78) - self.assertRaises(SystemError, fill, [], 0, 0, 0x78) - # CRASHES fill(s, 0, NULL, 0, 0) - # CRASHES fill(NULL, 0, 0, 0x78) - # TODO: Test PyUnicode_Fill() with non-modifiable unicode. - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_writechar(self): - """Test PyUnicode_WriteChar()""" - from _testlimitedcapi import unicode_writechar as writechar - - strings = [ - # one string for every kind - 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', - '\U0001f600\U0001f601\U0001f602' - ] - # one character for every kind + out of range code - chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000] - for i, s in enumerate(strings): - for j, c in enumerate(chars): - if j <= i: - self.assertEqual(writechar(s, 1, c), - (s[:1] + chr(c) + s[2:], 0)) - else: - self.assertRaises(ValueError, writechar, s, 1, c) - - self.assertRaises(IndexError, writechar, 'abc', 3, 0x78) - self.assertRaises(IndexError, writechar, 'abc', -1, 0x78) - self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MAX, 0x78) - self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MIN, 0x78) - self.assertRaises(TypeError, writechar, b'abc', 0, 0x78) - self.assertRaises(TypeError, writechar, [], 0, 0x78) - # CRASHES writechar(NULL, 0, 0x78) - # TODO: Test PyUnicode_WriteChar() with non-modifiable and legacy - # unicode. - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_resize(self): - """Test PyUnicode_Resize()""" - from _testlimitedcapi import unicode_resize as resize - - strings = [ - # all strings have exactly 3 characters - 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', - '\U0001f600\U0001f601\U0001f602' - ] - for s in strings: - self.assertEqual(resize(s, 3), (s, 0)) - self.assertEqual(resize(s, 2), (s[:2], 0)) - self.assertEqual(resize(s, 4), (s + '\0', 0)) - self.assertEqual(resize(s, 10), (s + '\0'*7, 0)) - self.assertEqual(resize(s, 0), ('', 0)) - self.assertRaises(MemoryError, resize, s, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, resize, s, -1) - self.assertRaises(SystemError, resize, s, PY_SSIZE_T_MIN) - self.assertRaises(SystemError, resize, b'abc', 0) - self.assertRaises(SystemError, resize, [], 0) - self.assertRaises(SystemError, resize, NULL, 0) - # TODO: Test PyUnicode_Resize() with non-modifiable and legacy unicode - # and with NULL as the address. - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_append(self): - """Test PyUnicode_Append()""" - from _testlimitedcapi import unicode_append as append - - strings = [ - 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', - '\U0001f600\U0001f601\U0001f602' - ] - for left in strings: - left = left[::-1] - for right in strings: - expected = left + right - self.assertEqual(append(left, right), expected) - - self.assertRaises(SystemError, append, 'abc', b'abc') - self.assertRaises(SystemError, append, b'abc', 'abc') - self.assertRaises(SystemError, append, b'abc', b'abc') - self.assertRaises(SystemError, append, 'abc', []) - self.assertRaises(SystemError, append, [], 'abc') - self.assertRaises(SystemError, append, [], []) - self.assertRaises(SystemError, append, NULL, 'abc') - self.assertRaises(SystemError, append, 'abc', NULL) - # TODO: Test PyUnicode_Append() with modifiable unicode - # and with NULL as the address. - # TODO: Check reference counts. - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_appendanddel(self): - """Test PyUnicode_AppendAndDel()""" - from _testlimitedcapi import unicode_appendanddel as appendanddel - - strings = [ - 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', - '\U0001f600\U0001f601\U0001f602' - ] - for left in strings: - left = left[::-1] - for right in strings: - self.assertEqual(appendanddel(left, right), left + right) - - self.assertRaises(SystemError, appendanddel, 'abc', b'abc') - self.assertRaises(SystemError, appendanddel, b'abc', 'abc') - self.assertRaises(SystemError, appendanddel, b'abc', b'abc') - self.assertRaises(SystemError, appendanddel, 'abc', []) - self.assertRaises(SystemError, appendanddel, [], 'abc') - self.assertRaises(SystemError, appendanddel, [], []) - self.assertRaises(SystemError, appendanddel, NULL, 'abc') - self.assertRaises(SystemError, appendanddel, 'abc', NULL) - # TODO: Test PyUnicode_AppendAndDel() with modifiable unicode - # and with NULL as the address. - # TODO: Check reference counts. - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_fromstringandsize(self): - """Test PyUnicode_FromStringAndSize()""" - from _testlimitedcapi import unicode_fromstringandsize as fromstringandsize - - self.assertEqual(fromstringandsize(b'abc'), 'abc') - self.assertEqual(fromstringandsize(b'abc', 2), 'ab') - self.assertEqual(fromstringandsize(b'abc\0def'), 'abc\0def') - self.assertEqual(fromstringandsize(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') - self.assertEqual(fromstringandsize(b'\xe4\xbd\xa0'), '\u4f60') - self.assertEqual(fromstringandsize(b'\xf0\x9f\x98\x80'), '\U0001f600') - self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xc2\xa1', 1) - self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xa1', 1) - self.assertEqual(fromstringandsize(b'', 0), '') - self.assertEqual(fromstringandsize(NULL, 0), '') - - self.assertRaises(MemoryError, fromstringandsize, b'abc', PY_SSIZE_T_MAX) - self.assertRaises(SystemError, fromstringandsize, b'abc', -1) - self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) - self.assertRaises(SystemError, fromstringandsize, NULL, -1) - self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) - self.assertRaises(SystemError, fromstringandsize, NULL, 3) - self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MAX) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_fromstring(self): - """Test PyUnicode_FromString()""" - from _testlimitedcapi import unicode_fromstring as fromstring - - self.assertEqual(fromstring(b'abc'), 'abc') - self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') - self.assertEqual(fromstring(b'\xe4\xbd\xa0'), '\u4f60') - self.assertEqual(fromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') - self.assertRaises(UnicodeDecodeError, fromstring, b'\xc2') - self.assertRaises(UnicodeDecodeError, fromstring, b'\xa1') - self.assertEqual(fromstring(b''), '') - - # CRASHES fromstring(NULL) - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_fromkindanddata(self): - """Test PyUnicode_FromKindAndData()""" - from _testcapi import unicode_fromkindanddata as fromkindanddata - - strings = [ - 'abcde', '\xa1\xa2\xa3\xa4\xa5', - '\u4f60\u597d\u4e16\u754c\uff01', - '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' - ] - enc1 = 'latin1' - for s in strings[:2]: - self.assertEqual(fromkindanddata(1, s.encode(enc1)), s) - enc2 = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' - for s in strings[:3]: - self.assertEqual(fromkindanddata(2, s.encode(enc2)), s) - enc4 = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' - for s in strings: - self.assertEqual(fromkindanddata(4, s.encode(enc4)), s) - self.assertEqual(fromkindanddata(2, '\U0001f600'.encode(enc2)), - '\ud83d\ude00') - for kind in 1, 2, 4: - self.assertEqual(fromkindanddata(kind, b''), '') - self.assertEqual(fromkindanddata(kind, b'\0'*kind), '\0') - self.assertEqual(fromkindanddata(kind, NULL, 0), '') - - for kind in -1, 0, 3, 5, 8: - self.assertRaises(SystemError, fromkindanddata, kind, b'') - self.assertRaises(ValueError, fromkindanddata, 1, b'abc', -1) - self.assertRaises(ValueError, fromkindanddata, 1, b'abc', PY_SSIZE_T_MIN) - self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1) - self.assertRaises(ValueError, fromkindanddata, 1, NULL, PY_SSIZE_T_MIN) - # CRASHES fromkindanddata(1, NULL, 1) - # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_substring(self): - """Test PyUnicode_Substring()""" - from _testlimitedcapi import unicode_substring as substring - - strings = [ - 'ab', 'ab\xa1\xa2', - 'ab\xa1\xa2\u4f60\u597d', - 'ab\xa1\xa2\u4f60\u597d\U0001f600\U0001f601' - ] - for s in strings: - for start in [*range(0, len(s) + 2), PY_SSIZE_T_MAX]: - for end in [*range(max(start-1, 0), len(s) + 2), PY_SSIZE_T_MAX]: - self.assertEqual(substring(s, start, end), s[start:end]) - - self.assertRaises(IndexError, substring, 'abc', -1, 0) - self.assertRaises(IndexError, substring, 'abc', PY_SSIZE_T_MIN, 0) - self.assertRaises(IndexError, substring, 'abc', 0, -1) - self.assertRaises(IndexError, substring, 'abc', 0, PY_SSIZE_T_MIN) - # CRASHES substring(b'abc', 0, 0) - # CRASHES substring([], 0, 0) - # CRASHES substring(NULL, 0, 0) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_getlength(self): - """Test PyUnicode_GetLength()""" - from _testlimitedcapi import unicode_getlength as getlength - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', - 'a\ud800b\udfffc', '\ud834\udd1e']: - self.assertEqual(getlength(s), len(s)) - - self.assertRaises(TypeError, getlength, b'abc') - self.assertRaises(TypeError, getlength, []) - # CRASHES getlength(NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_readchar(self): - """Test PyUnicode_ReadChar()""" - from _testlimitedcapi import unicode_readchar as readchar - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', - 'a\ud800b\udfffc', '\ud834\udd1e']: - for i, c in enumerate(s): - self.assertEqual(readchar(s, i), ord(c)) - self.assertRaises(IndexError, readchar, s, len(s)) - self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MAX) - self.assertRaises(IndexError, readchar, s, -1) - self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MIN) - - self.assertRaises(TypeError, readchar, b'abc', 0) - self.assertRaises(TypeError, readchar, [], 0) - # CRASHES readchar(NULL, 0) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_fromobject(self): - """Test PyUnicode_FromObject()""" - from _testlimitedcapi import unicode_fromobject as fromobject - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', - 'a\ud800b\udfffc', '\ud834\udd1e']: - self.assertEqual(fromobject(s), s) - o = Str(s) - s2 = fromobject(o) - self.assertEqual(s2, s) - self.assertIs(type(s2), str) - self.assertIsNot(s2, s) - - self.assertRaises(TypeError, fromobject, b'abc') - self.assertRaises(TypeError, fromobject, []) - # CRASHES fromobject(NULL) - - @unittest.skipIf(ctypes is None, 'need ctypes') - def test_from_format(self): - """Test PyUnicode_FromFormat()""" - # Length modifiers "j" and "t" are not tested here because ctypes does - # not expose types for intmax_t and ptrdiff_t. - # _testlimitedcapi.test_string_from_format() has a wider coverage of all - # formats. - from ctypes import ( - c_char_p, - pythonapi, py_object, sizeof, - c_int, c_long, c_longlong, c_ssize_t, - c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p, - c_wchar, c_wchar_p) - name = "PyUnicode_FromFormat" - _PyUnicode_FromFormat = getattr(pythonapi, name) - _PyUnicode_FromFormat.argtypes = (c_char_p,) - _PyUnicode_FromFormat.restype = py_object - - def PyUnicode_FromFormat(format, *args): - cargs = tuple( - py_object(arg) if isinstance(arg, str) else arg - for arg in args) - return _PyUnicode_FromFormat(format, *cargs) - - def check_format(expected, format, *args): - text = PyUnicode_FromFormat(format, *args) - self.assertEqual(expected, text) - - # ascii format, non-ascii argument - check_format('ascii\x7f=unicode\xe9', - b'ascii\x7f=%U', 'unicode\xe9') - - # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() - # raises an error - self.assertRaisesRegex(ValueError, - r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' - 'string, got a non-ASCII byte: 0xe9$', - PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') - - # test "%c" - check_format('\uabcd', - b'%c', c_int(0xabcd)) - check_format('\U0010ffff', - b'%c', c_int(0x10ffff)) - with self.assertRaises(OverflowError): - PyUnicode_FromFormat(b'%c', c_int(0x110000)) - # Issue #18183 - check_format('\U00010000\U00100000', - b'%c%c', c_int(0x10000), c_int(0x100000)) - - # test "%" - check_format('%', - b'%%') - check_format('%s', - b'%%s') - check_format('[%]', - b'[%%]') - check_format('%abc', - b'%%%s', b'abc') - - # truncated string - check_format('abc', - b'%.3s', b'abcdef') - check_format('abc[', - b'%.6s', 'abc[\u20ac]'.encode('utf8')) - check_format('abc[\u20ac', - b'%.7s', 'abc[\u20ac]'.encode('utf8')) - check_format('abc[\ufffd', - b'%.5s', b'abc[\xff]') - check_format('abc[', - b'%.6s', b'abc[\xe2\x82]') - check_format('abc[\ufffd]', - b'%.7s', b'abc[\xe2\x82]') - check_format('abc[\ufffd', - b'%.7s', b'abc[\xe2\x82\0') - check_format(' abc[', - b'%10.6s', 'abc[\u20ac]'.encode('utf8')) - check_format(' abc[\u20ac', - b'%10.7s', 'abc[\u20ac]'.encode('utf8')) - check_format(' abc[\ufffd', - b'%10.5s', b'abc[\xff]') - check_format(' abc[', - b'%10.6s', b'abc[\xe2\x82]') - check_format(' abc[\ufffd]', - b'%10.7s', b'abc[\xe2\x82]') - - check_format("'\\u20acABC'", - b'%A', '\u20acABC') - check_format("'\\u20", - b'%.5A', '\u20acABCDEF') - check_format("'\u20acABC'", - b'%R', '\u20acABC') - check_format("'\u20acA", - b'%.3R', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3S', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3U', '\u20acABCDEF') - - check_format('\u20acAB', - b'%.3V', '\u20acABCDEF', None) - check_format('abc[', - b'%.6V', None, 'abc[\u20ac]'.encode('utf8')) - check_format('abc[\u20ac', - b'%.7V', None, 'abc[\u20ac]'.encode('utf8')) - check_format('abc[\ufffd', - b'%.5V', None, b'abc[\xff]') - check_format('abc[', - b'%.6V', None, b'abc[\xe2\x82]') - check_format('abc[\ufffd]', - b'%.7V', None, b'abc[\xe2\x82]') - check_format(' abc[', - b'%10.6V', None, 'abc[\u20ac]'.encode('utf8')) - check_format(' abc[\u20ac', - b'%10.7V', None, 'abc[\u20ac]'.encode('utf8')) - check_format(' abc[\ufffd', - b'%10.5V', None, b'abc[\xff]') - check_format(' abc[', - b'%10.6V', None, b'abc[\xe2\x82]') - check_format(' abc[\ufffd]', - b'%10.7V', None, b'abc[\xe2\x82]') - check_format(' abc[\ufffd', - b'%10.7V', None, b'abc[\xe2\x82\0') - - # following tests comes from #7330 - # test width modifier and precision modifier with %S - check_format("repr= abc", - b'repr=%5S', 'abc') - check_format("repr=ab", - b'repr=%.2S', 'abc') - check_format("repr= ab", - b'repr=%5.2S', 'abc') - - # test width modifier and precision modifier with %R - check_format("repr= 'abc'", - b'repr=%8R', 'abc') - check_format("repr='ab", - b'repr=%.3R', 'abc') - check_format("repr= 'ab", - b'repr=%5.3R', 'abc') - - # test width modifier and precision modifier with %A - check_format("repr= 'abc'", - b'repr=%8A', 'abc') - check_format("repr='ab", - b'repr=%.3A', 'abc') - check_format("repr= 'ab", - b'repr=%5.3A', 'abc') - - # test width modifier and precision modifier with %s - check_format("repr= abc", - b'repr=%5s', b'abc') - check_format("repr=ab", - b'repr=%.2s', b'abc') - check_format("repr= ab", - b'repr=%5.2s', b'abc') - - # test width modifier and precision modifier with %U - check_format("repr= abc", - b'repr=%5U', 'abc') - check_format("repr=ab", - b'repr=%.2U', 'abc') - check_format("repr= ab", - b'repr=%5.2U', 'abc') - - # test width modifier and precision modifier with %V - check_format("repr= abc", - b'repr=%5V', 'abc', b'123') - check_format("repr=ab", - b'repr=%.2V', 'abc', b'123') - check_format("repr= ab", - b'repr=%5.2V', 'abc', b'123') - check_format("repr= 123", - b'repr=%5V', None, b'123') - check_format("repr=12", - b'repr=%.2V', None, b'123') - check_format("repr= 12", - b'repr=%5.2V', None, b'123') - - # test integer formats (%i, %d, %u, %o, %x, %X) - check_format('010', - b'%03i', c_int(10)) - check_format('0010', - b'%0.4i', c_int(10)) - for conv, signed, value, expected in [ - (b'i', True, -123, '-123'), - (b'd', True, -123, '-123'), - (b'u', False, 123, '123'), - (b'o', False, 0o123, '123'), - (b'x', False, 0xabc, 'abc'), - (b'X', False, 0xabc, 'ABC'), - ]: - for mod, ctype in [ - (b'', c_int if signed else c_uint), - (b'l', c_long if signed else c_ulong), - (b'll', c_longlong if signed else c_ulonglong), - (b'z', c_ssize_t if signed else c_size_t), - ]: - with self.subTest(format=b'%' + mod + conv): - check_format(expected, - b'%' + mod + conv, ctype(value)) - - # test long output - min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) - max_longlong = -min_longlong - 1 - check_format(str(min_longlong), - b'%lld', c_longlong(min_longlong)) - check_format(str(max_longlong), - b'%lld', c_longlong(max_longlong)) - max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 - check_format(str(max_ulonglong), - b'%llu', c_ulonglong(max_ulonglong)) - PyUnicode_FromFormat(b'%p', c_void_p(-1)) - - # test padding (width and/or precision) - check_format('123', b'%2i', c_int(123)) - check_format(' 123', b'%10i', c_int(123)) - check_format('0000000123', b'%010i', c_int(123)) - check_format('123 ', b'%-10i', c_int(123)) - check_format('123 ', b'%-010i', c_int(123)) - check_format('123', b'%.2i', c_int(123)) - check_format('0000123', b'%.7i', c_int(123)) - check_format(' 123', b'%10.2i', c_int(123)) - check_format(' 0000123', b'%10.7i', c_int(123)) - check_format('0000000123', b'%010.7i', c_int(123)) - check_format('0000123 ', b'%-10.7i', c_int(123)) - check_format('0000123 ', b'%-010.7i', c_int(123)) - - check_format('-123', b'%2i', c_int(-123)) - check_format(' -123', b'%10i', c_int(-123)) - check_format('-000000123', b'%010i', c_int(-123)) - check_format('-123 ', b'%-10i', c_int(-123)) - check_format('-123 ', b'%-010i', c_int(-123)) - check_format('-123', b'%.2i', c_int(-123)) - check_format('-0000123', b'%.7i', c_int(-123)) - check_format(' -123', b'%10.2i', c_int(-123)) - check_format(' -0000123', b'%10.7i', c_int(-123)) - check_format('-000000123', b'%010.7i', c_int(-123)) - check_format('-0000123 ', b'%-10.7i', c_int(-123)) - check_format('-0000123 ', b'%-010.7i', c_int(-123)) - - check_format('123', b'%2u', c_uint(123)) - check_format(' 123', b'%10u', c_uint(123)) - check_format('0000000123', b'%010u', c_uint(123)) - check_format('123 ', b'%-10u', c_uint(123)) - check_format('123 ', b'%-010u', c_uint(123)) - check_format('123', b'%.2u', c_uint(123)) - check_format('0000123', b'%.7u', c_uint(123)) - check_format(' 123', b'%10.2u', c_uint(123)) - check_format(' 0000123', b'%10.7u', c_uint(123)) - check_format('0000000123', b'%010.7u', c_uint(123)) - check_format('0000123 ', b'%-10.7u', c_uint(123)) - check_format('0000123 ', b'%-010.7u', c_uint(123)) - - check_format('123', b'%2o', c_uint(0o123)) - check_format(' 123', b'%10o', c_uint(0o123)) - check_format('0000000123', b'%010o', c_uint(0o123)) - check_format('123 ', b'%-10o', c_uint(0o123)) - check_format('123 ', b'%-010o', c_uint(0o123)) - check_format('123', b'%.2o', c_uint(0o123)) - check_format('0000123', b'%.7o', c_uint(0o123)) - check_format(' 123', b'%10.2o', c_uint(0o123)) - check_format(' 0000123', b'%10.7o', c_uint(0o123)) - check_format('0000000123', b'%010.7o', c_uint(0o123)) - check_format('0000123 ', b'%-10.7o', c_uint(0o123)) - check_format('0000123 ', b'%-010.7o', c_uint(0o123)) - - check_format('abc', b'%2x', c_uint(0xabc)) - check_format(' abc', b'%10x', c_uint(0xabc)) - check_format('0000000abc', b'%010x', c_uint(0xabc)) - check_format('abc ', b'%-10x', c_uint(0xabc)) - check_format('abc ', b'%-010x', c_uint(0xabc)) - check_format('abc', b'%.2x', c_uint(0xabc)) - check_format('0000abc', b'%.7x', c_uint(0xabc)) - check_format(' abc', b'%10.2x', c_uint(0xabc)) - check_format(' 0000abc', b'%10.7x', c_uint(0xabc)) - check_format('0000000abc', b'%010.7x', c_uint(0xabc)) - check_format('0000abc ', b'%-10.7x', c_uint(0xabc)) - check_format('0000abc ', b'%-010.7x', c_uint(0xabc)) - - check_format('ABC', b'%2X', c_uint(0xabc)) - check_format(' ABC', b'%10X', c_uint(0xabc)) - check_format('0000000ABC', b'%010X', c_uint(0xabc)) - check_format('ABC ', b'%-10X', c_uint(0xabc)) - check_format('ABC ', b'%-010X', c_uint(0xabc)) - check_format('ABC', b'%.2X', c_uint(0xabc)) - check_format('0000ABC', b'%.7X', c_uint(0xabc)) - check_format(' ABC', b'%10.2X', c_uint(0xabc)) - check_format(' 0000ABC', b'%10.7X', c_uint(0xabc)) - check_format('0000000ABC', b'%010.7X', c_uint(0xabc)) - check_format('0000ABC ', b'%-10.7X', c_uint(0xabc)) - check_format('0000ABC ', b'%-010.7X', c_uint(0xabc)) - - # test %A - check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", - b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') - - # test %V - check_format('abc', - b'%V', 'abc', b'xyz') - check_format('xyz', - b'%V', None, b'xyz') - - # test %ls - check_format('abc', b'%ls', c_wchar_p('abc')) - check_format('\u4eba\u6c11', b'%ls', c_wchar_p('\u4eba\u6c11')) - check_format('\U0001f4bb+\U0001f40d', - b'%ls', c_wchar_p('\U0001f4bb+\U0001f40d')) - check_format(' ab', b'%5.2ls', c_wchar_p('abc')) - check_format(' \u4eba\u6c11', b'%5ls', c_wchar_p('\u4eba\u6c11')) - check_format(' \U0001f4bb+\U0001f40d', - b'%5ls', c_wchar_p('\U0001f4bb+\U0001f40d')) - check_format('\u4eba', b'%.1ls', c_wchar_p('\u4eba\u6c11')) - check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', - b'%.1ls', c_wchar_p('\U0001f4bb+\U0001f40d')) - check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', - b'%.2ls', c_wchar_p('\U0001f4bb+\U0001f40d')) - - # test %lV - check_format('abc', - b'%lV', 'abc', c_wchar_p('xyz')) - check_format('xyz', - b'%lV', None, c_wchar_p('xyz')) - check_format('\u4eba\u6c11', - b'%lV', None, c_wchar_p('\u4eba\u6c11')) - check_format('\U0001f4bb+\U0001f40d', - b'%lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) - check_format(' ab', - b'%5.2lV', None, c_wchar_p('abc')) - check_format(' \u4eba\u6c11', - b'%5lV', None, c_wchar_p('\u4eba\u6c11')) - check_format(' \U0001f4bb+\U0001f40d', - b'%5lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) - check_format('\u4eba', - b'%.1lV', None, c_wchar_p('\u4eba\u6c11')) - check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', - b'%.1lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) - check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', - b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) - - # test %T - check_format('type: str', - b'type: %T', py_object("abc")) - check_format(f'type: st', - b'type: %.2T', py_object("abc")) - check_format(f'type: str', - b'type: %10T', py_object("abc")) - - class LocalType: - pass - obj = LocalType() - fullname = f'{__name__}.{LocalType.__qualname__}' - check_format(f'type: {fullname}', - b'type: %T', py_object(obj)) - fullname_alt = f'{__name__}:{LocalType.__qualname__}' - check_format(f'type: {fullname_alt}', - b'type: %#T', py_object(obj)) - - # test %N - check_format('type: str', - b'type: %N', py_object(str)) - check_format(f'type: st', - b'type: %.2N', py_object(str)) - check_format(f'type: str', - b'type: %10N', py_object(str)) - - check_format(f'type: {fullname}', - b'type: %N', py_object(type(obj))) - check_format(f'type: {fullname_alt}', - b'type: %#N', py_object(type(obj))) - with self.assertRaisesRegex(TypeError, "%N argument must be a type"): - check_format('type: str', - b'type: %N', py_object("abc")) - - # test variable width and precision - check_format(' abc', b'%*s', c_int(5), b'abc') - check_format('ab', b'%.*s', c_int(2), b'abc') - check_format(' ab', b'%*.*s', c_int(5), c_int(2), b'abc') - check_format(' abc', b'%*U', c_int(5), 'abc') - check_format('ab', b'%.*U', c_int(2), 'abc') - check_format(' ab', b'%*.*U', c_int(5), c_int(2), 'abc') - check_format(' ab', b'%*.*V', c_int(5), c_int(2), None, b'abc') - check_format(' ab', b'%*.*lV', c_int(5), c_int(2), - None, c_wchar_p('abc')) - check_format(' 123', b'%*i', c_int(8), c_int(123)) - check_format('00123', b'%.*i', c_int(5), c_int(123)) - check_format(' 00123', b'%*.*i', c_int(8), c_int(5), c_int(123)) - - # test %p - # We cannot test the exact result, - # because it returns a hex representation of a C pointer, - # which is going to be different each time. But, we can test the format. - p_format_regex = r'^0x[a-zA-Z0-9]{3,}$' - p_format1 = PyUnicode_FromFormat(b'%p', 'abc') - self.assertIsInstance(p_format1, str) - self.assertRegex(p_format1, p_format_regex) - - p_format2 = PyUnicode_FromFormat(b'%p %p', '123456', b'xyz') - self.assertIsInstance(p_format2, str) - self.assertRegex(p_format2, - r'0x[a-zA-Z0-9]{3,} 0x[a-zA-Z0-9]{3,}') - - # Extra args are ignored: - p_format3 = PyUnicode_FromFormat(b'%p', '123456', None, b'xyz') - self.assertIsInstance(p_format3, str) - self.assertRegex(p_format3, p_format_regex) - - # Test string decode from parameter of %s using utf-8. - # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of - # '\u4eba\u6c11' - check_format('repr=\u4eba\u6c11', - b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') - - #Test replace error handler. - check_format('repr=abc\ufffd', - b'repr=%V', None, b'abc\xff') - - # Issue #33817: empty strings - check_format('', - b'') - check_format('', - b'%s', b'') - - # test invalid format strings. these tests are just here - # to check for crashes and should not be considered as specifications - for fmt in (b'%', b'%0', b'%01', b'%.', b'%.1', - b'%0%s', b'%1%s', b'%.%s', b'%.1%s', b'%1abc', - b'%l', b'%ll', b'%z', b'%lls', b'%zs'): - with self.subTest(fmt=fmt): - self.assertRaisesRegex(SystemError, 'invalid format string', - PyUnicode_FromFormat, fmt, b'abc') - self.assertRaisesRegex(SystemError, 'invalid format string', - PyUnicode_FromFormat, b'%+i', c_int(10)) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_interninplace(self): - """Test PyUnicode_InternInPlace()""" - from _testlimitedcapi import unicode_interninplace as interninplace - - s = b'abc'.decode() - r = interninplace(s) - self.assertEqual(r, 'abc') - - # CRASHES interninplace(b'abc') - # CRASHES interninplace(NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_internfromstring(self): - """Test PyUnicode_InternFromString()""" - from _testlimitedcapi import unicode_internfromstring as internfromstring - - self.assertEqual(internfromstring(b'abc'), 'abc') - self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') - self.assertRaises(UnicodeDecodeError, internfromstring, b'\xc2') - self.assertRaises(UnicodeDecodeError, internfromstring, b'\xa1') - self.assertEqual(internfromstring(b''), '') - - # CRASHES internfromstring(NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_fromwidechar(self): - """Test PyUnicode_FromWideChar()""" - from _testlimitedcapi import unicode_fromwidechar as fromwidechar - from _testcapi import SIZEOF_WCHAR_T - - if SIZEOF_WCHAR_T == 2: - encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' - elif SIZEOF_WCHAR_T == 4: - encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' - - for s in '', 'abc', '\xa1\xa2', '\u4f60', '\U0001f600': - b = s.encode(encoding) - self.assertEqual(fromwidechar(b), s) - self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) - for s in '\ud83d', '\ude00': - b = s.encode(encoding, 'surrogatepass') - self.assertEqual(fromwidechar(b), s) - self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) - - self.assertEqual(fromwidechar('abc'.encode(encoding), 2), 'ab') - if SIZEOF_WCHAR_T == 2: - self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d') - - self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX) - self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2) - self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN) - self.assertEqual(fromwidechar(NULL, 0), '') - self.assertRaises(SystemError, fromwidechar, NULL, 1) - self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, fromwidechar, NULL, -1) - self.assertRaises(SystemError, fromwidechar, NULL, -2) - self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_aswidechar(self): - """Test PyUnicode_AsWideChar()""" - from _testlimitedcapi import unicode_aswidechar - from _testlimitedcapi import unicode_aswidechar_null - from _testcapi import SIZEOF_WCHAR_T - - wchar, size = unicode_aswidechar('abcdef', 2) - self.assertEqual(size, 2) - self.assertEqual(wchar, 'ab') - - wchar, size = unicode_aswidechar('abc', 3) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc') - self.assertEqual(unicode_aswidechar_null('abc', 10), 4) - self.assertEqual(unicode_aswidechar_null('abc', 0), 4) - - wchar, size = unicode_aswidechar('abc', 4) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidechar('abc', 10) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidechar('abc\0def', 20) - self.assertEqual(size, 7) - self.assertEqual(wchar, 'abc\0def\0') - self.assertEqual(unicode_aswidechar_null('abc\0def', 20), 8) - - nonbmp = chr(0x10ffff) - if SIZEOF_WCHAR_T == 2: - nchar = 2 - else: # SIZEOF_WCHAR_T == 4 - nchar = 1 - wchar, size = unicode_aswidechar(nonbmp, 10) - self.assertEqual(size, nchar) - self.assertEqual(wchar, nonbmp + '\0') - self.assertEqual(unicode_aswidechar_null(nonbmp, 10), nchar + 1) - - self.assertRaises(TypeError, unicode_aswidechar, b'abc', 10) - self.assertRaises(TypeError, unicode_aswidechar, [], 10) - self.assertRaises(SystemError, unicode_aswidechar, NULL, 10) - self.assertRaises(TypeError, unicode_aswidechar_null, b'abc', 10) - self.assertRaises(TypeError, unicode_aswidechar_null, [], 10) - self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_aswidecharstring(self): - """Test PyUnicode_AsWideCharString()""" - from _testlimitedcapi import unicode_aswidecharstring - from _testlimitedcapi import unicode_aswidecharstring_null - from _testcapi import SIZEOF_WCHAR_T - - wchar, size = unicode_aswidecharstring('abc') - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - self.assertEqual(unicode_aswidecharstring_null('abc'), 'abc') - - wchar, size = unicode_aswidecharstring('abc\0def') - self.assertEqual(size, 7) - self.assertEqual(wchar, 'abc\0def\0') - self.assertRaises(ValueError, unicode_aswidecharstring_null, 'abc\0def') - - nonbmp = chr(0x10ffff) - if SIZEOF_WCHAR_T == 2: - nchar = 2 - else: # SIZEOF_WCHAR_T == 4 - nchar = 1 - wchar, size = unicode_aswidecharstring(nonbmp) - self.assertEqual(size, nchar) - self.assertEqual(wchar, nonbmp + '\0') - self.assertEqual(unicode_aswidecharstring_null(nonbmp), nonbmp) - - self.assertRaises(TypeError, unicode_aswidecharstring, b'abc') - self.assertRaises(TypeError, unicode_aswidecharstring, []) - self.assertRaises(SystemError, unicode_aswidecharstring, NULL) - self.assertRaises(TypeError, unicode_aswidecharstring_null, b'abc') - self.assertRaises(TypeError, unicode_aswidecharstring_null, []) - self.assertRaises(SystemError, unicode_aswidecharstring_null, NULL) - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_asucs4(self): - """Test PyUnicode_AsUCS4()""" - from _testcapi import unicode_asucs4 - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', - 'a\ud800b\udfffc', '\ud834\udd1e']: - l = len(s) - self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') - self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') - self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') - self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') - self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) - self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) - s = '\0'.join([s, s]) - self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') - self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') - - # CRASHES unicode_asucs4(b'abc', 1, 0) - # CRASHES unicode_asucs4(b'abc', 1, 1) - # CRASHES unicode_asucs4([], 1, 1) - # CRASHES unicode_asucs4(NULL, 1, 0) - # CRASHES unicode_asucs4(NULL, 1, 1) - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_asucs4copy(self): - """Test PyUnicode_AsUCS4Copy()""" - from _testcapi import unicode_asucs4copy as asucs4copy - - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', - 'a\ud800b\udfffc', '\ud834\udd1e']: - self.assertEqual(asucs4copy(s), s+'\0') - s = '\0'.join([s, s]) - self.assertEqual(asucs4copy(s), s+'\0') - - # CRASHES asucs4copy(b'abc') - # CRASHES asucs4copy([]) - # CRASHES asucs4copy(NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_fromordinal(self): - """Test PyUnicode_FromOrdinal()""" - from _testlimitedcapi import unicode_fromordinal as fromordinal - - self.assertEqual(fromordinal(0x61), 'a') - self.assertEqual(fromordinal(0x20ac), '\u20ac') - self.assertEqual(fromordinal(0x1f600), '\U0001f600') - - self.assertRaises(ValueError, fromordinal, 0x110000) - self.assertRaises(ValueError, fromordinal, -1) - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_asutf8(self): - """Test PyUnicode_AsUTF8()""" - from _testcapi import unicode_asutf8 - - self.assertEqual(unicode_asutf8('abc', 4), b'abc\0') - self.assertEqual(unicode_asutf8('абв', 7), b'\xd0\xb0\xd0\xb1\xd0\xb2\0') - self.assertEqual(unicode_asutf8('\U0001f600', 5), b'\xf0\x9f\x98\x80\0') - self.assertEqual(unicode_asutf8('abc\0def', 8), b'abc\0def\0') - - self.assertRaises(UnicodeEncodeError, unicode_asutf8, '\ud8ff', 0) - self.assertRaises(TypeError, unicode_asutf8, b'abc', 0) - self.assertRaises(TypeError, unicode_asutf8, [], 0) - # CRASHES unicode_asutf8(NULL, 0) - - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - @threading_helper.requires_working_threading() - def test_asutf8_race(self): - """Test that there's no race condition in PyUnicode_AsUTF8()""" - unicode_asutf8 = _testcapi.unicode_asutf8 - from threading import Thread - - data = "😊" - - def worker(): - for _ in range(1000): - self.assertEqual(unicode_asutf8(data, 5), b'\xf0\x9f\x98\x8a\0') - - threads = [Thread(target=worker) for _ in range(10)] - with threading_helper.start_threads(threads): - pass - - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_asutf8andsize(self): - """Test PyUnicode_AsUTF8AndSize()""" - from _testlimitedcapi import unicode_asutf8andsize - from _testlimitedcapi import unicode_asutf8andsize_null - - self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) - self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) - self.assertEqual(unicode_asutf8andsize('\U0001f600', 5), (b'\xf0\x9f\x98\x80\0', 4)) - self.assertEqual(unicode_asutf8andsize('abc\0def', 8), (b'abc\0def\0', 7)) - self.assertEqual(unicode_asutf8andsize_null('abc', 4), b'abc\0') - self.assertEqual(unicode_asutf8andsize_null('abc\0def', 8), b'abc\0def\0') - - self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0) - self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0) - self.assertRaises(TypeError, unicode_asutf8andsize, [], 0) - self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize_null, '\ud8ff', 0) - self.assertRaises(TypeError, unicode_asutf8andsize_null, b'abc', 0) - self.assertRaises(TypeError, unicode_asutf8andsize_null, [], 0) - # CRASHES unicode_asutf8andsize(NULL, 0) - # CRASHES unicode_asutf8andsize_null(NULL, 0) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_getdefaultencoding(self): - """Test PyUnicode_GetDefaultEncoding()""" - from _testlimitedcapi import unicode_getdefaultencoding as getdefaultencoding - - self.assertEqual(getdefaultencoding(), b'utf-8') - - @support.cpython_only - @unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') - def test_transform_decimal_and_space(self): - """Test _PyUnicode_TransformDecimalAndSpaceToASCII()""" - from _testinternalcapi import _PyUnicode_TransformDecimalAndSpaceToASCII as transform_decimal - - self.assertEqual(transform_decimal('123'), - '123') - self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), - '3.14') - self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), - " 3.14 ") - self.assertEqual(transform_decimal('12\u20ac3'), - '12?') - self.assertEqual(transform_decimal(''), '') - - self.assertRaises(SystemError, transform_decimal, b'123') - self.assertRaises(SystemError, transform_decimal, []) - # CRASHES transform_decimal(NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_concat(self): - """Test PyUnicode_Concat()""" - from _testlimitedcapi import unicode_concat as concat - - self.assertEqual(concat('abc', 'def'), 'abcdef') - self.assertEqual(concat('abc', 'где'), 'abcгде') - self.assertEqual(concat('абв', 'def'), 'абвdef') - self.assertEqual(concat('абв', 'где'), 'абвгде') - self.assertEqual(concat('a\0b', 'c\0d'), 'a\0bc\0d') - - self.assertRaises(TypeError, concat, b'abc', 'def') - self.assertRaises(TypeError, concat, 'abc', b'def') - self.assertRaises(TypeError, concat, b'abc', b'def') - self.assertRaises(TypeError, concat, [], 'def') - self.assertRaises(TypeError, concat, 'abc', []) - self.assertRaises(TypeError, concat, [], []) - # CRASHES concat(NULL, 'def') - # CRASHES concat('abc', NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_split(self): - """Test PyUnicode_Split()""" - from _testlimitedcapi import unicode_split as split - - self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) - self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d']) - self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MAX), - ['a', 'b', 'c', 'd']) - self.assertEqual(split('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd']) - self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MIN), - ['a', 'b', 'c', 'd']) - self.assertEqual(split('a|b|c|d', '\u20ac'), ['a|b|c|d']) - self.assertEqual(split('a||b|c||d', '||'), ['a', 'b|c', 'd']) - self.assertEqual(split('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) - self.assertEqual(split('абабагаламага', 'а'), - ['', 'б', 'б', 'г', 'л', 'м', 'г', '']) - self.assertEqual(split(' a\tb\nc\rd\ve\f', NULL), - ['a', 'b', 'c', 'd', 'e']) - self.assertEqual(split('a\x85b\xa0c\u1680d\u2000e', NULL), - ['a', 'b', 'c', 'd', 'e']) - - self.assertRaises(ValueError, split, 'a|b|c|d', '') - self.assertRaises(TypeError, split, 'a|b|c|d', ord('|')) - self.assertRaises(TypeError, split, [], '|') - # CRASHES split(NULL, '|') - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_rsplit(self): - """Test PyUnicode_RSplit()""" - from _testlimitedcapi import unicode_rsplit as rsplit - - self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) - self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd']) - self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MAX), - ['a', 'b', 'c', 'd']) - self.assertEqual(rsplit('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd']) - self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MIN), - ['a', 'b', 'c', 'd']) - self.assertEqual(rsplit('a|b|c|d', '\u20ac'), ['a|b|c|d']) - self.assertEqual(rsplit('a||b|c||d', '||'), ['a', 'b|c', 'd']) - self.assertEqual(rsplit('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) - self.assertEqual(rsplit('абабагаламага', 'а'), - ['', 'б', 'б', 'г', 'л', 'м', 'г', '']) - self.assertEqual(rsplit('aжbжcжd', 'ж'), ['a', 'b', 'c', 'd']) - self.assertEqual(rsplit(' a\tb\nc\rd\ve\f', NULL), - ['a', 'b', 'c', 'd', 'e']) - self.assertEqual(rsplit('a\x85b\xa0c\u1680d\u2000e', NULL), - ['a', 'b', 'c', 'd', 'e']) - - self.assertRaises(ValueError, rsplit, 'a|b|c|d', '') - self.assertRaises(TypeError, rsplit, 'a|b|c|d', ord('|')) - self.assertRaises(TypeError, rsplit, [], '|') - # CRASHES rsplit(NULL, '|') - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_partition(self): - """Test PyUnicode_Partition()""" - from _testlimitedcapi import unicode_partition as partition - - self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c')) - self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c')) - self.assertEqual(partition('а|б|в', '|'), ('а', '|', 'б|в')) - self.assertEqual(partition('кабан', 'а'), ('к', 'а', 'бан')) - self.assertEqual(partition('aжbжc', 'ж'), ('a', 'ж', 'bжc')) - - self.assertRaises(ValueError, partition, 'a|b|c', '') - self.assertRaises(TypeError, partition, b'a|b|c', '|') - self.assertRaises(TypeError, partition, 'a|b|c', b'|') - self.assertRaises(TypeError, partition, 'a|b|c', ord('|')) - self.assertRaises(TypeError, partition, [], '|') - # CRASHES partition(NULL, '|') - # CRASHES partition('a|b|c', NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_rpartition(self): - """Test PyUnicode_RPartition()""" - from _testlimitedcapi import unicode_rpartition as rpartition - - self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c')) - self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c')) - self.assertEqual(rpartition('а|б|в', '|'), ('а|б', '|', 'в')) - self.assertEqual(rpartition('кабан', 'а'), ('каб', 'а', 'н')) - self.assertEqual(rpartition('aжbжc', 'ж'), ('aжb', 'ж', 'c')) - - self.assertRaises(ValueError, rpartition, 'a|b|c', '') - self.assertRaises(TypeError, rpartition, b'a|b|c', '|') - self.assertRaises(TypeError, rpartition, 'a|b|c', b'|') - self.assertRaises(TypeError, rpartition, 'a|b|c', ord('|')) - self.assertRaises(TypeError, rpartition, [], '|') - # CRASHES rpartition(NULL, '|') - # CRASHES rpartition('a|b|c', NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_splitlines(self): - """Test PyUnicode_SplitLines()""" - from _testlimitedcapi import unicode_splitlines as splitlines - - self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd']) - self.assertEqual(splitlines('a\nb\rc\r\nd', True), - ['a\n', 'b\r', 'c\r\n', 'd']) - self.assertEqual(splitlines('a\x85b\u2028c\u2029d'), - ['a', 'b', 'c', 'd']) - self.assertEqual(splitlines('a\x85b\u2028c\u2029d', True), - ['a\x85', 'b\u2028', 'c\u2029', 'd']) - self.assertEqual(splitlines('а\nб\rв\r\nг'), ['а', 'б', 'в', 'г']) - - self.assertRaises(TypeError, splitlines, b'a\nb\rc\r\nd') - # CRASHES splitlines(NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_translate(self): - """Test PyUnicode_Translate()""" - from _testlimitedcapi import unicode_translate as translate - - self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d') - self.assertEqual(translate('абвг', {ord('а'): 'А', ord('б'): ord('Б'), ord('в'): '<>'}), 'АБ<>г') - self.assertEqual(translate('abc', {}), 'abc') - self.assertEqual(translate('abc', []), 'abc') - self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}) - self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}, 'strict') - self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo') - self.assertEqual(translate('abc', {ord('b'): None}, 'ignore'), 'ac') - self.assertEqual(translate('abc', {ord('b'): None}, 'replace'), 'a\ufffdc') - self.assertEqual(translate('abc', {ord('b'): None}, 'backslashreplace'), r'a\x62c') - # XXX Other error handlers do not support UnicodeTranslateError - self.assertRaises(TypeError, translate, b'abc', []) - self.assertRaises(TypeError, translate, 123, []) - self.assertRaises(TypeError, translate, 'abc', {ord('a'): b'A'}) - self.assertRaises(TypeError, translate, 'abc', 123) - self.assertRaises(TypeError, translate, 'abc', NULL) - self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo') - # CRASHES translate(NULL, []) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_join(self): - """Test PyUnicode_Join()""" - from _testlimitedcapi import unicode_join as join - self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c') - self.assertEqual(join('|', ['a', '', 'c']), 'a||c') - self.assertEqual(join('', ['a', 'b', 'c']), 'abc') - self.assertEqual(join(NULL, ['a', 'b', 'c']), 'a b c') - self.assertEqual(join('|', ['а', 'б', 'в']), 'а|б|в') - self.assertEqual(join('ж', ['а', 'б', 'в']), 'ажбжв') - self.assertRaises(TypeError, join, b'|', ['a', 'b', 'c']) - self.assertRaises(TypeError, join, '|', [b'a', b'b', b'c']) - self.assertRaises(TypeError, join, NULL, [b'a', b'b', b'c']) - self.assertRaises(TypeError, join, '|', b'123') - self.assertRaises(TypeError, join, '|', 123) - self.assertRaises(SystemError, join, '|', NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_count(self): - """Test PyUnicode_Count()""" - from _testlimitedcapi import unicode_count - - for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": - for i, ch in enumerate(str): - self.assertEqual(unicode_count(str, ch, 0, len(str)), 1) - - str = "!>_= end - self.assertEqual(unicode_count(str, '!', 0, 0), 0) - self.assertEqual(unicode_count(str, '!', len(str), 0), 0) - # negative - self.assertEqual(unicode_count(str, '!', -len(str), -1), 1) - self.assertEqual(unicode_count(str, '!', -len(str)-1, -1), 1) - self.assertEqual(unicode_count(str, '!', PY_SSIZE_T_MIN, -1), 1) - # bad arguments - self.assertRaises(TypeError, unicode_count, str, b'!', 0, len(str)) - self.assertRaises(TypeError, unicode_count, b"!>__= end - self.assertEqual(find(str, '!', 0, 0, 1), -1) - self.assertEqual(find(str, '!', 0, 0, -1), -1) - self.assertEqual(find(str, '!', len(str), 0, 1), -1) - self.assertEqual(find(str, '!', len(str), 0, -1), -1) - # negative - self.assertEqual(find(str, '!', -len(str), -1, 1), 0) - self.assertEqual(find(str, '!', -len(str), -1, -1), 0) - self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, 1), 0) - self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, -1), 0) - self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0) - self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4) - # bad arguments - self.assertRaises(TypeError, find, str, b'!', 0, len(str), 1) - self.assertRaises(TypeError, find, b"!>__= end - self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) - self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, -1), -1) - self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) - self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, -1), -1) - # negative - self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) - self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) - self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, 1), 0) - self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, -1), 0) - self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0) - self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4) - # bad arguments - # CRASHES unicode_findchar(b"!>_'), '<>br<>c<>d<>br<>') - self.assertEqual(replace(str, 'abra', '='), '=cad=') - self.assertEqual(replace(str, 'a', '=', 2), '=br=cadabra') - self.assertEqual(replace(str, 'a', '=', 0), str) - self.assertEqual(replace(str, 'a', '=', PY_SSIZE_T_MAX), '=br=c=d=br=') - self.assertEqual(replace(str, 'a', '=', -1), '=br=c=d=br=') - self.assertEqual(replace(str, 'a', '=', PY_SSIZE_T_MIN), '=br=c=d=br=') - self.assertEqual(replace(str, 'z', '='), str) - self.assertEqual(replace(str, '', '='), '=a=b=r=a=c=a=d=a=b=r=a=') - self.assertEqual(replace(str, 'a', 'ж'), 'жbrжcжdжbrж') - self.assertEqual(replace('абабагаламага', 'а', '='), '=б=б=г=л=м=г=') - self.assertEqual(replace('Баден-Баден', 'Баден', 'Baden'), 'Baden-Baden') - # bad arguments - self.assertRaises(TypeError, replace, 'a', 'a', b'=') - self.assertRaises(TypeError, replace, 'a', b'a', '=') - self.assertRaises(TypeError, replace, b'a', 'a', '=') - self.assertRaises(TypeError, replace, 'a', 'a', ord('=')) - self.assertRaises(TypeError, replace, 'a', ord('a'), '=') - self.assertRaises(TypeError, replace, [], 'a', '=') - # CRASHES replace('a', 'a', NULL) - # CRASHES replace('a', NULL, '=') - # CRASHES replace(NULL, 'a', '=') - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_compare(self): - """Test PyUnicode_Compare()""" - from _testlimitedcapi import unicode_compare as compare - - self.assertEqual(compare('abc', 'abc'), 0) - self.assertEqual(compare('abc', 'def'), -1) - self.assertEqual(compare('def', 'abc'), 1) - self.assertEqual(compare('abc', 'abc\0def'), -1) - self.assertEqual(compare('abc\0def', 'abc\0def'), 0) - self.assertEqual(compare('абв', 'abc'), 1) - - self.assertRaises(TypeError, compare, b'abc', 'abc') - self.assertRaises(TypeError, compare, 'abc', b'abc') - self.assertRaises(TypeError, compare, b'abc', b'abc') - self.assertRaises(TypeError, compare, [], 'abc') - self.assertRaises(TypeError, compare, 'abc', []) - self.assertRaises(TypeError, compare, [], []) - # CRASHES compare(NULL, 'abc') - # CRASHES compare('abc', NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_comparewithasciistring(self): - """Test PyUnicode_CompareWithASCIIString()""" - from _testlimitedcapi import unicode_comparewithasciistring as comparewithasciistring - - self.assertEqual(comparewithasciistring('abc', b'abc'), 0) - self.assertEqual(comparewithasciistring('abc', b'def'), -1) - self.assertEqual(comparewithasciistring('def', b'abc'), 1) - self.assertEqual(comparewithasciistring('abc', b'abc\0def'), 0) - self.assertEqual(comparewithasciistring('abc\0def', b'abc\0def'), 1) - self.assertEqual(comparewithasciistring('абв', b'abc'), 1) - - # CRASHES comparewithasciistring(b'abc', b'abc') - # CRASHES comparewithasciistring([], b'abc') - # CRASHES comparewithasciistring(NULL, b'abc') - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_equaltoutf8(self): - # Test PyUnicode_EqualToUTF8() - from _testlimitedcapi import unicode_equaltoutf8 as equaltoutf8 - from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize - - strings = [ - 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', - '\U0001f600\U0001f601\U0001f602', - '\U0010ffff', - ] - for s in strings: - # Call PyUnicode_AsUTF8AndSize() which creates the UTF-8 - # encoded string cached in the Unicode object. - asutf8andsize(s, 0) - b = s.encode() - self.assertEqual(equaltoutf8(s, b), 1) # Use the UTF-8 cache. - s2 = b.decode() # New Unicode object without the UTF-8 cache. - self.assertEqual(equaltoutf8(s2, b), 1) - self.assertEqual(equaltoutf8(s + 'x', b + b'x'), 1) - self.assertEqual(equaltoutf8(s + 'x', b + b'y'), 0) - self.assertEqual(equaltoutf8(s, b + b'\0'), 1) - self.assertEqual(equaltoutf8(s2, b + b'\0'), 1) - self.assertEqual(equaltoutf8(s + '\0', b + b'\0'), 0) - self.assertEqual(equaltoutf8(s + '\0', b), 0) - self.assertEqual(equaltoutf8(s2, b + b'x'), 0) - self.assertEqual(equaltoutf8(s2, b[:-1]), 0) - self.assertEqual(equaltoutf8(s2, b[:-1] + b'x'), 0) - - self.assertEqual(equaltoutf8('', b''), 1) - self.assertEqual(equaltoutf8('', b'\0'), 1) - - # embedded null chars/bytes - self.assertEqual(equaltoutf8('abc', b'abc\0def\0'), 1) - self.assertEqual(equaltoutf8('a\0bc', b'abc'), 0) - self.assertEqual(equaltoutf8('abc', b'a\0bc'), 0) - - # Surrogate characters are always treated as not equal - self.assertEqual(equaltoutf8('\udcfe', - '\udcfe'.encode("utf8", "surrogateescape")), 0) - self.assertEqual(equaltoutf8('\udcfe', - '\udcfe'.encode("utf8", "surrogatepass")), 0) - self.assertEqual(equaltoutf8('\ud801', - '\ud801'.encode("utf8", "surrogatepass")), 0) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_equaltoutf8andsize(self): - # Test PyUnicode_EqualToUTF8AndSize() - from _testlimitedcapi import unicode_equaltoutf8andsize as equaltoutf8andsize - from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize - - strings = [ - 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', - '\U0001f600\U0001f601\U0001f602', - '\U0010ffff', - ] - for s in strings: - # Call PyUnicode_AsUTF8AndSize() which creates the UTF-8 - # encoded string cached in the Unicode object. - asutf8andsize(s, 0) - b = s.encode() - self.assertEqual(equaltoutf8andsize(s, b), 1) # Use the UTF-8 cache. - s2 = b.decode() # New Unicode object without the UTF-8 cache. - self.assertEqual(equaltoutf8andsize(s2, b), 1) - self.assertEqual(equaltoutf8andsize(s + 'x', b + b'x'), 1) - self.assertEqual(equaltoutf8andsize(s + 'x', b + b'y'), 0) - self.assertEqual(equaltoutf8andsize(s, b + b'\0'), 0) - self.assertEqual(equaltoutf8andsize(s2, b + b'\0'), 0) - self.assertEqual(equaltoutf8andsize(s + '\0', b + b'\0'), 1) - self.assertEqual(equaltoutf8andsize(s + '\0', b), 0) - self.assertEqual(equaltoutf8andsize(s2, b + b'x'), 0) - self.assertEqual(equaltoutf8andsize(s2, b[:-1]), 0) - self.assertEqual(equaltoutf8andsize(s2, b[:-1] + b'x'), 0) - # Not null-terminated, - self.assertEqual(equaltoutf8andsize(s, b + b'x', len(b)), 1) - self.assertEqual(equaltoutf8andsize(s2, b + b'x', len(b)), 1) - self.assertEqual(equaltoutf8andsize(s + '\0', b + b'\0x', len(b) + 1), 1) - self.assertEqual(equaltoutf8andsize(s2, b, len(b) - 1), 0) - self.assertEqual(equaltoutf8andsize(s, b, -1), 0) - self.assertEqual(equaltoutf8andsize(s, b, PY_SSIZE_T_MAX), 0) - self.assertEqual(equaltoutf8andsize(s, b, PY_SSIZE_T_MIN), 0) - - self.assertEqual(equaltoutf8andsize('', b''), 1) - self.assertEqual(equaltoutf8andsize('', b'\0'), 0) - self.assertEqual(equaltoutf8andsize('', b'x', 0), 1) - - # embedded null chars/bytes - self.assertEqual(equaltoutf8andsize('abc\0def', b'abc\0def'), 1) - self.assertEqual(equaltoutf8andsize('abc\0def\0', b'abc\0def\0'), 1) - - # Surrogate characters are always treated as not equal - self.assertEqual(equaltoutf8andsize('\udcfe', - '\udcfe'.encode("utf8", "surrogateescape")), 0) - self.assertEqual(equaltoutf8andsize('\udcfe', - '\udcfe'.encode("utf8", "surrogatepass")), 0) - self.assertEqual(equaltoutf8andsize('\ud801', - '\ud801'.encode("utf8", "surrogatepass")), 0) - - def check_not_equal_encoding(text, encoding): - self.assertEqual(equaltoutf8andsize(text, text.encode(encoding)), 0) - self.assertNotEqual(text.encode(encoding), text.encode("utf8")) - - # Strings encoded to other encodings are not equal to expected UTF8-encoding string - check_not_equal_encoding('Stéphane', 'latin1') - check_not_equal_encoding('Stéphane', 'utf-16-le') # embedded null characters - check_not_equal_encoding('北京市', 'gbk') - - # CRASHES equaltoutf8andsize('abc', b'abc', -1) - # CRASHES equaltoutf8andsize(b'abc', b'abc') - # CRASHES equaltoutf8andsize([], b'abc') - # CRASHES equaltoutf8andsize(NULL, b'abc') - # CRASHES equaltoutf8andsize('abc', NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_richcompare(self): - """Test PyUnicode_RichCompare()""" - from _testlimitedcapi import unicode_richcompare as richcompare - - LT, LE, EQ, NE, GT, GE = range(6) - strings = ('abc', 'абв', '\U0001f600', 'abc\0') - for s1 in strings: - for s2 in strings: - self.assertIs(richcompare(s1, s2, LT), s1 < s2) - self.assertIs(richcompare(s1, s2, LE), s1 <= s2) - self.assertIs(richcompare(s1, s2, EQ), s1 == s2) - self.assertIs(richcompare(s1, s2, NE), s1 != s2) - self.assertIs(richcompare(s1, s2, GT), s1 > s2) - self.assertIs(richcompare(s1, s2, GE), s1 >= s2) - - for op in LT, LE, EQ, NE, GT, GE: - self.assertIs(richcompare(b'abc', 'abc', op), NotImplemented) - self.assertIs(richcompare('abc', b'abc', op), NotImplemented) - self.assertIs(richcompare(b'abc', b'abc', op), NotImplemented) - self.assertIs(richcompare([], 'abc', op), NotImplemented) - self.assertIs(richcompare('abc', [], op), NotImplemented) - self.assertIs(richcompare([], [], op), NotImplemented) - - # CRASHES richcompare(NULL, 'abc', op) - # CRASHES richcompare('abc', NULL, op) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_format(self): - """Test PyUnicode_Format()""" - from _testlimitedcapi import unicode_format as format - - self.assertEqual(format('x=%d!', 42), 'x=42!') - self.assertEqual(format('x=%d!', (42,)), 'x=42!') - self.assertEqual(format('x=%d y=%s!', (42, [])), 'x=42 y=[]!') - - self.assertRaises(SystemError, format, 'x=%d!', NULL) - self.assertRaises(SystemError, format, NULL, 42) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_contains(self): - """Test PyUnicode_Contains()""" - from _testlimitedcapi import unicode_contains as contains - - self.assertEqual(contains('abcd', ''), 1) - self.assertEqual(contains('abcd', 'b'), 1) - self.assertEqual(contains('abcd', 'x'), 0) - self.assertEqual(contains('abcd', 'ж'), 0) - self.assertEqual(contains('abcd', '\0'), 0) - self.assertEqual(contains('abc\0def', '\0'), 1) - self.assertEqual(contains('abcd', 'bc'), 1) - - self.assertRaises(TypeError, contains, b'abcd', 'b') - self.assertRaises(TypeError, contains, 'abcd', b'b') - self.assertRaises(TypeError, contains, b'abcd', b'b') - self.assertRaises(TypeError, contains, [], 'b') - self.assertRaises(TypeError, contains, 'abcd', ord('b')) - # CRASHES contains(NULL, 'b') - # CRASHES contains('abcd', NULL) - - @support.cpython_only - @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') - def test_isidentifier(self): - """Test PyUnicode_IsIdentifier()""" - from _testlimitedcapi import unicode_isidentifier as isidentifier - - self.assertEqual(isidentifier("a"), 1) - self.assertEqual(isidentifier("b0"), 1) - self.assertEqual(isidentifier("µ"), 1) - self.assertEqual(isidentifier("𝔘𝔫𝔦𝔠𝔬𝔡𝔢"), 1) - - self.assertEqual(isidentifier(""), 0) - self.assertEqual(isidentifier(" "), 0) - self.assertEqual(isidentifier("["), 0) - self.assertEqual(isidentifier("©"), 0) - self.assertEqual(isidentifier("0"), 0) - self.assertEqual(isidentifier("32M"), 0) - - # CRASHES isidentifier(b"a") - # CRASHES isidentifier([]) - # CRASHES isidentifier(NULL) - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_copycharacters(self): - """Test PyUnicode_CopyCharacters()""" - from _testcapi import unicode_copycharacters - - strings = [ - # all strings have exactly 5 characters - 'abcde', '\xa1\xa2\xa3\xa4\xa5', - '\u4f60\u597d\u4e16\u754c\uff01', - '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' - ] - - for idx, from_ in enumerate(strings): - # wide -> narrow: exceed maxchar limitation - for to in strings[:idx]: - self.assertRaises( - SystemError, - unicode_copycharacters, to, 0, from_, 0, 5 - ) - # same kind - for from_start in range(5): - self.assertEqual( - unicode_copycharacters(from_, 0, from_, from_start, 5), - (from_[from_start:from_start+5].ljust(5, '\0'), - 5-from_start) - ) - for to_start in range(5): - self.assertEqual( - unicode_copycharacters(from_, to_start, from_, to_start, 5), - (from_[to_start:to_start+5].rjust(5, '\0'), - 5-to_start) - ) - # narrow -> wide - # Tests omitted since this creates invalid strings. - - s = strings[0] - self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, PY_SSIZE_T_MAX, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, PY_SSIZE_T_MIN, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, PY_SSIZE_T_MAX, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, PY_SSIZE_T_MIN, 5) - self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) - self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, PY_SSIZE_T_MAX) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, PY_SSIZE_T_MIN) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, [], 0, 0) - # CRASHES unicode_copycharacters(s, 0, NULL, 0, 0) - # TODO: Test PyUnicode_CopyCharacters() with non-unicode and - # non-modifiable unicode as "to". - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_pep393_utf8_caching_bug(self): - # Issue #25709: Problem with string concatenation and utf-8 cache - from _testcapi import getargs_s_hash - for k in 0x24, 0xa4, 0x20ac, 0x1f40d: - s = '' - for i in range(5): - # Due to CPython specific optimization the 's' string can be - # resized in-place. - s += chr(k) - # Parsing with the "s#" format code calls indirectly - # PyUnicode_AsUTF8AndSize() which creates the UTF-8 - # encoded string cached in the Unicode object. - self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - # Check that the second call returns the same result - self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - - -class PyUnicodeWriterTest(unittest.TestCase): - def create_writer(self, size): - return _testcapi.PyUnicodeWriter(size) - - def test_basic(self): - writer = self.create_writer(100) - - # test PyUnicodeWriter_WriteUTF8() - writer.write_utf8(b'var', -1) - - # test PyUnicodeWriter_WriteChar() - writer.write_char(ord('=')) - - # test PyUnicodeWriter_WriteSubstring() - writer.write_substring("[long]", 1, 5) - # CRASHES writer.write_substring(NULL, 0, 0) - - # test PyUnicodeWriter_WriteStr() - writer.write_str(" value ") - # CRASHES writer.write_str(NULL) - - # test PyUnicodeWriter_WriteRepr() - writer.write_repr("repr") - - self.assertEqual(writer.finish(), - "var=long value 'repr'") - - def test_repr_null(self): - writer = self.create_writer(0) - writer.write_utf8(b'var=', -1) - writer.write_repr(NULL) - self.assertEqual(writer.finish(), - "var=") - - def test_write_char(self): - writer = self.create_writer(0) - writer.write_char(0) - writer.write_char(ord('$')) - writer.write_char(0x20ac) - writer.write_char(0x10_ffff) - self.assertRaises(ValueError, writer.write_char, 0x11_0000) - self.assertRaises(ValueError, writer.write_char, 0xFFFF_FFFF) - self.assertEqual(writer.finish(), - "\0$\u20AC\U0010FFFF") - - def test_utf8(self): - writer = self.create_writer(0) - writer.write_utf8(b"ascii", -1) - writer.write_char(ord('-')) - writer.write_utf8(b"latin1=\xC3\xA9", -1) - writer.write_char(ord('-')) - writer.write_utf8(b"euro=\xE2\x82\xAC", -1) - writer.write_char(ord('.')) - writer.write_utf8(NULL, 0) - # CRASHES writer.write_utf8(NULL, 1) - # CRASHES writer.write_utf8(NULL, -1) - self.assertEqual(writer.finish(), - "ascii-latin1=\xE9-euro=\u20AC.") - - def test_ascii(self): - writer = self.create_writer(0) - writer.write_ascii(b"Hello ", -1) - writer.write_ascii(b"", 0) - writer.write_ascii(NULL, 0) - # CRASHES writer.write_ascii(NULL, 1) - # CRASHES writer.write_ascii(NULL, -1) - writer.write_ascii(b"Python! ", 6) - self.assertEqual(writer.finish(), "Hello Python") - - def test_invalid_utf8(self): - writer = self.create_writer(0) - with self.assertRaises(UnicodeDecodeError): - writer.write_utf8(b"invalid=\xFF", -1) - - def test_recover_utf8_error(self): - # test recovering from PyUnicodeWriter_WriteUTF8() error - writer = self.create_writer(0) - writer.write_utf8(b"value=", -1) - - # write fails with an invalid string - with self.assertRaises(UnicodeDecodeError): - writer.write_utf8(b"invalid\xFF", -1) - with self.assertRaises(UnicodeDecodeError): - s = "truncated\u20AC".encode() - writer.write_utf8(s, len(s) - 1) - - # retry write with a valid string - writer.write_utf8(b"valid", -1) - - self.assertEqual(writer.finish(), - "value=valid") - - def test_decode_utf8(self): - # test PyUnicodeWriter_DecodeUTF8Stateful() - writer = self.create_writer(0) - writer.decodeutf8stateful(b"ign\xFFore", -1, b"ignore") - writer.write_char(ord('-')) - writer.decodeutf8stateful(b"replace\xFF", -1, b"replace") - writer.write_char(ord('-')) - - # incomplete trailing UTF-8 sequence - writer.decodeutf8stateful(b"incomplete\xC3", -1, b"replace") - - writer.decodeutf8stateful(NULL, 0, b"replace") - # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace") - # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace") - with self.assertRaises(UnicodeDecodeError): - writer.decodeutf8stateful(b"default\xFF", -1, NULL) - - self.assertEqual(writer.finish(), - "ignore-replace\uFFFD-incomplete\uFFFD") - - def test_decode_utf8_consumed(self): - # test PyUnicodeWriter_DecodeUTF8Stateful() with consumed - writer = self.create_writer(0) - - # valid string - consumed = writer.decodeutf8stateful(b"text", -1, b"strict", True) - self.assertEqual(consumed, 4) - writer.write_char(ord('-')) - - # non-ASCII - consumed = writer.decodeutf8stateful(b"\xC3\xA9-\xE2\x82\xAC", 6, b"strict", True) - self.assertEqual(consumed, 6) - writer.write_char(ord('-')) - - # invalid UTF-8 (consumed is 0 on error) - with self.assertRaises(UnicodeDecodeError): - writer.decodeutf8stateful(b"invalid\xFF", -1, b"strict", True) - - # ignore error handler - consumed = writer.decodeutf8stateful(b"more\xFF", -1, b"ignore", True) - self.assertEqual(consumed, 5) - writer.write_char(ord('-')) - - # incomplete trailing UTF-8 sequence - consumed = writer.decodeutf8stateful(b"incomplete\xC3", -1, b"ignore", True) - self.assertEqual(consumed, 10) - writer.write_char(ord('-')) - - consumed = writer.decodeutf8stateful(NULL, 0, b"replace", True) - self.assertEqual(consumed, 0) - # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace", True) - # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace", True) - consumed = writer.decodeutf8stateful(b"default\xC3", -1, NULL, True) - self.assertEqual(consumed, 7) - - self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete-default") - - def test_widechar(self): - from _testcapi import SIZEOF_WCHAR_T - - if SIZEOF_WCHAR_T == 2: - encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' - elif SIZEOF_WCHAR_T == 4: - encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' - - writer = self.create_writer(0) - writer.write_widechar("latin1=\xE9".encode(encoding)) - writer.write_char(ord("-")) - writer.write_widechar("euro=\u20AC".encode(encoding)) - writer.write_char(ord("-")) - writer.write_widechar("max=\U0010ffff".encode(encoding)) - writer.write_char(ord("-")) - writer.write_widechar("zeroes=".encode(encoding).ljust(SIZEOF_WCHAR_T * 10, b'\0'), - 10) - writer.write_char(ord('.')) - - if SIZEOF_WCHAR_T == 4: - invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else - b'\x00\x11\x00\x00') - with self.assertRaises(ValueError): - writer.write_widechar("invalid=".encode(encoding) + invalid) - writer.write_widechar(b'', -5) - writer.write_widechar(NULL, 0) - # CRASHES writer.write_widechar(NULL, 1) - # CRASHES writer.write_widechar(NULL, -1) - - self.assertEqual(writer.finish(), - "latin1=\xE9-euro=\u20AC-max=\U0010ffff-zeroes=\0\0\0.") - - def test_ucs4(self): - encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' - - writer = self.create_writer(0) - writer.write_ucs4("ascii IGNORED".encode(encoding), 5) - writer.write_char(ord("-")) - writer.write_ucs4("latin1=\xe9".encode(encoding)) - writer.write_char(ord("-")) - writer.write_ucs4("euro=\u20ac".encode(encoding)) - writer.write_char(ord("-")) - writer.write_ucs4("max=\U0010ffff".encode(encoding)) - writer.write_char(ord(".")) - self.assertEqual(writer.finish(), - "ascii-latin1=\xE9-euro=\u20AC-max=\U0010ffff.") - - # Test some special characters - writer = self.create_writer(0) - # Lone surrogate character - writer.write_ucs4("lone\uDC80".encode(encoding, 'surrogatepass')) - writer.write_char(ord("-")) - # Surrogate pair - writer.write_ucs4("pair\uD83D\uDC0D".encode(encoding, 'surrogatepass')) - writer.write_char(ord("-")) - writer.write_ucs4("null[\0]".encode(encoding), 7) - invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else - b'\x00\x11\x00\x00') - # CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid) - writer.write_ucs4(NULL, 0) - # CRASHES writer.write_ucs4(NULL, 1) - self.assertEqual(writer.finish(), - "lone\udc80-pair\ud83d\udc0d-null[\x00]") - - # invalid size - writer = self.create_writer(0) - with self.assertRaises(ValueError): - writer.write_ucs4("text".encode(encoding), -1) - self.assertRaises(ValueError, writer.write_ucs4, b'', -1) - self.assertRaises(ValueError, writer.write_ucs4, NULL, -1) - - def test_substring_empty(self): - writer = self.create_writer(0) - writer.write_substring("abc", 1, 1) - self.assertEqual(writer.finish(), '') - - -@unittest.skipIf(ctypes is None, 'need ctypes') -class PyUnicodeWriterFormatTest(unittest.TestCase): - def create_writer(self, size): - return _testcapi.PyUnicodeWriter(size) - - def writer_format(self, writer, *args): - from ctypes import c_char_p, pythonapi, c_int, c_void_p - _PyUnicodeWriter_Format = getattr(pythonapi, "PyUnicodeWriter_Format") - _PyUnicodeWriter_Format.argtypes = (c_void_p, c_char_p,) - _PyUnicodeWriter_Format.restype = c_int - - if _PyUnicodeWriter_Format(writer.get_pointer(), *args) < 0: - raise ValueError("PyUnicodeWriter_Format failed") - - def test_format(self): - from ctypes import c_int - writer = self.create_writer(0) - self.writer_format(writer, b'%s %i', b'abc', c_int(123)) - writer.write_char(ord('.')) - self.assertEqual(writer.finish(), 'abc 123.') - - def test_recover_error(self): - # test recovering from PyUnicodeWriter_Format() error - writer = self.create_writer(0) - self.writer_format(writer, b"%s ", b"Hello") - - # PyUnicodeWriter_Format() fails with an invalid format string - with self.assertRaises(ValueError): - self.writer_format(writer, b"%s\xff", b"World") - - # Retry PyUnicodeWriter_Format() with a valid format string - self.writer_format(writer, b"%s.", b"World") - - self.assertEqual(writer.finish(), 'Hello World.') - - def test_unicode_equal(self): - unicode_equal = _testlimitedcapi.unicode_equal - - def copy(text): - return text.encode().decode() - - self.assertTrue(unicode_equal("", "")) - self.assertTrue(unicode_equal("abc", "abc")) - self.assertTrue(unicode_equal("abc", copy("abc"))) - self.assertTrue(unicode_equal("\u20ac", copy("\u20ac"))) - self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff"))) - - self.assertFalse(unicode_equal("abc", "abcd")) - self.assertFalse(unicode_equal("\u20ac", "\u20ad")) - self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe")) - - # str subclass - self.assertTrue(unicode_equal("abc", Str("abc"))) - self.assertTrue(unicode_equal(Str("abc"), "abc")) - self.assertFalse(unicode_equal("abc", Str("abcd"))) - self.assertFalse(unicode_equal(Str("abc"), "abcd")) - - # invalid type - for invalid_type in (b'bytes', 123, ("tuple",)): - with self.subTest(invalid_type=invalid_type): - with self.assertRaises(TypeError): - unicode_equal("abc", invalid_type) - with self.assertRaises(TypeError): - unicode_equal(invalid_type, "abc") - - # CRASHES unicode_equal("abc", NULL) - # CRASHES unicode_equal(NULL, "abc") - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py deleted file mode 100644 index 8644479d83d..00000000000 --- a/Lib/test/test_capi/test_watchers.py +++ /dev/null @@ -1,678 +0,0 @@ -import unittest -import contextvars - -from contextlib import contextmanager, ExitStack -from test.support import ( - catch_unraisable_exception, import_helper, - gc_collect) - - -# Skip this test if the _testcapi module isn't available. -_testcapi = import_helper.import_module('_testcapi') - - -class TestDictWatchers(unittest.TestCase): - # types of watchers testcapimodule can add: - EVENTS = 0 # appends dict events as strings to global event list - ERROR = 1 # unconditionally sets and signals a RuntimeException - SECOND = 2 # always appends "second" to global event list - - def add_watcher(self, kind=EVENTS): - return _testcapi.add_dict_watcher(kind) - - def clear_watcher(self, watcher_id): - _testcapi.clear_dict_watcher(watcher_id) - - @contextmanager - def watcher(self, kind=EVENTS): - wid = self.add_watcher(kind) - try: - yield wid - finally: - self.clear_watcher(wid) - - def assert_events(self, expected): - actual = _testcapi.get_dict_watcher_events() - self.assertEqual(actual, expected) - - def watch(self, wid, d): - _testcapi.watch_dict(wid, d) - - def unwatch(self, wid, d): - _testcapi.unwatch_dict(wid, d) - - def test_set_new_item(self): - d = {} - with self.watcher() as wid: - self.watch(wid, d) - d["foo"] = "bar" - self.assert_events(["new:foo:bar"]) - - def test_set_existing_item(self): - d = {"foo": "bar"} - with self.watcher() as wid: - self.watch(wid, d) - d["foo"] = "baz" - self.assert_events(["mod:foo:baz"]) - - def test_clone(self): - d = {} - d2 = {"foo": "bar"} - with self.watcher() as wid: - self.watch(wid, d) - d.update(d2) - self.assert_events(["clone"]) - - def test_no_event_if_not_watched(self): - d = {} - with self.watcher() as wid: - d["foo"] = "bar" - self.assert_events([]) - - def test_del(self): - d = {"foo": "bar"} - with self.watcher() as wid: - self.watch(wid, d) - del d["foo"] - self.assert_events(["del:foo"]) - - def test_pop(self): - d = {"foo": "bar"} - with self.watcher() as wid: - self.watch(wid, d) - d.pop("foo") - self.assert_events(["del:foo"]) - - def test_clear(self): - d = {"foo": "bar"} - with self.watcher() as wid: - self.watch(wid, d) - d.clear() - self.assert_events(["clear"]) - - def test_dealloc(self): - d = {"foo": "bar"} - with self.watcher() as wid: - self.watch(wid, d) - del d - self.assert_events(["dealloc"]) - - def test_object_dict(self): - class MyObj: pass - o = MyObj() - - with self.watcher() as wid: - self.watch(wid, o.__dict__) - o.foo = "bar" - o.foo = "baz" - del o.foo - self.assert_events(["new:foo:bar", "mod:foo:baz", "del:foo"]) - - with self.watcher() as wid: - self.watch(wid, o.__dict__) - for _ in range(100): - o.foo = "bar" - self.assert_events(["new:foo:bar"] + ["mod:foo:bar"] * 99) - - def test_unwatch(self): - d = {} - with self.watcher() as wid: - self.watch(wid, d) - d["foo"] = "bar" - self.unwatch(wid, d) - d["hmm"] = "baz" - self.assert_events(["new:foo:bar"]) - - def test_error(self): - d = {} - with self.watcher(kind=self.ERROR) as wid: - self.watch(wid, d) - with catch_unraisable_exception() as cm: - d["foo"] = "bar" - self.assertIn( - "Exception ignored in " - "PyDict_EVENT_ADDED watcher callback for Date: Sun, 17 May 2026 15:52:09 +0300 Subject: [PATCH 4/4] Remove `test_cext` --- Lib/test/test_cext/__init__.py | 127 ----------------------------- Lib/test/test_cext/extension.c | 142 --------------------------------- Lib/test/test_cext/setup.py | 135 ------------------------------- 3 files changed, 404 deletions(-) delete mode 100644 Lib/test/test_cext/__init__.py delete mode 100644 Lib/test/test_cext/extension.c delete mode 100644 Lib/test/test_cext/setup.py diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py deleted file mode 100644 index f75257f3d88..00000000000 --- a/Lib/test/test_cext/__init__.py +++ /dev/null @@ -1,127 +0,0 @@ -# gh-116869: Build a basic C test extension to check that the Python C API -# does not emit C compiler warnings. -# -# The Python C API must be compatible with building -# with the -Werror=declaration-after-statement compiler flag. - -import os.path -import shlex -import shutil -import subprocess -import unittest -from test import support - - -SOURCES = [ - os.path.join(os.path.dirname(__file__), 'extension.c'), -] -SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') - - -# With MSVC on a debug build, the linker fails with: cannot open file -# 'python311.lib', it should look 'python311_d.lib'. -@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG, - 'test fails on Windows debug build') -# Building and running an extension in clang sanitizing mode is not -# straightforward -@support.skip_if_sanitizer('test does not work with analyzing builds', - address=True, memory=True, ub=True, thread=True) -# the test uses venv+pip: skip if it's not available -@support.requires_venv_with_pip() -@support.requires_subprocess() -@support.requires_resource('cpu') -class BaseTests: - TEST_INTERNAL_C_API = False - - # Default build with no options - def test_build(self): - self.check_build('_test_cext') - - def check_build(self, extension_name, std=None, limited=False): - venv_dir = 'env' - with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: - self._check_build(extension_name, python_exe, - std=std, limited=limited) - - def _check_build(self, extension_name, python_exe, std, limited): - pkg_dir = 'pkg' - os.mkdir(pkg_dir) - shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) - for source in SOURCES: - dest = os.path.join(pkg_dir, os.path.basename(source)) - shutil.copy(source, dest) - - def run_cmd(operation, cmd): - env = os.environ.copy() - if std: - env['CPYTHON_TEST_STD'] = std - if limited: - env['CPYTHON_TEST_LIMITED'] = '1' - env['CPYTHON_TEST_EXT_NAME'] = extension_name - env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) - if support.verbose: - print('Run:', ' '.join(map(shlex.quote, cmd))) - subprocess.run(cmd, check=True, env=env) - else: - proc = subprocess.run(cmd, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True) - if proc.returncode: - print('Run:', ' '.join(map(shlex.quote, cmd))) - print(proc.stdout, end='') - self.fail( - f"{operation} failed with exit code {proc.returncode}") - - # Build and install the C extension - cmd = [python_exe, '-X', 'dev', - '-m', 'pip', 'install', '--no-build-isolation', - os.path.abspath(pkg_dir)] - if support.verbose: - cmd.append('-v') - run_cmd('Install', cmd) - - # Do a reference run. Until we test that running python - # doesn't leak references (gh-94755), run it so one can manually check - # -X showrefcount results against this baseline. - cmd = [python_exe, - '-X', 'dev', - '-X', 'showrefcount', - '-c', 'pass'] - run_cmd('Reference run', cmd) - - # Import the C extension - cmd = [python_exe, - '-X', 'dev', - '-X', 'showrefcount', - '-c', f"import {extension_name}"] - run_cmd('Import', cmd) - - -class TestPublicCAPI(BaseTests, unittest.TestCase): - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited(self): - self.check_build('_test_limited_cext', limited=True) - - @support.requires_gil_enabled('broken for now with Free Threading') - def test_build_limited_c11(self): - self.check_build('_test_limited_c11_cext', limited=True, std='c11') - - def test_build_c11(self): - self.check_build('_test_c11_cext', std='c11') - - @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") - def test_build_c99(self): - # In public docs, we say C API is compatible with C11. However, - # in practice we do maintain C99 compatibility in public headers. - # Please ask the C API WG before adding a new C11-only feature. - self.check_build('_test_c99_cext', std='c99') - - -class TestInteralCAPI(BaseTests, unittest.TestCase): - TEST_INTERNAL_C_API = True - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c deleted file mode 100644 index 56f40b354c6..00000000000 --- a/Lib/test/test_cext/extension.c +++ /dev/null @@ -1,142 +0,0 @@ -// gh-116869: Basic C test extension to check that the Python C API -// does not emit C compiler warnings. -// -// Test also the internal C API if the TEST_INTERNAL_C_API macro is defined. - -// Always enable assertions -#undef NDEBUG - -#ifdef TEST_INTERNAL_C_API -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include "Python.h" -#include "datetime.h" - -#ifdef TEST_INTERNAL_C_API - // gh-135906: Check for compiler warnings in the internal C API. - // - Cython uses pycore_critical_section.h, pycore_frame.h and - // pycore_template.h. - // - greenlet uses pycore_frame.h, pycore_interpframe_structs.h and - // pycore_interpframe.h. -# include "internal/pycore_critical_section.h" -# include "internal/pycore_frame.h" -# include "internal/pycore_gc.h" -# include "internal/pycore_interp.h" -# include "internal/pycore_interpframe.h" -# include "internal/pycore_interpframe_structs.h" -# include "internal/pycore_object.h" -# include "internal/pycore_pystate.h" -# include "internal/pycore_template.h" -#endif - -#ifndef MODULE_NAME -# error "MODULE_NAME macro must be defined" -#endif - -#define _STR(NAME) #NAME -#define STR(NAME) _STR(NAME) - -PyDoc_STRVAR(_testcext_add_doc, -"add(x, y)\n" -"\n" -"Return the sum of two integers: x + y."); - -static PyObject * -_testcext_add(PyObject *Py_UNUSED(module), PyObject *args) -{ - long i, j, res; - if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) { - return NULL; - } - res = i + j; - return PyLong_FromLong(res); -} - - -static PyObject * -test_datetime(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) -{ - // datetime.h is excluded from the limited C API -#ifndef Py_LIMITED_API - PyDateTime_IMPORT; - if (PyErr_Occurred()) { - return NULL; - } -#endif - - Py_RETURN_NONE; -} - - -static PyMethodDef _testcext_methods[] = { - {"add", _testcext_add, METH_VARARGS, _testcext_add_doc}, - {"test_datetime", test_datetime, METH_NOARGS, NULL}, - {NULL, NULL, 0, NULL} // sentinel -}; - - -static int -_testcext_exec(PyObject *module) -{ - PyObject *result; - -#ifdef __STDC_VERSION__ - if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { - return -1; - } -#endif - - result = PyObject_CallMethod(module, "test_datetime", ""); - if (!result) return -1; - Py_DECREF(result); - - // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() - Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); - assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); - - return 0; -} - -// Converting from function pointer to void* has undefined behavior, but -// works on all known platforms, and CPython's module and type slots currently -// need it. -// (GCC doesn't have a narrower category for this than -Wpedantic.) -_Py_COMP_DIAG_PUSH -#if defined(__GNUC__) -#pragma GCC diagnostic ignored "-Wpedantic" -#elif defined(__clang__) -#pragma clang diagnostic ignored "-Wpedantic" -#endif - -static PyModuleDef_Slot _testcext_slots[] = { - {Py_mod_exec, (void*)_testcext_exec}, - {0, NULL} -}; - -_Py_COMP_DIAG_POP - - -PyDoc_STRVAR(_testcext_doc, "C test extension."); - -static struct PyModuleDef _testcext_module = { - PyModuleDef_HEAD_INIT, // m_base - STR(MODULE_NAME), // m_name - _testcext_doc, // m_doc - 0, // m_size - _testcext_methods, // m_methods - _testcext_slots, // m_slots - NULL, // m_traverse - NULL, // m_clear - NULL, // m_free -}; - - -#define _FUNC_NAME(NAME) PyInit_ ## NAME -#define FUNC_NAME(NAME) _FUNC_NAME(NAME) - -PyMODINIT_FUNC -FUNC_NAME(MODULE_NAME)(void) -{ - return PyModuleDef_Init(&_testcext_module); -} diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py deleted file mode 100644 index f2a8c9f9381..00000000000 --- a/Lib/test/test_cext/setup.py +++ /dev/null @@ -1,135 +0,0 @@ -# gh-91321: Build a basic C test extension to check that the Python C API is -# compatible with C and does not emit C compiler warnings. -import os -import platform -import shlex -import sys -import sysconfig -from test import support - -from setuptools import setup, Extension - - -SOURCE = 'extension.c' - -if not support.MS_WINDOWS: - # C compiler flags for GCC and clang - BASE_CFLAGS = [ - # The purpose of test_cext extension is to check that building a C - # extension using the Python C API does not emit C compiler warnings. - '-Werror', - ] - - # C compiler flags for GCC and clang - PUBLIC_CFLAGS = [ - *BASE_CFLAGS, - - # gh-120593: Check the 'const' qualifier - '-Wcast-qual', - - # Ask for strict(er) compliance with the standard - '-pedantic-errors', - ] - if not support.Py_GIL_DISABLED: - PUBLIC_CFLAGS.append( - # gh-116869: The Python C API must be compatible with building - # with the -Werror=declaration-after-statement compiler flag. - '-Werror=declaration-after-statement', - ) - INTERNAL_CFLAGS = [*BASE_CFLAGS] -else: - # MSVC compiler flags - BASE_CFLAGS = [ - # Treat all compiler warnings as compiler errors - '/WX', - ] - PUBLIC_CFLAGS = [ - *BASE_CFLAGS, - # Display warnings level 1 to 4 - '/W4', - ] - INTERNAL_CFLAGS = [ - *BASE_CFLAGS, - # Display warnings level 1 to 3 - '/W3', - ] - - -def main(): - std = os.environ.get("CPYTHON_TEST_STD", "") - module_name = os.environ["CPYTHON_TEST_EXT_NAME"] - limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) - internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) - - sources = [SOURCE] - - if not internal: - cflags = list(PUBLIC_CFLAGS) - else: - cflags = list(INTERNAL_CFLAGS) - cflags.append(f'-DMODULE_NAME={module_name}') - - # Add -std=STD or /std:STD (MSVC) compiler flag - if std: - if support.MS_WINDOWS: - cflags.append(f'/std:{std}') - else: - cflags.append(f'-std={std}') - - # Remove existing -std or /std options from CC command line. - # Python adds -std=c11 option. - cmd = (sysconfig.get_config_var('CC') or '') - if cmd is not None: - if support.MS_WINDOWS: - std_prefix = '/std' - else: - std_prefix = '-std' - cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] - cmd = shlex.join(cmd) - # CC env var overrides sysconfig CC variable in setuptools - os.environ['CC'] = cmd - - # Define Py_LIMITED_API macro - if limited: - version = sys.hexversion - cflags.append(f'-DPy_LIMITED_API={version:#x}') - - if internal: - cflags.append('-DTEST_INTERNAL_C_API=1') - - # On Windows, add PCbuild\amd64\ to include and library directories - include_dirs = [] - library_dirs = [] - if support.MS_WINDOWS: - srcdir = sysconfig.get_config_var('srcdir') - machine = platform.uname().machine - pcbuild = os.path.join(srcdir, 'PCbuild', machine) - if os.path.exists(pcbuild): - # pyconfig.h is generated in PCbuild\amd64\ - include_dirs.append(pcbuild) - # python313.lib is generated in PCbuild\amd64\ - library_dirs.append(pcbuild) - print(f"Add PCbuild directory: {pcbuild}") - - # Display information to help debugging - for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): - if env_name in os.environ: - print(f"{env_name} env var: {os.environ[env_name]!r}") - else: - print(f"{env_name} env var: ") - print(f"extra_compile_args: {cflags!r}") - - ext = Extension( - module_name, - sources=sources, - extra_compile_args=cflags, - include_dirs=include_dirs, - library_dirs=library_dirs) - setup(name=f'internal_{module_name}', - version='0.0', - ext_modules=[ext]) - - -if __name__ == "__main__": - main()